refactor: drop code dups (#234)

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

.github/workflows/bump_deps.yaml

@@ -27,6 +27,12 @@ jobs:
           - repository: "go-skynet/go-bert.cpp"
             variable: "BERT_VERSION"
             branch: "master"
+          - repository: "go-skynet/bloomz.cpp"
+            variable: "BLOOMZ_VERSION"
+            branch: "main"
+          - repository: "go-skynet/gpt4all"
+            variable: "GPT4ALL_VERSION"
+            branch: "main"
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

Makefile

@@ -4,12 +4,14 @@ GOVET=$(GOCMD) vet
 BINARY_NAME=local-ai
 
 GOLLAMA_VERSION?=c03e8adbc45c866e0f6d876af1887d6b01d57eb4
-GOGPT4ALLJ_VERSION?=1f7bff57f66cb7062e40d0ac3abd2217815e5109
-GOGPT2_VERSION?=245a5bfe6708ab80dc5c733dcdbfbe3cfd2acdaa
+GPT4ALL_REPO?=https://github.com/go-skynet/gpt4all
+GPT4ALL_VERSION?=3657f9417e17edf378c27d0a9274a1bf41caa914
+GOGPT2_VERSION?=abf038a7d8efa4eefdc7c891f05ad33d4e59e49d
 RWKV_REPO?=https://github.com/donomii/go-rwkv.cpp
-RWKV_VERSION?=af62fcc432be2847acb6e0688b2c2491d6588d58
+RWKV_VERSION?=07166da10cb2a9e8854395a4f210464dcea76e47
 WHISPER_CPP_VERSION?=bf2449dfae35a46b2cd92ab22661ce81a48d4993
 BERT_VERSION?=ec771ec715576ac050263bb7bb74bfd616a5ba13
+BLOOMZ_VERSION?=e9366e82abdfe70565644fbfae9651976714efd1
 
 
 GREEN  := $(shell tput -Txterm setaf 2)
@@ -18,8 +20,8 @@ WHITE  := $(shell tput -Txterm setaf 7)
 CYAN   := $(shell tput -Txterm setaf 6)
 RESET  := $(shell tput -Txterm sgr0)
 
-C_INCLUDE_PATH=$(shell pwd)/go-llama:$(shell pwd)/go-gpt4all-j:$(shell pwd)/go-gpt2:$(shell pwd)/go-rwkv:$(shell pwd)/whisper.cpp:$(shell pwd)/go-bert
-LIBRARY_PATH=$(shell pwd)/go-llama:$(shell pwd)/go-gpt4all-j:$(shell pwd)/go-gpt2:$(shell pwd)/go-rwkv:$(shell pwd)/whisper.cpp:$(shell pwd)/go-bert
+C_INCLUDE_PATH=$(shell pwd)/go-llama:$(shell pwd)/gpt4all/gpt4all-bindings/golang/:$(shell pwd)/go-gpt2:$(shell pwd)/go-rwkv:$(shell pwd)/whisper.cpp:$(shell pwd)/go-bert:$(shell pwd)/bloomz
+LIBRARY_PATH=$(shell pwd)/go-llama:$(shell pwd)/gpt4all/gpt4all-bindings/golang/:$(shell pwd)/go-gpt2:$(shell pwd)/go-rwkv:$(shell pwd)/whisper.cpp:$(shell pwd)/go-bert:$(shell pwd)/bloomz
 
 # Use this if you want to set the default behavior
 ifndef BUILD_TYPE
@@ -36,19 +38,26 @@ endif
 
 all: help
 
-## GPT4ALL-J
-go-gpt4all-j:
-	git clone --recurse-submodules https://github.com/go-skynet/go-gpt4all-j.cpp go-gpt4all-j
-	cd go-gpt4all-j && git checkout -b build $(GOGPT4ALLJ_VERSION) && git submodule update --init --recursive --depth 1
+## GPT4ALL
+gpt4all:
+	git clone --recurse-submodules $(GPT4ALL_REPO) gpt4all
+	cd gpt4all && git checkout -b build $(GPT4ALL_VERSION) && git submodule update --init --recursive --depth 1
 	# This is hackish, but needed as both go-llama and go-gpt4allj have their own version of ggml..
-	@find ./go-gpt4all-j -type f -name "*.c" -exec sed -i'' -e 's/ggml_/ggml_gptj_/g' {} +
-	@find ./go-gpt4all-j -type f -name "*.cpp" -exec sed -i'' -e 's/ggml_/ggml_gptj_/g' {} +
-	@find ./go-gpt4all-j -type f -name "*.h" -exec sed -i'' -e 's/ggml_/ggml_gptj_/g' {} +
-	@find ./go-gpt4all-j -type f -name "*.cpp" -exec sed -i'' -e 's/gpt_/gptj_/g' {} +
-	@find ./go-gpt4all-j -type f -name "*.h" -exec sed -i'' -e 's/gpt_/gptj_/g' {} +
-	@find ./go-gpt4all-j -type f -name "*.cpp" -exec sed -i'' -e 's/json_/json_gptj_/g' {} +
-	@find ./go-gpt4all-j -type f -name "*.cpp" -exec sed -i'' -e 's/void replace/void json_gptj_replace/g' {} +
-	@find ./go-gpt4all-j -type f -name "*.cpp" -exec sed -i'' -e 's/::replace/::json_gptj_replace/g' {} +
+	@find ./gpt4all -type f -name "*.c" -exec sed -i'' -e 's/ggml_/ggml_gptj_/g' {} +
+	@find ./gpt4all -type f -name "*.cpp" -exec sed -i'' -e 's/ggml_/ggml_gptj_/g' {} +
+	@find ./gpt4all -type f -name "*.h" -exec sed -i'' -e 's/ggml_/ggml_gptj_/g' {} +
+	@find ./gpt4all -type f -name "*.cpp" -exec sed -i'' -e 's/gpt_/gptj_/g' {} +
+	@find ./gpt4all -type f -name "*.h" -exec sed -i'' -e 's/gpt_/gptj_/g' {} +
+	@find ./gpt4all -type f -name "*.h" -exec sed -i'' -e 's/set_console_color/set_gptj_console_color/g' {} +
+	@find ./gpt4all -type f -name "*.cpp" -exec sed -i'' -e 's/set_console_color/set_gptj_console_color/g' {} +	
+	@find ./gpt4all -type f -name "*.cpp" -exec sed -i'' -e 's/llama_/gptjllama_/g' {} +
+	@find ./gpt4all -type f -name "*.go" -exec sed -i'' -e 's/llama_/gptjllama_/g' {} +
+	@find ./gpt4all -type f -name "*.h" -exec sed -i'' -e 's/llama_/gptjllama_/g' {} +
+	@find ./gpt4all -type f -name "*.txt" -exec sed -i'' -e 's/llama_/gptjllama_/g' {} +
+	@find ./gpt4all -type f -name "*.cpp" -exec sed -i'' -e 's/json_/json_gptj_/g' {} +
+	@find ./gpt4all -type f -name "*.cpp" -exec sed -i'' -e 's/void replace/void json_gptj_replace/g' {} +
+	@find ./gpt4all -type f -name "*.cpp" -exec sed -i'' -e 's/::replace/::json_gptj_replace/g' {} +
+	mv ./gpt4all/gpt4all-backend/llama.cpp/llama_util.h ./gpt4all/gpt4all-backend/llama.cpp/gptjllama_util.h
 
 ## BERT embeddings
 go-bert:
@@ -69,11 +78,23 @@ go-rwkv:
 go-rwkv/librwkv.a: go-rwkv
 	cd go-rwkv && cd rwkv.cpp &&	cmake . -DRWKV_BUILD_SHARED_LIBRARY=OFF &&	cmake --build . && 	cp librwkv.a .. && cp ggml/src/libggml.a ..
 
+## bloomz
+bloomz:
+	git clone --recurse-submodules https://github.com/go-skynet/bloomz.cpp bloomz
+	@find ./bloomz -type f -name "*.c" -exec sed -i'' -e 's/ggml_/ggml_bloomz_/g' {} +
+	@find ./bloomz -type f -name "*.cpp" -exec sed -i'' -e 's/ggml_/ggml_bloomz_/g' {} +
+	@find ./bloomz -type f -name "*.h" -exec sed -i'' -e 's/ggml_/ggml_bloomz_/g' {} +
+	@find ./bloomz -type f -name "*.cpp" -exec sed -i'' -e 's/gpt_/gpt_bloomz_/g' {} +
+	@find ./bloomz -type f -name "*.h" -exec sed -i'' -e 's/gpt_/gpt_bloomz_/g' {} +
+
+bloomz/libbloomz.a: bloomz
+	cd bloomz && make libbloomz.a
+
 go-bert/libgobert.a: go-bert
 	$(MAKE) -C go-bert libgobert.a
 
-go-gpt4all-j/libgptj.a: go-gpt4all-j
-	$(MAKE) -C go-gpt4all-j $(GENERIC_PREFIX)libgptj.a
+gpt4all/gpt4all-bindings/golang/libgpt4all.a: gpt4all
+	$(MAKE) -C gpt4all/gpt4all-bindings/golang/ $(GENERIC_PREFIX)libgpt4all.a
 
 ## CEREBRAS GPT
 go-gpt2: 
@@ -106,33 +127,36 @@ go-llama/libbinding.a: go-llama
 
 replace:
 	$(GOCMD) mod edit -replace github.com/go-skynet/go-llama.cpp=$(shell pwd)/go-llama
-	$(GOCMD) mod edit -replace github.com/go-skynet/go-gpt4all-j.cpp=$(shell pwd)/go-gpt4all-j
+	$(GOCMD) mod edit -replace github.com/nomic/gpt4all/gpt4all-bindings/golang=$(shell pwd)/gpt4all/gpt4all-bindings/golang
 	$(GOCMD) mod edit -replace github.com/go-skynet/go-gpt2.cpp=$(shell pwd)/go-gpt2
 	$(GOCMD) mod edit -replace github.com/donomii/go-rwkv.cpp=$(shell pwd)/go-rwkv
 	$(GOCMD) mod edit -replace github.com/ggerganov/whisper.cpp=$(shell pwd)/whisper.cpp
 	$(GOCMD) mod edit -replace github.com/go-skynet/go-bert.cpp=$(shell pwd)/go-bert
+	$(GOCMD) mod edit -replace github.com/go-skynet/bloomz.cpp=$(shell pwd)/bloomz
 
-prepare-sources: go-llama go-gpt2 go-gpt4all-j go-rwkv whisper.cpp go-bert
+prepare-sources: go-llama go-gpt2 gpt4all go-rwkv whisper.cpp go-bert bloomz replace
 	$(GOCMD) mod download
 
 ## GENERIC
 rebuild: ## Rebuilds the project
 	$(MAKE) -C go-llama clean
-	$(MAKE) -C go-gpt4all-j clean
+	$(MAKE) -C gpt4all/gpt4all-bindings/golang/ clean
 	$(MAKE) -C go-gpt2 clean
 	$(MAKE) -C go-rwkv clean
 	$(MAKE) -C whisper.cpp clean
 	$(MAKE) -C go-bert clean
+	$(MAKE) -C bloomz clean
 	$(MAKE) build
 
-prepare: prepare-sources go-llama/libbinding.a go-gpt4all-j/libgptj.a go-bert/libgobert.a go-gpt2/libgpt2.a go-rwkv/librwkv.a whisper.cpp/libwhisper.a replace ## Prepares for building
+prepare: prepare-sources gpt4all/gpt4all-bindings/golang/libgpt4all.a go-llama/libbinding.a go-bert/libgobert.a go-gpt2/libgpt2.a go-rwkv/librwkv.a whisper.cpp/libwhisper.a bloomz/libbloomz.a  ## Prepares for building
 
 clean: ## Remove build related file
 	rm -fr ./go-llama
-	rm -rf ./go-gpt4all-j
+	rm -rf ./gpt4all
 	rm -rf ./go-gpt2
 	rm -rf ./go-rwkv
 	rm -rf ./go-bert
+	rm -rf ./bloomz
 	rm -rf $(BINARY_NAME)
 
 ## Build:
@@ -140,7 +164,7 @@ clean: ## Remove build related file
 build: prepare ## Build the project
 	$(info ${GREEN}I local-ai build info:${RESET})
 	$(info ${GREEN}I BUILD_TYPE: ${YELLOW}$(BUILD_TYPE)${RESET})
-	C_INCLUDE_PATH=${C_INCLUDE_PATH} LIBRARY_PATH=${LIBRARY_PATH} $(GOCMD) build -o $(BINARY_NAME) ./
+	C_INCLUDE_PATH=${C_INCLUDE_PATH} LIBRARY_PATH=${LIBRARY_PATH} $(GOCMD) build -x -o $(BINARY_NAME) ./
 
 generic-build: ## Build the project using generic
 	BUILD_TYPE="generic" $(MAKE) build

README.md

@@ -25,7 +25,7 @@ See [examples on how to integrate LocalAI](https://github.com/go-skynet/LocalAI/
 
 ## News
 
-- 10-05-2023: Added support for fast and accurate embeddings with `bert.cpp` ( https://github.com/go-skynet/LocalAI/pull/222 )
+- 10-05-2023: __1.8.0__ released! 🔥 Added support for fast and accurate embeddings with `bert.cpp` ( https://github.com/go-skynet/LocalAI/pull/222 )
 - 09-05-2023: Added experimental support for transcriptions endpoint ( https://github.com/go-skynet/LocalAI/pull/211 )
 - 08-05-2023: Support for embeddings with models using the `llama.cpp` backend ( https://github.com/go-skynet/LocalAI/pull/207 )
 - 02-05-2023: Support for `rwkv.cpp` models ( https://github.com/go-skynet/LocalAI/pull/158 ) and for `/edits` endpoint
@@ -73,7 +73,7 @@ Note: You might need to convert older models to the new format, see [here](https
 
 A full example on how to run a rwkv model is in the [examples](https://github.com/go-skynet/LocalAI/tree/master/examples/rwkv).
 
-Note: rwkv models have an associated tokenizer along that needs to be provided with it:
+Note: rwkv models needs to specify the backend `rwkv` in the YAML config files and have an associated tokenizer along that needs to be provided with it:
 
 ```
 36464540 -rw-r--r--  1 mudler mudler 1.2G May  3 10:51 rwkv_small
@@ -545,6 +545,7 @@ name: text-embedding-ada-002
 parameters:
   model: bert
 embeddings: true
+backend: "bert-embeddings"
 ```
 
 There is an example available [here](https://github.com/go-skynet/LocalAI/tree/master/examples/query_data/).
@@ -563,6 +564,7 @@ Download one of the models from https://huggingface.co/ggerganov/whisper.cpp/tre
 
 ```yaml
 name: whisper-1
+backend: whisper
 parameters:
   model: whisper-en
 ```

api/api_test.go

@@ -79,7 +79,7 @@ var _ = Describe("API test", func() {
 		It("returns errors", func() {
 			_, err := client.CreateCompletion(context.TODO(), openai.CompletionRequest{Model: "foomodel", Prompt: "abcdedfghikl"})
 			Expect(err).To(HaveOccurred())
-			Expect(err.Error()).To(ContainSubstring("error, status code: 500, message: could not load model - all backends returned error: 6 errors occurred:"))
+			Expect(err.Error()).To(ContainSubstring("error, status code: 500, message: could not load model - all backends returned error: 9 errors occurred:"))
 		})
 
 	})

api/config.go

@@ -285,5 +285,10 @@ func readConfig(cm ConfigMerger, c *fiber.Ctx, loader *model.ModelLoader, debug
 		}
 	}
 
+	// Enforce debug flag if passed from CLI
+	if debug {
+		config.Debug = true
+	}
+
 	return config, input, nil
 }

api/openai.go

@@ -5,16 +5,18 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
 
+	"github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper"
 	model "github.com/go-skynet/LocalAI/pkg/model"
-	"github.com/go-skynet/LocalAI/pkg/whisper"
+	whisperutil "github.com/go-skynet/LocalAI/pkg/whisper"
+	llama "github.com/go-skynet/go-llama.cpp"
 	"github.com/gofiber/fiber/v2"
-	"github.com/otiai10/copy"
 	"github.com/rs/zerolog/log"
 	"github.com/valyala/fasthttp"
 )
@@ -409,24 +411,43 @@ func transcriptEndpoint(cm ConfigMerger, debug bool, loader *model.ModelLoader,
 		if err != nil {
 			return c.Status(http.StatusBadRequest).JSON(fiber.Map{"error": err.Error()})
 		}
+		f, err := file.Open()
+		if err != nil {
+			return c.Status(http.StatusBadRequest).JSON(fiber.Map{"error": err.Error()})
+		}
+		defer f.Close()
 		log.Debug().Msgf("Audio file: %+v", file)
 
 		dir, err := os.MkdirTemp("", "whisper")
+
 		if err != nil {
 			return err
 		}
 		defer os.RemoveAll(dir)
 
 		dst := filepath.Join(dir, path.Base(file.Filename))
-		if err := copy.Copy(file.Filename, dst); err != nil {
+		dstFile, err := os.Create(dst)
+		if err != nil {
+			return c.Status(http.StatusBadRequest).JSON(fiber.Map{"error": err.Error()})
+		}
+
+		if _, err := io.Copy(dstFile, f); err != nil {
+			log.Debug().Msgf("Audio file %+v - %+v - err %+v", file.Filename, dst, err)
 			return err
 		}
 
 		log.Debug().Msgf("Audio file copied to: %+v", dst)
 
-		tr, err := whisper.Transcript(filepath.Join(loader.ModelPath, config.Model), dst, input.Language)
+		whisperModel, err := loader.BackendLoader("whisper", config.Model, []llama.ModelOption{}, uint32(config.Threads))
 		if err != nil {
-			return err
+			return c.Status(http.StatusBadRequest).JSON(fiber.Map{"error": err.Error()})
+		}
+
+		w := whisperModel.(whisper.Model)
+
+		tr, err := whisperutil.Transcript(w, dst, input.Language)
+		if err != nil {
+			return c.Status(http.StatusBadRequest).JSON(fiber.Map{"error": err.Error()})
 		}
 
 		log.Debug().Msgf("Trascribed: %+v", tr)

api/prediction.go

@@ -8,10 +8,11 @@ import (
 
 	"github.com/donomii/go-rwkv.cpp"
 	model "github.com/go-skynet/LocalAI/pkg/model"
+	"github.com/go-skynet/bloomz.cpp"
 	bert "github.com/go-skynet/go-bert.cpp"
 	gpt2 "github.com/go-skynet/go-gpt2.cpp"
-	gptj "github.com/go-skynet/go-gpt4all-j.cpp"
 	llama "github.com/go-skynet/go-llama.cpp"
+	gpt4all "github.com/nomic/gpt4all/gpt4all-bindings/golang"
 )
 
 // mutex still needed, see: https://github.com/ggerganov/llama.cpp/discussions/784
@@ -198,6 +199,50 @@ func ModelInference(s string, loader *model.ModelLoader, c Config, tokenCallback
 
 			return response, nil
 		}
+	case *gpt2.RedPajama:
+		fn = func() (string, error) {
+			// Generate the prediction using the language model
+			predictOptions := []gpt2.PredictOption{
+				gpt2.SetTemperature(c.Temperature),
+				gpt2.SetTopP(c.TopP),
+				gpt2.SetTopK(c.TopK),
+				gpt2.SetTokens(c.Maxtokens),
+				gpt2.SetThreads(c.Threads),
+			}
+
+			if c.Batch != 0 {
+				predictOptions = append(predictOptions, gpt2.SetBatch(c.Batch))
+			}
+
+			if c.Seed != 0 {
+				predictOptions = append(predictOptions, gpt2.SetSeed(c.Seed))
+			}
+
+			return model.Predict(
+				s,
+				predictOptions...,
+			)
+		}
+	case *bloomz.Bloomz:
+		fn = func() (string, error) {
+			// Generate the prediction using the language model
+			predictOptions := []bloomz.PredictOption{
+				bloomz.SetTemperature(c.Temperature),
+				bloomz.SetTopP(c.TopP),
+				bloomz.SetTopK(c.TopK),
+				bloomz.SetTokens(c.Maxtokens),
+				bloomz.SetThreads(c.Threads),
+			}
+
+			if c.Seed != 0 {
+				predictOptions = append(predictOptions, bloomz.SetSeed(c.Seed))
+			}
+
+			return model.Predict(
+				s,
+				predictOptions...,
+			)
+		}
 	case *gpt2.StableLM:
 		fn = func() (string, error) {
 			// Generate the prediction using the language model
@@ -222,6 +267,30 @@ func ModelInference(s string, loader *model.ModelLoader, c Config, tokenCallback
 				predictOptions...,
 			)
 		}
+	case *gpt2.Dolly:
+		fn = func() (string, error) {
+			// Generate the prediction using the language model
+			predictOptions := []gpt2.PredictOption{
+				gpt2.SetTemperature(c.Temperature),
+				gpt2.SetTopP(c.TopP),
+				gpt2.SetTopK(c.TopK),
+				gpt2.SetTokens(c.Maxtokens),
+				gpt2.SetThreads(c.Threads),
+			}
+
+			if c.Batch != 0 {
+				predictOptions = append(predictOptions, gpt2.SetBatch(c.Batch))
+			}
+
+			if c.Seed != 0 {
+				predictOptions = append(predictOptions, gpt2.SetSeed(c.Seed))
+			}
+
+			return model.Predict(
+				s,
+				predictOptions...,
+			)
+		}
 	case *gpt2.GPT2:
 		fn = func() (string, error) {
 			// Generate the prediction using the language model
@@ -246,29 +315,35 @@ func ModelInference(s string, loader *model.ModelLoader, c Config, tokenCallback
 				predictOptions...,
 			)
 		}
-	case *gptj.GPTJ:
+	case *gpt4all.Model:
+		supportStreams = true
+
 		fn = func() (string, error) {
+			if tokenCallback != nil {
+				model.SetTokenCallback(tokenCallback)
+			}
+
 			// Generate the prediction using the language model
-			predictOptions := []gptj.PredictOption{
-				gptj.SetTemperature(c.Temperature),
-				gptj.SetTopP(c.TopP),
-				gptj.SetTopK(c.TopK),
-				gptj.SetTokens(c.Maxtokens),
-				gptj.SetThreads(c.Threads),
+			predictOptions := []gpt4all.PredictOption{
+				gpt4all.SetTemperature(c.Temperature),
+				gpt4all.SetTopP(c.TopP),
+				gpt4all.SetTopK(c.TopK),
+				gpt4all.SetTokens(c.Maxtokens),
 			}
 
 			if c.Batch != 0 {
-				predictOptions = append(predictOptions, gptj.SetBatch(c.Batch))
+				predictOptions = append(predictOptions, gpt4all.SetBatch(c.Batch))
 			}
 
-			if c.Seed != 0 {
-				predictOptions = append(predictOptions, gptj.SetSeed(c.Seed))
-			}
-
-			return model.Predict(
+			str, er := model.Predict(
 				s,
 				predictOptions...,
 			)
+			// Seems that if we don't free the callback explicitly we leave functions registered (that might try to send on closed channels)
+			// For instance otherwise the API returns: {"error":{"code":500,"message":"send on closed channel","type":""}}
+			// after a stream event has occurred
+			model.SetTokenCallback(nil)
+			return str, er
 		}
 	case *llama.LLama:
 		supportStreams = true

go.mod

@@ -4,12 +4,13 @@ go 1.19
 
 require (
 	github.com/donomii/go-rwkv.cpp v0.0.0-20230503112711-af62fcc432be
-	github.com/ggerganov/whisper.cpp/bindings/go v0.0.0-20230508180809-bf2449dfae35
+	github.com/ggerganov/whisper.cpp/bindings/go v0.0.0-20230509153812-1d17cd5bb37a
 	github.com/go-audio/wav v1.1.0
+	github.com/go-skynet/bloomz.cpp v0.0.0-20230510195113-ad7e89a0885f
 	github.com/go-skynet/go-bert.cpp v0.0.0-20230510101404-7bb183b147ea
-	github.com/go-skynet/go-gpt2.cpp v0.0.0-20230422085954-245a5bfe6708
+	github.com/go-skynet/go-gpt2.cpp v0.0.0-20230509180201-d49823284cc6
 	github.com/go-skynet/go-gpt4all-j.cpp v0.0.0-20230422090028-1f7bff57f66c
-	github.com/go-skynet/go-llama.cpp v0.0.0-20230509080828-f4d26f43f1d3
+	github.com/go-skynet/go-llama.cpp v0.0.0-20230510072905-70593fccbe4b
 	github.com/gofiber/fiber/v2 v2.45.0
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/onsi/ginkgo/v2 v2.9.4
@@ -48,6 +49,7 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.18 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/nomic/gpt4all/gpt4all-bindings/golang v0.0.0-00010101000000-000000000000 // indirect
 	github.com/philhofer/fwd v1.1.2 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect

go.sum

@@ -16,8 +16,12 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/donomii/go-rwkv.cpp v0.0.0-20230503112711-af62fcc432be h1:3Hic97PY6hcw/SY44RuR7kyONkxd744RFeRrqckzwNQ=
+github.com/donomii/go-rwkv.cpp v0.0.0-20230503112711-af62fcc432be/go.mod h1:gWy7FIWioqYmYxkaoFyBnaKApeZVrUkHhv9EV9pz4dM=
 github.com/ggerganov/whisper.cpp/bindings/go v0.0.0-20230508180809-bf2449dfae35 h1:sMg/SgnMPS/HNUO/2kGm72vl8R9TmNIwgLFr2TNwR3g=
 github.com/ggerganov/whisper.cpp/bindings/go v0.0.0-20230508180809-bf2449dfae35/go.mod h1:QIjZ9OktHFG7p+/m3sMvrAJKKdWrr1fZIK0rM6HZlyo=
+github.com/ggerganov/whisper.cpp/bindings/go v0.0.0-20230509153812-1d17cd5bb37a h1:MlyiDLNCM/wjbv8U5Elj18NvaAgl61SGiRUpqQz5dfs=
+github.com/ggerganov/whisper.cpp/bindings/go v0.0.0-20230509153812-1d17cd5bb37a/go.mod h1:QIjZ9OktHFG7p+/m3sMvrAJKKdWrr1fZIK0rM6HZlyo=
 github.com/go-audio/audio v1.0.0 h1:zS9vebldgbQqktK4H0lUqWrG8P0NxCJVqcj7ZpNnwd4=
 github.com/go-audio/audio v1.0.0/go.mod h1:6uAu0+H2lHkwdGsAY+j2wHPNPpPoeg5AaEFh9FlA+Zs=
 github.com/go-audio/riff v1.0.0 h1:d8iCGbDvox9BfLagY94fBynxSPHO80LmZCaOsmKxokA=
@@ -36,6 +40,16 @@ github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
 github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
+github.com/go-skynet/bloomz.cpp v0.0.0-20230510195113-ad7e89a0885f h1:GW8RQa1RVeDF1dOuAP/y6xWVC+BRtf9tJOuEza6Asbg=
+github.com/go-skynet/bloomz.cpp v0.0.0-20230510195113-ad7e89a0885f/go.mod h1:wc0fJ9V04yiYTfgKvE5RUUSRQ5Kzi0Bo4I+U3nNOUuA=
+github.com/go-skynet/go-bert.cpp v0.0.0-20230510101404-7bb183b147ea h1:8Isk9D+Auth5OuXVAQPC3MO+5zF/2S7mvs2JZLw6a+8=
+github.com/go-skynet/go-bert.cpp v0.0.0-20230510101404-7bb183b147ea/go.mod h1:NHwIVvsg7Jh6p0M4uBLVmSMEaPUia6O6yjXUpLWVJmQ=
+github.com/go-skynet/go-gpt2.cpp v0.0.0-20230509180201-d49823284cc6 h1:XshpypO6ekU09CI19vuzke2a1Es1lV5ZaxA7CUehu0E=
+github.com/go-skynet/go-gpt2.cpp v0.0.0-20230509180201-d49823284cc6/go.mod h1:1Wj/xbkMfwQSOrhNYK178IzqQHstZbRfhx4s8p1M5VM=
+github.com/go-skynet/go-gpt4all-j.cpp v0.0.0-20230422090028-1f7bff57f66c h1:48I7jpLNGiQeBmF0SFVVbREh8vlG0zN13v9LH5ctXis=
+github.com/go-skynet/go-gpt4all-j.cpp v0.0.0-20230422090028-1f7bff57f66c/go.mod h1:5VZ9XbcINI0XcHhkcX8GPK8TplFGAzu1Hrg4tNiMCtI=
+github.com/go-skynet/go-llama.cpp v0.0.0-20230510072905-70593fccbe4b h1:qqxrjY8fYDXQahmCMTCACahm1tbiqHLPUHALkFLyBfo=
+github.com/go-skynet/go-llama.cpp v0.0.0-20230510072905-70593fccbe4b/go.mod h1:DLfsPD7tYYnpksERH83HSf7qVNW3FIwmz7/zfYO0/6I=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -187,9 +201,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
 google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U=
-gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

pkg/model/initializers.go

@@ -0,0 +1,158 @@
+package model
+
+import (
+	"fmt"
+	"strings"
+
+	rwkv "github.com/donomii/go-rwkv.cpp"
+	whisper "github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper"
+	bloomz "github.com/go-skynet/bloomz.cpp"
+	bert "github.com/go-skynet/go-bert.cpp"
+	gpt2 "github.com/go-skynet/go-gpt2.cpp"
+	llama "github.com/go-skynet/go-llama.cpp"
+	"github.com/hashicorp/go-multierror"
+	gpt4all "github.com/nomic/gpt4all/gpt4all-bindings/golang"
+	"github.com/rs/zerolog/log"
+)
+
+const tokenizerSuffix = ".tokenizer.json"
+
+const (
+	LlamaBackend          = "llama"
+	BloomzBackend         = "bloomz"
+	StableLMBackend       = "stablelm"
+	DollyBackend          = "dolly"
+	RedPajamaBackend      = "redpajama"
+	Gpt2Backend           = "gpt2"
+	Gpt4AllLlamaBackend   = "gpt4all-llama"
+	Gpt4AllMptBackend     = "gpt4all-mpt"
+	Gpt4AllJBackend       = "gpt4all-j"
+	BertEmbeddingsBackend = "bert-embeddings"
+	RwkvBackend           = "rwkv"
+	WhisperBackend        = "whisper"
+)
+
+var backends []string = []string{
+	LlamaBackend,
+	Gpt4AllLlamaBackend,
+	Gpt4AllMptBackend,
+	Gpt4AllJBackend,
+	Gpt2Backend,
+	WhisperBackend,
+	RwkvBackend,
+	BloomzBackend,
+	StableLMBackend,
+	DollyBackend,
+	RedPajamaBackend,
+	BertEmbeddingsBackend,
+}
+
+var redPajama = func(modelFile string) (interface{}, error) {
+	return gpt2.NewRedPajama(modelFile)
+}
+
+var dolly = func(modelFile string) (interface{}, error) {
+	return gpt2.NewDolly(modelFile)
+}
+
+var stableLM = func(modelFile string) (interface{}, error) {
+	return gpt2.NewStableLM(modelFile)
+}
+
+var bertEmbeddings = func(modelFile string) (interface{}, error) {
+	return bert.New(modelFile)
+}
+
+var bloomzLM = func(modelFile string) (interface{}, error) {
+	return bloomz.New(modelFile)
+}
+var gpt2LM = func(modelFile string) (interface{}, error) {
+	return gpt2.New(modelFile)
+}
+
+var whisperModel = func(modelFile string) (interface{}, error) {
+	return whisper.New(modelFile)
+}
+
+func llamaLM(opts ...llama.ModelOption) func(string) (interface{}, error) {
+	return func(s string) (interface{}, error) {
+		return llama.New(s, opts...)
+	}
+}
+
+func gpt4allLM(opts ...gpt4all.ModelOption) func(string) (interface{}, error) {
+	return func(s string) (interface{}, error) {
+		return gpt4all.New(s, opts...)
+	}
+}
+
+func rwkvLM(tokenFile string, threads uint32) func(string) (interface{}, error) {
+	return func(s string) (interface{}, error) {
+		model := rwkv.LoadFiles(s, tokenFile, threads)
+		if model == nil {
+			return nil, fmt.Errorf("could not load model")
+		}
+		return model, nil
+	}
+}
+
+func (ml *ModelLoader) BackendLoader(backendString string, modelFile string, llamaOpts []llama.ModelOption, threads uint32) (model interface{}, err error) {
+	switch strings.ToLower(backendString) {
+	case LlamaBackend:
+		return ml.LoadModel(modelFile, llamaLM(llamaOpts...))
+	case BloomzBackend:
+		return ml.LoadModel(modelFile, bloomzLM)
+	case StableLMBackend:
+		return ml.LoadModel(modelFile, stableLM)
+	case DollyBackend:
+		return ml.LoadModel(modelFile, dolly)
+	case RedPajamaBackend:
+		return ml.LoadModel(modelFile, redPajama)
+	case Gpt2Backend:
+		return ml.LoadModel(modelFile, gpt2LM)
+	case Gpt4AllLlamaBackend:
+		return ml.LoadModel(modelFile, gpt4allLM(gpt4all.SetThreads(int(threads)), gpt4all.SetModelType(gpt4all.LLaMAType)))
+	case Gpt4AllMptBackend:
+		return ml.LoadModel(modelFile, gpt4allLM(gpt4all.SetThreads(int(threads)), gpt4all.SetModelType(gpt4all.MPTType)))
+	case Gpt4AllJBackend:
+		return ml.LoadModel(modelFile, gpt4allLM(gpt4all.SetThreads(int(threads)), gpt4all.SetModelType(gpt4all.GPTJType)))
+	case BertEmbeddingsBackend:
+		return ml.LoadModel(modelFile, bertEmbeddings)
+	case RwkvBackend:
+		return ml.LoadModel(modelFile, rwkvLM(modelFile+tokenizerSuffix, threads))
+	case WhisperBackend:
+		return ml.LoadModel(modelFile, whisperModel)
+	default:
+		return nil, fmt.Errorf("backend unsupported: %s", backendString)
+	}
+}
+
+func (ml *ModelLoader) GreedyLoader(modelFile string, llamaOpts []llama.ModelOption, threads uint32) (interface{}, error) {
+	log.Debug().Msgf("Loading models greedly")
+
+	ml.mu.Lock()
+	m, exists := ml.models[modelFile]
+	if exists {
+		ml.mu.Unlock()
+		return m, nil
+	}
+	ml.mu.Unlock()
+	var err error
+
+	for _, b := range backends {
+		if b == BloomzBackend || b == WhisperBackend || b == RwkvBackend { // do not autoload bloomz/whisper/rwkv
+			continue
+		}
+		log.Debug().Msgf("[%s] Attempting to load", b)
+		model, modelerr := ml.BackendLoader(b, modelFile, llamaOpts, threads)
+		if modelerr == nil && model != nil {
+			log.Debug().Msgf("[%s] Loads OK", b)
+			return model, nil
+		} else if modelerr != nil {
+			err = multierror.Append(err, modelerr)
+			log.Debug().Msgf("[%s] Fails: %s", b, modelerr.Error())
+		}
+	}
+
+	return nil, fmt.Errorf("could not load model - all backends returned error: %s", err.Error())
+}

pkg/model/loader.go

@@ -10,40 +10,22 @@ import (
 	"sync"
 	"text/template"
 
-	"github.com/hashicorp/go-multierror"
 	"github.com/rs/zerolog/log"
-
-	rwkv "github.com/donomii/go-rwkv.cpp"
-	bert "github.com/go-skynet/go-bert.cpp"
-	gpt2 "github.com/go-skynet/go-gpt2.cpp"
-	gptj "github.com/go-skynet/go-gpt4all-j.cpp"
-	llama "github.com/go-skynet/go-llama.cpp"
 )
 
 type ModelLoader struct {
 	ModelPath string
 	mu        sync.Mutex
 	// TODO: this needs generics
-	models            map[string]*llama.LLama
-	gptmodels         map[string]*gptj.GPTJ
-	gpt2models        map[string]*gpt2.GPT2
-	gptstablelmmodels map[string]*gpt2.StableLM
-	rwkv              map[string]*rwkv.RwkvState
-	bert              map[string]*bert.Bert
-
+	models           map[string]interface{}
 	promptsTemplates map[string]*template.Template
 }
 
 func NewModelLoader(modelPath string) *ModelLoader {
 	return &ModelLoader{
-		ModelPath:         modelPath,
-		gpt2models:        make(map[string]*gpt2.GPT2),
-		gptmodels:         make(map[string]*gptj.GPTJ),
-		gptstablelmmodels: make(map[string]*gpt2.StableLM),
-		models:            make(map[string]*llama.LLama),
-		rwkv:              make(map[string]*rwkv.RwkvState),
-		bert:              make(map[string]*bert.Bert),
-		promptsTemplates:  make(map[string]*template.Template),
+		ModelPath:        modelPath,
+		models:           make(map[string]interface{}),
+		promptsTemplates: make(map[string]*template.Template),
 	}
 }
 
@@ -128,175 +110,11 @@ func (ml *ModelLoader) loadTemplateIfExists(modelName, modelFile string) error {
 	return nil
 }
 
-func (ml *ModelLoader) LoadStableLMModel(modelName string) (*gpt2.StableLM, error) {
+func (ml *ModelLoader) LoadModel(modelName string, loader func(string) (interface{}, error)) (interface{}, error) {
 	ml.mu.Lock()
 	defer ml.mu.Unlock()
 
 	// Check if we already have a loaded model
-	if !ml.ExistsInModelPath(modelName) {
-		return nil, fmt.Errorf("model does not exist")
-	}
-
-	if m, ok := ml.gptstablelmmodels[modelName]; ok {
-		log.Debug().Msgf("Model already loaded in memory: %s", modelName)
-		return m, nil
-	}
-
-	// Load the model and keep it in memory for later use
-	modelFile := filepath.Join(ml.ModelPath, modelName)
-	log.Debug().Msgf("Loading model in memory from file: %s", modelFile)
-
-	model, err := gpt2.NewStableLM(modelFile)
-	if err != nil {
-		return nil, err
-	}
-
-	// If there is a prompt template, load it
-	if err := ml.loadTemplateIfExists(modelName, modelFile); err != nil {
-		return nil, err
-	}
-
-	ml.gptstablelmmodels[modelName] = model
-	return model, err
-}
-
-func (ml *ModelLoader) LoadBERT(modelName string) (*bert.Bert, error) {
-	ml.mu.Lock()
-	defer ml.mu.Unlock()
-
-	// Check if we already have a loaded model
-	if !ml.ExistsInModelPath(modelName) {
-		return nil, fmt.Errorf("model does not exist")
-	}
-
-	if m, ok := ml.bert[modelName]; ok {
-		log.Debug().Msgf("Model already loaded in memory: %s", modelName)
-		return m, nil
-	}
-
-	// Load the model and keep it in memory for later use
-	modelFile := filepath.Join(ml.ModelPath, modelName)
-	log.Debug().Msgf("Loading model in memory from file: %s", modelFile)
-
-	model, err := bert.New(modelFile)
-	if err != nil {
-		return nil, err
-	}
-
-	// If there is a prompt template, load it
-	if err := ml.loadTemplateIfExists(modelName, modelFile); err != nil {
-		return nil, err
-	}
-
-	ml.bert[modelName] = model
-	return model, err
-}
-
-func (ml *ModelLoader) LoadGPT2Model(modelName string) (*gpt2.GPT2, error) {
-	ml.mu.Lock()
-	defer ml.mu.Unlock()
-
-	// Check if we already have a loaded model
-	if !ml.ExistsInModelPath(modelName) {
-		return nil, fmt.Errorf("model does not exist")
-	}
-
-	if m, ok := ml.gpt2models[modelName]; ok {
-		log.Debug().Msgf("Model already loaded in memory: %s", modelName)
-		return m, nil
-	}
-
-	// Load the model and keep it in memory for later use
-	modelFile := filepath.Join(ml.ModelPath, modelName)
-	log.Debug().Msgf("Loading model in memory from file: %s", modelFile)
-
-	model, err := gpt2.New(modelFile)
-	if err != nil {
-		return nil, err
-	}
-
-	// If there is a prompt template, load it
-	if err := ml.loadTemplateIfExists(modelName, modelFile); err != nil {
-		return nil, err
-	}
-
-	ml.gpt2models[modelName] = model
-	return model, err
-}
-
-func (ml *ModelLoader) LoadGPTJModel(modelName string) (*gptj.GPTJ, error) {
-	ml.mu.Lock()
-	defer ml.mu.Unlock()
-
-	// Check if we already have a loaded model
-	if !ml.ExistsInModelPath(modelName) {
-		return nil, fmt.Errorf("model does not exist")
-	}
-
-	if m, ok := ml.gptmodels[modelName]; ok {
-		log.Debug().Msgf("Model already loaded in memory: %s", modelName)
-		return m, nil
-	}
-
-	// Load the model and keep it in memory for later use
-	modelFile := filepath.Join(ml.ModelPath, modelName)
-	log.Debug().Msgf("Loading model in memory from file: %s", modelFile)
-
-	model, err := gptj.New(modelFile)
-	if err != nil {
-		return nil, err
-	}
-
-	// If there is a prompt template, load it
-	if err := ml.loadTemplateIfExists(modelName, modelFile); err != nil {
-		return nil, err
-	}
-
-	ml.gptmodels[modelName] = model
-	return model, err
-}
-
-func (ml *ModelLoader) LoadRWKV(modelName, tokenFile string, threads uint32) (*rwkv.RwkvState, error) {
-	ml.mu.Lock()
-	defer ml.mu.Unlock()
-
-	log.Debug().Msgf("Loading model name: %s", modelName)
-
-	// Check if we already have a loaded model
-	if !ml.ExistsInModelPath(modelName) {
-		return nil, fmt.Errorf("model does not exist")
-	}
-
-	if m, ok := ml.rwkv[modelName]; ok {
-		log.Debug().Msgf("Model already loaded in memory: %s", modelName)
-		return m, nil
-	}
-
-	// Load the model and keep it in memory for later use
-	modelFile := filepath.Join(ml.ModelPath, modelName)
-	tokenPath := filepath.Join(ml.ModelPath, tokenFile)
-	log.Debug().Msgf("Loading model in memory from file: %s", modelFile)
-
-	model := rwkv.LoadFiles(modelFile, tokenPath, threads)
-	if model == nil {
-		return nil, fmt.Errorf("could not load model")
-	}
-
-	ml.rwkv[modelName] = model
-	return model, nil
-}
-
-func (ml *ModelLoader) LoadLLaMAModel(modelName string, opts ...llama.ModelOption) (*llama.LLama, error) {
-	ml.mu.Lock()
-	defer ml.mu.Unlock()
-
-	log.Debug().Msgf("Loading model name: %s", modelName)
-
-	// Check if we already have a loaded model
-	if !ml.ExistsInModelPath(modelName) {
-		return nil, fmt.Errorf("model does not exist")
-	}
-
 	if m, ok := ml.models[modelName]; ok {
 		log.Debug().Msgf("Model already loaded in memory: %s", modelName)
 		return m, nil
@@ -306,7 +124,7 @@ func (ml *ModelLoader) LoadLLaMAModel(modelName string, opts ...llama.ModelOptio
 	modelFile := filepath.Join(ml.ModelPath, modelName)
 	log.Debug().Msgf("Loading model in memory from file: %s", modelFile)
 
-	model, err := llama.New(modelFile, opts...)
+	model, err := loader(modelFile)
 	if err != nil {
 		return nil, err
 	}
@@ -317,95 +135,5 @@ func (ml *ModelLoader) LoadLLaMAModel(modelName string, opts ...llama.ModelOptio
 	}
 
 	ml.models[modelName] = model
-	return model, err
-}
-
-const tokenizerSuffix = ".tokenizer.json"
-
-var loadedModels map[string]interface{} = map[string]interface{}{}
-var muModels sync.Mutex
-
-func (ml *ModelLoader) BackendLoader(backendString string, modelFile string, llamaOpts []llama.ModelOption, threads uint32) (model interface{}, err error) {
-	switch strings.ToLower(backendString) {
-	case "llama":
-		return ml.LoadLLaMAModel(modelFile, llamaOpts...)
-	case "stablelm":
-		return ml.LoadStableLMModel(modelFile)
-	case "gpt2":
-		return ml.LoadGPT2Model(modelFile)
-	case "gptj":
-		return ml.LoadGPTJModel(modelFile)
-	case "bert-embeddings":
-		return ml.LoadBERT(modelFile)
-	case "rwkv":
-		return ml.LoadRWKV(modelFile, modelFile+tokenizerSuffix, threads)
-	default:
-		return nil, fmt.Errorf("backend unsupported: %s", backendString)
-	}
-}
-
-func (ml *ModelLoader) GreedyLoader(modelFile string, llamaOpts []llama.ModelOption, threads uint32) (model interface{}, err error) {
-	updateModels := func(model interface{}) {
-		muModels.Lock()
-		defer muModels.Unlock()
-		loadedModels[modelFile] = model
-	}
-
-	muModels.Lock()
-	m, exists := loadedModels[modelFile]
-	if exists {
-		muModels.Unlock()
-		return m, nil
-	}
-	muModels.Unlock()
-
-	model, modelerr := ml.LoadLLaMAModel(modelFile, llamaOpts...)
-	if modelerr == nil {
-		updateModels(model)
-		return model, nil
-	} else {
-		err = multierror.Append(err, modelerr)
-	}
-
-	model, modelerr = ml.LoadGPTJModel(modelFile)
-	if modelerr == nil {
-		updateModels(model)
-		return model, nil
-	} else {
-		err = multierror.Append(err, modelerr)
-	}
-
-	model, modelerr = ml.LoadGPT2Model(modelFile)
-	if modelerr == nil {
-		updateModels(model)
-		return model, nil
-	} else {
-		err = multierror.Append(err, modelerr)
-	}
-
-	model, modelerr = ml.LoadStableLMModel(modelFile)
-	if modelerr == nil {
-		updateModels(model)
-		return model, nil
-	} else {
-		err = multierror.Append(err, modelerr)
-	}
-
-	model, modelerr = ml.LoadRWKV(modelFile, modelFile+tokenizerSuffix, threads)
-	if modelerr == nil {
-		updateModels(model)
-		return model, nil
-	} else {
-		err = multierror.Append(err, modelerr)
-	}
-
-	model, modelerr = ml.LoadBERT(modelFile)
-	if modelerr == nil {
-		updateModels(model)
-		return model, nil
-	} else {
-		err = multierror.Append(err, modelerr)
-	}
-
-	return nil, fmt.Errorf("could not load model - all backends returned error: %s", err.Error())
+	return model, nil
 }

pkg/whisper/whisper.go

@@ -28,7 +28,7 @@ func audioToWav(src, dst string) error {
 	return nil
 }
 
-func Transcript(modelpath, audiopath, language string) (string, error) {
+func Transcript(model whisper.Model, audiopath, language string) (string, error) {
 
 	dir, err := os.MkdirTemp("", "whisper")
 	if err != nil {
@@ -58,13 +58,6 @@ func Transcript(modelpath, audiopath, language string) (string, error) {
 
 	data := buf.AsFloat32Buffer().Data
 
-	// Load the model
-	model, err := whisper.New(modelpath)
-	if err != nil {
-		return "", err
-	}
-	defer model.Close()
-
 	// Process samples
 	context, err := model.NewContext()
 	if err != nil {