diff --git a/.agents/ci-caching.md b/.agents/ci-caching.md index 6742049e6..aaac05f2d 100644 --- a/.agents/ci-caching.md +++ b/.agents/ci-caching.md @@ -176,7 +176,7 @@ Deliberately **not** applied to: | Workflow | Why it must keep seeing `backend/**` | |---|---| -| `test.yml` | `TEST_PATHS` explicitly includes `./backend/go/cloud-proxy/...`, `./backend/go/local-store/...` and `./backend/go/valkey-store/...` | +| `test.yml` | `TEST_PATHS` explicitly includes `./backend/go/cloud-proxy/...`, `./backend/go/local-store/...`, `./backend/go/valkey-store/...` and `./backend/go/qdrant-store/...` | | `lint.yml` | `.golangci.yml` carries `backend/`-scoped rules, so golangci-lint covers that tree | | `tests-e2e.yml` | The e2e suite drives real backends over gRPC | | `backend_pr.yml` | This is the workflow whose entire job is to rebuild the changed backend | diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index fa61fa514..fa40632f1 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -5977,6 +5977,35 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + # qdrant-store + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + platform-tag: 'amd64' + tag-latest: 'auto' + tag-suffix: '-cpu-qdrant-store' + runs-on: 'ubuntu-latest' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "qdrant-store" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' + - build-type: '' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/arm64' + platform-tag: 'arm64' + tag-latest: 'auto' + tag-suffix: '-cpu-qdrant-store' + runs-on: 'ubuntu-24.04-arm' + base-image: "ubuntu:24.04" + skip-drivers: 'false' + backend: "qdrant-store" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' # rfdetr - build-type: '' cuda-major-version: "" @@ -6757,6 +6786,10 @@ includeDarwin: tag-suffix: "-metal-darwin-arm64-valkey-store" build-type: "metal" lang: "go" + - backend: "qdrant-store" + tag-suffix: "-metal-darwin-arm64-qdrant-store" + build-type: "metal" + lang: "go" - backend: "llama-cpp-quantization" tag-suffix: "-metal-darwin-arm64-llama-cpp-quantization" build-type: "mps" diff --git a/.gitignore b/.gitignore index a4c84a0e8..7e02884d1 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ LocalAI /cloud-proxy /local-store /valkey-store +/qdrant-store # prevent above rules from omitting the helm chart !charts/* # prevent above rules from omitting the api/localai folder diff --git a/Makefile b/Makefile index e39c5dfa0..67171f38d 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Disable parallel execution for backend builds -.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin +.NOTPARALLEL: backends/diffusers backends/llama-cpp backends/turboquant backends/bonsai backends/outetts backends/piper backends/stablediffusion-ggml backends/trellis2cpp backends/trellis2cpp-darwin backends/whisper backends/crispasr backends/parakeet-cpp backends/moss-transcribe-cpp backends/nemo-speech-cpp backends/faster-whisper backends/silero-vad backends/local-store backends/valkey-store backends/qdrant-store backends/cloud-proxy backends/huggingface backends/rfdetr backends/rfdetr-cpp backends/insightface backends/speaker-recognition backends/kitten-tts backends/kokoro backends/chatterbox backends/llama-cpp-darwin backends/neutts build-darwin-python-backend build-darwin-go-backend backends/mlx backends/mlx-video backends/diffuser-darwin backends/mlx-vlm backends/mlx-audio backends/mlx-distributed backends/stablediffusion-ggml-darwin backends/vllm backends/vllm-omni backends/longcat-video backends/sglang backends/moonshine backends/pocket-tts backends/qwen-tts backends/faster-qwen3-tts backends/qwen-asr backends/nemo backends/voxcpm backends/whisperx backends/ace-step backends/acestep-cpp backends/fish-speech backends/voxtral backends/opus backends/trl backends/llama-cpp-quantization backends/kokoros backends/sam3-cpp backends/qwen3-tts-cpp backends/moss-tts-cpp backends/magpie-tts-cpp backends/vllm-cpp backends/omnivoice-cpp backends/vibevoice-cpp backends/localvqe backends/tinygrad backends/sherpa-onnx backends/ds4 backends/ds4-darwin backends/liquid-audio backends/supertonic backends/depth-anything-cpp backends/privacy-filter backends/privacy-filter-darwin backends/audio-cpp backends/audio-cpp-darwin .NOTPARALLEL: backends/whisper-medusa .NOTPARALLEL: backends/funasr @@ -76,7 +76,7 @@ else GORELEASER=$(shell which goreleaser) endif -TEST_PATHS?=./api/... ./pkg/... ./core/... ./backend/go/cloud-proxy/... ./backend/go/local-store/... ./backend/go/valkey-store/... +TEST_PATHS?=./api/... ./pkg/... ./core/... ./backend/go/cloud-proxy/... ./backend/go/local-store/... ./backend/go/valkey-store/... ./backend/go/qdrant-store/... ## Coverage output and the committed baseline that CI compares against. ## The gate is strict: total coverage must never decrease (no tolerance). @@ -427,6 +427,11 @@ test-valkey-store: backends/valkey-store BACKENDS_PATH=$(abspath ./)/backends \ $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) --label-filter='valkey' -v -r tests/integration +## Needs a Qdrant gRPC endpoint in $$QDRANT_ADDR, e.g. localhost:6334. +test-qdrant-store: backends/qdrant-store + BACKENDS_PATH=$(abspath ./)/backends \ + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) --label-filter='qdrant' -v -r tests/integration + test-opus: @echo 'Running opus backend tests' $(MAKE) -C backend/go/opus libopusshim.so @@ -641,6 +646,7 @@ prepare-test-extra: protogen-python $(MAKE) -C backend/go/trellis2cpp $(MAKE) -C backend/go/kimodocpp $(MAKE) -C backend/go/valkey-store + $(MAKE) -C backend/go/qdrant-store test-extra: prepare-test-extra $(MAKE) -C backend/python/transformers test @@ -680,6 +686,7 @@ test-extra: prepare-test-extra $(MAKE) -C backend/go/trellis2cpp test $(MAKE) -C backend/go/kimodocpp test $(MAKE) -C backend/go/valkey-store test + $(MAKE) -C backend/go/qdrant-store test ## ## End-to-end gRPC tests that exercise a built backend container image. @@ -1329,6 +1336,7 @@ BACKEND_AUDIO_CPP = audio-cpp|audio-cpp|.|false|false BACKEND_PIPER = piper|golang|.|false|true BACKEND_LOCAL_STORE = local-store|golang|.|false|true BACKEND_VALKEY_STORE = valkey-store|golang|.|false|true +BACKEND_QDRANT_STORE = qdrant-store|golang|.|false|true BACKEND_CLOUD_PROXY = cloud-proxy|golang|.|false|true BACKEND_HUGGINGFACE = huggingface|golang|.|false|true BACKEND_SILERO_VAD = silero-vad|golang|.|false|true @@ -1435,6 +1443,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_AUDIO_CPP))) $(eval $(call generate-docker-build-target,$(BACKEND_PIPER))) $(eval $(call generate-docker-build-target,$(BACKEND_LOCAL_STORE))) $(eval $(call generate-docker-build-target,$(BACKEND_VALKEY_STORE))) +$(eval $(call generate-docker-build-target,$(BACKEND_QDRANT_STORE))) $(eval $(call generate-docker-build-target,$(BACKEND_CLOUD_PROXY))) $(eval $(call generate-docker-build-target,$(BACKEND_HUGGINGFACE))) $(eval $(call generate-docker-build-target,$(BACKEND_SILERO_VAD))) @@ -1511,7 +1520,7 @@ $(eval $(call generate-docker-build-target,$(BACKEND_SUPERTONIC))) docker-save-%: backend-images docker save local-ai-backend:$* -o backend-images/$*.tar -docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-nemo-speech-cpp docker-build-privacy-filter docker-build-trellis2cpp docker-build-valkey-store docker-build-audio-cpp +docker-build-backends: docker-build-llama-cpp docker-build-ik-llama-cpp docker-build-turboquant docker-build-bonsai docker-build-ds4 docker-build-rerankers docker-build-vllm docker-build-vllm-omni docker-build-longcat-video docker-build-sglang docker-build-transformers docker-build-outetts docker-build-diffusers docker-build-kokoro docker-build-faster-whisper docker-build-crispasr docker-build-coqui docker-build-chatterbox docker-build-vibevoice docker-build-liquid-audio docker-build-moonshine docker-build-pocket-tts docker-build-qwen-tts docker-build-fish-speech docker-build-faster-qwen3-tts docker-build-qwen-asr docker-build-nemo docker-build-voxcpm docker-build-whisperx docker-build-ace-step docker-build-acestep-cpp docker-build-voxtral docker-build-mlx-distributed docker-build-trl docker-build-llama-cpp-quantization docker-build-tinygrad docker-build-kokoros docker-build-sam3-cpp docker-build-rfdetr-cpp docker-build-qwen3-tts-cpp docker-build-moss-tts-cpp docker-build-magpie-tts-cpp docker-build-vllm-cpp docker-build-omnivoice-cpp docker-build-vibevoice-cpp docker-build-localvqe docker-build-insightface docker-build-speaker-recognition docker-build-sherpa-onnx docker-build-cloud-proxy docker-build-supertonic docker-build-depth-anything-cpp docker-build-moss-transcribe-cpp docker-build-nemo-speech-cpp docker-build-privacy-filter docker-build-trellis2cpp docker-build-valkey-store docker-build-qdrant-store docker-build-audio-cpp docker-build-backends: docker-build-whisper-medusa docker-build-backends: docker-build-funasr diff --git a/backend/README.md b/backend/README.md index 41c7e215e..5e7f40d16 100644 --- a/backend/README.md +++ b/backend/README.md @@ -57,6 +57,7 @@ The backend system provides language-specific Dockerfiles that handle the build - **piper**: Text-to-speech synthesis Golang with C bindings using rhaspy/piper - **local-store**: Vector storage backend - **valkey-store**: Durable vector storage backend backed by Valkey Search (FT.*) +- **qdrant-store**: Durable vector storage backend backed by Qdrant (self-hosted or Qdrant Cloud) #### C++ Backends (`cpp/`) - **llama-cpp**: Llama.cpp integration diff --git a/backend/go/qdrant-store/Makefile b/backend/go/qdrant-store/Makefile new file mode 100644 index 000000000..7f1a385ef --- /dev/null +++ b/backend/go/qdrant-store/Makefile @@ -0,0 +1,16 @@ +GOCMD=go +TEST_FLAKES?=5 + +qdrant-store: + CGO_ENABLED=0 $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o qdrant-store ./ + +package: + bash package.sh + +build: qdrant-store package + +test: + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./ + +clean: + rm -f qdrant-store diff --git a/backend/go/qdrant-store/config.go b/backend/go/qdrant-store/config.go new file mode 100644 index 000000000..f223ecc62 --- /dev/null +++ b/backend/go/qdrant-store/config.go @@ -0,0 +1,105 @@ +package main + +import ( + "fmt" + "net" + "os" + "strconv" + "strings" + "time" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/qdrant/go-client/qdrant" +) + +type Config struct { + Host string + Port int + APIKey string + UseTLS bool + TLSSkipVerify bool + TLSCACert string + Collection string + Distance qdrant.Distance + RequestTimeout time.Duration +} + +func (c Config) Addr() string { + return net.JoinHostPort(c.Host, strconv.Itoa(c.Port)) +} + +func loadConfig(opts *pb.ModelOptions) (Config, error) { + cfg := Config{ + Host: "localhost", + Port: 6334, + Distance: qdrant.Distance_Cosine, + RequestTimeout: 5 * time.Second, + } + var apiKeyEnv string + + for _, o := range opts.GetOptions() { + key, value, _ := strings.Cut(o, ":") + key, value = strings.ToLower(strings.TrimSpace(key)), strings.TrimSpace(value) + var err error + switch key { + case "addr": + cfg.Host, cfg.Port, err = parseAddr(value) + case "api_key": + cfg.APIKey = value + case "api_key_env": + apiKeyEnv = value + case "tls": + cfg.UseTLS, err = strconv.ParseBool(value) + case "tls_skip_verify": + cfg.TLSSkipVerify, err = strconv.ParseBool(value) + case "tls_ca_cert": + cfg.TLSCACert = value + case "collection": + cfg.Collection = value + case "distance_metric": + cfg.Distance, err = parseDistance(value) + case "request_timeout_ms": + var ms uint64 + ms, err = strconv.ParseUint(value, 10, 31) + if ms > 0 { + cfg.RequestTimeout = time.Duration(ms) * time.Millisecond + } + } + if err != nil { + return Config{}, fmt.Errorf("qdrant-store: invalid option %q: %w", o, err) + } + } + + if cfg.TLSCACert != "" || cfg.TLSSkipVerify { + cfg.UseTLS = true + } + + if cfg.APIKey == "" && apiKeyEnv != "" { + cfg.APIKey = os.Getenv(apiKeyEnv) + } + return cfg, nil +} + +func parseAddr(addr string) (string, int, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + if !strings.Contains(err.Error(), "missing port") { + return "", 0, err + } + return strings.Trim(addr, "[]"), 6334, nil + } + p, err := strconv.Atoi(port) + if err != nil || p < 1 || p > 65535 { + return "", 0, fmt.Errorf("bad port %q", port) + } + return host, p, nil +} + +func parseDistance(name string) (qdrant.Distance, error) { + for n, v := range qdrant.Distance_value { + if v != int32(qdrant.Distance_UnknownDistance) && strings.EqualFold(n, name) { + return qdrant.Distance(v), nil + } + } + return 0, fmt.Errorf("want COSINE, EUCLID, DOT or MANHATTAN") +} diff --git a/backend/go/qdrant-store/main.go b/backend/go/qdrant-store/main.go new file mode 100644 index 000000000..393aabf45 --- /dev/null +++ b/backend/go/qdrant-store/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "flag" + "os" + + grpc "github.com/mudler/LocalAI/pkg/grpc" + "github.com/mudler/xlog" +) + +var ( + addr = flag.String("addr", "localhost:50051", "the address to connect to") +) + +func main() { + xlog.SetLogger(xlog.NewLogger(xlog.LogLevel(os.Getenv("LOCALAI_LOG_LEVEL")), os.Getenv("LOCALAI_LOG_FORMAT"))) + + flag.Parse() + + if err := grpc.StartServer(*addr, &QdrantStore{}); err != nil { + panic(err) + } +} diff --git a/backend/go/qdrant-store/package.sh b/backend/go/qdrant-store/package.sh new file mode 100755 index 000000000..ea688a5d9 --- /dev/null +++ b/backend/go/qdrant-store/package.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +CURDIR=$(dirname "$(realpath "$0")") + +mkdir -p "$CURDIR/package" +cp -avf "$CURDIR/qdrant-store" "$CURDIR/package/" +cp -rfv "$CURDIR/run.sh" "$CURDIR/package/" diff --git a/backend/go/qdrant-store/run.sh b/backend/go/qdrant-store/run.sh new file mode 100755 index 000000000..932ccf020 --- /dev/null +++ b/backend/go/qdrant-store/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -ex + +CURDIR=$(dirname "$(realpath "$0")") + +exec "$CURDIR"/qdrant-store "$@" diff --git a/backend/go/qdrant-store/store.go b/backend/go/qdrant-store/store.go new file mode 100644 index 000000000..a3459e340 --- /dev/null +++ b/backend/go/qdrant-store/store.go @@ -0,0 +1,347 @@ +package main + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "fmt" + "math" + "os" + "strings" + + "github.com/google/uuid" + "github.com/mudler/LocalAI/pkg/grpc/base" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/store" + "github.com/qdrant/go-client/qdrant" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + payloadKey = "key" + payloadValue = "value" + + // Results carry up to 50 MiB of base64-encoded values. + maxRecvMsgBytes = 128 << 20 +) + +var pointNamespace = uuid.NewSHA1(uuid.NameSpaceURL, []byte("https://localai.io/qdrant-store")) + +func newClient(cfg Config) (*qdrant.Client, error) { + qcfg := &qdrant.Config{ + Host: cfg.Host, + Port: cfg.Port, + APIKey: cfg.APIKey, + UseTLS: cfg.UseTLS, + SkipCompatibilityCheck: true, + GrpcOptions: []grpc.DialOption{grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxRecvMsgBytes))}, + } + if cfg.UseTLS { + qcfg.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS13, InsecureSkipVerify: cfg.TLSSkipVerify} + if cfg.TLSCACert != "" { + pem, err := os.ReadFile(cfg.TLSCACert) + if err != nil { + return nil, fmt.Errorf("read tls_ca_cert: %w", err) + } + qcfg.TLSConfig.RootCAs = x509.NewCertPool() + if !qcfg.TLSConfig.RootCAs.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("tls_ca_cert %q has no valid certificate", cfg.TLSCACert) + } + } + } + return qdrant.NewClient(qcfg) +} + +type QdrantStore struct { + base.SingleThread + + client *qdrant.Client + cfg Config + collection string + checked bool +} + +func (s *QdrantStore) Load(opts *pb.ModelOptions) error { + // The loader's greedy autoload probes every backend with real model names. + if !strings.HasPrefix(opts.GetModel(), store.NamespacePrefix) { + return fmt.Errorf("qdrant-store: refusing to load %q: not a store namespace", opts.GetModel()) + } + cfg, err := loadConfig(opts) + if err != nil { + return err + } + client, err := newClient(cfg) + if err != nil { + return fmt.Errorf("qdrant-store: connect to %s: %w", cfg.Addr(), err) + } + s.client, s.cfg, s.collection = client, cfg, collectionName(cfg, opts.GetModel()) + + ctx, cancel := s.ctx() + defer cancel() + if err := s.checkCollection(ctx); err != nil && !isNotFound(err) { + _ = client.Close() + return fmt.Errorf("qdrant-store: %s: %w", cfg.Addr(), err) + } + return nil +} + +func (s *QdrantStore) checkCollection(ctx context.Context) error { + info, err := s.client.GetCollectionInfo(ctx, s.collection) + if err != nil { + return err + } + params := info.GetConfig().GetParams().GetVectorsConfig().GetParams() + if params == nil { + return fmt.Errorf("collection %s does not use a single unnamed vector", s.collection) + } + if params.GetDistance() != s.cfg.Distance { + return fmt.Errorf("collection %s uses distance %s but distance_metric is %s", s.collection, params.GetDistance(), s.cfg.Distance) + } + s.checked = true + return nil +} + +func (s *QdrantStore) Free() error { + if s.client == nil { + return nil + } + return s.client.Close() +} + +func (s *QdrantStore) StoresSet(opts *pb.StoresSetOptions) error { + if len(opts.Keys) == 0 || len(opts.Keys) != len(opts.Values) { + return fmt.Errorf("qdrant-store: Set: need matching keys and values, got %d and %d", len(opts.Keys), len(opts.Values)) + } + points := make([]*qdrant.PointStruct, len(opts.Keys)) + for i, k := range opts.Keys { + if err := checkFinite(k.Floats); err != nil { + return fmt.Errorf("qdrant-store: Set: key %d: %w", i, err) + } + points[i] = &qdrant.PointStruct{ + Id: pointID(k.Floats), + Vectors: qdrant.NewVectorsDense(k.Floats), + // Qdrant normalises COSINE vectors, so the exact key goes in the payload. + Payload: qdrant.NewValueMap(map[string]any{ + payloadKey: base64.StdEncoding.EncodeToString(vecToBytes(k.Floats)), + payloadValue: base64.StdEncoding.EncodeToString(opts.Values[i].Bytes), + }), + } + } + + err := s.upsert(points) + if isNotFound(err) { + if err := s.createCollection(len(opts.Keys[0].Floats)); err != nil { + return fmt.Errorf("qdrant-store: Set: %w", err) + } + err = s.upsert(points) + } + if err != nil { + return fmt.Errorf("qdrant-store: Set: %w", err) + } + return nil +} + +func (s *QdrantStore) upsert(points []*qdrant.PointStruct) error { + ctx, cancel := s.ctx() + defer cancel() + _, err := s.client.Upsert(ctx, &qdrant.UpsertPoints{ + CollectionName: s.collection, + Wait: qdrant.PtrOf(true), + Points: points, + }) + return err +} + +func (s *QdrantStore) createCollection(dim int) error { + ctx, cancel := s.ctx() + defer cancel() + err := s.client.CreateCollection(ctx, &qdrant.CreateCollection{ + CollectionName: s.collection, + VectorsConfig: qdrant.NewVectorsConfig(&qdrant.VectorParams{Size: uint64(dim), Distance: s.cfg.Distance}), + }) + if err != nil && status.Code(err) != codes.AlreadyExists { + return fmt.Errorf("create collection %s: %w", s.collection, err) + } + return nil +} + +func (s *QdrantStore) StoresGet(opts *pb.StoresGetOptions) (pb.StoresGetResult, error) { + if len(opts.Keys) == 0 { + return pb.StoresGetResult{}, nil + } + ctx, cancel := s.ctx() + defer cancel() + points, err := s.client.Get(ctx, &qdrant.GetPoints{ + CollectionName: s.collection, + Ids: pointIDs(opts.Keys), + WithPayload: qdrant.NewWithPayloadInclude(payloadKey, payloadValue), + }) + if isNotFound(err) { + return pb.StoresGetResult{}, nil + } + if err != nil { + return pb.StoresGetResult{}, fmt.Errorf("qdrant-store: Get: %w", err) + } + + var keys [][]float32 + var values [][]byte + for _, p := range points { + key, value, err := decodePayload(p.GetPayload()) + if err != nil { + return pb.StoresGetResult{}, fmt.Errorf("qdrant-store: Get: %w", err) + } + keys = append(keys, key) + values = append(values, value) + } + return pb.StoresGetResult{Keys: store.WrapKeys(keys), Values: store.WrapValues(values)}, nil +} + +func (s *QdrantStore) StoresDelete(opts *pb.StoresDeleteOptions) error { + if len(opts.Keys) == 0 { + return fmt.Errorf("qdrant-store: Delete: no keys to delete") + } + ctx, cancel := s.ctx() + defer cancel() + _, err := s.client.Delete(ctx, &qdrant.DeletePoints{ + CollectionName: s.collection, + Wait: qdrant.PtrOf(true), + Points: qdrant.NewPointsSelector(pointIDs(opts.Keys)...), + }) + if err != nil && !isNotFound(err) { + return fmt.Errorf("qdrant-store: Delete: %w", err) + } + return nil +} + +func (s *QdrantStore) StoresFind(opts *pb.StoresFindOptions) (pb.StoresFindResult, error) { + query := opts.GetKey().GetFloats() + if opts.TopK < 1 { + return pb.StoresFindResult{}, fmt.Errorf("qdrant-store: Find: topK = %d, must be >= 1", opts.TopK) + } + if err := checkFinite(query); err != nil { + return pb.StoresFindResult{}, fmt.Errorf("qdrant-store: Find: %w", err) + } + + ctx, cancel := s.ctx() + defer cancel() + if !s.checked { + err := s.checkCollection(ctx) + if isNotFound(err) { + return pb.StoresFindResult{}, nil + } + if err != nil { + return pb.StoresFindResult{}, fmt.Errorf("qdrant-store: Find: %w", err) + } + } + points, err := s.client.Query(ctx, &qdrant.QueryPoints{ + CollectionName: s.collection, + Query: qdrant.NewQueryDense(query), + Limit: qdrant.PtrOf(uint64(opts.TopK)), + WithPayload: qdrant.NewWithPayloadInclude(payloadKey, payloadValue), + }) + if isNotFound(err) { + return pb.StoresFindResult{}, nil + } + if err != nil { + return pb.StoresFindResult{}, fmt.Errorf("qdrant-store: Find: %w", err) + } + + var keys [][]float32 + var values [][]byte + var sims []float32 + for _, p := range points { + key, value, err := decodePayload(p.GetPayload()) + if err != nil { + return pb.StoresFindResult{}, fmt.Errorf("qdrant-store: Find: %w", err) + } + keys = append(keys, key) + values = append(values, value) + sims = append(sims, p.GetScore()) + } + return pb.StoresFindResult{Keys: store.WrapKeys(keys), Values: store.WrapValues(values), Similarities: sims}, nil +} + +func (s *QdrantStore) ctx() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), s.cfg.RequestTimeout) +} + +func pointID(vec []float32) *qdrant.PointId { + return qdrant.NewID(uuid.NewSHA1(pointNamespace, vecToBytes(vec)).String()) +} + +func pointIDs(keys []*pb.StoresKey) []*qdrant.PointId { + ids := make([]*qdrant.PointId, len(keys)) + for i, k := range keys { + ids[i] = pointID(k.Floats) + } + return ids +} + +func vecToBytes(vec []float32) []byte { + b := make([]byte, 4*len(vec)) + for i, x := range vec { + binary.LittleEndian.PutUint32(b[4*i:], math.Float32bits(x)) + } + return b +} + +func decodePayload(payload map[string]*qdrant.Value) ([]float32, []byte, error) { + k, okKey := payload[payloadKey] + v, okValue := payload[payloadValue] + if !okKey || !okValue { + return nil, nil, fmt.Errorf("point has no %q/%q payload; not written by qdrant-store", payloadKey, payloadValue) + } + keyBytes, err := base64.StdEncoding.DecodeString(k.GetStringValue()) + if err != nil { + return nil, nil, fmt.Errorf("decode key payload: %w", err) + } + if len(keyBytes)%4 != 0 { + return nil, nil, fmt.Errorf("key payload of %d bytes is not a float32 vector", len(keyBytes)) + } + value, err := base64.StdEncoding.DecodeString(v.GetStringValue()) + if err != nil { + return nil, nil, fmt.Errorf("decode value payload: %w", err) + } + key := make([]float32, len(keyBytes)/4) + for i := range key { + key[i] = math.Float32frombits(binary.LittleEndian.Uint32(keyBytes[4*i:])) + } + return key, value, nil +} + +func collectionName(cfg Config, model string) string { + if cfg.Collection != "" { + return cfg.Collection + } + ns := strings.TrimPrefix(model, store.NamespacePrefix) + readable := strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_' || r == '-' || r == '.' { + return r + } + return '_' + }, ns) + if len(readable) > 64 { + readable = readable[:64] + } + sum := sha256.Sum256([]byte(ns)) + return "localai_" + readable + "-" + hex.EncodeToString(sum[:4]) +} + +func checkFinite(vec []float32) error { + for i, x := range vec { + if math.IsNaN(float64(x)) || math.IsInf(float64(x), 0) { + return fmt.Errorf("component %d is %v; vectors must be finite", i, x) + } + } + return nil +} + +func isNotFound(err error) bool { + return status.Code(err) == codes.NotFound +} diff --git a/backend/go/qdrant-store/store_suite_test.go b/backend/go/qdrant-store/store_suite_test.go new file mode 100644 index 000000000..928d3aa99 --- /dev/null +++ b/backend/go/qdrant-store/store_suite_test.go @@ -0,0 +1,13 @@ +package main + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestQdrantStore(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "qdrant-store test suite") +} diff --git a/backend/go/qdrant-store/store_test.go b/backend/go/qdrant-store/store_test.go new file mode 100644 index 000000000..e91f627a7 --- /dev/null +++ b/backend/go/qdrant-store/store_test.go @@ -0,0 +1,75 @@ +package main + +import ( + "math" + "time" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/store" + "github.com/qdrant/go-client/qdrant" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("loadConfig", func() { + It("applies defaults and parses every option", func() { + cfg, err := loadConfig(&pb.ModelOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).To(Equal(Config{Host: "localhost", Port: 6334, Distance: qdrant.Distance_Cosine, RequestTimeout: 5 * time.Second})) + + GinkgoT().Setenv("QDRANT_STORE_TEST_KEY", "from-env") + cfg, err = loadConfig(&pb.ModelOptions{Options: []string{"addr:[::1]", "api_key_env:QDRANT_STORE_TEST_KEY", "tls:true", + "tls_skip_verify:true", "tls_ca_cert:/ca.pem", "collection:c", "distance_metric:dot", "request_timeout_ms:250"}}) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).To(Equal(Config{Host: "::1", Port: 6334, APIKey: "from-env", UseTLS: true, TLSSkipVerify: true, + TLSCACert: "/ca.pem", Collection: "c", Distance: qdrant.Distance_Dot, RequestTimeout: 250 * time.Millisecond})) + }) + + It("turns TLS on for tls_ca_cert and tls_skip_verify", func() { + for _, o := range []string{"tls_ca_cert:/ca.pem", "tls_skip_verify:true"} { + cfg, err := loadConfig(&pb.ModelOptions{Options: []string{o}}) + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.UseTLS).To(BeTrue(), o) + } + }) + + It("rejects malformed values", func() { + for _, o := range []string{"addr:host:0", "addr:https://x:6333", "tls:maybe", "distance_metric:L2", "request_timeout_ms:soon"} { + _, err := loadConfig(&pb.ModelOptions{Options: []string{o}}) + Expect(err).To(HaveOccurred(), o) + } + }) +}) + +var _ = Describe("QdrantStore", func() { + It("derives deterministic point IDs and distinct collection names", func() { + Expect(pointID([]float32{1, 2})).To(Equal(pointID([]float32{1, 2}))) + Expect(pointID([]float32{1, 2})).NotTo(Equal(pointID([]float32{2, 1}))) + Expect(collectionName(Config{}, "store://faces")).To(HavePrefix("localai_faces-")) + Expect(collectionName(Config{}, "store://a b")).NotTo(Equal(collectionName(Config{}, "store://a/b"))) + Expect(collectionName(Config{Collection: "mine"}, "store://faces")).To(Equal("mine")) + }) + + It("fails to load a non-store name, a bad tls_ca_cert or an unreachable server", func() { + load := func(model string, opts ...string) error { + return (&QdrantStore{}).Load(&pb.ModelOptions{Model: model, Options: opts}) + } + Expect(load("llama-3.gguf")).To(MatchError(ContainSubstring("not a store namespace"))) + Expect(load("store://x", "tls_ca_cert:/nonexistent.pem")).To(MatchError(ContainSubstring("read tls_ca_cert"))) + Expect(load("store://x", "addr:127.0.0.1:1")).To(MatchError(ContainSubstring("127.0.0.1:1"))) + }) + + It("rejects invalid input before calling Qdrant", func() { + s := &QdrantStore{} + set := func(keys [][]float32, values ...[]byte) error { + return s.StoresSet(&pb.StoresSetOptions{Keys: store.WrapKeys(keys), Values: store.WrapValues(values)}) + } + Expect(set(nil)).To(HaveOccurred()) + Expect(set([][]float32{{1, 2}})).To(HaveOccurred()) + Expect(set([][]float32{{float32(math.NaN()), 1}}, []byte("a"))).To(MatchError(ContainSubstring("must be finite"))) + Expect(s.StoresDelete(&pb.StoresDeleteOptions{})).To(HaveOccurred()) + _, err := s.StoresFind(&pb.StoresFindOptions{Key: &pb.StoresKey{Floats: []float32{1}}, TopK: 0}) + Expect(err).To(MatchError(ContainSubstring("topK"))) + }) +}) diff --git a/backend/index.yaml b/backend/index.yaml index 8b2efd417..5344a6e53 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -2055,6 +2055,25 @@ capabilities: default: "cpu-valkey-store" metal: "metal-valkey-store" +- &qdrant-store + name: "qdrant-store" + urls: + - https://github.com/mudler/LocalAI + - https://github.com/qdrant/qdrant + description: | + Qdrant Store is a Qdrant-backed vector store for LocalAI. It persists + vectors across restarts in HNSW-indexed collections and works with + self-hosted Qdrant (single node or cluster) and Qdrant Cloud. Requires a + reachable Qdrant server (gRPC port, 6334 by default). + tags: + - vector-database + - qdrant + - open-source + - CPU + license: MIT + capabilities: + default: "cpu-qdrant-store" + metal: "metal-qdrant-store" - &kitten-tts name: "kitten-tts" urls: @@ -2652,6 +2671,35 @@ uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-valkey-store" mirrors: - localai/localai-backends:master-metal-darwin-arm64-valkey-store +- !!merge <<: *qdrant-store + name: "cpu-qdrant-store" + alias: "qdrant-store" + uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-qdrant-store" + mirrors: + - localai/localai-backends:latest-cpu-qdrant-store +- !!merge <<: *qdrant-store + name: "cpu-qdrant-store-development" + alias: "qdrant-store" + uri: "quay.io/go-skynet/local-ai-backends:master-cpu-qdrant-store" + mirrors: + - localai/localai-backends:master-cpu-qdrant-store +- !!merge <<: *qdrant-store + name: "qdrant-store-development" + alias: "qdrant-store" + capabilities: + default: "cpu-qdrant-store-development" + metal: "metal-qdrant-store-development" +- !!merge <<: *qdrant-store + name: "metal-qdrant-store" + uri: "quay.io/go-skynet/local-ai-backends:latest-metal-darwin-arm64-qdrant-store" + mirrors: + - localai/localai-backends:latest-metal-darwin-arm64-qdrant-store +- !!merge <<: *qdrant-store + name: "metal-qdrant-store-development" + alias: "qdrant-store" + uri: "quay.io/go-skynet/local-ai-backends:master-metal-darwin-arm64-qdrant-store" + mirrors: + - localai/localai-backends:master-metal-darwin-arm64-qdrant-store - !!merge <<: *opus name: "cpu-opus" uri: "quay.io/go-skynet/local-ai-backends:latest-cpu-opus" diff --git a/core/backend/stores.go b/core/backend/stores.go index 6a566af53..778d5f3c8 100644 --- a/core/backend/stores.go +++ b/core/backend/stores.go @@ -32,8 +32,8 @@ type Neighbor struct { Payload []byte } -// NewVectorStore returns a VectorStore backed by the local-store -// gRPC backend, namespaced by storeName so two routers don't collide. +// NewVectorStore returns a VectorStore backed by the store's gRPC backend, +// namespaced by storeName so two routers don't collide. // cl resolves the per-store model config (backend + options); it may be nil, // in which case the store falls back to the default backend and its built-in // defaults. @@ -51,8 +51,8 @@ type localVectorStore struct { storeName string } -func (s *localVectorStore) backend(_ context.Context) (grpc.Backend, error) { - return StoreBackend(s.loader, s.appConfig, s.cl, s.storeName, "") +func (s *localVectorStore) backend(_ context.Context) (grpc.Backend, string, error) { + return storeBackend(s.loader, s.cl, s.storeName, "") } // Search is the top-1 special case of SearchK; delegating keeps the @@ -69,11 +69,11 @@ func (s *localVectorStore) Search(ctx context.Context, vec []float32) (float64, func (s *localVectorStore) SearchK(ctx context.Context, vec []float32, k int) (neighbors []Neighbor, err error) { outcome := "hit" sim := 0.0 - be, berr := s.backend(ctx) + be, backendName, berr := s.backend(ctx) if berr != nil { outcome = "backend_load_error" err = fmt.Errorf("vector store load: %w", berr) - s.recordTrace("", time.Now(), "search", len(vec), 0, outcome, err) + s.recordTrace("", backendName, time.Now(), "search", len(vec), 0, outcome, err) return nil, err } release, err := AcquireGlobalBackendSlot() @@ -82,9 +82,9 @@ func (s *localVectorStore) SearchK(ctx context.Context, vec []float32, k int) (n } defer release() start := time.Now() - traceID := s.beginTrace(start, "search") + traceID := s.beginTrace(backendName, start, "search") defer func() { - s.recordTrace(traceID, start, "search", len(vec), sim, outcome, err) + s.recordTrace(traceID, backendName, start, "search", len(vec), sim, outcome, err) }() _, values, similarities, ferr := store.Find(ctx, be, vec, k) if ferr != nil { @@ -105,11 +105,11 @@ func (s *localVectorStore) SearchK(ctx context.Context, vec []float32, k int) (n func (s *localVectorStore) Insert(ctx context.Context, vec []float32, payload []byte) (err error) { outcome := "ok" - be, berr := s.backend(ctx) + be, backendName, berr := s.backend(ctx) if berr != nil { outcome = "backend_load_error" err = fmt.Errorf("vector store load: %w", berr) - s.recordTrace("", time.Now(), "insert", len(vec), 0, outcome, err) + s.recordTrace("", backendName, time.Now(), "insert", len(vec), 0, outcome, err) return err } release, err := AcquireGlobalBackendSlot() @@ -118,9 +118,9 @@ func (s *localVectorStore) Insert(ctx context.Context, vec []float32, payload [] } defer release() start := time.Now() - traceID := s.beginTrace(start, "insert") + traceID := s.beginTrace(backendName, start, "insert") defer func() { - s.recordTrace(traceID, start, "insert", len(vec), 0, outcome, err) + s.recordTrace(traceID, backendName, start, "insert", len(vec), 0, outcome, err) }() if serr := store.SetSingle(ctx, be, vec, payload); serr != nil { outcome = "insert_error" @@ -138,11 +138,11 @@ func (s *localVectorStore) InsertBatch(ctx context.Context, vecs [][]float32, pa if len(vecs) > 0 { dim = len(vecs[0]) } - be, berr := s.backend(ctx) + be, backendName, berr := s.backend(ctx) if berr != nil { outcome = "backend_load_error" err = fmt.Errorf("vector store load: %w", berr) - s.recordTrace("", time.Now(), "insert_batch", dim, 0, outcome, err) + s.recordTrace("", backendName, time.Now(), "insert_batch", dim, 0, outcome, err) return err } release, err := AcquireGlobalBackendSlot() @@ -151,9 +151,9 @@ func (s *localVectorStore) InsertBatch(ctx context.Context, vecs [][]float32, pa } defer release() start := time.Now() - traceID := s.beginTrace(start, "insert_batch") + traceID := s.beginTrace(backendName, start, "insert_batch") defer func() { - s.recordTrace(traceID, start, "insert_batch", dim, 0, outcome, err) + s.recordTrace(traceID, backendName, start, "insert_batch", dim, 0, outcome, err) }() if serr := store.SetCols(ctx, be, vecs, payloads); serr != nil { outcome = "insert_error" @@ -171,11 +171,11 @@ func (s *localVectorStore) Delete(ctx context.Context, vecs [][]float32) (err er if len(vecs) > 0 { dim = len(vecs[0]) } - be, berr := s.backend(ctx) + be, backendName, berr := s.backend(ctx) if berr != nil { outcome = "backend_load_error" err = fmt.Errorf("vector store load: %w", berr) - s.recordTrace("", time.Now(), "delete", dim, 0, outcome, err) + s.recordTrace("", backendName, time.Now(), "delete", dim, 0, outcome, err) return err } release, err := AcquireGlobalBackendSlot() @@ -184,9 +184,9 @@ func (s *localVectorStore) Delete(ctx context.Context, vecs [][]float32) (err er } defer release() start := time.Now() - traceID := s.beginTrace(start, "delete") + traceID := s.beginTrace(backendName, start, "delete") defer func() { - s.recordTrace(traceID, start, "delete", dim, 0, outcome, err) + s.recordTrace(traceID, backendName, start, "delete", dim, 0, outcome, err) }() if serr := store.DeleteCols(ctx, be, vecs); serr != nil { outcome = "delete_error" @@ -198,17 +198,16 @@ func (s *localVectorStore) Delete(ctx context.Context, vecs [][]float32) (err er // recordTrace surfaces vector-store calls in /api/backend-traces, including // the backend-load-failure path that otherwise vanishes into an xlog.Warn. // modelName uses the store namespace (e.g. "router-cache-smart-router") so -// admins can tell which router's cache misbehaved; the backend is always -// "local-store" and can't disambiguate. -func (s *localVectorStore) beginTrace(start time.Time, op string) string { +// admins can tell which router's cache misbehaved. +func (s *localVectorStore) beginTrace(backendName string, start time.Time, op string) string { if s.appConfig == nil || !s.appConfig.EnableTracing { return "" } trace.InitBackendTracingIfEnabled(s.appConfig.TracingMaxItems, s.appConfig.TracingMaxBodyBytes) - return trace.BeginBackendTrace(trace.BackendTrace{Timestamp: start, Type: trace.BackendTraceVectorStore, ModelName: s.storeName, Backend: model.LocalStoreBackend, Summary: op}) + return trace.BeginBackendTrace(trace.BackendTrace{Timestamp: start, Type: trace.BackendTraceVectorStore, ModelName: s.storeName, Backend: backendName, Summary: op}) } -func (s *localVectorStore) recordTrace(traceID string, start time.Time, op string, vecDim int, sim float64, outcome string, err error) { +func (s *localVectorStore) recordTrace(traceID, backendName string, start time.Time, op string, vecDim int, sim float64, outcome string, err error) { if s.appConfig == nil || !s.appConfig.EnableTracing { return } @@ -237,7 +236,7 @@ func (s *localVectorStore) recordTrace(traceID string, start time.Time, op strin Duration: time.Since(start), Type: trace.BackendTraceVectorStore, ModelName: s.storeName, - Backend: model.LocalStoreBackend, + Backend: backendName, Summary: summary, Error: errStr, Data: data, @@ -245,26 +244,43 @@ func (s *localVectorStore) recordTrace(traceID string, start time.Time, op strin } func StoreBackend(sl *model.ModelLoader, appConfig *config.ApplicationConfig, cl *config.ModelConfigLoader, storeName string, backend string) (grpc.Backend, error) { - // Resolve the per-store model config (keyed by the store namespace, which - // is the model ID for a store). This is the LocalAI-native config surface: - // a store's backend selection and its backend-specific settings live in a - // model YAML's `backend:` and `options:` fields, so different stores can - // point at different servers/indexes. When no config exists for the store, - // we fall back to the default backend and let the backend apply its own - // built-in defaults — preserving the zero-config experience. - var loadOpts []string + be, _, err := storeBackend(sl, cl, storeName, backend) + return be, err +} + +func storeBackend(sl *model.ModelLoader, cl *config.ModelConfigLoader, storeName string, backend string) (grpc.Backend, string, error) { + backend, loadOpts, modelID := resolveStoreBackend(cl, storeName, backend) + be, err := loadStoreBackend(sl, modelID, storeName, backend, loadOpts) + return be, model.CanonicalBackend(backend), err +} + +// The loader caches processes by ID alone, so an explicit backend other than +// the store's own needs its own ID. +func resolveStoreBackend(cl *config.ModelConfigLoader, storeName string, backend string) (string, []string, string) { + home := model.LocalStoreBackend + var cfgBackend string + var opts []string if cl != nil { if cfg, ok := cl.GetModelConfig(storeName); ok { - if backend == "" { - backend = cfg.Backend + cfgBackend, opts = cfg.Backend, cfg.Options + if cfgBackend != "" { + home = cfgBackend } - loadOpts = cfg.Options } } - if backend == "" { - backend = model.LocalStoreBackend + backend = home } + if model.CanonicalBackend(backend) == model.CanonicalBackend(home) { + return backend, opts, storeName + } + if cfgBackend != "" { + opts = nil + } + return backend, opts, storeName + "@" + model.CanonicalBackend(backend) +} + +func loadStoreBackend(sl *model.ModelLoader, modelID, storeName, backend string, loadOpts []string) (grpc.Backend, error) { // ModelLoader caches backend processes by `modelID`, not by the `model` // passed via WithModel. Without a distinct modelID, every StoreBackend // call collapses to the same `modelID=""` cache slot — face (512-D) and @@ -281,7 +297,7 @@ func StoreBackend(sl *model.ModelLoader, appConfig *config.ApplicationConfig, cl // convention upgrades in lockstep). sc := []model.Option{ model.WithBackendString(backend), - model.WithModelID(storeName), + model.WithModelID(modelID), model.WithModel(store.NamespacePrefix + storeName), } diff --git a/core/backend/stores_test.go b/core/backend/stores_test.go index e9d5208a3..f887c594d 100644 --- a/core/backend/stores_test.go +++ b/core/backend/stores_test.go @@ -70,6 +70,36 @@ var _ = Describe("localVectorStore tracing", func() { Expect(bt.Error).To(ContainSubstring("vector store load")) }) + It("records the backend selected by the store's model config, not local-store", func() { + appCfg := &config.ApplicationConfig{ + EnableTracing: true, + TracingMaxItems: 16, + TracingMaxBodyBytes: 1024, + } + cl := config.NewModelConfigLoader(GinkgoT().TempDir()) + cl.ReplaceModelConfigs([]config.ModelConfig{ + {Name: "router-cache-valkey", Backend: "valkey"}, + }) + s := &localVectorStore{ + loader: model.NewModelLoader(&system.SystemState{}), + appConfig: appCfg, + cl: cl, + storeName: "router-cache-valkey", + } + + err := s.Insert(context.Background(), []float32{0.1, 0.2}, []byte("payload")) + Expect(err).To(HaveOccurred()) + + Eventually(func() *trace.BackendTrace { + return findVectorStoreTrace("router-cache-valkey") + }).ShouldNot(BeNil()) + + bt := findVectorStoreTrace("router-cache-valkey") + Expect(bt.Backend).To(Equal(model.ValkeyStoreBackend)) + Expect(bt.Data["op"]).To(Equal("insert")) + Expect(bt.Data["outcome"]).To(Equal("backend_load_error")) + }) + It("does not record a trace when tracing is disabled", func() { // Opt-out path: appConfig.EnableTracing=false must short-circuit // before InitBackendTracingIfEnabled, so a workload with tracing @@ -86,3 +116,78 @@ var _ = Describe("localVectorStore tracing", func() { }).Should(BeNil()) }) }) + +var _ = Describe("resolveStoreBackend", func() { + It("defaults to local-store with no config and no explicit backend", func() { + name, opts, id := resolveStoreBackend(nil, "some-store", "") + Expect(name).To(Equal(model.LocalStoreBackend)) + Expect(opts).To(BeEmpty()) + Expect(id).To(Equal("some-store")) + }) + + It("uses the store's model config backend and options", func() { + cl := config.NewModelConfigLoader(GinkgoT().TempDir()) + cl.ReplaceModelConfigs([]config.ModelConfig{{ + Name: "faces", + Backend: model.QdrantStoreBackend, + Options: []string{"addr:qdrant:6334"}, + }}) + name, opts, id := resolveStoreBackend(cl, "faces", "") + Expect(name).To(Equal(model.QdrantStoreBackend)) + Expect(opts).To(Equal([]string{"addr:qdrant:6334"})) + Expect(id).To(Equal("faces")) + }) + + It("keeps the config's options and slot when the explicit backend is the configured one (aliases included)", func() { + cl := config.NewModelConfigLoader(GinkgoT().TempDir()) + cl.ReplaceModelConfigs([]config.ModelConfig{{ + Name: "faces", + Backend: model.QdrantStoreBackend, + Options: []string{"addr:qdrant:6334"}, + }}) + name, opts, id := resolveStoreBackend(cl, "faces", "qdrant") + Expect(name).To(Equal("qdrant")) + Expect(opts).To(Equal([]string{"addr:qdrant:6334"})) + Expect(id).To(Equal("faces")) + }) + + It("gives an explicit override of another backend its own slot and not the config's options", func() { + cl := config.NewModelConfigLoader(GinkgoT().TempDir()) + cl.ReplaceModelConfigs([]config.ModelConfig{{ + Name: "faces", + Backend: model.ValkeyStoreBackend, + Options: []string{"addr:valkey:6379"}, + }}) + name, opts, id := resolveStoreBackend(cl, "faces", model.QdrantStoreBackend) + Expect(name).To(Equal(model.QdrantStoreBackend)) + Expect(opts).To(BeEmpty()) + Expect(id).To(Equal("faces@qdrant-store")) + }) + + It("passes the options of a config without backend: to an explicit backend", func() { + cl := config.NewModelConfigLoader(GinkgoT().TempDir()) + cl.ReplaceModelConfigs([]config.ModelConfig{{ + Name: "vectors", + Options: []string{"addr:valkey:6379"}, + }}) + name, opts, id := resolveStoreBackend(cl, "vectors", model.ValkeyStoreBackend) + Expect(name).To(Equal(model.ValkeyStoreBackend)) + Expect(opts).To(Equal([]string{"addr:valkey:6379"})) + Expect(id).To(Equal("vectors@valkey-store")) + }) + + It("separates an override from the default local-store slot when there is no config", func() { + _, _, id := resolveStoreBackend(nil, "mix", "qdrant") + Expect(id).To(Equal("mix@qdrant-store")) + _, _, id = resolveStoreBackend(nil, "mix", "embedded-store") + Expect(id).To(Equal("mix")) + }) + + It("compares backends by canonical name (aliases, case)", func() { + Expect(model.CanonicalBackend("qdrant")).To(Equal(model.QdrantStoreBackend)) + Expect(model.CanonicalBackend("Valkey")).To(Equal(model.ValkeyStoreBackend)) + Expect(model.CanonicalBackend("embedded-store")).To(Equal(model.LocalStoreBackend)) + Expect(model.CanonicalBackend(model.QdrantStoreBackend)).To(Equal(model.QdrantStoreBackend)) + Expect(model.CanonicalBackend("Qdrant-Store")).To(Equal(model.QdrantStoreBackend)) + }) +}) diff --git a/docs/content/features/backends.md b/docs/content/features/backends.md index 69ae0cf0f..b68b74d2d 100644 --- a/docs/content/features/backends.md +++ b/docs/content/features/backends.md @@ -218,7 +218,7 @@ LocalAI supports various types of backends: - **Vision & Detection Backends**: For object detection, segmentation, depth, and face/voice recognition (e.g., rf-detr.cpp, locate-anything.cpp, sam3.cpp, insightface) - **Audio Processing Backends**: For voice activity detection and audio enhancement (e.g., Silero VAD, LocalVQE, [audio.cpp]({{%relref "features/audio-cpp" %}})) - **Source Separation & Voice Conversion Backends**: For splitting a mix into named stems (vocals, drums, bass) and for converting speech or singing to a target voice (e.g., [audio.cpp]({{%relref "features/audio-cpp" %}})) -- **Utility Backends**: For reranking, PII/NER token classification, fine-tuning, quantization, and vector storage (e.g., rerankers, privacy-filter.cpp, TRL, local-store, valkey-store) +- **Utility Backends**: For reranking, PII/NER token classification, fine-tuning, quantization, and vector storage (e.g., rerankers, privacy-filter.cpp, TRL, local-store, valkey-store, qdrant-store) See the [Backend & Model Compatibility Table]({{%relref "reference/compatibility-table" %}}) for the full catalog. diff --git a/docs/content/features/stores.md b/docs/content/features/stores.md index 1d2a3d851..a28b44a39 100644 --- a/docs/content/features/stores.md +++ b/docs/content/features/stores.md @@ -51,12 +51,28 @@ but you can select a different store backend per request (see [Backends](#backen ## Backends Each `/stores/*` request accepts an optional `backend` field selecting the store implementation. -Two backends ship with LocalAI: +Three backends ship with LocalAI: | Backend | `backend` value | Persistence | Notes | |---------|-----------------|-------------|-------| | Local (default) | `local-store` (alias `embedded-store`) | In-memory, lost on restart | Exact cosine similarity, zero configuration. | | Valkey Search | `valkey-store` (alias `valkey`) | Durable (Valkey RDB/AOF) | Backed by a Valkey Search (`FT.*`) server; survives restarts and supports opt-in HNSW. | +| Qdrant | `qdrant-store` (alias `qdrant`) | Durable (Qdrant storage) | Backed by a Qdrant server or cluster, or Qdrant Cloud; HNSW-indexed; on a single node a `find` right after a `set` sees the new vectors. | + +Internal features that use stores (face and voice recognition, the router's KNN classifier and +embedding cache) don't send a `backend` field. To move the router's stores (`router-cache-` +and `router-corpus-`) to a durable backend, create a model config named after the store with +the `backend:` and `options:` shown below. Keep the default cosine `distance_metric` for them: they +compare similarities against cosine thresholds. Keep face and voice recognition on `local-store` +for now: their registries remember enrolled IDs only in memory, so after a restart entries in a +durable store could no longer be forgotten. With tracing enabled, the router's vector-store calls +appear in `/api/backend-traces` under the backend each store resolved to. + +A `backend` field that names a different backend than the store's own (its model config's +`backend:`, or `local-store` when there is none) is served by a separate backend instance, so it +never lands on the store's usual backend. It gets the model config's `options:` only if the config +has no `backend:`; otherwise those options belong to the configured backend, and it gets its own +defaults. ### Valkey store backend @@ -146,6 +162,83 @@ bundle; `tls_skip_verify:true` disables verification entirely and should only be local testing. {{% /notice %}} +### Qdrant store backend + +The `qdrant-store` backend keeps vectors in a [Qdrant](https://qdrant.tech/) collection, so the +data survives a LocalAI restart. It works with a single Qdrant node, a Qdrant cluster and Qdrant +Cloud. It talks to Qdrant's **gRPC** port (`6334` by default), not the `6333` REST port. + +Select it by passing `"backend": "qdrant-store"` (or the `"qdrant"` alias) on any `/stores/*` request: + +``` +curl -X POST http://localhost:8080/stores/set \ + -H "Content-Type: application/json" \ + -d '{"backend": "qdrant-store", "store": "my-vectors", "keys": [[0.1, 0.2], [0.3, 0.4]], "values": ["foo", "bar"]}' +``` + +As with Valkey, the connection and collection are configured through a **model config** named +after the store, with the settings in `options:` as `key:value` strings: + +```yaml +name: my-vectors +backend: qdrant-store +options: + - addr:xyz-example.eu-central.aws.cloud.qdrant.io:6334 + - tls:true + - api_key_env:QDRANT_API_KEY + - distance_metric:COSINE +``` + +When no config exists for a store, the backend connects to `localhost:6334` with the defaults +below. + +Each store gets its own collection, named `localai_-` (for example +`localai_my-vectors-1a2b3c4d`) unless you set `collection`. It is created on the first `set` with +the dimension of the first key, and Qdrant rejects keys and queries of any other dimension, even +after every key is deleted. To change the dimension (for example after switching embedding +models), delete the collection in Qdrant or set a new `collection`. The backend never deletes +collections itself, since with several writers that could lose another writer's data. + +Keys are matched exactly, and a repeated key keeps the last value. A `set` is a single Qdrant +request, so a rejected `set` writes nothing. `NaN` and `Inf` components are rejected. + +| Option | Default | Description | +|--------|---------|-------------| +| `addr` | `localhost:6334` | Qdrant gRPC address (`host:port`; the port defaults to `6334`). Not a URL — use `tls:true` for HTTPS endpoints. | +| `api_key` | *(empty)* | Qdrant API key (plaintext in config). | +| `api_key_env` | *(empty)* | Name of an env var that holds the API key. Preferred over `api_key` — keeps the secret out of the model YAML. | +| `tls` | `false` | Enable TLS (required by Qdrant Cloud). | +| `tls_ca_cert` | *(empty)* | Path to a PEM CA bundle used to verify the server certificate (self-signed / private CA). Turns on `tls`. | +| `tls_skip_verify` | `false` | Skip TLS certificate verification. Insecure — for testing only. Turns on `tls`. | +| `collection` | *(derived)* | Use this exact collection name instead of the derived one. | +| `distance_metric` | `COSINE` | `COSINE`, `EUCLID`, `DOT` or `MANHATTAN` (case-insensitive). | +| `request_timeout_ms` | `5000` | Per-request timeout in milliseconds. | + +For `COSINE` the returned `similarities` follow the local store's convention (`1.0` = identical, +`-1.0` = opposite). For `DOT` they are dot products (larger is closer). For `EUCLID` and `MANHATTAN` +they are distances, so smaller is closer, which is the opposite ordering to `COSINE`. In every case +results come back nearest-first. If a collection already exists with a different distance than +`distance_metric`, loading the store (or, if the collection appeared later, the first `find`) fails +rather than returning scores with the wrong meaning. + +{{% notice note %}} +The backend creates collections with Qdrant's default settings. For custom settings (sharding and +replication on a cluster, HNSW parameters, on-disk vectors, quantization), create the collection +in Qdrant yourself and point `collection` at it: an existing collection is used as long as it has a +single unnamed vector with the configured `distance_metric`. +{{% /notice %}} + +{{% notice note %}} +Small collections are searched exactly. Once Qdrant builds the HNSW index (when a segment passes +its indexing threshold, 10 MB of vectors per segment by default, so typically tens of thousands of +embeddings), `find` is approximate, with recall set by the collection's HNSW settings. +{{% /notice %}} + +{{% notice warning %}} +`tls` defaults to `false` (plaintext). Set `tls:true` whenever Qdrant is not on `localhost` or an +API key is configured, otherwise the key and the stored vectors travel the network unencrypted. +{{% /notice %}} + ## Set To set some keys you can do diff --git a/docs/content/reference/compatibility-table.md b/docs/content/reference/compatibility-table.md index 7e6bfd73d..5ce56e3f4 100644 --- a/docs/content/reference/compatibility-table.md +++ b/docs/content/reference/compatibility-table.md @@ -129,6 +129,7 @@ All backends listed here can be installed on demand from the [Backend Gallery]({ | [privacy-filter.cpp](https://github.com/localai-org/privacy-filter.cpp) | Standalone GGML engine for the openai-privacy-filter PII/NER token-classification model family (powers LocalAI's PII redaction tier) | CPU, CUDA 13, Vulkan | | [local-store](https://github.com/mudler/LocalAI) | Local-first vector database for embeddings | CPU, Metal | | [valkey-store](https://github.com/mudler/LocalAI) | Durable vector store for embeddings backed by Valkey Search (FLAT or HNSW) | CPU, Metal | +| [qdrant-store](https://github.com/mudler/LocalAI) | Durable vector store for embeddings backed by Qdrant (HNSW; self-hosted, clustered or Qdrant Cloud) | CPU, Metal | | [TRL](https://github.com/huggingface/trl) | Fine-tuning (SFT, DPO, GRPO, RLOO, KTO, ORPO) | CPU, CUDA 12/13 | | [llama.cpp quantization](https://github.com/ggml-org/llama.cpp) | HuggingFace → GGUF model conversion and quantization | CPU, Metal | diff --git a/go.mod b/go.mod index 1d4025b89..70ee82b71 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/otiai10/openaigo v1.7.0 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 github.com/prometheus/client_golang v1.23.2 + github.com/qdrant/go-client v1.17.1 github.com/robfig/cron/v3 v3.0.1 github.com/russross/blackfriday v1.6.0 github.com/sashabaranov/go-openai v1.41.2 @@ -457,7 +458,7 @@ require ( github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/zeroconf/v2 v2.2.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect - github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect + github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.14 // indirect diff --git a/go.sum b/go.sum index 891a60d05..7d9b69642 100644 --- a/go.sum +++ b/go.sum @@ -921,6 +921,8 @@ github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQ github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3 h1:PwQumkgq4/acIiZhtifTV5OUqqiP82UAl0h87xj/l9k= +github.com/lufia/plan9stats v0.0.0-20251013123823-9fd1530e3ec3/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= @@ -1221,6 +1223,8 @@ github.com/prometheus/otlptranslator v1.0.0 h1:s0LJW/iN9dkIH+EnhiD3BlkkP5QVIUVEo github.com/prometheus/otlptranslator v1.0.0/go.mod h1:vRYWnXvI6aWGpsdY/mOT/cbeVRBlPWtBNDb7kGR3uKM= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= +github.com/qdrant/go-client v1.17.1 h1:7QmPwDddrHL3hC4NfycwtQlraVKRLcRi++BX6TTm+3g= +github.com/qdrant/go-client v1.17.1/go.mod h1:n1h6GhkdAzcohoXt/5Z19I2yxbCkMA6Jejob3S6NZT8= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go index cbfb7840c..3946fcad2 100644 --- a/pkg/model/initializers.go +++ b/pkg/model/initializers.go @@ -29,6 +29,7 @@ var Aliases = map[string]string{ "ik-llama": IKLLamaCPP, "embedded-store": LocalStoreBackend, "valkey": ValkeyStoreBackend, + "qdrant": QdrantStoreBackend, "huggingface-embeddings": TransformersBackend, "transformers-musicgen": TransformersBackend, "sentencetransformers": TransformersBackend, @@ -36,6 +37,14 @@ var Aliases = map[string]string{ "stablediffusion": StableDiffusionGGMLBackend, } +func CanonicalBackend(name string) string { + b := strings.ToLower(name) + if real, ok := Aliases[b]; ok { + return real + } + return b +} + var TypeAlias = map[string]string{ "sentencetransformers": "SentenceTransformer", "huggingface-embeddings": "SentenceTransformer", @@ -51,6 +60,7 @@ const ( TransformersBackend = "transformers" LocalStoreBackend = "local-store" ValkeyStoreBackend = "valkey-store" + QdrantStoreBackend = "qdrant-store" ) // starts the grpcModelProcess for the backend, and returns a grpc client @@ -320,17 +330,9 @@ func (ml *ModelLoader) backendLoader(opts ...Option) (client grpc.Backend, err e "f16", opt.F16Memory) } - backend := strings.ToLower(o.backendString) - if realBackend, exists := Aliases[backend]; exists { - typeAlias, exists := TypeAlias[backend] - if exists { - xlog.Debug("alias is a type alias", "alias", backend, "realBackend", realBackend, "type", typeAlias) - o.gRPCOptions.Type = typeAlias - } else { - xlog.Debug("alias", "alias", backend, "realBackend", realBackend) - } - - backend = realBackend + backend := CanonicalBackend(o.backendString) + if typeAlias, ok := TypeAlias[strings.ToLower(o.backendString)]; ok { + o.gRPCOptions.Type = typeAlias } modelFileName := o.modelFile diff --git a/tests/integration/qdrant_stores_test.go b/tests/integration/qdrant_stores_test.go new file mode 100644 index 000000000..dcdaf5533 --- /dev/null +++ b/tests/integration/qdrant_stores_test.go @@ -0,0 +1,198 @@ +package integration_test + +import ( + "context" + "fmt" + "net" + "os" + "strconv" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/qdrant/go-client/qdrant" + + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/grpc" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/store" + "github.com/mudler/LocalAI/pkg/system" +) + +var _ = Describe("Integration tests for the qdrant-store backend", Label("stores"), Label("qdrant"), func() { + Context("Qdrant get, set, delete and find", func() { + var sl *model.ModelLoader + var sc grpc.Backend + var tmpdir string + var namespace string + var qdrantAddr string + var backendsPath string + + tryLoadStore := func(ns string, extra ...string) (grpc.Backend, error) { + storeOpts := []model.Option{ + model.WithBackendString(model.QdrantStoreBackend), + model.WithModelID(ns), + model.WithModel(store.NamespacePrefix + ns), + model.WithLoadGRPCLoadModelOpts(&pb.ModelOptions{ + Options: append([]string{"addr:" + qdrantAddr}, extra...), + }), + } + return sl.Load(storeOpts...) + } + + loadStore := func(ns string, extra ...string) grpc.Backend { + backend, err := tryLoadStore(ns, extra...) + Expect(err).ToNot(HaveOccurred()) + return backend + } + + initLoader := func() { + systemState, err := system.GetSystemState( + system.WithModelPath(tmpdir), + system.WithBackendPath(backendsPath), + ) + Expect(err).ToNot(HaveOccurred()) + + sl = model.NewModelLoader(systemState) + Expect(gallery.RegisterBackends(systemState, sl)).To(Succeed()) + } + + BeforeEach(func() { + qdrantAddr = os.Getenv("QDRANT_ADDR") + if qdrantAddr == "" { + Skip("QDRANT_ADDR is not set; skipping Qdrant integration tests") + } + backendsPath = os.Getenv("BACKENDS_PATH") + if backendsPath == "" { + Skip("BACKENDS_PATH is not set; build the backend and point BACKENDS_PATH at it (see make test-qdrant-store)") + } + + var err error + tmpdir, err = os.MkdirTemp("", "") + Expect(err).ToNot(HaveOccurred()) + + namespace = fmt.Sprintf("it-%d-%d", GinkgoRandomSeed(), namespaceCounter.Add(1)) + + initLoader() + sc = loadStore(namespace) + }) + + AfterEach(func() { + if sl != nil { + Expect(sl.StopAllGRPC()).To(Succeed()) + } + if tmpdir != "" { + _ = os.RemoveAll(tmpdir) + } + dropNamespaceCollections(qdrantAddr, namespace) + }) + + It("should set, get and delete keys", func() { + err := store.SetCols(context.Background(), sc, [][]float32{{0.1, 0.2, 0.3}, {0.4, 0.5, 0.6}, {0.7, 0.8, 0.9}}, [][]byte{[]byte("test1"), []byte("test2"), []byte("test3")}) + Expect(err).ToNot(HaveOccurred()) + + keys, vals, err := store.GetCols(context.Background(), sc, [][]float32{{0.7, 0.8, 0.9}, {9, 9, 9}, {0.1, 0.2, 0.3}}) + Expect(err).ToNot(HaveOccurred()) + Expect(keys).To(Equal([][]float32{{0.7, 0.8, 0.9}, {0.1, 0.2, 0.3}})) + Expect(vals).To(Equal([][]byte{[]byte("test3"), []byte("test1")})) + + err = store.DeleteCols(context.Background(), sc, [][]float32{{0.1, 0.2, 0.3}, {0.7, 0.8, 0.9}, {5, 5, 5}}) + Expect(err).ToNot(HaveOccurred()) + + keys, vals, err = store.GetCols(context.Background(), sc, [][]float32{{0.1, 0.2, 0.3}, {0.4, 0.5, 0.6}, {0.7, 0.8, 0.9}}) + Expect(err).ToNot(HaveOccurred()) + Expect(keys).To(Equal([][]float32{{0.4, 0.5, 0.6}})) + Expect(vals).To(Equal([][]byte{[]byte("test2")})) + }) + + It("should find similar keys, returning the exact unnormalised keys", func() { + err := store.SetCols(context.Background(), sc, [][]float32{{0.5, 0.5, 0.5}, {0.6, 0.6, -0.6}, {0.7, -0.7, -0.7}}, [][]byte{[]byte("test1"), []byte("test2"), []byte("test3")}) + Expect(err).ToNot(HaveOccurred()) + + keys, vals, sims, err := store.Find(context.Background(), sc, []float32{0.1, 0.3, 0.5}, 2) + Expect(err).ToNot(HaveOccurred()) + Expect(keys).To(HaveLen(2)) + Expect(sims).To(HaveLen(2)) + Expect(keys[0]).To(Equal([]float32{0.5, 0.5, 0.5})) + Expect(vals[0]).To(Equal([]byte("test1"))) + Expect(keys[1]).To(Equal([]float32{0.6, 0.6, -0.6})) + }) + + It("persists data across a backend restart", func() { + err := store.SetCols(context.Background(), sc, + [][]float32{{0.1, 0.2, 0.3}, {0.4, 0.5, 0.6}}, + [][]byte{[]byte("persisted1"), []byte("persisted2")}) + Expect(err).ToNot(HaveOccurred()) + + Expect(sl.StopAllGRPC()).To(Succeed()) + + initLoader() + sc = loadStore(namespace) + + val, err := store.GetSingle(context.Background(), sc, []float32{0.1, 0.2, 0.3}) + Expect(err).ToNot(HaveOccurred()) + Expect(val).To(Equal([]byte("persisted1"))) + + keys, _, _, err := store.Find(context.Background(), sc, []float32{0.1, 0.2, 0.3}, 2) + Expect(err).ToNot(HaveOccurred()) + Expect(keys).To(HaveLen(2)) + + Expect(store.SetSingle(context.Background(), sc, []float32{0.1, 0.2}, []byte("bad"))).ToNot(Succeed()) + }) + + It("creates, recreates and checks a named collection on the server", func() { + Expect(sl.StopAllGRPC()).To(Succeed()) + initLoader() + collection := "localai_it_" + namespace + sc = loadStore(namespace, "collection:"+collection, "distance_metric:DOT") + Expect(store.SetSingle(context.Background(), sc, []float32{1, 2, 3, 4}, []byte("a"))).To(Succeed()) + + host, port := splitAddr(qdrantAddr) + client, err := qdrant.NewClient(&qdrant.Config{Host: host, Port: port, SkipCompatibilityCheck: true}) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(client.Close) + DeferCleanup(func() { _ = client.DeleteCollection(context.Background(), collection) }) + + info, err := client.GetCollectionInfo(context.Background(), collection) + Expect(err).ToNot(HaveOccurred()) + params := info.GetConfig().GetParams().GetVectorsConfig().GetParams() + Expect(params.GetSize()).To(Equal(uint64(4))) + Expect(params.GetDistance()).To(Equal(qdrant.Distance_Dot)) + Expect(info.GetPointsCount()).To(Equal(uint64(1))) + + Expect(client.DeleteCollection(context.Background(), collection)).To(Succeed()) + Expect(store.SetSingle(context.Background(), sc, []float32{1, 2}, []byte("b"))).To(Succeed()) + + Expect(sl.StopAllGRPC()).To(Succeed()) + initLoader() + _, err = tryLoadStore(namespace, "collection:"+collection, "distance_metric:COSINE") + Expect(err).To(MatchError(ContainSubstring("distance_metric is Cosine"))) + }) + }) +}) + +func splitAddr(addr string) (string, int) { + host, portStr, err := net.SplitHostPort(addr) + Expect(err).ToNot(HaveOccurred()) + port, err := strconv.Atoi(portStr) + Expect(err).ToNot(HaveOccurred()) + return host, port +} + +func dropNamespaceCollections(addr, namespace string) { + if addr == "" || namespace == "" { + return + } + host, port := splitAddr(addr) + client, err := qdrant.NewClient(&qdrant.Config{Host: host, Port: port, SkipCompatibilityCheck: true}) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = client.Close() }() + names, err := client.ListCollections(context.Background()) + Expect(err).ToNot(HaveOccurred()) + for _, n := range names { + if strings.HasPrefix(n, "localai_"+namespace+"-") { + Expect(client.DeleteCollection(context.Background(), n)).To(Succeed()) + } + } +} diff --git a/website/data/engines.yaml b/website/data/engines.yaml index 96f8f1948..36c714a88 100644 --- a/website/data/engines.yaml +++ b/website/data/engines.yaml @@ -267,7 +267,7 @@ engines: - Ships in-tree and is the default, so embeddings work on a fresh install - Exact cosine similarity, zero configuration - Backs RAG, the face and voice registries and the semantic router cache - - Swap in valkey-store per request when you need durability + - Swap in valkey-store or qdrant-store per request when you need durability - name: apex-quant tagline: Fit a 35B mixture-of-experts model on a card you already own, and watch it run faster than the full-size build.