feat(search): semantic image search with CLIP embeddings

semantic:"..." in KQL embeds the query text (immich-ml, multilingual CLIP)
and ranks image vectors by cosine similarity: bleve via faiss KNN behind the
new vectors build tag (RRF fusion, vector round-trip through a stored-only
field), OpenSearch via knn_vector plus client-side RRF. The filter part of
the query keeps its meaning and stays the only source of totals and facets.

opencloud/Makefile

@@ -7,6 +7,12 @@ ifdef ENABLE_VIPS
 	TAGS := ${TAGS},enable_vips
 endif
 
+# bleve KNN support; needs libfaiss_c at build and run time (see the vectors
+# stage in docker/Dockerfile.multiarch)
+ifdef ENABLE_VECTORS
+	TAGS := ${TAGS},vectors
+endif
+
 ifneq (, $(shell command -v go 2> /dev/null)) # suppress `command not found warnings` for non go targets in CI
 include ../.bingo/Variables.mk
 endif

opencloud/docker/Dockerfile.multiarch

@@ -1,3 +1,24 @@
+FROM quay.io/opencloudeu/golang-ci:1.25 AS faiss
+# pinned to the last bleve-branch commit before the size-API change in
+# Index_c_ex.h, matching the vendored go-faiss v1.1.0
+ARG FAISS_REF=3ea23cd9bfdc9b0ca6d48beca0c85cb208a18119
+RUN apk add --no-cache cmake g++ make git openblas-dev && \
+    git clone https://github.com/blevesearch/faiss.git /faiss && \
+    git -C /faiss checkout --detach "$FAISS_REF" && \
+    cmake -S /faiss -B /faiss/build \
+      -DFAISS_ENABLE_GPU=OFF \
+      -DFAISS_ENABLE_PYTHON=OFF \
+      -DFAISS_ENABLE_C_API=ON \
+      -DBUILD_SHARED_LIBS=ON \
+      -DBUILD_TESTING=OFF \
+      -DFAISS_OPT_LEVEL=generic \
+      -DCMAKE_BUILD_TYPE=Release && \
+    make -C /faiss/build -j"$(nproc)" faiss faiss_c && \
+    make -C /faiss/build install && \
+    cp /faiss/build/c_api/libfaiss_c.so /usr/local/lib/ && \
+    mkdir -p /usr/local/include/faiss/c_api && \
+    cd /faiss/c_api && find . -name '*.h' -exec cp --parents {} /usr/local/include/faiss/c_api/ \;
+
 FROM quay.io/opencloudeu/golang-ci:1.25 AS build
 ARG TARGETOS
 ARG TARGETARCH
@@ -6,13 +27,18 @@ ARG STRING
 ARG EDITION="dev"
 ARG SRCDIR
 
+# openblas resolves the BLAS/LAPACK symbols libfaiss references at link time
+RUN apk add --no-cache openblas
+COPY --from=faiss /usr/local/lib/libfaiss.so /usr/local/lib/libfaiss_c.so /usr/local/lib/
+COPY --from=faiss /usr/local/include/faiss /usr/local/include/faiss
+
 WORKDIR /build
 RUN --mount=type=bind,target=/build,rw \
     --mount=type=cache,target=/go/pkg/mod \
     --mount=type=cache,target=/root/.cache \
     GOOS="${TARGETOS:-linux}" GOARCH="${TARGETARCH:-amd64}" ; \
     make -C ${SRCDIR:-.}/opencloud release-linux-docker-${TARGETARCH} \
-        ENABLE_VIPS=true DIST=/dist \
+        ENABLE_VIPS=true ENABLE_VECTORS=true DIST=/dist \
         VERSION="${VERSION}" EDITION="${EDITION}" \
         ${STRING:+STRING="${STRING}"}
 
@@ -23,10 +49,12 @@ ARG TARGETOS
 ARG TARGETARCH
 
 RUN apk add --no-cache attr bash ca-certificates curl imagemagick \
-                       inotify-tools libc6-compat mailcap tree vips \
-		       vips-magick patch && \
+                       inotify-tools libc6-compat libgomp libstdc++ mailcap \
+                       openblas tree vips vips-magick patch && \
     echo 'hosts: files dns' >| /etc/nsswitch.conf
 
+COPY --from=faiss /usr/local/lib/libfaiss.so /usr/local/lib/libfaiss_c.so /usr/lib/
+
 LABEL maintainer="openCloud GmbH <devops@opencloud.eu>" \
   org.opencontainers.image.title="OpenCloud" \
   org.opencontainers.image.vendor="OpenCloud GmbH" \

services/search/pkg/bleve/backend.go

@@ -2,6 +2,7 @@ package bleve
 
 import (
 	"context"
+	"fmt"
 	"math"
 	"time"
 
@@ -24,45 +25,82 @@ import (
 
 const defaultBatchSize = 50
 
+// semanticK picks the number of nearest neighbors for a semantic clause: at
+// least a fusion-friendly window, at most a sane cap (huge page sizes stand
+// for "everything", but a similarity ranking beyond 1000 hits carries no
+// signal, see the KNNRequest semantics: k results per clause, no threshold).
+func semanticK(size int) int64 {
+	const minK, maxK = 200, 1000
+	switch {
+	case size >= maxK:
+		return maxK
+	case size < minK:
+		return minK
+	default:
+		return int64(size)
+	}
+}
+
 var _ search.Engine = (*Backend)(nil) // ensure Backend implements Engine
 
 type Backend struct {
 	index        bleve.Index
 	queryCreator searchQuery.Creator[query.Query]
+	vectorizer   search.TextVectorizer
 	log          log.Logger
 }
 
-func NewBackend(index bleve.Index, queryCreator searchQuery.Creator[query.Query], log log.Logger) *Backend {
-	return &Backend{
+// Option configures a Backend.
+type Option func(*Backend)
+
+// WithTextVectorizer enables semantic queries (`semantic:"..."`); without it
+// they are rejected.
+func WithTextVectorizer(v search.TextVectorizer) Option {
+	return func(b *Backend) {
+		b.vectorizer = v
+	}
+}
+
+func NewBackend(index bleve.Index, queryCreator searchQuery.Creator[query.Query], log log.Logger, opts ...Option) *Backend {
+	b := &Backend{
 		index:        index,
 		queryCreator: queryCreator,
 		log:          log,
 	}
+	for _, opt := range opts {
+		opt(b)
+	}
+	return b
 }
 
 // Search executes a search request operation within the index.
 // Returns a SearchIndexResponse object or an error.
-func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) {
-	createdQuery, err := b.queryCreator.Create(sir.Query)
+func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) {
+	// the semantic clause ranks, the remaining query filters: it scopes the
+	// vector search and stays the only source of totals. The creator splits
+	// the clause off the parsed KQL tree.
+	createdQuery, semanticText, err := b.queryCreator.CreateWithSemantic(sir.Query)
 	if err != nil {
 		if kql.IsValidationError(err) {
 			return nil, errtypes.BadRequest(err.Error())
 		}
 		return nil, err
 	}
+	pureSemantic := createdQuery == nil && semanticText != ""
 
-	q := bleve.NewConjunctionQuery(
+	// scope holds the restrictions every hit must satisfy; it also pre-filters
+	// the vector search
+	scope := bleve.NewConjunctionQuery(
 		// Skip documents that have been marked as deleted
 		&query.BoolFieldQuery{
 			Bool:     false,
 			FieldVal: "Deleted",
 		},
-		createdQuery,
 	)
 
 	if sir.Ref != nil {
-		q.Conjuncts = append(
-			q.Conjuncts,
+		scope.Conjuncts = append(
+			scope.Conjuncts,
 			&query.TermQuery{
 				FieldVal: "RootID",
 				Term: storagespace.FormatResourceID(
@@ -79,13 +117,25 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
 		// (paths act as references, /Foo and /foo are distinct), so the exact
 		// folder or the folder prefix matches all of, and only, the scope.
 		if requestedPath := utils.MakeRelativePath(sir.Ref.Path); requestedPath != "." {
-			q.Conjuncts = append(q.Conjuncts, query.NewDisjunctionQuery([]query.Query{
+			scope.Conjuncts = append(scope.Conjuncts, query.NewDisjunctionQuery([]query.Query{
 				&query.TermQuery{FieldVal: "Path", Term: requestedPath},
 				&query.PrefixQuery{FieldVal: "Path", Prefix: requestedPath + "/"},
 			}))
 		}
 	}
 
+	// A purely semantic query has no filter part: the hits come exclusively
+	// from the KNN clause (base match_none), so the reported total is the
+	// number of semantic hits, not a library count (a similarity search has
+	// no result set, only a ranking). With a filter part the hits are the
+	// filter matches, re-ranked by the fusion.
+	var q query.Query
+	if pureSemantic {
+		q = query.NewMatchNoneQuery()
+	} else {
+		q = bleve.NewConjunctionQuery(append([]query.Query{}, append(scope.Conjuncts, createdQuery)...)...)
+	}
+
 	bleveReq := bleve.NewSearchRequest(q)
 	bleveReq.Highlight = bleve.NewHighlight()
 
@@ -98,12 +148,43 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
 		bleveReq.Size = int(sir.PageSize)
 	}
 
+	if semanticText != "" {
+		if b.vectorizer == nil {
+			return nil, errtypes.BadRequest("semantic search is not configured")
+		}
+		vector, err := b.vectorizer.VectorizeText(ctx, semanticText)
+		if err != nil {
+			return nil, fmt.Errorf("failed to vectorize the semantic query: %w", err)
+		}
+		// pre-filter the vector search: scope only for a purely semantic
+		// query, the full filter conjunction otherwise
+		knnFilter := query.Query(scope)
+		if !pureSemantic {
+			knnFilter = q
+		}
+		if err := addSemanticKNN(bleveReq, vector, knnFilter, semanticK(bleveReq.Size), !pureSemantic); err != nil {
+			return nil, err
+		}
+	}
+
 	bleveReq.Fields = []string{"*"}
 	res, err := b.index.Search(bleveReq)
 	if err != nil {
 		return nil, err
 	}
 
+	// hybrid semantic: the fusion reports the fused-hit count as Total, so the
+	// filter-part total (the only meaningful one) needs its own cheap count
+	if semanticText != "" && !pureSemantic {
+		countReq := bleve.NewSearchRequest(q)
+		countReq.Size = 0
+		countRes, err := b.index.Search(countReq)
+		if err != nil {
+			return nil, err
+		}
+		res.Total = countRes.Total
+	}
+
 	matches := make([]*searchMessage.Match, 0, len(res.Hits))
 	totalMatches := res.Total
 	for _, hit := range res.Hits {

services/search/pkg/bleve/backend_novectors.go

@@ -0,0 +1,16 @@
+//go:build !vectors
+
+package bleve
+
+import (
+	"github.com/blevesearch/bleve/v2"
+	"github.com/blevesearch/bleve/v2/search/query"
+
+	"github.com/opencloud-eu/reva/v2/pkg/errtypes"
+)
+
+// addSemanticKNN rejects semantic clauses: this build carries no bleve vector
+// support (see the vectors build tag).
+func addSemanticKNN(*bleve.SearchRequest, []float32, query.Query, int64, bool) error {
+	return errtypes.BadRequest("semantic search is not supported by this build")
+}

services/search/pkg/bleve/backend_vectors.go

@@ -0,0 +1,32 @@
+//go:build vectors
+
+package bleve
+
+import (
+	"github.com/blevesearch/bleve/v2"
+	"github.com/blevesearch/bleve/v2/search/query"
+
+	"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
+)
+
+// addSemanticKNN attaches the semantic clause to the request: a KNN search on
+// the _faiss sibling, pre-filtered by the regular query. With a lexical part
+// (hybrid) the two rankings are fused via reciprocal rank fusion; without an
+// explicit Score bleve would naively sum BM25 and vector scores, whose scales
+// are not comparable. A purely semantic query keeps the raw similarity
+// scores: they are comparable across the per-space searches the service layer
+// merges, while rank-based fusion scores are not.
+func addSemanticKNN(req *bleve.SearchRequest, vector []float32, filter query.Query, k int64, hybrid bool) error {
+	req.AddKNNWithFilter("imageVector"+mapping.VectorIndexSuffix, vector, k, 1.0, filter)
+	if hybrid {
+		req.Score = "rrf"
+		// the fusion window defaults to From+Size, which would truncate the
+		// ranking depth to the page size
+		window := req.From + req.Size
+		if window < int(k) {
+			window = int(k)
+		}
+		req.Params = &bleve.RequestParams{ScoreWindowSize: window}
+	}
+	return nil
+}

services/search/pkg/bleve/batch.go

@@ -49,6 +49,10 @@ func (b *Batch) indexResource(id string, r search.Resource) error {
 	if err != nil {
 		return err
 	}
+	// vectors build: copy vector fields to their indexable _faiss sibling
+	// (the field itself is mapped stored-only, see VectorIndexSuffix);
+	// !vectors build: drop them
+	prepareVectorFields(doc, r.SearchFieldOverrides())
 	return b.batch.Index(id, doc)
 }
 

services/search/pkg/bleve/novectors.go

@@ -0,0 +1,18 @@
+//go:build !vectors
+
+package bleve
+
+import (
+	"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
+)
+
+// prepareVectorFields drops vector fields from the document: this build cannot
+// search them, so storing them would only cost space. Enabling vectors later
+// requires a reindex anyway (the faiss part of the index was never populated).
+func prepareVectorFields(doc map[string]any, overrides map[string]mapping.FieldOpts) {
+	for key, opts := range overrides {
+		if opts.Type == mapping.TypeVector {
+			delete(doc, key)
+		}
+	}
+}

services/search/pkg/bleve/semantic_vectors_test.go

@@ -0,0 +1,145 @@
+//go:build vectors
+
+package bleve_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	bleveSearch "github.com/blevesearch/bleve/v2"
+	"github.com/blevesearch/bleve/v2/index/scorch"
+	sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/opencloud-eu/opencloud/pkg/log"
+	searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
+	searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
+	"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
+	"github.com/opencloud-eu/opencloud/services/search/pkg/content"
+	bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve"
+	"github.com/opencloud-eu/opencloud/services/search/pkg/search"
+)
+
+// axisVector returns a one-hot vector of the schema dimensionality: distinct
+// axes are orthogonal (cosine 0), same axes identical (cosine 1).
+func axisVector(axis int) []float32 {
+	v := make([]float32, content.ImageVectorDims)
+	v[axis] = 1
+	return v
+}
+
+// fakeVectorizer maps query texts to fixed vectors.
+type fakeVectorizer struct{ byText map[string][]float32 }
+
+func (f fakeVectorizer) VectorizeText(_ context.Context, text string) ([]float32, error) {
+	v, ok := f.byText[strings.ToLower(text)]
+	if !ok {
+		return nil, fmt.Errorf("no vector for %q", text)
+	}
+	return v, nil
+}
+
+var _ = Describe("Semantic search (vectors build)", func() {
+	var (
+		eng *bleve.Backend
+
+		doSearch = func(query string) *searchsvc.SearchIndexResponse {
+			res, err := eng.Search(context.Background(), &searchsvc.SearchIndexRequest{
+				Query: query,
+				Ref: &searchmsg.Reference{
+					ResourceId: &searchmsg.ResourceID{StorageId: "1", SpaceId: "2", OpaqueId: "2"},
+				},
+			})
+			ExpectWithOffset(1, err).ToNot(HaveOccurred())
+			return res
+		}
+
+		names = func(res *searchsvc.SearchIndexResponse) []string {
+			out := make([]string, 0, len(res.Matches))
+			for _, m := range res.Matches {
+				out = append(out, m.GetEntity().GetName())
+			}
+			return out
+		}
+	)
+
+	BeforeEach(func() {
+		mapping, err := bleve.NewMapping()
+		Expect(err).ToNot(HaveOccurred())
+
+		tmpDir, err := os.MkdirTemp("", "bleve-semantic-test-")
+		Expect(err).ToNot(HaveOccurred())
+		idx, err := bleveSearch.NewUsing(tmpDir, mapping, scorch.Name, bleveSearch.Config.DefaultKVStore, nil)
+		Expect(err).ToNot(HaveOccurred())
+		// close before removing: the background persister still writes into
+		// the store directory otherwise
+		DeferCleanup(func() error {
+			if err := idx.Close(); err != nil {
+				return err
+			}
+			return os.RemoveAll(tmpDir)
+		})
+
+		eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}, bleve.WithTextVectorizer(fakeVectorizer{
+			byText: map[string][]float32{
+				"cat": axisVector(0),
+				"dog": axisVector(1),
+			},
+		}))
+
+		upsert := func(id, path, name string, vector []float32) {
+			r := search.Resource{
+				ID:       id,
+				RootID:   "1$2!2",
+				ParentID: "1$2!2",
+				Path:     path,
+				Type:     uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
+				Document: content.Document{Name: name, MimeType: "image/jpeg", ImageVector: vector},
+			}
+			Expect(eng.Upsert(id, r)).To(Succeed())
+		}
+		upsert("1$2!10", "./cat.jpg", "cat.jpg", axisVector(0))
+		upsert("1$2!11", "./dog.jpg", "dog.jpg", axisVector(1))
+		Expect(eng.Upsert("1$2!12", search.Resource{
+			ID: "1$2!12", RootID: "1$2!2", ParentID: "1$2!2", Path: "./note.txt",
+			Type:     uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
+			Document: content.Document{Name: "note.txt", MimeType: "text/plain"},
+		})).To(Succeed())
+	})
+
+	It("ranks the nearest image first on a purely semantic query", func() {
+		res := doSearch(`semantic:"cat"`)
+		Expect(len(res.Matches)).To(BeNumerically(">=", 1))
+		Expect(res.Matches[0].GetEntity().GetName()).To(Equal("cat.jpg"))
+	})
+
+	It("combines the semantic clause with a filter part", func() {
+		res := doSearch(`Name:dog* AND semantic:"cat"`)
+		Expect(names(res)).To(Equal([]string{"dog.jpg"}))
+	})
+
+	It("keeps the vector across move and restore round-trips", func() {
+		Expect(eng.Move("1$2!10", "1$2!2", "./moved-cat.jpg")).To(Succeed())
+		res := doSearch(`semantic:"cat"`)
+		Expect(res.Matches[0].GetEntity().GetName()).To(Equal("moved-cat.jpg"))
+
+		Expect(eng.Delete("1$2!10")).To(Succeed())
+		res = doSearch(`semantic:"cat"`)
+		for _, n := range names(res) {
+			Expect(n).ToNot(Equal("moved-cat.jpg"))
+		}
+
+		Expect(eng.Restore("1$2!10")).To(Succeed())
+		res = doSearch(`semantic:"cat"`)
+		Expect(res.Matches[0].GetEntity().GetName()).To(Equal("moved-cat.jpg"))
+	})
+
+	It("rejects semantic queries without a vectorizer", func() {
+		bare := bleve.NewBackend(nil, bleveQuery.DefaultCreator, log.Logger{})
+		_, err := bare.Search(context.Background(), &searchsvc.SearchIndexRequest{Query: `semantic:"cat"`})
+		Expect(err).To(MatchError(ContainSubstring("not configured")))
+	})
+})

services/search/pkg/bleve/testdata/mapping.golden.json

@@ -702,6 +702,16 @@
           }
         }
       },
+      "imageVector": {
+        "enabled": true,
+        "dynamic": true,
+        "fields": [
+          {
+            "type": "number",
+            "store": true
+          }
+        ]
+      },
       "location": {
         "enabled": true,
         "dynamic": true,

services/search/pkg/bleve/vectors.go

@@ -0,0 +1,13 @@
+//go:build vectors
+
+package bleve
+
+import (
+	"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
+)
+
+// prepareVectorFields copies vector fields to their _faiss siblings, which
+// carry the searchable vector in this build (see mapping.VectorIndexSuffix).
+func prepareVectorFields(doc map[string]any, overrides map[string]mapping.FieldOpts) {
+	mapping.AddVectorIndexSiblings(doc, overrides)
+}

services/search/pkg/clip/client.go

@@ -0,0 +1,117 @@
+// Package clip talks to a CLIP inference service (immich machine-learning API).
+// Images and texts are embedded into the same vector space, so a text query
+// vector can rank image vectors by cosine similarity.
+package clip
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"time"
+)
+
+// DefaultModel is a multilingual CLIP model: German (and other non-English)
+// queries match image content without a translation step.
+const DefaultModel = "XLM-Roberta-Large-Vit-B-32"
+
+// Client is a minimal client for the immich machine-learning /predict API.
+type Client struct {
+	url   string
+	model string
+	hc    *http.Client
+}
+
+// NewClient creates a Client for the inference service at url. A zero timeout
+// falls back to 30 seconds, an empty model to DefaultModel.
+func NewClient(url, model string, timeout time.Duration) *Client {
+	if model == "" {
+		model = DefaultModel
+	}
+	if timeout == 0 {
+		timeout = 30 * time.Second
+	}
+	return &Client{
+		url:   url,
+		model: model,
+		hc:    &http.Client{Timeout: timeout},
+	}
+}
+
+// VectorizeText embeds a text query.
+func (c *Client) VectorizeText(ctx context.Context, text string) ([]float32, error) {
+	return c.predict(ctx, "textual", func(w *multipart.Writer) error {
+		return w.WriteField("text", text)
+	})
+}
+
+// VectorizeImage embeds an image from r.
+func (c *Client) VectorizeImage(ctx context.Context, r io.Reader) ([]float32, error) {
+	return c.predict(ctx, "visual", func(w *multipart.Writer) error {
+		fw, err := w.CreateFormFile("image", "image")
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(fw, r)
+		return err
+	})
+}
+
+// predict posts a multipart request to /predict. The entries field declares
+// which model runs; the response carries the embedding as a JSON-encoded array
+// inside a JSON string field.
+func (c *Client) predict(ctx context.Context, task string, addPayload func(*multipart.Writer) error) ([]float32, error) {
+	var body bytes.Buffer
+	w := multipart.NewWriter(&body)
+
+	entries, err := json.Marshal(map[string]any{
+		"clip": map[string]any{task: map[string]any{"modelName": c.model}},
+	})
+	if err != nil {
+		return nil, err
+	}
+	if err := w.WriteField("entries", string(entries)); err != nil {
+		return nil, err
+	}
+	if err := addPayload(w); err != nil {
+		return nil, err
+	}
+	if err := w.Close(); err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.url+"/predict", &body)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", w.FormDataContentType())
+
+	res, err := c.hc.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		msg, _ := io.ReadAll(io.LimitReader(res.Body, 512))
+		return nil, fmt.Errorf("clip inference failed: %s: %s", res.Status, msg)
+	}
+
+	var payload struct {
+		Clip string `json:"clip"`
+	}
+	if err := json.NewDecoder(res.Body).Decode(&payload); err != nil {
+		return nil, fmt.Errorf("failed to decode clip inference response: %w", err)
+	}
+	var vector []float32
+	if err := json.Unmarshal([]byte(payload.Clip), &vector); err != nil {
+		return nil, fmt.Errorf("failed to decode clip embedding: %w", err)
+	}
+	if len(vector) == 0 {
+		return nil, fmt.Errorf("clip inference returned an empty embedding")
+	}
+	return vector, nil
+}

services/search/pkg/clip/querycache.go

@@ -0,0 +1,82 @@
+package clip
+
+import (
+	"container/list"
+	"context"
+	"strings"
+	"sync"
+
+	"golang.org/x/sync/singleflight"
+)
+
+// QueryCache caches text embeddings behind an LRU: users type the same terms
+// repeatedly, and every miss is a network round-trip to the inference service.
+// A singleflight collapses the concurrent misses the per-space search fan-out
+// produces for one and the same query.
+type QueryCache struct {
+	next interface {
+		VectorizeText(ctx context.Context, text string) ([]float32, error)
+	}
+	group singleflight.Group
+
+	mu      sync.Mutex
+	maxSize int
+	entries map[string]*list.Element
+	order   *list.List // front = most recently used
+}
+
+type cacheEntry struct {
+	key    string
+	vector []float32
+}
+
+// NewQueryCache wraps a text vectorizer with an LRU of the given size.
+func NewQueryCache(next *Client, size int) *QueryCache {
+	if size <= 0 {
+		size = 512
+	}
+	return &QueryCache{
+		next:    next,
+		maxSize: size,
+		entries: map[string]*list.Element{},
+		order:   list.New(),
+	}
+}
+
+// VectorizeText returns the cached embedding for text or fetches and caches it.
+func (c *QueryCache) VectorizeText(ctx context.Context, text string) ([]float32, error) {
+	key := strings.Join(strings.Fields(strings.ToLower(text)), " ")
+
+	c.mu.Lock()
+	if el, ok := c.entries[key]; ok {
+		c.order.MoveToFront(el)
+		vector := el.Value.(*cacheEntry).vector
+		c.mu.Unlock()
+		return vector, nil
+	}
+	c.mu.Unlock()
+
+	v, err, _ := c.group.Do(key, func() (any, error) {
+		vector, err := c.next.VectorizeText(ctx, text)
+		if err != nil {
+			return nil, err
+		}
+		c.mu.Lock()
+		defer c.mu.Unlock()
+		if el, ok := c.entries[key]; ok {
+			c.order.MoveToFront(el)
+			return el.Value.(*cacheEntry).vector, nil
+		}
+		c.entries[key] = c.order.PushFront(&cacheEntry{key: key, vector: vector})
+		if c.order.Len() > c.maxSize {
+			oldest := c.order.Back()
+			c.order.Remove(oldest)
+			delete(c.entries, oldest.Value.(*cacheEntry).key)
+		}
+		return vector, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return v.([]float32), nil
+}

services/search/pkg/command/server.go

@@ -15,6 +15,7 @@ import (
 	"github.com/opencloud-eu/opencloud/pkg/tracing"
 	"github.com/opencloud-eu/opencloud/pkg/version"
 	"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
+	"github.com/opencloud-eu/opencloud/services/search/pkg/clip"
 	"github.com/opencloud-eu/opencloud/services/search/pkg/config"
 	"github.com/opencloud-eu/opencloud/services/search/pkg/config/parser"
 	"github.com/opencloud-eu/opencloud/services/search/pkg/content"
@@ -63,6 +64,15 @@ func Server(cfg *config.Config) *cobra.Command {
 			mtrcs := metrics.New()
 			mtrcs.BuildInfo.WithLabelValues(version.GetString()).Set(1)
 
+			// one clip client serves both sides: the extractor embeds images at
+			// index time, the engines embed query texts at search time
+			var clipClient *clip.Client
+			var vectorizer search.TextVectorizer
+			if cfg.Extractor.Clip.URL != "" {
+				clipClient = clip.NewClient(cfg.Extractor.Clip.URL, cfg.Extractor.Clip.Model, time.Duration(cfg.Extractor.Clip.Timeout)*time.Second)
+				vectorizer = clip.NewQueryCache(clipClient, 512)
+			}
+
 			// initialize search engine
 			var eng search.Engine
 			switch cfg.Engine.Type {
@@ -78,16 +88,25 @@ func Server(cfg *config.Config) *cobra.Command {
 					}
 				}()
 
-				eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, logger)
+				var opts []bleve.Option
+				if vectorizer != nil {
+					opts = append(opts, bleve.WithTextVectorizer(vectorizer))
+				}
+				eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, logger, opts...)
 			case "open-search":
 				client, err := opensearch.NewClient(cfg.Engine.OpenSearch.Client)
 				if err != nil {
 					return err
 				}
 
+				var opts []opensearch.Option
+				if vectorizer != nil {
+					opts = append(opts, opensearch.WithTextVectorizer(vectorizer))
+				}
+
 				// a hung cluster must fail the start, not block it forever
 				startupCtx, cancelStartup := context.WithTimeout(ctx, time.Minute)
-				openSearchBackend, err := opensearch.NewBackend(startupCtx, cfg.Engine.OpenSearch.ResourceIndex.Name, client, logger)
+				openSearchBackend, err := opensearch.NewBackend(startupCtx, cfg.Engine.OpenSearch.ResourceIndex.Name, client, logger, opts...)
 				cancelStartup()
 				if err != nil {
 					return fmt.Errorf("failed to create OpenSearch backend: %w", err)
@@ -120,6 +139,14 @@ func Server(cfg *config.Config) *cobra.Command {
 				return fmt.Errorf("unknown search extractor: %s", cfg.Extractor.Type)
 			}
 
+			// vectorization is orthogonal to text extraction: decorate the
+			// chosen extractor instead of adding an exclusive type
+			if clipClient != nil {
+				if extractor, err = content.NewClipExtractor(extractor, clipClient, selector, logger, cfg); err != nil {
+					return err
+				}
+			}
+
 			ss := search.NewService(selector, eng, extractor, mtrcs, logger, cfg)
 
 			// setup the servers

services/search/pkg/config/content.go

@@ -5,6 +5,7 @@ type Extractor struct {
 	Type             string        `yaml:"type" env:"SEARCH_EXTRACTOR_TYPE" desc:"Defines the content extraction engine. Defaults to 'basic'. Supported values are: 'basic' and 'tika'." introductionVersion:"1.0.0"`
 	CS3AllowInsecure bool          `yaml:"cs3_allow_insecure" env:"OC_INSECURE;SEARCH_EXTRACTOR_CS3SOURCE_INSECURE" desc:"Ignore untrusted SSL certificates when connecting to the CS3 source." introductionVersion:"1.0.0"`
 	Tika             ExtractorTika `yaml:"tika"`
+	Clip             ExtractorClip `yaml:"clip"`
 }
 
 // ExtractorTika configures the Tika extractor
@@ -12,3 +13,15 @@ type ExtractorTika struct {
 	TikaURL        string `yaml:"tika_url" env:"SEARCH_EXTRACTOR_TIKA_TIKA_URL" desc:"URL of the tika server." introductionVersion:"1.0.0"`
 	CleanStopWords bool   `yaml:"clean_stop_words" env:"SEARCH_EXTRACTOR_TIKA_CLEAN_STOP_WORDS" desc:"Defines if stop words should be cleaned or not. See the documentation for more details." introductionVersion:"1.0.0"`
 }
+
+// ExtractorClip configures the CLIP inference service used for semantic image
+// search. It decorates the configured extractor, so it combines with 'basic'
+// and 'tika'. The vector dimensionality is part of the index schema and
+// therefore not configurable; the service verifies at startup that the
+// configured model matches.
+type ExtractorClip struct {
+	URL      string `yaml:"url" env:"SEARCH_EXTRACTOR_CLIP_URL" desc:"URL of the CLIP inference service (immich machine-learning API). When set, image files are embedded during indexing to enable semantic search." introductionVersion:"7.5.0"`
+	Model    string `yaml:"model" env:"SEARCH_EXTRACTOR_CLIP_MODEL" desc:"Name of the CLIP model to use. Must be a multilingual model producing 512-dimensional embeddings. Changing the model requires a full reindex." introductionVersion:"7.5.0"`
+	MaxBytes uint64 `yaml:"max_bytes" env:"SEARCH_EXTRACTOR_CLIP_MAX_BYTES" desc:"Maximum image size in bytes to send to the inference service. Larger images are skipped." introductionVersion:"7.5.0"`
+	Timeout  int    `yaml:"timeout" env:"SEARCH_EXTRACTOR_CLIP_TIMEOUT" desc:"Timeout in seconds for requests to the inference service." introductionVersion:"7.5.0"`
+}

services/search/pkg/config/defaults/defaultconfig.go

@@ -52,6 +52,11 @@ func DefaultConfig() *config.Config {
 				TikaURL:        "http://127.0.0.1:9998",
 				CleanStopWords: false,
 			},
+			Clip: config.ExtractorClip{
+				Model:    "XLM-Roberta-Large-Vit-B-32",
+				MaxBytes: 50 * 1024 * 1024,
+				Timeout:  30,
+			},
 		},
 		Events: config.Events{
 			Endpoint:         "127.0.0.1:9233",

services/search/pkg/content/clip.go

@@ -0,0 +1,91 @@
+package content
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
+
+	"github.com/opencloud-eu/opencloud/pkg/log"
+	"github.com/opencloud-eu/opencloud/services/search/pkg/clip"
+	"github.com/opencloud-eu/opencloud/services/search/pkg/config"
+)
+
+// Clip decorates another extractor with image vectorization: it delegates
+// Extract and adds an ImageVector to image documents. Vectorization is
+// orthogonal to text extraction, so it wraps 'basic' as well as 'tika'.
+type Clip struct {
+	next Extractor
+	Retriever
+	client   *clip.Client
+	logger   log.Logger
+	maxBytes uint64
+}
+
+// NewClipExtractor wraps next with a CLIP vectorization step. It probes the
+// inference service once to verify that the configured model produces vectors
+// of the schema dimensionality (ImageVectorDims) and refuses to start
+// otherwise: mismatched vectors would be dropped silently by the index.
+func NewClipExtractor(next Extractor, client *clip.Client, gatewaySelector pool.Selectable[gateway.GatewayAPIClient], logger log.Logger, cfg *config.Config) (*Clip, error) {
+	probe, err := client.VectorizeText(context.Background(), "startup probe")
+	if err != nil {
+		return nil, fmt.Errorf("clip inference service unavailable: %w", err)
+	}
+	if len(probe) != ImageVectorDims {
+		return nil, fmt.Errorf("clip model produces %d-dimensional vectors, the index schema requires %d; use a matching model", len(probe), ImageVectorDims)
+	}
+	logger.Info().Str("url", cfg.Extractor.Clip.URL).Int("dims", len(probe)).Msg("clip inference service connected")
+
+	maxBytes := cfg.Extractor.Clip.MaxBytes
+	if maxBytes == 0 {
+		maxBytes = 50 * 1024 * 1024
+	}
+
+	return &Clip{
+		next:      next,
+		Retriever: newCS3Retriever(gatewaySelector, logger, cfg.Extractor.CS3AllowInsecure),
+		client:    client,
+		logger:    logger,
+		maxBytes:  maxBytes,
+	}, nil
+}
+
+// Extract delegates to the wrapped extractor and adds the image embedding.
+// Vectorization failures are not fatal: the document is indexed without a
+// vector and the rest of the search keeps working.
+func (c *Clip) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document, error) {
+	doc, err := c.next.Extract(ctx, ri)
+	if err != nil {
+		return doc, err
+	}
+
+	if ri.Type != provider.ResourceType_RESOURCE_TYPE_FILE ||
+		!strings.HasPrefix(ri.GetMimeType(), "image/") ||
+		ri.Size == 0 || ri.Size > c.maxBytes {
+		return doc, nil
+	}
+
+	data, err := c.Retrieve(ctx, ri.Id)
+	if err != nil {
+		c.logger.Warn().Err(err).Interface("ResourceID", ri.Id).Str("Name", ri.Name).Msg("clip: failed to retrieve image, indexing without vector")
+		return doc, nil
+	}
+	defer data.Close()
+
+	vector, err := c.client.VectorizeImage(ctx, data)
+	if err != nil {
+		c.logger.Warn().Err(err).Interface("ResourceID", ri.Id).Str("Name", ri.Name).Msg("clip: vectorization failed, indexing without vector")
+		return doc, nil
+	}
+	if len(vector) != ImageVectorDims {
+		// the index would drop a mismatched vector silently, make it visible
+		c.logger.Warn().Int("dims", len(vector)).Int("want", ImageVectorDims).Interface("ResourceID", ri.Id).Str("Name", ri.Name).Msg("clip: vector dimensionality mismatch, indexing without vector")
+		return doc, nil
+	}
+
+	doc.ImageVector = vector
+	return doc, nil
+}

services/search/pkg/content/clip_test.go

@@ -0,0 +1,191 @@
+package content_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/stretchr/testify/mock"
+
+	"github.com/opencloud-eu/opencloud/pkg/log"
+	"github.com/opencloud-eu/opencloud/services/search/pkg/clip"
+	conf "github.com/opencloud-eu/opencloud/services/search/pkg/config/defaults"
+	"github.com/opencloud-eu/opencloud/services/search/pkg/content"
+	contentMocks "github.com/opencloud-eu/opencloud/services/search/pkg/content/mocks"
+)
+
+// clipResponse renders the immich-ml /predict response for a vector of n dims.
+func clipResponse(n int) string {
+	vec := make([]float32, n)
+	for i := range vec {
+		vec[i] = float32(i) / float32(n)
+	}
+	inner, _ := json.Marshal(vec)
+	outer, _ := json.Marshal(map[string]string{"clip": string(inner)})
+	return string(outer)
+}
+
+var _ = Describe("Clip", func() {
+	var (
+		srv        *httptest.Server
+		dims       int
+		failNext   atomic.Bool
+		imageCalls atomic.Int32
+		inner      *contentMocks.Extractor
+		retriever  *contentMocks.Retriever
+	)
+
+	BeforeEach(func() {
+		dims = content.ImageVectorDims
+		failNext.Store(false)
+		imageCalls.Store(0)
+		srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			if req.URL.Path != "/predict" {
+				w.WriteHeader(http.StatusNotFound)
+				return
+			}
+			if failNext.Load() {
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+			Expect(req.ParseMultipartForm(1 << 20)).To(Succeed())
+			if _, _, err := req.FormFile("image"); err == nil {
+				imageCalls.Add(1)
+			}
+			_, _ = w.Write([]byte(clipResponse(dims)))
+		}))
+
+		inner = &contentMocks.Extractor{}
+		retriever = &contentMocks.Retriever{}
+	})
+
+	AfterEach(func() {
+		srv.Close()
+	})
+
+	newClip := func() (*content.Clip, error) {
+		cfg := conf.DefaultConfig()
+		cfg.Extractor.Clip.URL = srv.URL
+
+		client := clip.NewClient(srv.URL, cfg.Extractor.Clip.Model, 0)
+		extractor, err := content.NewClipExtractor(inner, client, nil, log.NewLogger(), cfg)
+		if err != nil {
+			return nil, err
+		}
+		extractor.Retriever = retriever
+		return extractor, nil
+	}
+
+	imageResource := func(size uint64) *provider.ResourceInfo {
+		return &provider.ResourceInfo{
+			Type:     provider.ResourceType_RESOURCE_TYPE_FILE,
+			Name:     "photo.jpg",
+			MimeType: "image/jpeg",
+			Size:     size,
+		}
+	}
+
+	Describe("NewClipExtractor", func() {
+		It("fails when the inference service is unreachable", func() {
+			srv.Close()
+			_, err := newClip()
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("fails when the model dimensionality does not match the schema", func() {
+			dims = 768
+			_, err := newClip()
+			Expect(err).To(MatchError(ContainSubstring("768")))
+		})
+	})
+
+	Describe("Extract", func() {
+		var extractor *content.Clip
+
+		BeforeEach(func() {
+			var err error
+			extractor, err = newClip()
+			Expect(err).ToNot(HaveOccurred())
+
+			inner.On("Extract", mock.Anything, mock.Anything).Return(content.Document{Name: "photo.jpg"}, nil)
+			retriever.On("Retrieve", mock.Anything, mock.Anything).Return(io.NopCloser(strings.NewReader("fakeimagebytes")), nil)
+		})
+
+		It("adds a vector to image documents", func() {
+			doc, err := extractor.Extract(context.TODO(), imageResource(1024))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(doc.Name).To(Equal("photo.jpg"))
+			Expect(doc.ImageVector).To(HaveLen(content.ImageVectorDims))
+			Expect(imageCalls.Load()).To(Equal(int32(1)))
+		})
+
+		It("skips non-image files", func() {
+			doc, err := extractor.Extract(context.TODO(), &provider.ResourceInfo{
+				Type:     provider.ResourceType_RESOURCE_TYPE_FILE,
+				Name:     "doc.pdf",
+				MimeType: "application/pdf",
+				Size:     1024,
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(doc.ImageVector).To(BeNil())
+			Expect(imageCalls.Load()).To(Equal(int32(0)))
+		})
+
+		It("skips directories", func() {
+			doc, err := extractor.Extract(context.TODO(), &provider.ResourceInfo{
+				Type:     provider.ResourceType_RESOURCE_TYPE_CONTAINER,
+				MimeType: "image/jpeg",
+				Size:     1024,
+			})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(doc.ImageVector).To(BeNil())
+		})
+
+		It("skips images above the size limit", func() {
+			doc, err := extractor.Extract(context.TODO(), imageResource(51*1024*1024))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(doc.ImageVector).To(BeNil())
+			Expect(imageCalls.Load()).To(Equal(int32(0)))
+		})
+
+		It("indexes without a vector when inference fails", func() {
+			failNext.Store(true)
+			doc, err := extractor.Extract(context.TODO(), imageResource(1024))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(doc.Name).To(Equal("photo.jpg"))
+			Expect(doc.ImageVector).To(BeNil())
+		})
+
+		It("indexes without a vector when retrieval fails", func() {
+			failingRetriever := &contentMocks.Retriever{}
+			failingRetriever.On("Retrieve", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("boom"))
+			extractor.Retriever = failingRetriever
+
+			doc, err := extractor.Extract(context.TODO(), imageResource(1024))
+			Expect(err).ToNot(HaveOccurred())
+			Expect(doc.ImageVector).To(BeNil())
+		})
+
+		It("propagates inner extractor errors", func() {
+			failingInner := &contentMocks.Extractor{}
+			failingInner.On("Extract", mock.Anything, mock.Anything).Return(content.Document{}, fmt.Errorf("inner boom"))
+			cfg := conf.DefaultConfig()
+			cfg.Extractor.Clip.URL = srv.URL
+			client := clip.NewClient(srv.URL, cfg.Extractor.Clip.Model, 0)
+			failing, err := content.NewClipExtractor(failingInner, client, nil, log.NewLogger(), cfg)
+			Expect(err).ToNot(HaveOccurred())
+			failing.Retriever = retriever
+
+			_, err = failing.Extract(context.TODO(), imageResource(1024))
+			Expect(err).To(MatchError(ContainSubstring("inner boom")))
+		})
+	})
+})

services/search/pkg/content/content.go

@@ -12,6 +12,13 @@ func init() {
 	stopwords.OverwriteWordSegmenter(`[^ ]+`)
 }
 
+// ImageVectorDims is the dimensionality of the ImageVector field. It is part
+// of the index schema (baked into the index mapping on creation), not a
+// configuration value: vectors of any other length cannot be indexed, and a
+// model of another size requires a new index. The CLIP extractor verifies at
+// startup that the configured model produces vectors of this length.
+const ImageVectorDims = 512
+
 // Document wraps all resource meta fields,
 // it is used as a content extraction result.
 type Document struct {
@@ -29,6 +36,7 @@ type Document struct {
 	Photo       *libregraph.Photo          `json:"photo,omitempty"`
 	Video       *libregraph.Video          `json:"video,omitempty"`
 	MotionPhoto *libregraph.MotionPhoto    `json:"motionPhoto,omitempty"`
+	ImageVector []float32                  `json:"imageVector,omitempty"`
 }
 
 func CleanString(content, langCode string) string {

services/search/pkg/mapping/bleve.go

@@ -74,6 +74,29 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix
 			return nil
 		}
 
+		if fieldType == TypeVector {
+			// The field itself is stored-only: it round-trips through the
+			// regular deserialization (bleve cannot return vector-typed fields
+			// as stored fields, see VectorIndexSuffix). Both builds.
+			nf := bleve.NewNumericFieldMapping()
+			nf.Index = false
+			nf.Store = true
+			nf.IncludeInAll = false
+			nf.DocValues = false
+			doc.AddFieldMappingsAt(fi.Name, nf)
+			// The _faiss sibling carries the searchable vector; without the
+			// vectors build tag bleve cannot index it and the field is left
+			// out of the mapping entirely (nil, nil from the !vectors twin).
+			fm, err := bleveVectorFieldMapping(opts)
+			if err != nil {
+				return fmt.Errorf("mapping: field %q: %w", key, err)
+			}
+			if fm != nil {
+				doc.AddFieldMappingsAt(fi.Name+VectorIndexSuffix, fm)
+			}
+			return nil
+		}
+
 		fm, err := bleveFieldMapping(fieldType, opts)
 		if err != nil {
 			return fmt.Errorf("mapping: field %q: %w", key, err)

services/search/pkg/mapping/bleve_novectors.go

@@ -0,0 +1,14 @@
+//go:build !vectors
+
+package mapping
+
+import (
+	bleveMapping "github.com/blevesearch/bleve/v2/mapping"
+)
+
+// bleveVectorFieldMapping is the !vectors twin: bleve has no vector support in
+// this build, so TypeVector fields are left out of the mapping (nil, nil means
+// "skip the field").
+func bleveVectorFieldMapping(FieldOpts) (*bleveMapping.FieldMapping, error) {
+	return nil, nil
+}

services/search/pkg/mapping/bleve_vectors.go

@@ -0,0 +1,25 @@
+//go:build vectors
+
+package mapping
+
+import (
+	"fmt"
+
+	"github.com/blevesearch/bleve/v2"
+	bleveMapping "github.com/blevesearch/bleve/v2/mapping"
+)
+
+// bleveVectorFieldMapping builds the bleve field mapping for a TypeVector
+// field. Only compiled in with the vectors build tag (bleve gates its KNN
+// support behind it); the !vectors twin drops the field from the mapping.
+func bleveVectorFieldMapping(opts FieldOpts) (*bleveMapping.FieldMapping, error) {
+	if opts.Dims <= 0 {
+		return nil, fmt.Errorf("vector field needs Dims")
+	}
+	fm := bleve.NewVectorFieldMapping()
+	fm.Dims = opts.Dims
+	// cosine: bleve normalizes the vectors at index time itself
+	fm.Similarity = "cosine"
+	fm.VectorIndexOptimizedFor = "recall"
+	return fm, nil
+}

services/search/pkg/mapping/opensearch.go

@@ -79,7 +79,7 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p
 			return nil
 		}
 
-		fm, err := openSearchFieldMapping(fieldType, fi.GoField.Type)
+		fm, err := openSearchFieldMapping(fieldType, opts, fi.GoField.Type)
 		if err != nil {
 			return fmt.Errorf("mapping: field %q: %w", key, err)
 		}
@@ -91,7 +91,7 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p
 
 // openSearchFieldMapping handles the non-keyword/path types; keyword and path
 // are emitted (with their cased forms) by buildOpenSearchProperties directly.
-func openSearchFieldMapping(fieldType string, goType reflect.Type) (map[string]any, error) {
+func openSearchFieldMapping(fieldType string, opts FieldOpts, goType reflect.Type) (map[string]any, error) {
 	switch fieldType {
 	case TypeFulltext:
 		return map[string]any{
@@ -112,6 +112,23 @@ func openSearchFieldMapping(fieldType string, goType reflect.Type) (map[string]a
 		return map[string]any{"type": "date"}, nil
 	case TypeGeopoint:
 		return map[string]any{"type": "geo_point"}, nil
+	case TypeVector:
+		if opts.Dims <= 0 {
+			return nil, fmt.Errorf("vector field needs Dims")
+		}
+		// lucene engine: native efficient pre-filtering, no plugin config
+		return map[string]any{
+			"type":      "knn_vector",
+			"dimension": opts.Dims,
+			"method": map[string]any{
+				"name":       "hnsw",
+				"engine":     "lucene",
+				"space_type": "cosinesimil",
+				// OpenSearch adds this on read, emit it so the reconcile
+				// comparison sees local and remote as equal
+				"parameters": map[string]any{},
+			},
+		}, nil
 	case "":
 		return nil, fmt.Errorf("no type inferred and no override")
 	}

services/search/pkg/mapping/opts.go

@@ -15,6 +15,7 @@ const (
 	TypeBool     = "bool"
 	TypeObject   = "object"
 	TypeGeopoint = "geopoint"
+	TypeVector   = "vector"
 )
 
 // LowercaseSuffix names the lowercased sibling of a keyword/path field.
@@ -51,6 +52,11 @@ type FieldOpts struct {
 	// IncludeInAll controls bleve's _all field inclusion. Nil means "use the
 	// bleve default for this field type". Has no effect on OpenSearch.
 	IncludeInAll *bool
+
+	// Dims is the dimensionality of a TypeVector field. It is index schema:
+	// vectors of any other length are not indexed (bleve drops them silently),
+	// and changing it requires a new index. Similarity is always cosine.
+	Dims int
 }
 
 func (o FieldOpts) caseInsensitive() bool { return o.CaseInsensitive == nil || *o.CaseInsensitive }

services/search/pkg/mapping/vectordata.go

@@ -0,0 +1,44 @@
+package mapping
+
+import (
+	"strings"
+)
+
+// VectorIndexSuffix is appended to a TypeVector field's name to produce the
+// bleve sibling field that faiss actually indexes (e.g. "imageVector" ->
+// "imageVector_faiss"), mirroring the "location" / "location_geopoint" split.
+// bleve hands vector fields to faiss and cannot return them as stored fields
+// (verified: even Store:true yields nothing), so the original field is mapped
+// stored-only instead and round-trips like any other field, keeping vectors
+// alive across Move / Delete / Restore. OpenSearch keeps vectors in _source
+// and indexes the original field directly, no sibling there.
+const VectorIndexSuffix = "_faiss"
+
+// AddVectorIndexSiblings writes, for each TypeVector override present in m,
+// the vector again under the suffixed sibling key. bleve vectors-build upsert
+// path only: without the vectors tag the sibling has no mapping and bleve's
+// dynamic mapping would index the raw floats.
+func AddVectorIndexSiblings(m map[string]any, overrides map[string]FieldOpts) {
+	for key, opts := range overrides {
+		if opts.Type != TypeVector {
+			continue
+		}
+		parts := strings.Split(key, ".")
+		parent := m
+		for _, p := range parts[:len(parts)-1] {
+			next, ok := parent[p].(map[string]any)
+			if !ok {
+				parent = nil
+				break
+			}
+			parent = next
+		}
+		if parent == nil {
+			continue
+		}
+		leaf := parts[len(parts)-1]
+		if vec, ok := parent[leaf].([]any); ok && len(vec) > 0 {
+			parent[leaf+VectorIndexSuffix] = vec
+		}
+	}
+}

services/search/pkg/opensearch/backend.go

@@ -3,6 +3,7 @@ package opensearch
 import (
 	"context"
 	"fmt"
+	"sort"
 	"time"
 
 	storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
@@ -29,12 +30,24 @@ var (
 )
 
 type Backend struct {
-	index  string
-	client *opensearchgoAPI.Client
+	index      string
+	client     *opensearchgoAPI.Client
+	vectorizer search.TextVectorizer
+}
+
+// Option configures a Backend.
+type Option func(*Backend)
+
+// WithTextVectorizer enables semantic queries (`semantic:"..."`); without it
+// they are rejected.
+func WithTextVectorizer(v search.TextVectorizer) Option {
+	return func(b *Backend) {
+		b.vectorizer = v
+	}
 }
 
 // NewBackend creates a backend on the versioned generation of the named index.
-func NewBackend(ctx context.Context, name string, client *opensearchgoAPI.Client, logger log.Logger) (*Backend, error) {
+func NewBackend(ctx context.Context, name string, client *opensearchgoAPI.Client, logger log.Logger, opts ...Option) (*Backend, error) {
 	index := VersionedIndexName(name)
 
 	pingResp, err := client.Ping(ctx, &opensearchgoAPI.PingReq{})
@@ -68,17 +81,29 @@ func NewBackend(ctx context.Context, name string, client *opensearchgoAPI.Client
 		return nil, fmt.Errorf("%w, cluster health is not green or yellow: %s", ErrUnhealthyCluster, resp.Status)
 	}
 
-	return &Backend{index: index, client: client}, nil
+	b := &Backend{index: index, client: client}
+	for _, opt := range opts {
+		opt(b)
+	}
+	return b, nil
 }
 
 func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) {
-	boolQuery, err := convert.KQLToOpenSearchBoolQuery(sir.Query)
+	// the semantic clause ranks, the remaining query filters: it scopes the
+	// vector search and stays the only source of totals. The converter splits
+	// the clause off the parsed KQL tree.
+	boolQuery, semanticText, err := convert.KQLToOpenSearchBoolQueryWithSemantic(sir.Query, true)
 	switch {
 	case kql.IsValidationError(err):
 		return nil, errtypes.BadRequest(err.Error())
 	case err != nil:
 		return nil, fmt.Errorf("failed to convert KQL query to OpenSearch bool query: %w", err)
 	}
+	pureSemantic := semanticText != "" && isEmptyBoolQuery(boolQuery)
+
+	if semanticText != "" && b.vectorizer == nil {
+		return nil, errtypes.BadRequest("semantic search is not configured")
+	}
 
 	// filter out deleted resources
 	boolQuery.Filter(
@@ -110,7 +135,9 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
 	}
 
 	searchParams := opensearchgoAPI.SearchParams{
-		SourceExcludes: []string{"Content"}, // Do not send back the full content in the search response, as it is only needed for highlighting and can be large. The highlighted snippets will be sent back in the response instead.
+		// Do not send back the full content (only needed for highlighting, the
+		// snippets come back instead) or the raw image vectors (ranking data).
+		SourceExcludes: []string{"Content", "imageVector"},
 	}
 
 	switch {
@@ -122,9 +149,15 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
 		searchParams.Size = conversions.ToPointer(int(sir.PageSize))
 	}
 
+	// the filter request carries the totals and the lexical ranking; for a
+	// purely semantic query it only supplies the totals
+	filterParams := searchParams
+	if pureSemantic {
+		filterParams.Size = conversions.ToPointer(0)
+	}
 	req, err := osu.BuildSearchReq(&opensearchgoAPI.SearchReq{
 		Indices: []string{b.index},
-		Params:  searchParams,
+		Params:  filterParams,
 	},
 		boolQuery,
 		osu.SearchBodyParams{
@@ -151,15 +184,47 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
 		return nil, fmt.Errorf("failed to search: %w", err)
 	}
 
-	matches := make([]*searchMessage.Match, 0, len(resp.Hits.Hits))
+	matches, err := convertHits(resp.Hits.Hits)
+	if err != nil {
+		return nil, err
+	}
 	totalMatches := resp.Hits.Total.Value
-	for _, hit := range resp.Hits.Hits {
-		match, err := convert.OpenSearchHitToMatch(hit)
+
+	if semanticText != "" {
+		vector, err := b.vectorizer.VectorizeText(ctx, semanticText)
 		if err != nil {
-			return nil, fmt.Errorf("failed to convert hit to match: %w", err)
+			return nil, fmt.Errorf("failed to vectorize the semantic query: %w", err)
+		}
+		k := semanticK(searchParams.Size)
+		knnParams := searchParams
+		knnParams.Size = conversions.ToPointer(k)
+		// the full filter query also pre-filters the neighbor search
+		knnQuery := osu.NewKnnQuery("imageVector").Vector(vector).K(k).Filter(boolQuery)
+		knnReq, err := osu.BuildSearchReq(&opensearchgoAPI.SearchReq{
+			Indices: []string{b.index},
+			Params:  knnParams,
+		}, knnQuery)
+		if err != nil {
+			return nil, fmt.Errorf("failed to build knn request: %w", err)
+		}
+		knnResp, err := b.client.Search(ctx, knnReq)
+		if err != nil {
+			return nil, fmt.Errorf("failed to run knn search: %w", err)
+		}
+		knnMatches, err := convertHits(knnResp.Hits.Hits)
+		if err != nil {
+			return nil, err
 		}
 
-		matches = append(matches, match)
+		if pureSemantic {
+			// the knn result is the ranking, and the only honest total is the
+			// number of semantic hits (a similarity search has no result set,
+			// only a ranking)
+			matches = knnMatches
+			totalMatches = len(knnMatches)
+		} else {
+			matches = fuseRRF(matches, knnMatches, searchParams.Size)
+		}
 	}
 
 	return &searchService.SearchIndexResponse{
@@ -168,6 +233,80 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
 	}, nil
 }
 
+// isEmptyBoolQuery reports whether q carries no clauses (yet): an empty map
+// render means the parsed query had no filter part.
+func isEmptyBoolQuery(q *osu.BoolQuery) bool {
+	m, err := q.Map()
+	return err == nil && len(m) == 0
+}
+
+// convertHits converts OpenSearch hits to matches.
+func convertHits(hits []opensearchgoAPI.SearchHit) ([]*searchMessage.Match, error) {
+	matches := make([]*searchMessage.Match, 0, len(hits))
+	for _, hit := range hits {
+		match, err := convert.OpenSearchHitToMatch(hit)
+		if err != nil {
+			return nil, fmt.Errorf("failed to convert hit to match: %w", err)
+		}
+		matches = append(matches, match)
+	}
+	return matches, nil
+}
+
+// semanticK picks the number of nearest neighbors for a semantic clause: at
+// least a fusion-friendly window, at most a sane cap (huge page sizes stand
+// for "everything", but a similarity ranking beyond 1000 hits carries no
+// signal).
+func semanticK(size *int) int {
+	const minK, maxK = 200, 1000
+	switch {
+	case size == nil || *size >= maxK:
+		return maxK
+	case *size < minK:
+		return minK
+	default:
+		return *size
+	}
+}
+
+// fuseRRF merges the lexical and the semantic ranking via reciprocal rank
+// fusion (rank constant 60, like bleve's default) and trims to limit.
+func fuseRRF(lexical, semantic []*searchMessage.Match, limit *int) []*searchMessage.Match {
+	const rankConst = 60
+	type entry struct {
+		match *searchMessage.Match
+		score float64
+	}
+	byID := map[string]*entry{}
+	order := make([]*entry, 0, len(lexical)+len(semantic))
+	add := func(list []*searchMessage.Match) {
+		for i, m := range list {
+			id := m.GetEntity().GetId()
+			key := id.GetStorageId() + "$" + id.GetSpaceId() + "!" + id.GetOpaqueId()
+			e, ok := byID[key]
+			if !ok {
+				e = &entry{match: m}
+				byID[key] = e
+				order = append(order, e)
+			}
+			e.score += 1.0 / float64(rankConst+i+1)
+		}
+	}
+	add(lexical)
+	add(semantic)
+	sort.SliceStable(order, func(i, j int) bool { return order[i].score > order[j].score })
+
+	out := make([]*searchMessage.Match, 0, len(order))
+	for _, e := range order {
+		e.match.Score = float32(e.score)
+		out = append(out, e.match)
+	}
+	if limit != nil && len(out) > *limit {
+		out = out[:*limit]
+	}
+	return out
+}
+
 func (b *Backend) DocCount() (uint64, error) {
 	req, err := osu.BuildIndicesCountReq(
 		&opensearchgoAPI.IndicesCountReq{

services/search/pkg/opensearch/backend_test.go

@@ -2,6 +2,8 @@ package opensearch_test
 
 import (
 	"context"
+	"fmt"
+	"strings"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -10,7 +12,9 @@ import (
 	opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
 
 	"github.com/opencloud-eu/opencloud/pkg/log"
+	searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
 	"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
+	"github.com/opencloud-eu/opencloud/services/search/pkg/content"
 	"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
 )
 
@@ -80,4 +84,92 @@ var _ = Describe("Backend", func() {
 		})
 	})
 
+	Describe("Semantic search", func() {
+		const indexName = "opencloud-test-engine-semantic"
+
+		var (
+			tc      *opensearchtest.TestClient
+			backend *opensearch.Backend
+		)
+
+		BeforeEach(func() {
+			// the backend versions the physical index by schema generation
+			physical := opensearch.VersionedIndexName(indexName)
+
+			tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
+			tc.Require.IndicesReset([]string{physical})
+			deleteIndexOnCleanup(tc, physical)
+
+			var err error
+			backend, err = opensearch.NewBackend(context.Background(), indexName, tc.Client(), log.NopLogger(), opensearch.WithTextVectorizer(fakeVectorizer{
+				byText: map[string][]float32{
+					"cat": axisVector(0),
+					"dog": axisVector(1),
+				},
+			}))
+			Expect(err).ToNot(HaveOccurred())
+
+			upsert := func(id, path, name string, vector []float32) {
+				r := opensearchtest.Testdata.Resources.File
+				r.ID = id
+				r.Path = path
+				r.Name = name
+				r.ImageVector = vector
+				Expect(backend.Upsert(id, r)).To(Succeed())
+			}
+			upsert("1$1!10", "./cat.jpg", "cat.jpg", axisVector(0))
+			upsert("1$1!11", "./dog.jpg", "dog.jpg", axisVector(1))
+			upsert("1$1!12", "./note.txt", "note.txt", nil)
+			tc.Require.IndicesRefresh([]string{physical}, nil)
+		})
+
+		It("ranks the nearest image first for a purely semantic query", func() {
+			resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `semantic:"cat"`})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(resp.Matches).ToNot(BeEmpty())
+			Expect(resp.Matches[0].Entity.Name).To(Equal("cat.jpg"))
+			// the only honest total is the number of semantic hits
+			Expect(resp.TotalMatches).To(Equal(int32(2)))
+		})
+
+		It("combines a semantic clause with a filter part", func() {
+			resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `Name:dog* AND semantic:"cat"`})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(resp.Matches).To(HaveLen(1))
+			Expect(resp.Matches[0].Entity.Name).To(Equal("dog.jpg"))
+			Expect(resp.TotalMatches).To(Equal(int32(1)))
+		})
+
+		It("keeps semantic inside a quoted value literal", func() {
+			resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `Name:"*semantic:cat*"`})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(resp.Matches).To(BeEmpty())
+		})
+
+		It("rejects semantic queries without a vectorizer", func() {
+			bare, err := opensearch.NewBackend(context.Background(), indexName, tc.Client(), log.NopLogger())
+			Expect(err).ToNot(HaveOccurred())
+			_, err = bare.Search(context.Background(), &searchService.SearchIndexRequest{Query: `semantic:"cat"`})
+			Expect(err).To(MatchError(ContainSubstring("not configured")))
+		})
+	})
 })
+
+// axisVector returns a one-hot vector of the schema dimensionality: distinct
+// axes are orthogonal (cosine 0), same axes identical (cosine 1).
+func axisVector(axis int) []float32 {
+	v := make([]float32, content.ImageVectorDims)
+	v[axis] = 1
+	return v
+}
+
+// fakeVectorizer maps query texts to fixed vectors.
+type fakeVectorizer struct{ byText map[string][]float32 }
+
+func (f fakeVectorizer) VectorizeText(_ context.Context, text string) ([]float32, error) {
+	v, ok := f.byText[strings.ToLower(text)]
+	if !ok {
+		return nil, fmt.Errorf("no vector for %q", text)
+	}
+	return v, nil
+}

services/search/pkg/opensearch/index.go

@@ -79,6 +79,9 @@ func buildResourceMapping() ([]byte, error) {
 		"settings": map[string]any{
 			"number_of_shards":   "1",
 			"number_of_replicas": "1",
+			// static setting, must be set at creation time: enables the knn
+			// query type for the imageVector field
+			"knn": "true",
 			"analysis": map[string]any{
 				// path_hierarchy is case-preserving; casing lives in the value.
 				"analyzer": map[string]any{

services/search/pkg/opensearch/internal/convert/kql_query.go

@@ -13,9 +13,25 @@ var (
 )
 
 func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) {
+	q, _, err := KQLToOpenSearchBoolQueryWithSemantic(kqlQuery, false)
+	return q, err
+}
+
+// KQLToOpenSearchBoolQueryWithSemantic additionally splits the semantic
+// free-text clause off the parsed tree (see query.ExtractSemantic) when
+// withSemantic is set. A purely semantic query yields an empty bool query.
+func KQLToOpenSearchBoolQueryWithSemantic(kqlQuery string, withSemantic bool) (*osu.BoolQuery, string, error) {
 	kqlAst, err := kql.Builder{}.Build(kqlQuery)
 	if err != nil {
-		return nil, err
+		return nil, "", err
+	}
+
+	var semanticText string
+	if withSemantic {
+		semanticText = query.ExtractSemantic(kqlAst)
+		if semanticText != "" && len(kqlAst.Nodes) == 0 {
+			return osu.NewBoolQuery(), semanticText, nil
+		}
 	}
 
 	// shared lowering: field resolution, media-type expansion, value lowercasing.
@@ -23,12 +39,12 @@ func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) {
 
 	builder, err := TranspileKQLToOpenSearch(kqlAst.Nodes)
 	if err != nil {
-		return nil, fmt.Errorf("failed to compile query: %w", err)
+		return nil, "", fmt.Errorf("failed to compile query: %w", err)
 	}
 
 	if q, ok := builder.(*osu.BoolQuery); !ok {
-		return osu.NewBoolQuery().Must(builder), nil
+		return osu.NewBoolQuery().Must(builder), semanticText, nil
 	} else {
-		return q, nil
+		return q, semanticText, nil
 	}
 }

services/search/pkg/opensearch/internal/osu/query_knn.go

@@ -0,0 +1,65 @@
+package osu
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// KnnQuery builds an approximate k-NN query on a knn_vector field, optionally
+// pre-filtered (the engine restricts the neighbor search to the filter
+// matches).
+type KnnQuery struct {
+	field  string
+	vector []float32
+	k      int
+	filter Builder
+}
+
+func NewKnnQuery(field string) *KnnQuery {
+	return &KnnQuery{field: field}
+}
+
+func (q *KnnQuery) Vector(v []float32) *KnnQuery {
+	q.vector = v
+	return q
+}
+
+func (q *KnnQuery) K(k int) *KnnQuery {
+	q.k = k
+	return q
+}
+
+func (q *KnnQuery) Filter(f Builder) *KnnQuery {
+	q.filter = f
+	return q
+}
+
+func (q *KnnQuery) Map() (map[string]any, error) {
+	if q.field == "" || len(q.vector) == 0 || q.k <= 0 {
+		return nil, fmt.Errorf("knn query needs a field, a vector and k > 0")
+	}
+	inner := map[string]any{
+		"vector": q.vector,
+		"k":      q.k,
+	}
+	if q.filter != nil {
+		f, err := q.filter.Map()
+		if err != nil {
+			return nil, err
+		}
+		inner["filter"] = f
+	}
+	return map[string]any{
+		"knn": map[string]any{
+			q.field: inner,
+		},
+	}, nil
+}
+
+func (q *KnnQuery) MarshalJSON() ([]byte, error) {
+	data, err := q.Map()
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(data)
+}

services/search/pkg/opensearch/internal/osu/query_knn_test.go

@@ -0,0 +1,63 @@
+package osu_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
+	"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
+)
+
+func TestKnnQuery(t *testing.T) {
+	tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
+		{
+			Name: "vector and k",
+			Got:  osu.NewKnnQuery("imageVector").Vector([]float32{0.5, 0.25}).K(10),
+			Want: map[string]any{
+				"knn": map[string]any{
+					"imageVector": map[string]any{
+						"vector": []any{0.5, 0.25},
+						"k":      10,
+					},
+				},
+			},
+		},
+		{
+			Name: "with filter",
+			Got: osu.NewKnnQuery("imageVector").Vector([]float32{1, 0}).K(3).Filter(
+				osu.NewBoolQuery().Filter(osu.NewTermQuery[bool]("Deleted").Value(false)),
+			),
+			Want: map[string]any{
+				"knn": map[string]any{
+					"imageVector": map[string]any{
+						"vector": []any{1.0, 0.0},
+						"k":      3,
+						"filter": map[string]any{
+							"bool": map[string]any{
+								"filter": []any{
+									map[string]any{
+										"term": map[string]any{
+											"Deleted": map[string]any{"value": false},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Name, func(t *testing.T) {
+			assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, test.Got))
+		})
+	}
+
+	t.Run("incomplete queries error", func(t *testing.T) {
+		_, err := osu.NewKnnQuery("imageVector").Map()
+		assert.Error(t, err)
+	})
+}

services/search/pkg/opensearch/testdata/resource.golden.json

@@ -188,6 +188,16 @@
           }
         }
       },
+      "imageVector": {
+        "dimension": 512,
+        "method": {
+          "engine": "lucene",
+          "name": "hnsw",
+          "parameters": {},
+          "space_type": "cosinesimil"
+        },
+        "type": "knn_vector"
+      },
       "location": {
         "properties": {
           "altitude": {
@@ -348,6 +358,7 @@
         }
       }
     },
+    "knn": "true",
     "number_of_replicas": "1",
     "number_of_shards": "1"
   }

services/search/pkg/query/bleve/bleve.go

@@ -34,5 +34,29 @@ func (c Creator[T]) Create(qs string) (T, error) {
 	return t, nil
 }
 
+// CreateWithSemantic implements the Creator interface: the semantic clause is
+// split off the parsed tree before lowering, the rest compiles as usual. A
+// purely semantic query returns the zero query.
+func (c Creator[T]) CreateWithSemantic(qs string) (T, string, error) {
+	var t T
+	builderAst, err := c.builder.Build(qs)
+	if err != nil {
+		return t, "", err
+	}
+
+	text := query.ExtractSemantic(builderAst)
+	if text != "" && len(builderAst.Nodes) == 0 {
+		return t, text, nil
+	}
+
+	builderAst = query.Normalize(builderAst, query.ResolveField)
+	t, err = c.compiler.Compile(builderAst)
+	if err != nil {
+		return t, "", err
+	}
+
+	return t, text, nil
+}
+
 // DefaultCreator exposes a kql to bleve query creator.
 var DefaultCreator = Creator[bQuery.Query]{kql.Builder{}, Compiler{}}

services/search/pkg/query/query.go

@@ -16,4 +16,9 @@ type Compiler[T any] interface {
 // Creator is the interface that wraps the basic Create method.
 type Creator[T any] interface {
 	Create(qs string) (T, error)
+
+	// CreateWithSemantic splits the semantic free-text clause off the parsed
+	// query (see ExtractSemantic). A purely semantic query yields the zero
+	// value for the compiled query.
+	CreateWithSemantic(qs string) (T, string, error)
 }

services/search/pkg/query/semantic.go

@@ -0,0 +1,83 @@
+package query
+
+import (
+	"strings"
+
+	"github.com/opencloud-eu/opencloud/pkg/ast"
+	"github.com/opencloud-eu/opencloud/pkg/kql"
+)
+
+// ExtractSemantic splits the semantic free-text clause (`semantic:"..."`) off
+// a parsed KQL query. The clause is removed from the tree (together with the
+// operator it hung on) and its text returned; the remaining query is the
+// filter part. Working on the parsed tree means quoted values elsewhere are
+// never touched: name:"*semantic:x*" is a name search for a literal string.
+func ExtractSemantic(a *ast.Ast) (text string) {
+	a.Nodes = extractSemanticNodes(a.Nodes, &text)
+	return text
+}
+
+// extractSemanticNodes rewrites nodes, dropping the first semantic restriction
+// found. Keyed groups (e.g. name:(...)) are skipped: their bare children are
+// values of the group key, not restrictions of their own.
+func extractSemanticNodes(nodes []ast.Node, text *string) []ast.Node {
+	out := make([]ast.Node, 0, len(nodes))
+	for _, n := range nodes {
+		n = toPointer(n) // parser emits some nodes by value
+		switch node := n.(type) {
+		case *ast.StringNode:
+			if *text == "" && strings.EqualFold(node.Key, "semantic") && node.Value != "" {
+				*text = node.Value
+				continue
+			}
+		case *ast.GroupNode:
+			if node.Key == "" {
+				node.Nodes = extractSemanticNodes(node.Nodes, text)
+				if len(node.Nodes) == 0 {
+					continue // the group only held the semantic clause
+				}
+			}
+		}
+		out = append(out, n)
+	}
+	if len(out) == len(nodes) {
+		// nothing was removed, the operators are the parser's and stay untouched
+		return out
+	}
+	return sanitizeOperators(out)
+}
+
+// sanitizeOperators repairs a node sequence after removals: operators must
+// only stand between operands, so leading, trailing and stacked operators
+// left behind by a removed operand are dropped.
+func sanitizeOperators(nodes []ast.Node) []ast.Node {
+	out := make([]ast.Node, 0, len(nodes))
+	for _, n := range nodes {
+		op, isOp := n.(*ast.OperatorNode)
+		if !isOp {
+			out = append(out, n)
+			continue
+		}
+		if len(out) == 0 && op.Value != kql.BoolNOT {
+			continue // binary operator without a left operand
+		}
+		if len(out) > 0 {
+			// two operators in a row: the operand between them was removed. A
+			// binary operator followed by NOT is not that case, "a AND NOT b"
+			// is how a negation is written.
+			if prev, prevIsOp := out[len(out)-1].(*ast.OperatorNode); prevIsOp && prev.Value != kql.BoolNOT && op.Value != kql.BoolNOT {
+				out[len(out)-1] = n
+				continue
+			}
+		}
+		out = append(out, n)
+	}
+	// drop a trailing operator (its right operand was removed)
+	for len(out) > 0 {
+		if _, isOp := out[len(out)-1].(*ast.OperatorNode); !isOp {
+			break
+		}
+		out = out[:len(out)-1]
+	}
+	return out
+}

services/search/pkg/query/semantic_test.go

@@ -0,0 +1,87 @@
+package query_test
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/opencloud-eu/opencloud/pkg/ast"
+	"github.com/opencloud-eu/opencloud/pkg/kql"
+	"github.com/opencloud-eu/opencloud/services/search/pkg/query"
+)
+
+// render flattens a node sequence into comparable tokens.
+func render(nodes []ast.Node) []string {
+	out := make([]string, 0, len(nodes))
+	for _, n := range nodes {
+		switch node := n.(type) {
+		case *ast.StringNode:
+			out = append(out, node.Key+":"+node.Value)
+		case *ast.OperatorNode:
+			out = append(out, node.Value)
+		case *ast.GroupNode:
+			out = append(out, "("+node.Key, fmt.Sprint(render(node.Nodes)), ")")
+		default:
+			out = append(out, fmt.Sprintf("%T", n))
+		}
+	}
+	return out
+}
+
+var _ = DescribeTable("ExtractSemantic",
+	func(input, wantText string, wantRemaining []string) {
+		parsed, err := kql.Builder{}.Build(input)
+		Expect(err).ToNot(HaveOccurred())
+
+		text := query.ExtractSemantic(parsed)
+
+		Expect(text).To(Equal(wantText))
+		Expect(render(parsed.Nodes)).To(Equal(wantRemaining))
+	},
+	Entry("no semantic part",
+		`mediatype:image AND Tags:foo`,
+		``,
+		[]string{"mediatype:image", "AND", "Tags:foo"},
+	),
+	Entry("purely semantic",
+		`semantic:"Hund am Strand"`,
+		`Hund am Strand`,
+		[]string{},
+	),
+	Entry("unquoted single word",
+		`semantic:Kirche`,
+		`Kirche`,
+		[]string{},
+	),
+	Entry("case-insensitive key",
+		`Semantic:"Meer"`,
+		`Meer`,
+		[]string{},
+	),
+	Entry("semantic combined with a filter",
+		`semantic:"Kirche" AND Tags:foo`,
+		`Kirche`,
+		[]string{"Tags:foo"},
+	),
+	Entry("semantic in the middle of the query",
+		`mediatype:image AND semantic:"Meer" AND Tags:urlaub`,
+		`Meer`,
+		[]string{"mediatype:image", "AND", "Tags:urlaub"},
+	),
+	Entry("group that only held the semantic clause collapses",
+		`(semantic:"Berge") AND mediatype:image`,
+		`Berge`,
+		[]string{"mediatype:image"},
+	),
+	Entry("semantic inside a quoted value stays a literal (web name wrapping)",
+		`(name:"*semantic:konzert*" OR content:"semantic:konzert")`,
+		``,
+		[]string{"(", `[name:*semantic:konzert* OR content:semantic:konzert]`, ")"},
+	),
+	Entry("semantic value containing a colon",
+		`semantic:"Kirche: innen" AND Tags:foo`,
+		`Kirche: innen`,
+		[]string{"Tags:foo"},
+	),
+)

services/search/pkg/search/search.go

@@ -48,6 +48,13 @@ type Engine interface {
 	NewBatch(batchSize int) (BatchOperator, error)
 }
 
+// TextVectorizer embeds a query text into the image vector space, so the
+// backends can rank image vectors against it. nil means semantic search is not
+// configured.
+type TextVectorizer interface {
+	VectorizeText(ctx context.Context, text string) ([]float32, error)
+}
+
 type BatchOperator interface {
 	Upsert(id string, r Resource) error
 	Move(id string, parentID string, targetPath string) error
@@ -79,15 +86,16 @@ var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts
 		// every keyword field searches case-insensitively and by word (name,
 		// title, the facets) unless opted out: ids are opaque, paths are POSIX,
 		// the mime type is normalized already, a tag is one label
-		"ID":        {CaseInsensitive: &False, NoWordBreaker: &True},
-		"RootID":    {CaseInsensitive: &False, NoWordBreaker: &True},
-		"ParentID":  {CaseInsensitive: &False, NoWordBreaker: &True},
-		"Path":      {Type: mapping.TypePath, CaseInsensitive: &False},
-		"MimeType":  {CaseInsensitive: &False, NoWordBreaker: &True},
-		"Content":   {Type: mapping.TypeFulltext},
-		"Tags":      {NoWordBreaker: &True, IncludeInAll: &False},
-		"Favorites": {NoWordBreaker: &True, IncludeInAll: &False, CaseInsensitive: &False}, // opaque user ids
-		"location":  {Type: mapping.TypeGeopoint},
+		"ID":          {CaseInsensitive: &False, NoWordBreaker: &True},
+		"RootID":      {CaseInsensitive: &False, NoWordBreaker: &True},
+		"ParentID":    {CaseInsensitive: &False, NoWordBreaker: &True},
+		"Path":        {Type: mapping.TypePath, CaseInsensitive: &False},
+		"MimeType":    {CaseInsensitive: &False, NoWordBreaker: &True},
+		"Content":     {Type: mapping.TypeFulltext},
+		"Tags":        {NoWordBreaker: &True, IncludeInAll: &False},
+		"Favorites":   {NoWordBreaker: &True, IncludeInAll: &False, CaseInsensitive: &False}, // opaque user ids
+		"location":    {Type: mapping.TypeGeopoint},
+		"imageVector": {Type: mapping.TypeVector, Dims: content.ImageVectorDims},
 	}
 })