From 4d58fa0056e7cfcdda3886f307fd4918a31fe90f Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 4 Nov 2026 17:30:51 +0000 Subject: [PATCH] feat(studio): search all chat text with a real index Search read the newest 500 chats and scanned them in memory, so older chats dropped out silently. The chat store now keeps a search index of the text of user and assistant messages: FTS5 with the trigram tokenizer on SQLite, a pg_trgm GIN index on PostgreSQL, and a scan of the same table when neither is there. The file store scans its in-memory transcripts for up to two seconds, newest chat first, and says when it stopped early. The index row is written in the transaction that writes the entry and removed in the one that deletes the chat or truncates the transcript. A query joins the chat row for the owner, so one user never sees the chats of another, and sub-agent chats stay out. Messages written before the index existed are added in the background with a resumable cursor and progress in the log; startup does not wait for it. GET /api/chats/search keeps its fields and adds the role, time, match count and highlight offsets of the hit, and filters for agent, origin, role and date range. A phrase in double quotes is one word. Hits rank by match quality and recency. group=message lists every matching line. Assisted-by: Claude:claude-sonnet-5-5 --- core/http/endpoints/studio/chatfeatures.go | 86 ++- core/http/routes/studio_search_test.go | 89 ++++ core/schema/chats_features.go | 27 +- core/services/studio/chatexport.go | 131 ++--- core/services/studio/chatsearch/chatsearch.go | 492 ++++++++++++++++++ .../chatsearch/chatsearch_suite_test.go | 13 + .../studio/chatsearch/chatsearch_test.go | 168 ++++++ core/services/studio/store/gorm.go | 28 +- .../studio/store/gorm_postgres_test.go | 40 +- core/services/studio/store/gormsearch.go | 429 +++++++++++++++ .../studio/store/gormsearch_internal_test.go | 38 ++ core/services/studio/store/gormsearch_test.go | 112 ++++ core/services/studio/store/jsonsearch.go | 87 ++++ .../studio/store/storetest/contract.go | 1 + .../services/studio/store/storetest/search.go | 229 ++++++++ docs/content/features/chats.md | 13 +- swagger/docs.go | 98 +++- swagger/swagger.json | 98 +++- swagger/swagger.yaml | 89 +++- 19 files changed, 2158 insertions(+), 110 deletions(-) create mode 100644 core/http/routes/studio_search_test.go create mode 100644 core/services/studio/chatsearch/chatsearch.go create mode 100644 core/services/studio/chatsearch/chatsearch_suite_test.go create mode 100644 core/services/studio/chatsearch/chatsearch_test.go create mode 100644 core/services/studio/store/gormsearch.go create mode 100644 core/services/studio/store/gormsearch_internal_test.go create mode 100644 core/services/studio/store/gormsearch_test.go create mode 100644 core/services/studio/store/jsonsearch.go create mode 100644 core/services/studio/store/storetest/search.go diff --git a/core/http/endpoints/studio/chatfeatures.go b/core/http/endpoints/studio/chatfeatures.go index 683393ddfb..4d1a264959 100644 --- a/core/http/endpoints/studio/chatfeatures.go +++ b/core/http/endpoints/studio/chatfeatures.go @@ -3,6 +3,7 @@ package studio import ( "errors" "net/http" + "slices" "strconv" "strings" "time" @@ -12,6 +13,7 @@ import ( "github.com/mudler/LocalAI/core/http/auth" "github.com/mudler/LocalAI/core/schema" studiosvc "github.com/mudler/LocalAI/core/services/studio" + "github.com/mudler/LocalAI/core/services/studio/chatsearch" "github.com/mudler/LocalAI/core/services/studio/session" ) @@ -154,31 +156,95 @@ func (h *Handlers) Fork(c echo.Context) error { return c.JSON(http.StatusCreated, out) } +// searchRoles and searchOrigins are what the search filters accept. +var ( + searchRoles = []string{session.RoleUser, session.RoleAssistant} + searchOrigins = []string{chatsearch.OriginPerson, chatsearch.OriginAutomated, session.OriginTrigger, session.OriginConnector, session.OriginJob, session.OriginPage} +) + +// searchTime reads a date (YYYY-MM-DD, UTC) or a time (RFC 3339). A date given +// as the end of a range includes that whole day. +func searchTime(v string, endOfDay bool) (time.Time, error) { + if t, err := time.Parse(time.DateOnly, v); err == nil { + if endOfDay { + t = t.AddDate(0, 0, 1) + } + return t, nil + } + return time.Parse(time.RFC3339, v) +} + // Search finds chats. // @Summary Search your chats. -// @Description Finds chats whose title or messages contain the text, ignoring case. The newest chats come first. It reads the newest 500 chats. +// @Description Finds the messages and titles of your chats that hold every word of the text (a phrase in double quotes counts as one word), ignoring case. It searches all your chats, not only the newest, except the runs of sub-agents. The best match comes first: whole words, the whole phrase and newer messages rank higher. By default each chat appears once, with its best line; group=message returns every matching line. A hit has the line (seq), who wrote it, when, a snippet and the parts of the snippet that match (highlights, UTF-16 offsets). Messages written before the search index existed are added in the background after an upgrade (indexing is true meanwhile). A server that keeps chats in files has no index: it scans for a few seconds and says when it stopped early (partial). // @Tags chats -// @Param q query string true "text to find" -// @Param limit query int false "most chats to return (default 50, at most 200)" +// @Param q query string true "text to find: words that must all be in the message" +// @Param limit query int false "most hits to return (default 50, at most 200)" +// @Param agent query string false "only chats of this saved agent" +// @Param origin query string false "only chats of one origin: person, automated, trigger, connector, job or page" +// @Param role query string false "only lines written by user or assistant" +// @Param since query string false "only lines at or after this date (YYYY-MM-DD) or time (RFC 3339)" +// @Param until query string false "only lines before this time, or on or before this date (YYYY-MM-DD)" +// @Param group query string false "chat (default): one hit per chat; message: every matching line" // @Success 200 {object} schema.ChatsChatSearchResponse "The matching chats" +// @Failure 400 {object} schema.ErrorResponse "The request is not valid" // @Router /api/chats/search [get] func (h *Handlers) Search(c echo.Context) error { q := strings.TrimSpace(c.QueryParam("q")) if q == "" { return fail(c, http.StatusBadRequest, "invalid_request_error", "q is required") } + o := studiosvc.SearchOptions{Query: q, Limit: intParam(c, "limit", defaultListLimit, maxListLimit), Titles: true, + AgentID: c.QueryParam("agent"), Origin: c.QueryParam("origin"), Role: c.QueryParam("role"), PerChat: true} + if o.Role != "" && !slices.Contains(searchRoles, o.Role) { + return fail(c, http.StatusBadRequest, "invalid_request_error", "role must be user or assistant") + } + if o.Origin != "" && !slices.Contains(searchOrigins, o.Origin) { + return fail(c, http.StatusBadRequest, "invalid_request_error", "origin must be person, automated, trigger, connector, job or page") + } + switch c.QueryParam("group") { + case "", "chat": + case "message": + o.PerChat = false + default: + return fail(c, http.StatusBadRequest, "invalid_request_error", "group must be chat or message") + } + var err error + if v := c.QueryParam("since"); v != "" { + if o.Since, err = searchTime(v, false); err != nil { + return fail(c, http.StatusBadRequest, "invalid_request_error", "since must be YYYY-MM-DD or an RFC 3339 time") + } + } + if v := c.QueryParam("until"); v != "" { + if o.Until, err = searchTime(v, true); err != nil { + return fail(c, http.StatusBadRequest, "invalid_request_error", "until must be YYYY-MM-DD or an RFC 3339 time") + } + } user := userID(c) - hits, err := h.svc.Search(c.Request().Context(), user, q, intParam(c, "limit", defaultListLimit, maxListLimit)) + res, err := h.svc.SearchChats(c.Request().Context(), user, o) if err != nil { return failFor(c, err) } - out := schema.ChatsChatSearchResponse{Hits: make([]schema.ChatsChatHit, 0, len(hits))} - for _, hit := range hits { - conv, err := h.describe(c, user, hit.Chat) - if err != nil { - return failFor(c, err) + out := schema.ChatsChatSearchResponse{Hits: make([]schema.ChatsChatHit, 0, len(res.Hits)), Engine: res.Engine, + Indexing: res.Progress.Indexing, Partial: res.Partial, SearchedChats: res.Searched, TotalChats: res.Total} + described := map[string]schema.ChatsConversation{} + for _, hit := range res.Hits { + conv, ok := described[hit.Chat.ID] + if !ok { + if conv, err = h.describe(c, user, hit.Chat); err != nil { + return failFor(c, err) + } + described[hit.Chat.ID] = conv } - out.Hits = append(out.Hits, schema.ChatsChatHit{Chat: conv, InTitle: hit.InTitle, Seq: hit.Seq, Snippet: hit.Snippet}) + item := schema.ChatsChatHit{Chat: conv, InTitle: hit.InTitle, Seq: hit.Seq, Snippet: hit.Snippet, Role: hit.Role, Matches: hit.Matches} + if !hit.At.IsZero() { + at := hit.At + item.At = &at + } + for _, r := range hit.Highlights { + item.Highlights = append(item.Highlights, schema.ChatsSearchRange{Start: r.Start, End: r.End}) + } + out.Hits = append(out.Hits, item) } return c.JSON(http.StatusOK, out) } diff --git a/core/http/routes/studio_search_test.go b/core/http/routes/studio_search_test.go new file mode 100644 index 0000000000..d805c1c3a1 --- /dev/null +++ b/core/http/routes/studio_search_test.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: MIT +//go:build auth + +package routes_test + +import ( + "context" + "net/http" + "net/url" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/studio/session" +) + +var _ = Describe("Chats search endpoint", func() { + var s *studioServer + + BeforeEach(func() { s = newStudioServer() }) + + say := func(id, role, text string, at time.Time) { + _, err := s.store.AppendEntry(context.Background(), id, session.Entry{Role: role, Text: text, At: at}) + Expect(err).NotTo(HaveOccurred()) + } + search := func(user string, params url.Values) map[string]any { + rec := s.do(user, http.MethodGet, "/api/chats/search?"+params.Encode(), nil) + Expect(rec.Code).To(Equal(http.StatusOK), rec.Body.String()) + return decodeJSON(rec) + } + hits := func(body map[string]any) []map[string]any { + var out []map[string]any + for _, h := range body["hits"].([]any) { + out = append(out, h.(map[string]any)) + } + return out + } + + It("returns the old fields and the new ones, with highlights", func() { + id := s.create("alice") + say(id, session.RoleUser, "What is the Qdrant port?", time.Now().Add(-time.Hour)) + body := search("alice", url.Values{"q": {"qdrant"}}) + Expect(body).To(HaveKey("engine")) + h := hits(body) + Expect(h).To(HaveLen(1)) + Expect(h[0]["chat"]).To(HaveKeyWithValue("id", id)) + Expect(h[0]).To(HaveKeyWithValue("seq", BeNumerically("==", 1))) + Expect(h[0]).To(HaveKeyWithValue("role", "user")) + Expect(h[0]).To(HaveKeyWithValue("snippet", "What is the Qdrant port?")) + Expect(h[0]).To(HaveKey("at")) + Expect(h[0]["highlights"]).To(Equal([]any{map[string]any{"start": float64(12), "end": float64(18)}})) + Expect(h[0]).To(HaveKeyWithValue("in_title", false)) + }) + + It("never returns the chats of another user", func() { + a, b := s.create("alice"), s.create("bob") + say(a, session.RoleUser, "zebra stripes", time.Now()) + say(b, session.RoleUser, "zebra stripes and secrets", time.Now()) + Expect(hits(search("alice", url.Values{"q": {"zebra"}}))).To(HaveLen(1)) + Expect(hits(search("alice", url.Values{"q": {"secrets"}}))).To(BeEmpty()) + Expect(hits(search("bob", url.Values{"q": {"secrets"}}))).To(HaveLen(1)) + }) + + It("groups by chat unless asked for messages, and filters by role and date", func() { + id := s.create("alice") + old := time.Date(2026, 1, 5, 10, 0, 0, 0, time.UTC) + say(id, session.RoleUser, "ocelot question", old) + say(id, session.RoleAssistant, "ocelot answer", old.Add(48*time.Hour)) + Expect(hits(search("alice", url.Values{"q": {"ocelot"}}))).To(HaveLen(1)) + Expect(hits(search("alice", url.Values{"q": {"ocelot"}, "group": {"message"}}))).To(HaveLen(2)) + Expect(hits(search("alice", url.Values{"q": {"ocelot"}, "group": {"message"}, "role": {"assistant"}}))).To(HaveLen(1)) + Expect(hits(search("alice", url.Values{"q": {"ocelot"}, "group": {"message"}, "since": {"2026-01-06"}}))).To(HaveLen(1)) + // until is inclusive of the whole day. + Expect(hits(search("alice", url.Values{"q": {"ocelot"}, "group": {"message"}, "until": {"2026-01-05"}}))).To(HaveLen(1)) + Expect(hits(search("alice", url.Values{"q": {"ocelot"}, "group": {"message"}, "since": {"2026-01-05T09:00:00Z"}, "until": {"2026-01-07"}}))).To(HaveLen(2)) + }) + + It("refuses a bad parameter", func() { + for _, p := range []url.Values{{"q": {"x"}, "role": {"tool"}}, {"q": {"x"}, "origin": {"nope"}}, {"q": {"x"}, "since": {"yesterday"}}, {"q": {"x"}, "group": {"all"}}, {}} { + Expect(s.do("alice", http.MethodGet, "/api/chats/search?"+p.Encode(), nil).Code).To(Equal(http.StatusBadRequest), p.Encode()) + } + }) + + It("is refused to a user whose chats feature was revoked", func() { + Expect(s.do("carol", http.MethodGet, "/api/chats/search?q=x", nil).Code).To(Equal(http.StatusForbidden)) + Expect(s.do("", http.MethodGet, "/api/chats/search?q=x", nil).Code).To(Equal(http.StatusUnauthorized)) + }) +}) diff --git a/core/schema/chats_features.go b/core/schema/chats_features.go index 682f698326..49bbae7a46 100644 --- a/core/schema/chats_features.go +++ b/core/schema/chats_features.go @@ -42,20 +42,45 @@ type ChatsForkRequest struct { Title string `json:"title,omitempty"` } +// ChatsSearchRange marks a part of a snippet. Start and End are offsets in +// UTF-16 code units (the way a JavaScript string counts); End is exclusive. +type ChatsSearchRange struct { + Start int `json:"start"` + End int `json:"end"` +} + // ChatsChatHit is a chat that matches a search. type ChatsChatHit struct { Chat ChatsConversation `json:"chat"` // InTitle is true when the title matches. InTitle bool `json:"in_title"` - // Seq is the first matching transcript line, or 0 when only the title matches. + // Seq is the matching transcript line, or 0 when only the title matches. Seq int64 `json:"seq,omitempty"` // Snippet is the matching text with a little around it. Snippet string `json:"snippet,omitempty"` + // Role is user or assistant: who wrote the matching line. + Role string `json:"role,omitempty"` + // At is when the matching line was written. + At *time.Time `json:"at,omitempty"` + // Highlights are the parts of Snippet that match, in order. + Highlights []ChatsSearchRange `json:"highlights,omitempty"` + // Matches is how many lines of the chat match, among the newest matches. + Matches int `json:"matches,omitempty"` } // ChatsChatSearchResponse is the response of the chat search. type ChatsChatSearchResponse struct { Hits []ChatsChatHit `json:"hits"` + // Engine is how the server searched: fts5, pg_trgm, scan or memory. + Engine string `json:"engine,omitempty"` + // Indexing is true while the server still adds earlier chats to its + // search index, so older chats may be missing from the hits. + Indexing bool `json:"indexing,omitempty"` + // Partial is true when a server without an index stopped before it had + // read every chat; SearchedChats of TotalChats were read. + Partial bool `json:"partial,omitempty"` + SearchedChats int `json:"searched_chats,omitempty"` + TotalChats int `json:"total_chats,omitempty"` } // ChatsExport is a chat as a file. Version is 1. diff --git a/core/services/studio/chatexport.go b/core/services/studio/chatexport.go index 2e480c2e47..e7e05f14c0 100644 --- a/core/services/studio/chatexport.go +++ b/core/services/studio/chatexport.go @@ -2,12 +2,14 @@ package studio import ( "context" + "errors" "fmt" "path" "strings" "time" "unicode/utf8" + "github.com/mudler/LocalAI/core/services/studio/chatsearch" "github.com/mudler/LocalAI/core/services/studio/session" ) @@ -199,91 +201,60 @@ func checkImportAttachments(atts []session.Attachment) error { return nil } -// SearchHit is a chat that matches a search. -type SearchHit struct { - Chat session.ChatRecord - // InTitle is true when the title matches. - InTitle bool - // Seq is the first matching transcript line, or 0 when only the title matches. - Seq int64 - // Snippet is the matching text with a little around it. - Snippet string +// SearchHit is a message (or a chat title) that matches a search. Chat, InTitle, +// Seq and Snippet are what the first version of the search gave. +type SearchHit = chatsearch.Hit + +// SearchOptions is a search of the chats of one user. +type SearchOptions struct { + // Query is the text to find: words that must all be in a message, or a + // phrase in double quotes. + Query string + // Limit is the most hits; 0 is 50. + Limit int + // AgentID, Origin, Role, Since and Until narrow the search; see + // chatsearch.Query. + AgentID, Origin, Role string + Since, Until time.Time + // ChatID limits the search to one chat. + ChatID string + // PerChat returns the best message of each chat. + PerChat bool + // Titles also matches chat titles. + Titles bool + // SnippetRunes is the width of a snippet; 0 is the default. + SnippetRunes int + // Exclude drops the chats it returns true for. + Exclude func(session.ChatRecord) bool } -// Limits of a search. -const ( - maxSearchChats = 500 - maxSearchBytes = 64 << 20 -) +// ErrSearchUnavailable means the chat store cannot search text. +var ErrSearchUnavailable = errors.New("studio: this chat store cannot search text") + +// SearchChats finds the user's chats by their messages (and titles) as the +// store's index allows, newest and best match first. It looks at every chat of +// the user except sub-agent chats; the owner is applied by the store in the +// query itself. +func (s *Service) SearchChats(ctx context.Context, user string, o SearchOptions) (chatsearch.Result, error) { + if len(chatsearch.Terms(o.Query)) == 0 { + return chatsearch.Result{}, fmt.Errorf("%w: empty query", session.ErrInvalid) + } + idx, ok := s.d.Store.(chatsearch.Index) + if !ok { + return chatsearch.Result{}, ErrSearchUnavailable + } + return idx.SearchText(ctx, chatsearch.Query{ + Principal: principal(user), Text: o.Query, Limit: o.Limit, AgentID: o.AgentID, Origin: o.Origin, + Role: o.Role, Since: o.Since, Until: o.Until, ChatID: o.ChatID, PerChat: o.PerChat, Titles: o.Titles, + SnippetRunes: o.SnippetRunes, Exclude: o.Exclude, + }) +} // Search finds the user's chats whose title or user and assistant messages -// contain query, ignoring case. The newest chats come first. It reads at most -// the newest maxSearchChats chats and maxSearchBytes of text. +// contain every word of query, ignoring case: one hit per chat. func (s *Service) Search(ctx context.Context, user, query string, limit int) ([]SearchHit, error) { - q := strings.ToLower(strings.TrimSpace(query)) - if q == "" { - return nil, fmt.Errorf("%w: empty query", session.ErrInvalid) - } - chats, err := s.ListChats(ctx, user, maxSearchChats) - if err != nil { - return nil, err - } - var ( - hits []SearchHit - spent int - ) - for _, rec := range chats { - if limit > 0 && len(hits) >= limit { - break - } - hit := SearchHit{Chat: rec} - if strings.Contains(strings.ToLower(rec.Title), q) { - hit.InTitle = true - } - if spent < maxSearchBytes { - entries, err := s.d.Store.Entries(ctx, rec.ID, 0, 0) - if err != nil { - return nil, err - } - for _, e := range entries { - spent += len(e.Text) - if e.Role != session.RoleUser && e.Role != session.RoleAssistant { - continue - } - // The words the person reads, not the list of files the model gets. - text := session.DisplayEntry(e).Text - if i := strings.Index(strings.ToLower(text), q); i >= 0 { - hit.Seq, hit.Snippet = e.Seq, snippet(text, i, len(q)) - break - } - } - } - if hit.InTitle || hit.Seq != 0 { - hits = append(hits, hit) - } - } - return hits, nil -} - -// snippet returns the text around a match, about 120 bytes wide, on rune -// boundaries. -func snippet(text string, at, n int) string { - at = min(at, len(text)) - from, to := max(0, at-50), min(len(text), at+n+70) - for from > 0 && !utf8.RuneStart(text[from]) { - from-- - } - for to < len(text) && !utf8.RuneStart(text[to]) { - to++ - } - out := strings.Join(strings.Fields(text[from:to]), " ") - if from > 0 { - out = "…" + out - } - if to < len(text) { - out += "…" - } - return out + res, err := s.SearchChats(ctx, user, SearchOptions{Query: query, Limit: limit, PerChat: true, Titles: true}) + return res.Hits, err } // cleanInvocation keeps the invocation of an imported message only when it is diff --git a/core/services/studio/chatsearch/chatsearch.go b/core/services/studio/chatsearch/chatsearch.go new file mode 100644 index 0000000000..d5e440aeab --- /dev/null +++ b/core/services/studio/chatsearch/chatsearch.go @@ -0,0 +1,492 @@ +// Package chatsearch finds text in chat transcripts. A store keeps an index +// (or scans) and returns candidate messages; Build then checks them, ranks +// them and cuts the snippets, so every store gives the same answers. +// +// Matching is substring matching on text folded to lower case rune by rune +// (Fold), so the index, the query and the snippet agree on every script. A +// query is split in words and a message matches when it holds every word; a +// phrase in double quotes is one word. +package chatsearch + +import ( + "context" + "math" + "slices" + "sort" + "strings" + "time" + "unicode" + "unicode/utf16" + "unicode/utf8" + + "github.com/mudler/LocalAI/core/services/studio/session" +) + +// Limits of a search. +const ( + // MinTermRunes is the length under which the trigram index finds nothing; + // a shorter word is checked on the rows the longer words select. + MinTermRunes = 3 + // MaxTerms is the most words a query may have; the rest is ignored. + MaxTerms = 6 + // MaxTermRunes is the longest word; a longer one is cut. + MaxTermRunes = 64 + // PoolSize is the most candidate messages a store returns, newest first. + // Ranking works on this pool. + PoolSize = 600 + // DefaultSnippetRunes is the width of a snippet unless the query says. + DefaultSnippetRunes = 160 + // ScanBudget is the time a store without an index spends on one search. + ScanBudget = 2 * time.Second +) + +// Origin filters for Query.Origin besides a kind of origin (trigger, +// connector, job, page). +const ( + OriginPerson = "person" // chats a person started + OriginAutomated = "automated" // every other chat +) + +// Query is a search. +type Query struct { + // Principal owns the chats. A store applies it in the query itself. + Principal string + // Text is what to find. + Text string + + // ChatID limits the search to one chat. + ChatID string + // AgentID limits it to the chats of one saved agent. + AgentID string + // Origin limits it to chats of one origin: OriginPerson, OriginAutomated + // or a kind of origin. + Origin string + // Role limits it to messages of one role (user or assistant). + Role string + // Since and Until bound the time of the message; the zero time is no bound. + Since, Until time.Time + // Exclude drops a chat after it was loaded (a private chat, a chat the + // caller may not read). It may be nil. + Exclude func(session.ChatRecord) bool + + // Limit is the most hits to return; 0 is 50. + Limit int + // PerChat keeps the best message of each chat. + PerChat bool + // Titles also matches the title of the chats. + Titles bool + // SnippetRunes is the width of a snippet; 0 is DefaultSnippetRunes. + SnippetRunes int + // Now is the time recency is measured from; zero is the clock. + Now time.Time +} + +// Terms returns the words of the query, folded. +func (q Query) Terms() []string { return Terms(q.Text) } + +// Range marks a part of a snippet. Start and End are offsets in UTF-16 code +// units, the way a JavaScript string counts, End exclusive. +type Range struct { + Start int `json:"start"` + End int `json:"end"` +} + +// Hit is one matching message, or a chat whose title matches (Seq 0). +type Hit struct { + Chat session.ChatRecord + InTitle bool + // Seq, Role and At say which message matched; Seq is 0 for a title hit. + Seq int64 + Role string + At time.Time + // Snippet is the text around the match, with the words marked by + // Highlights. + Snippet string + Highlights []Range + // Matches is how many messages of the chat matched, among the candidates. + Matches int + Score float64 +} + +// Progress says how complete the index is. +type Progress struct { + // Indexing is true while the messages written before the index existed + // are still being added. + Indexing bool + // Done and Total count chats; Total is 0 when unknown. + Done, Total int +} + +// Result is the answer of a search. +type Result struct { + Hits []Hit + // Engine is how the store searched: fts5, pg_trgm, scan or memory. + Engine string + // Partial is true when a time budget ended the search before every chat + // was read (a store without an index). + Partial bool + // Searched and Total count the chats a store without an index read, and + // the chats it could have read; both 0 for an indexed store. + Searched, Total int + Progress Progress +} + +// Index is what a store implements to be searched. +type Index interface { + // SearchText finds the messages of q.Principal's chats that match q. A + // chat of another principal, a sub-agent chat and a deleted chat never + // appear. + SearchText(ctx context.Context, q Query) (Result, error) +} + +// Candidate is a message a store found: it may match. +type Candidate struct { + Chat session.ChatRecord + Seq int64 + Role string + // Text is what the person reads (see IndexText). + Text string + At time.Time +} + +// Fold lowers s rune by rune, so a folded string has as many runes as s. +func Fold(s string) string { return strings.Map(unicode.ToLower, s) } + +// Terms splits text into folded words. A phrase in double quotes is one word. +func Terms(text string) []string { + var out []string + add := func(w string) { + w = strings.TrimSpace(Fold(w)) + if r := []rune(w); len(r) > MaxTermRunes { + w = string(r[:MaxTermRunes]) + } + if w != "" && !slices.Contains(out, w) && len(out) < MaxTerms { + out = append(out, w) + } + } + for len(text) > 0 { + text = strings.TrimLeftFunc(text, unicode.IsSpace) + if strings.HasPrefix(text, `"`) { + if end := strings.Index(text[1:], `"`); end >= 0 { + add(text[1 : 1+end]) + text = text[end+2:] + continue + } + text = text[1:] + continue + } + end := strings.IndexFunc(text, unicode.IsSpace) + if end < 0 { + end = len(text) + } + add(text[:end]) + text = text[end:] + } + return out +} + +// IndexText is the text of a message that search covers, folded for the +// index: the words the person reads, not the list of files the model gets. It +// returns false for a line that is not a user or assistant message or has no +// text. hasAttachments says the entry keeps its files apart already, so its +// text has no list to remove. +func IndexText(role, text string, hasAttachments bool) (display string, ok bool) { + if role != session.RoleUser && role != session.RoleAssistant { + return "", false + } + e := session.Entry{Role: role, Text: text} + if hasAttachments { + e.Attachments = []session.Attachment{{}} + } + display = session.DisplayEntry(e).Text + return display, strings.TrimSpace(display) != "" +} + +// Matches reports whether folded holds every term. +func Matches(folded string, terms []string) bool { + if len(terms) == 0 { + return false + } + for _, t := range terms { + if !strings.Contains(folded, t) { + return false + } + } + return true +} + +// ChatMatches reports whether the chat passes the filters of q that do not +// depend on the message: the kind, the agent, the origin and the chat. +func (q Query) ChatMatches(rec session.ChatRecord) bool { + if rec.Principal != q.Principal || rec.Kind == session.KindSubAgent { + return false + } + if q.ChatID != "" && rec.ID != q.ChatID { + return false + } + if q.AgentID != "" && rec.AgentID != q.AgentID { + return false + } + switch q.Origin { + case "": + case OriginPerson: + if rec.Origin != "" { + return false + } + case OriginAutomated: + if rec.Origin == "" { + return false + } + default: + if rec.Origin != q.Origin { + return false + } + } + return q.Exclude == nil || !q.Exclude(rec) +} + +// TimeMatches reports whether t is inside Since and Until. +func (q Query) TimeMatches(t time.Time) bool { + if !q.Since.IsZero() && t.Before(q.Since) { + return false + } + return q.Until.IsZero() || t.Before(q.Until) +} + +// Build checks the candidates against q, ranks them and returns up to +// q.Limit hits. titles are chats whose title may match; a chat whose title +// matches but that has no matching message comes back as a title hit. +func Build(q Query, cands []Candidate, titles []session.ChatRecord) []Hit { + terms := q.Terms() + if len(terms) == 0 { + return nil + } + now := q.Now + if now.IsZero() { + now = time.Now() + } + width := q.SnippetRunes + if width <= 0 { + width = DefaultSnippetRunes + } + phrase := Fold(strings.Join(strings.Fields(q.Text), " ")) + titleHit := map[string]bool{} + if q.Titles { + for _, rec := range titles { + if q.ChatMatches(rec) && Matches(Fold(rec.Title), terms) { + titleHit[rec.ID] = true + } + } + } + + byChat := map[string][]Hit{} + var order []string + for _, c := range cands { + if !q.ChatMatches(c.Chat) || !q.TimeMatches(c.At) || (q.Role != "" && c.Role != q.Role) { + continue + } + folded := Fold(c.Text) + if !Matches(folded, terms) { + continue + } + snip, marks := cut(c.Text, folded, terms, width) + h := Hit{Chat: c.Chat, Seq: c.Seq, Role: c.Role, At: c.At, Snippet: snip, Highlights: marks, + InTitle: titleHit[c.Chat.ID]} + h.Score = quality(folded, terms, phrase) + recency(now, c.At) + if h.InTitle { + h.Score += 1.5 + } + if _, seen := byChat[c.Chat.ID]; !seen { + order = append(order, c.Chat.ID) + } + byChat[c.Chat.ID] = append(byChat[c.Chat.ID], h) + } + + var hits []Hit + for _, id := range order { + group := byChat[id] + sort.SliceStable(group, func(i, j int) bool { return better(group[i], group[j]) }) + for i := range group { + group[i].Matches = len(group) + } + if q.PerChat { + group = group[:1] + } + hits = append(hits, group...) + } + // A chat whose title matches and that has no matching message. + if q.Titles { + for _, rec := range titles { + if _, has := byChat[rec.ID]; has || !titleHit[rec.ID] || !q.TimeMatches(rec.Updated) { + continue + } + hits = append(hits, Hit{Chat: rec, InTitle: true, Score: 1.5 + 2 + recency(now, rec.Updated)}) + } + } + sort.SliceStable(hits, func(i, j int) bool { return better(hits[i], hits[j]) }) + limit := q.Limit + if limit <= 0 { + limit = 50 + } + if len(hits) > limit { + hits = hits[:limit] + } + return hits +} + +// better orders hits: score, then the newest, then the chat and line. +func better(a, b Hit) bool { + if a.Score != b.Score { + return a.Score > b.Score + } + ta, tb := a.At, b.At + if !ta.Equal(tb) { + return ta.After(tb) + } + if a.Chat.ID != b.Chat.ID { + return a.Chat.ID > b.Chat.ID + } + return a.Seq < b.Seq +} + +// quality scores how well folded matches: the whole query as one phrase, each +// word as a whole word, and repeats. +func quality(folded string, terms []string, phrase string) float64 { + var s float64 + if len(terms) > 1 && strings.Contains(folded, phrase) { + s += 2 + } + repeats := 0 + for _, t := range terms { + if wholeWord(folded, t) { + s += 1 + } + repeats += min(strings.Count(folded, t), 4) - 1 + } + return s + 0.1*float64(repeats) +} + +// wholeWord reports whether t occurs in s between non-letters. +func wholeWord(s, t string) bool { + for from := 0; ; { + i := strings.Index(s[from:], t) + if i < 0 { + return false + } + i += from + before, _ := utf8.DecodeLastRuneInString(s[:i]) + after, _ := utf8.DecodeRuneInString(s[i+len(t):]) + if (i == 0 || !isWord(before)) && (i+len(t) == len(s) || !isWord(after)) { + return true + } + from = i + len(t) + } +} + +func isWord(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' } + +// recency is 1 for now and falls to 0.5 after a month, 0.25 after two. +func recency(now, at time.Time) float64 { + if at.IsZero() { + return 0 + } + days := now.Sub(at).Hours() / 24 + if days < 0 { + days = 0 + } + return math.Exp2(-days / 30) +} + +// cut returns the text around the first match, about width runes, with white +// space folded to single spaces, and the ranges of the words in it. +func cut(text, folded string, terms []string, width int) (string, []Range) { + tr, fr := []rune(text), []rune(folded) + if len(tr) != len(fr) { // Fold keeps the rune count; stay safe anyway + tr = fr + } + first := -1 + for _, t := range terms { + if i := indexRunes(fr, []rune(t), 0); i >= 0 && (first < 0 || i < first) { + first = i + } + } + if first < 0 { + first = 0 + } + from := max(0, first-width/3) + to := min(len(tr), from+width) + from = max(0, min(from, to-width)) + window, wf := tr[from:to], fr[from:to] + + // Collapse white space, keeping the folded copy in step. + var out, outF []rune + space := true + for i, r := range window { + if unicode.IsSpace(r) { + if !space { + out, outF = append(out, ' '), append(outF, ' ') + } + space = true + continue + } + space = false + out, outF = append(out, r), append(outF, wf[i]) + } + if n := len(out); n > 0 && out[n-1] == ' ' { + out, outF = out[:n-1], outF[:n-1] + } + pre := 0 + if from > 0 { + out, outF = append([]rune("…"), out...), append([]rune("…"), outF...) + pre = 1 + } + if to < len(tr) { + out, outF = append(out, '…'), append(outF, '…') + } + + var marks []Range + for _, t := range terms { + tt := []rune(t) + for at := indexRunes(outF, tt, pre); at >= 0; at = indexRunes(outF, tt, at+len(tt)) { + marks = append(marks, Range{Start: at, End: at + len(tt)}) + } + } + marks = mergeRanges(marks) + for i, m := range marks { + marks[i] = Range{Start: utf16Len(out[:m.Start]), End: utf16Len(out[:m.End])} + } + return string(out), marks +} + +func utf16Len(r []rune) int { return len(utf16.Encode(r)) } + +// indexRunes finds sub in s at or after from, or -1. +func indexRunes(s, sub []rune, from int) int { + if len(sub) == 0 { + return -1 + } + for i := max(from, 0); i+len(sub) <= len(s); i++ { + if s[i] == sub[0] && slices.Equal(s[i:i+len(sub)], sub) { + return i + } + } + return -1 +} + +// mergeRanges sorts the ranges and joins the ones that touch or overlap. +func mergeRanges(r []Range) []Range { + if len(r) == 0 { + return nil + } + sort.Slice(r, func(i, j int) bool { return r[i].Start < r[j].Start }) + out := []Range{r[0]} + for _, x := range r[1:] { + last := &out[len(out)-1] + if x.Start <= last.End { + last.End = max(last.End, x.End) + continue + } + out = append(out, x) + } + return out +} diff --git a/core/services/studio/chatsearch/chatsearch_suite_test.go b/core/services/studio/chatsearch/chatsearch_suite_test.go new file mode 100644 index 0000000000..f69c3c6626 --- /dev/null +++ b/core/services/studio/chatsearch/chatsearch_suite_test.go @@ -0,0 +1,13 @@ +package chatsearch_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestChatSearch(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Chat search suite") +} diff --git a/core/services/studio/chatsearch/chatsearch_test.go b/core/services/studio/chatsearch/chatsearch_test.go new file mode 100644 index 0000000000..549ac11bba --- /dev/null +++ b/core/services/studio/chatsearch/chatsearch_test.go @@ -0,0 +1,168 @@ +package chatsearch_test + +import ( + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/studio/chatsearch" + "github.com/mudler/LocalAI/core/services/studio/session" +) + +var now = time.Date(2026, 11, 4, 12, 0, 0, 0, time.UTC) + +func rec(id, title string) session.ChatRecord { + return session.ChatRecord{ID: id, Principal: "alice", Title: title, Updated: now} +} + +func cand(c session.ChatRecord, seq int64, role, text string, age time.Duration) chatsearch.Candidate { + return chatsearch.Candidate{Chat: c, Seq: seq, Role: role, Text: text, At: now.Add(-age)} +} + +var _ = Describe("Terms", func() { + It("folds, splits and keeps a quoted phrase as one word", func() { + Expect(chatsearch.Terms(` Qdrant "Port Number" qdrant `)).To(Equal([]string{"qdrant", "port number"})) + }) + It("caps the number and the length of the words", func() { + Expect(chatsearch.Terms("a b c d e f g h")).To(HaveLen(chatsearch.MaxTerms)) + Expect([]rune(chatsearch.Terms(strings.Repeat("x", 500))[0])).To(HaveLen(chatsearch.MaxTermRunes)) + }) + It("returns nothing for blank text", func() { + Expect(chatsearch.Terms(" ")).To(BeEmpty()) + }) +}) + +var _ = Describe("IndexText", func() { + It("skips lines that are not messages", func() { + _, ok := chatsearch.IndexText(session.RoleCompacted, "x", false) + Expect(ok).To(BeFalse()) + }) + It("leaves out the list of files of an old message", func() { + txt, ok := chatsearch.IndexText(session.RoleUser, "see this\n\nThe user attached these files:\n- /workspace/home/a", false) + Expect(ok).To(BeTrue()) + Expect(txt).To(Equal("see this")) + }) +}) + +var _ = Describe("Build", func() { + a, b := rec("a", "Qdrant setup"), rec("b", "Other") + q := chatsearch.Query{Principal: "alice", Text: "qdrant port", Now: now, Titles: true} + + It("needs every word in the message", func() { + hits := chatsearch.Build(q, []chatsearch.Candidate{ + cand(a, 1, "user", "the qdrant port is 6333", time.Hour), + cand(a, 2, "user", "only qdrant here", time.Hour), + }, nil) + Expect(hits).To(HaveLen(1)) + Expect(hits[0].Seq).To(Equal(int64(1))) + }) + + It("ranks a whole-word phrase above a newer partial match", func() { + hits := chatsearch.Build(q, []chatsearch.Candidate{ + cand(b, 1, "user", "xqdrantx and reporting", time.Hour), + cand(a, 1, "assistant", "the Qdrant port is 6333", 20*24*time.Hour), + }, nil) + Expect(hits).To(HaveLen(2)) + Expect(hits[0].Chat.ID).To(Equal("a")) + }) + + It("prefers the newer message when quality is equal", func() { + hits := chatsearch.Build(q, []chatsearch.Candidate{ + cand(a, 1, "user", "qdrant port", 40*24*time.Hour), + cand(b, 1, "user", "qdrant port", time.Hour), + }, nil) + Expect(hits[0].Chat.ID).To(Equal("b")) + }) + + It("keeps one hit per chat when asked and counts the matches", func() { + pq := q + pq.PerChat = true + hits := chatsearch.Build(pq, []chatsearch.Candidate{ + cand(a, 1, "user", "qdrant port", time.Hour), + cand(a, 3, "user", "qdrant port again", time.Hour), + }, nil) + Expect(hits).To(HaveLen(1)) + Expect(hits[0].Matches).To(Equal(2)) + }) + + It("returns a chat whose title matches and has no matching message", func() { + tq := q + tq.Text = "qdrant" + hits := chatsearch.Build(tq, nil, []session.ChatRecord{a}) + Expect(hits).To(HaveLen(1)) + Expect(hits[0].InTitle).To(BeTrue()) + Expect(hits[0].Seq).To(BeZero()) + }) + + It("filters by role, time, agent and origin", func() { + x := rec("x", "t") + x.AgentID, x.Origin = "ag", "trigger" + cs := []chatsearch.Candidate{ + cand(x, 1, "user", "qdrant port", 2*time.Hour), + cand(x, 2, "assistant", "qdrant port", 48*time.Hour), + } + with := func(f func(*chatsearch.Query)) []chatsearch.Hit { + qq := q + f(&qq) + return chatsearch.Build(qq, cs, nil) + } + Expect(with(func(qq *chatsearch.Query) { qq.Role = "assistant" })).To(HaveLen(1)) + Expect(with(func(qq *chatsearch.Query) { qq.Since = now.Add(-24 * time.Hour) })).To(HaveLen(1)) + Expect(with(func(qq *chatsearch.Query) { qq.Until = now.Add(-24 * time.Hour) })).To(HaveLen(1)) + Expect(with(func(qq *chatsearch.Query) { qq.AgentID = "other" })).To(BeEmpty()) + Expect(with(func(qq *chatsearch.Query) { qq.Origin = chatsearch.OriginPerson })).To(BeEmpty()) + Expect(with(func(qq *chatsearch.Query) { qq.Origin = chatsearch.OriginAutomated })).To(HaveLen(2)) + Expect(with(func(qq *chatsearch.Query) { qq.Origin = "trigger" })).To(HaveLen(2)) + Expect(with(func(qq *chatsearch.Query) { qq.Origin = "connector" })).To(BeEmpty()) + }) + + It("never returns another principal's chat, a sub-agent chat or an excluded chat", func() { + other := rec("o", "t") + other.Principal = "bob" + sub := rec("s", "t") + sub.Kind = session.KindSubAgent + priv := rec("p", "t") + qq := q + qq.Exclude = func(r session.ChatRecord) bool { return r.ID == "p" } + Expect(chatsearch.Build(qq, []chatsearch.Candidate{ + cand(other, 1, "user", "qdrant port", time.Hour), + cand(sub, 1, "user", "qdrant port", time.Hour), + cand(priv, 1, "user", "qdrant port", time.Hour), + }, nil)).To(BeEmpty()) + }) + + It("cuts a snippet with highlight offsets in UTF-16 units", func() { + text := "😀 prefix " + strings.Repeat("filler ", 40) + "the Qdrant port is 6333 " + strings.Repeat("tail ", 40) + hits := chatsearch.Build(q, []chatsearch.Candidate{cand(a, 1, "user", text, time.Hour)}, nil) + Expect(hits).To(HaveLen(1)) + h := hits[0] + Expect(h.Snippet).To(HavePrefix("…")) + Expect(h.Snippet).To(HaveSuffix("…")) + Expect(len([]rune(h.Snippet))).To(BeNumerically("<=", chatsearch.DefaultSnippetRunes+2)) + Expect(h.Highlights).To(HaveLen(2)) + units := []rune(h.Snippet) // the snippet has no astral rune here + got := []string{} + for _, m := range h.Highlights { + got = append(got, string(units[m.Start:m.End])) + } + Expect(got).To(Equal([]string{"Qdrant", "port"})) + }) + + It("counts an astral rune as two units in the offsets", func() { + hits := chatsearch.Build(chatsearch.Query{Principal: "alice", Text: "needle", Now: now}, + []chatsearch.Candidate{cand(a, 1, "user", "😀 needle", time.Hour)}, nil) + Expect(hits[0].Highlights).To(Equal([]chatsearch.Range{{Start: 3, End: 9}})) + }) + + It("limits the hits", func() { + var cs []chatsearch.Candidate + for i := range 5 { + cs = append(cs, cand(a, int64(i+1), "user", "qdrant port", time.Duration(i)*time.Hour)) + } + qq := q + qq.Limit = 2 + Expect(chatsearch.Build(qq, cs, nil)).To(HaveLen(2)) + }) +}) diff --git a/core/services/studio/store/gorm.go b/core/services/studio/store/gorm.go index 316144f7bc..122055cc02 100644 --- a/core/services/studio/store/gorm.go +++ b/core/services/studio/store/gorm.go @@ -136,14 +136,17 @@ type pendingRow struct { func (pendingRow) TableName() string { return "studio_pending" } -type gormStore struct{ db *gorm.DB } +type gormStore struct { + db *gorm.DB + text textIndex +} // NewGORM returns a Store on db (SQLite or PostgreSQL) and creates its tables. func NewGORM(db *gorm.DB) (Store, error) { if db == nil { return nil, errors.New("studio store: nil *gorm.DB") } - if err := db.AutoMigrate(&chatRow{}, &entryRow{}, &pendingRow{}); err != nil { + if err := db.AutoMigrate(&chatRow{}, &entryRow{}, &pendingRow{}, &textRow{}, &textStateRow{}); err != nil { return nil, fmt.Errorf("studio store: migrate: %w", err) } s := &gormStore{db: db} @@ -151,6 +154,11 @@ func NewGORM(db *gorm.DB) (Store, error) { // Reads normalize the markers in memory, so the store works anyway. xlog.Warn("studio store: could not rewrite the legacy skill markers", "error", err) } + s.setupTextIndex() + // Indexing is true from here, so a search never reports the index as + // complete before the first look at the history. + s.text.indexing.Store(true) + go s.backfillTexts() return s, nil } @@ -662,6 +670,9 @@ func (s *gormStore) TruncateEntries(ctx context.Context, principal, id string, k if err := tx.Where("chat_id = ? AND seq > ?", id, keep).Delete(&entryRow{}).Error; err != nil { return err } + if err := tx.Where("chat_id = ? AND seq > ?", id, keep).Delete(&textRow{}).Error; err != nil { + return err + } // lock bumped the counter; the next entry is keep+1. return tx.Model(&chatRow{}).Where("id = ?", id).UpdateColumn("next_entry", keep).Error }) @@ -679,6 +690,9 @@ func (s *gormStore) DeleteChat(ctx context.Context, principal, id string) error if err := tx.Where("chat_id = ?", id).Delete(&entryRow{}).Error; err != nil { return err } + if err := tx.Where("chat_id = ?", id).Delete(&textRow{}).Error; err != nil { + return err + } return tx.Where("chat_id = ?", id).Delete(&pendingRow{}).Error }) } @@ -722,6 +736,9 @@ func (s *gormStore) AppendEntry(ctx context.Context, id string, e session.Entry) if err := tx.Create(&row).Error; err != nil { return err } + if err := putTexts(tx, textRows(id, seq, []session.Entry{e})); err != nil { + return err + } stored = seq return nil }) @@ -747,17 +764,22 @@ func (s *gormStore) AppendEntries(ctx context.Context, id string, entries []sess } } rows := make([]entryRow, len(entries)) + stamped := make([]session.Entry, len(entries)) for i, e := range entries { if e.At.IsZero() { e.At = time.Now().UTC() } + stamped[i] = e row, err := toEntryRow(id, first+int64(i), e) if err != nil { return err } rows[i] = row } - return tx.CreateInBatches(&rows, 100).Error + if err := tx.CreateInBatches(&rows, 100).Error; err != nil { + return err + } + return putTexts(tx, textRows(id, first, stamped)) }) } diff --git a/core/services/studio/store/gorm_postgres_test.go b/core/services/studio/store/gorm_postgres_test.go index 7d50c63556..9c21bbf646 100644 --- a/core/services/studio/store/gorm_postgres_test.go +++ b/core/services/studio/store/gorm_postgres_test.go @@ -1,13 +1,16 @@ package store_test import ( + "context" "os" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "gorm.io/driver/postgres" "gorm.io/gorm" + "github.com/mudler/LocalAI/core/services/studio/chatsearch" "github.com/mudler/LocalAI/core/services/studio/store" "github.com/mudler/LocalAI/core/services/studio/store/storetest" ) @@ -29,9 +32,44 @@ var _ = Describe("the GORM store on PostgreSQL", func() { DeferCleanup(func() { _ = sqlDB.Close() }) s, err := store.NewGORM(db) Expect(err).NotTo(HaveOccurred()) - for _, tbl := range []string{"studio_pending", "studio_entries", "studio_chats"} { + for _, tbl := range []string{"studio_pending", "studio_entry_texts", "studio_entry_text_state", "studio_entries", "studio_chats"} { Expect(db.Exec("DELETE FROM " + tbl).Error).NotTo(HaveOccurred()) } return s }) }) + +var _ = Describe("the text index of the GORM store on PostgreSQL", func() { + url := os.Getenv("LOCALAI_TEST_POSTGRES_URL") + if url == "" { + It("indexes and backfills", func() { Skip("LOCALAI_TEST_POSTGRES_URL is not set") }) + return + } + open := func() *gorm.DB { + db, err := gorm.Open(postgres.Open(url), &gorm.Config{}) + Expect(err).NotTo(HaveOccurred()) + sqlDB, _ := db.DB() + DeferCleanup(func() { _ = sqlDB.Close() }) + for _, tbl := range []string{"studio_pending", "studio_entry_texts", "studio_entry_text_state", "studio_entries", "studio_chats"} { + _ = db.Exec("DELETE FROM " + tbl).Error + } + return db + } + + It("uses the trigram index when the extension is there", func() { + s, err := store.NewGORM(open()) + Expect(err).NotTo(HaveOccurred()) + Expect(store.TextEngine(s)).To(Equal("pg_trgm")) + }) + + It("indexes the messages of a database from before the index", func() { + db := open() + seedLegacy(db, 40) + s, err := store.NewGORM(db) + Expect(err).NotTo(HaveOccurred()) + waitIndexed(s) + res, err := s.(chatsearch.Index).SearchText(context.Background(), chatsearch.Query{Principal: "alice", Text: "marmot", Limit: 100, Now: time.Now()}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Hits).To(HaveLen(40)) + }) +}) diff --git a/core/services/studio/store/gormsearch.go b/core/services/studio/store/gormsearch.go new file mode 100644 index 0000000000..608e7a273e --- /dev/null +++ b/core/services/studio/store/gormsearch.go @@ -0,0 +1,429 @@ +package store + +import ( + "context" + "fmt" + "strings" + "sync/atomic" + "time" + "unicode/utf8" + + "github.com/mudler/xlog" + "gorm.io/gorm" + "gorm.io/gorm/clause" + + "github.com/mudler/LocalAI/core/services/studio/chatsearch" + "github.com/mudler/LocalAI/core/services/studio/session" +) + +// textRow is the search index of one message: the text the person reads, +// folded to lower case. It carries no principal: a query joins the chat row, +// so the owner of a chat can never be out of step with its index rows. The +// row is written in the transaction that writes the entry and removed in the +// one that removes it. +type textRow struct { + ID int64 `gorm:"primaryKey;autoIncrement"` + ChatID string `gorm:"size:128;not null;uniqueIndex:idx_studio_entry_texts_pos,priority:1"` + Seq int64 `gorm:"not null;uniqueIndex:idx_studio_entry_texts_pos,priority:2"` + Role string `gorm:"size:16"` + At time.Time `gorm:"index:idx_studio_entry_texts_at"` + Folded string `gorm:"type:text"` +} + +func (textRow) TableName() string { return "studio_entry_texts" } + +// textStateRow remembers how far the backfill of the messages written before +// the index existed has come, so it resumes after a restart. +type textStateRow struct { + ID int `gorm:"primaryKey;autoIncrement:false"` + LastChat string `gorm:"size:128"` + Done bool +} + +func (textStateRow) TableName() string { return "studio_entry_text_state" } + +// Engines of the text search. +const ( + engineScan = "scan" // instr or LIKE over the folded column + engineFTS5 = "fts5" // SQLite FTS5 with the trigram tokenizer + enginePGTrgm = "pg_trgm" // PostgreSQL LIKE with a trigram GIN index + + ftsTable = "studio_entry_texts_fts" + pgTrgmIndex = "idx_studio_entry_texts_trgm" + titlePool = 200 + backfillPage = 200 + backfillStep = 50 + likeEscape = `\` +) + +// textIndex is the state of the search of a GORM store. +type textIndex struct { + engine string + postgres bool + indexing atomic.Bool + done atomic.Int64 + total atomic.Int64 +} + +func (s *gormStore) setupTextIndex() { + s.text.postgres = strings.Contains(s.db.Dialector.Name(), "postgres") + s.text.engine = engineScan + if s.text.postgres { + s.setupPGTrgm() + } else { + s.setupFTS5() + } +} + +func (s *gormStore) setupFTS5() { + var n int64 + s.db.Raw("SELECT count(*) FROM sqlite_master WHERE type = 'table' AND name = ?", ftsTable).Scan(&n) + fresh := n == 0 + stmts := []string{ + `CREATE VIRTUAL TABLE IF NOT EXISTS ` + ftsTable + ` USING fts5(folded, content='studio_entry_texts', content_rowid='id', tokenize='trigram')`, + `CREATE TRIGGER IF NOT EXISTS studio_entry_texts_ai AFTER INSERT ON studio_entry_texts BEGIN + INSERT INTO ` + ftsTable + `(rowid, folded) VALUES (new.id, new.folded); END`, + `CREATE TRIGGER IF NOT EXISTS studio_entry_texts_ad AFTER DELETE ON studio_entry_texts BEGIN + INSERT INTO ` + ftsTable + `(` + ftsTable + `, rowid, folded) VALUES ('delete', old.id, old.folded); END`, + `CREATE TRIGGER IF NOT EXISTS studio_entry_texts_au AFTER UPDATE OF folded ON studio_entry_texts BEGIN + INSERT INTO ` + ftsTable + `(` + ftsTable + `, rowid, folded) VALUES ('delete', old.id, old.folded); + INSERT INTO ` + ftsTable + `(rowid, folded) VALUES (new.id, new.folded); END`, + } + for _, q := range stmts { + if err := s.db.Exec(q).Error; err != nil { + xlog.Info("Chat search: SQLite has no FTS5 trigram search, using a scan", "error", err) + return + } + } + if fresh { + // Rows written before the FTS table existed. + if err := s.db.Exec(`INSERT INTO ` + ftsTable + `(` + ftsTable + `) VALUES ('rebuild')`).Error; err != nil { + xlog.Warn("Chat search: could not build the FTS5 table, using a scan", "error", err) + return + } + } + s.text.engine = engineFTS5 +} + +func (s *gormStore) setupPGTrgm() { + // Creating the extension needs a privilege the role may not have; a + // failure only means the slower scan. + _ = s.db.Exec(`CREATE EXTENSION IF NOT EXISTS pg_trgm`).Error + var n int64 + if err := s.db.Raw(`SELECT count(*) FROM pg_extension WHERE extname = 'pg_trgm'`).Scan(&n).Error; err != nil || n == 0 { + xlog.Info("Chat search: PostgreSQL has no pg_trgm extension, using a scan") + return + } + if err := s.db.Exec(`CREATE INDEX IF NOT EXISTS ` + pgTrgmIndex + ` ON studio_entry_texts USING gin (folded gin_trgm_ops)`).Error; err != nil { + xlog.Warn("Chat search: could not create the trigram index, using a scan", "error", err) + return + } + s.text.engine = enginePGTrgm +} + +// TextEngine says how a GORM store searches text: fts5, pg_trgm or scan. Any +// other store gives "". +func TextEngine(s Store) string { + if g, ok := s.(*gormStore); ok { + return g.text.engine + } + return "" +} + +// textRows is the index rows of the entries es, which have the sequence +// numbers first, first+1, and so on. +func textRows(id string, first int64, es []session.Entry) []textRow { + var rows []textRow + for i, e := range es { + display, ok := chatsearch.IndexText(e.Role, e.Text, len(e.Attachments) > 0) + if !ok { + continue + } + rows = append(rows, textRow{ChatID: id, Seq: first + int64(i), Role: e.Role, At: e.At.UTC(), Folded: chatsearch.Fold(display)}) + } + return rows +} + +// putTexts writes index rows. A row of the same message is replaced, so a +// row left by a racing backfill never blocks the entry written now. +func putTexts(tx *gorm.DB, rows []textRow) error { + if len(rows) == 0 { + return nil + } + return tx.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "chat_id"}, {Name: "seq"}}, + DoUpdates: clause.AssignmentColumns([]string{"role", "at", "folded"}), + }).CreateInBatches(&rows, 100).Error +} + +// SearchText implements chatsearch.Index. +func (s *gormStore) SearchText(ctx context.Context, q chatsearch.Query) (chatsearch.Result, error) { + res := chatsearch.Result{Engine: s.text.engine, Progress: s.textProgress()} + terms := q.Terms() + if len(terms) == 0 { + return res, nil + } + cands, err := s.candidates(ctx, q, terms) + if err != nil { + return res, err + } + var titles []session.ChatRecord + if q.Titles { + if titles, err = s.titleChats(ctx, q, terms); err != nil { + return res, err + } + } + res.Hits = chatsearch.Build(q, cands, titles) + return res, nil +} + +func (s *gormStore) textProgress() chatsearch.Progress { + return chatsearch.Progress{Indexing: s.text.indexing.Load(), Done: int(s.text.done.Load()), Total: int(s.text.total.Load())} +} + +// chatFilter adds the conditions on the chat row: the owner first. +func chatFilter(db *gorm.DB, q chatsearch.Query, alias string) *gorm.DB { + db = db.Where(alias+".principal = ? AND COALESCE("+alias+".kind, '') <> ?", q.Principal, session.KindSubAgent) + if q.ChatID != "" { + db = db.Where(alias+".id = ?", q.ChatID) + } + if q.AgentID != "" { + db = db.Where(alias+".agent_id = ?", q.AgentID) + } + switch q.Origin { + case "": + case chatsearch.OriginPerson: + db = db.Where("COALESCE(" + alias + ".origin, '') = ''") + case chatsearch.OriginAutomated: + db = db.Where("COALESCE(" + alias + ".origin, '') <> ''") + default: + db = db.Where(alias+".origin = ?", q.Origin) + } + return db +} + +func likeEscaped(s string) string { + return strings.NewReplacer(likeEscape, likeEscape+likeEscape, "%", likeEscape+"%", "_", likeEscape+"_").Replace(s) +} + +// matchTerms keeps the index rows whose folded text holds every term. +func (s *gormStore) matchTerms(db *gorm.DB, terms []string) *gorm.DB { + var fts []string + for _, t := range terms { + switch { + case s.text.engine == engineFTS5 && utf8.RuneCountInString(t) >= chatsearch.MinTermRunes: + fts = append(fts, `"`+strings.ReplaceAll(t, `"`, `""`)+`"`) + case s.text.postgres: + // The trigram index serves LIKE; a short word is a scan of the rows left. + db = db.Where(`t.folded LIKE ? ESCAPE '\'`, "%"+likeEscaped(t)+"%") + default: + db = db.Where("instr(t.folded, ?) > 0", t) + } + } + if len(fts) > 0 { + db = db.Where("t.id IN (SELECT rowid FROM "+ftsTable+" WHERE "+ftsTable+" MATCH ?)", strings.Join(fts, " AND ")) + } + return db +} + +type candidateRow struct { + ChatID string + Seq int64 + Role string + At time.Time + Text string + Attachments string +} + +// candidates reads the newest messages that may match, with their chats. +func (s *gormStore) candidates(ctx context.Context, q chatsearch.Query, terms []string) ([]chatsearch.Candidate, error) { + db := s.db.WithContext(ctx).Table("studio_entry_texts AS t"). + Select("t.chat_id, t.seq, t.role, t.at, e.text, e.attachments"). + Joins("JOIN studio_chats c ON c.id = t.chat_id"). + // The entry must still exist: a row left by a race is not a hit. + Joins("JOIN studio_entries e ON e.chat_id = t.chat_id AND e.seq = t.seq") + db = chatFilter(db, q, "c") + if q.Role != "" { + db = db.Where("t.role = ?", q.Role) + } + if !q.Since.IsZero() { + db = db.Where("t.at >= ?", q.Since.UTC()) + } + if !q.Until.IsZero() { + db = db.Where("t.at < ?", q.Until.UTC()) + } + db = s.matchTerms(db, terms) + var rows []candidateRow + if err := db.Order("t.at DESC, t.id DESC").Limit(chatsearch.PoolSize).Scan(&rows).Error; err != nil { + return nil, err + } + ids := make([]string, 0, len(rows)) + seen := map[string]bool{} + for _, r := range rows { + if !seen[r.ChatID] { + seen[r.ChatID] = true + ids = append(ids, r.ChatID) + } + } + chats, err := s.chatsByID(ctx, q.Principal, ids) + if err != nil { + return nil, err + } + out := make([]chatsearch.Candidate, 0, len(rows)) + for _, r := range rows { + rec, ok := chats[r.ChatID] + if !ok { + continue + } + text, ok := chatsearch.IndexText(r.Role, r.Text, r.Attachments != "") + if !ok { + continue + } + out = append(out, chatsearch.Candidate{Chat: rec, Seq: r.Seq, Role: r.Role, Text: text, At: r.At}) + } + return out, nil +} + +// chatsByID loads the chats of a principal by ID. +func (s *gormStore) chatsByID(ctx context.Context, principal string, ids []string) (map[string]session.ChatRecord, error) { + out := make(map[string]session.ChatRecord, len(ids)) + const chunk = 400 + for len(ids) > 0 { + n := min(chunk, len(ids)) + var rows []chatRow + if err := s.db.WithContext(ctx).Where("principal = ? AND id IN ?", principal, ids[:n]).Find(&rows).Error; err != nil { + return nil, err + } + for _, r := range rows { + rec, err := r.record() + if err != nil { + return nil, err + } + out[rec.ID] = rec + } + ids = ids[n:] + } + return out, nil +} + +// titleChats reads the newest chats whose title may hold every term. +func (s *gormStore) titleChats(ctx context.Context, q chatsearch.Query, terms []string) ([]session.ChatRecord, error) { + db := chatFilter(s.db.WithContext(ctx).Table("studio_chats AS c").Select("c.*"), q, "c") + for _, t := range terms { + db = db.Where(`lower(c.title) LIKE ? ESCAPE '\'`, "%"+likeEscaped(t)+"%") + } + if !q.Since.IsZero() { + db = db.Where("c.updated >= ?", q.Since) + } + if !q.Until.IsZero() { + db = db.Where("c.updated < ?", q.Until) + } + var rows []chatRow + if err := db.Order("c.updated DESC, c.id DESC").Limit(titlePool).Find(&rows).Error; err != nil { + return nil, err + } + out := make([]session.ChatRecord, 0, len(rows)) + for _, r := range rows { + rec, err := r.record() + if err != nil { + return nil, err + } + out = append(out, rec) + } + return out, nil +} + +// backfillTexts indexes the messages written before the index existed, a few +// chats at a time, so a large history does not stall the start. It resumes +// from the chat it reached. A message written while it runs is indexed by the +// write itself, and a row that backfill adds for a message that was just +// removed is ignored by every search (it joins the entry). +func (s *gormStore) backfillTexts() { + defer s.text.indexing.Store(false) + ctx := context.Background() + var st textStateRow + if err := s.db.Where("id = ?", 1).Limit(1).Find(&st).Error; err != nil { + xlog.Warn("Chat search: could not read the backfill state", "error", err) + return + } + if st.Done { + return + } + var total int64 + if err := s.db.Model(&chatRow{}).Count(&total).Error; err != nil { + return + } + s.text.total.Store(total) + if total > 0 { + xlog.Info("Chat search: indexing earlier chats in the background", "chats", total) + } + last, started, logged := st.LastChat, time.Now(), time.Now() + for { + var ids []string + if err := s.db.Model(&chatRow{}).Where("id > ?", last).Order("id").Limit(backfillStep).Pluck("id", &ids).Error; err != nil { + xlog.Debug("Chat search: backfill stopped", "error", err) + return + } + if len(ids) == 0 { + break + } + for _, id := range ids { + if err := s.backfillChat(ctx, id); err != nil { + xlog.Debug("Chat search: backfill stopped", "error", err) + return + } + s.text.done.Add(1) + } + last = ids[len(ids)-1] + if err := s.db.Save(&textStateRow{ID: 1, LastChat: last}).Error; err != nil { + xlog.Debug("Chat search: backfill stopped", "error", err) + return + } + if time.Since(logged) > 10*time.Second { + logged = time.Now() + xlog.Info("Chat search: indexing earlier chats", "done", s.text.done.Load(), "of", total) + } + time.Sleep(10 * time.Millisecond) // leave the database to the chats being written + } + // Rows of a chat deleted while the backfill read it. + _ = s.db.Exec(`DELETE FROM studio_entry_texts WHERE NOT EXISTS (SELECT 1 FROM studio_chats c WHERE c.id = studio_entry_texts.chat_id)`).Error + if err := s.db.Save(&textStateRow{ID: 1, LastChat: last, Done: true}).Error; err != nil { + xlog.Debug("Chat search: could not save the backfill state", "error", err) + return + } + if total > 0 { + xlog.Info("Chat search: indexed earlier chats", "chats", s.text.done.Load(), "took", time.Since(started).Round(time.Second).String()) + } +} + +// backfillChat indexes the messages of one chat that have no row yet. +func (s *gormStore) backfillChat(ctx context.Context, id string) error { + var after int64 + for { + var rows []entryRow + err := s.db.WithContext(ctx).Select("seq", "role", "text", "attachments", "at"). + Where("chat_id = ? AND seq > ? AND role IN ?", id, after, []string{session.RoleUser, session.RoleAssistant}). + Order("seq").Limit(backfillPage).Find(&rows).Error + if err != nil { + return err + } + if len(rows) == 0 { + return nil + } + out := make([]textRow, 0, len(rows)) + for _, r := range rows { + if display, ok := chatsearch.IndexText(r.Role, r.Text, r.Attachments != ""); ok { + out = append(out, textRow{ChatID: id, Seq: r.Seq, Role: r.Role, At: r.At.UTC(), Folded: chatsearch.Fold(display)}) + } + } + if len(out) > 0 { + err = s.db.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).CreateInBatches(&out, 100).Error + if err != nil { + return fmt.Errorf("chat %s: %w", id, err) + } + } + after = rows[len(rows)-1].Seq + } +} + +var _ chatsearch.Index = (*gormStore)(nil) diff --git a/core/services/studio/store/gormsearch_internal_test.go b/core/services/studio/store/gormsearch_internal_test.go new file mode 100644 index 0000000000..8419a5bd03 --- /dev/null +++ b/core/services/studio/store/gormsearch_internal_test.go @@ -0,0 +1,38 @@ +package store + +import ( + "context" + "path/filepath" + "time" + + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/http/auth" + "github.com/mudler/LocalAI/core/services/studio/chatsearch" + "github.com/mudler/LocalAI/core/services/studio/session" +) + +var _ = ginkgo.Describe("the text search without FTS5 or pg_trgm", func() { + ginkgo.It("scans the folded column and gives the same answers", func() { + db, err := auth.OpenSQLiteFile(filepath.Join(ginkgo.GinkgoT().TempDir(), "studio.db")) + Expect(err).NotTo(HaveOccurred()) + sqlDB, _ := db.DB() + ginkgo.DeferCleanup(func() { _ = sqlDB.Close() }) + st, err := NewGORM(db) + Expect(err).NotTo(HaveOccurred()) + s := st.(*gormStore) + s.text.engine = engineScan + ctx := context.Background() + now := time.Now() + Expect(s.CreateChat(ctx, session.ChatRecord{ID: "c1", Principal: "alice", Title: "t", Created: now, Updated: now})).To(Succeed()) + _, err = s.AppendEntry(ctx, "c1", session.Entry{Role: session.RoleUser, Text: "Qdrant port 6333, go is fun"}) + Expect(err).NotTo(HaveOccurred()) + for text, want := range map[string]int{"QDRANT port": 1, "go fun": 1, "qdrant nope": 0, "6333": 1} { + res, err := s.SearchText(ctx, chatsearch.Query{Principal: "alice", Text: text, Now: now}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Engine).To(Equal(engineScan)) + Expect(res.Hits).To(HaveLen(want), text) + } + }) +}) diff --git a/core/services/studio/store/gormsearch_test.go b/core/services/studio/store/gormsearch_test.go new file mode 100644 index 0000000000..35731a4b25 --- /dev/null +++ b/core/services/studio/store/gormsearch_test.go @@ -0,0 +1,112 @@ +package store_test + +import ( + "context" + "fmt" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gorm.io/gorm" + + "github.com/mudler/LocalAI/core/services/studio/chatsearch" + "github.com/mudler/LocalAI/core/services/studio/session" + "github.com/mudler/LocalAI/core/services/studio/store" +) + +// waitIndexed waits for the background backfill of the text index. +func waitIndexed(s store.Store) { + GinkgoHelper() + Eventually(func() bool { + r, err := s.(chatsearch.Index).SearchText(context.Background(), chatsearch.Query{Principal: "alice", Text: "zzzzzz"}) + Expect(err).NotTo(HaveOccurred()) + return !r.Progress.Indexing + }, "20s", "20ms").Should(BeTrue()) +} + +// seedLegacy makes chats with messages and then removes every trace of the +// text index, as a database from before the index looks. +func seedLegacy(db *gorm.DB, chats int) { + GinkgoHelper() + s, err := store.NewGORM(db) + Expect(err).NotTo(HaveOccurred()) + waitIndexed(s) + ctx := context.Background() + at := time.Date(2026, 10, 1, 9, 0, 0, 0, time.UTC) + for i := range chats { + id := fmt.Sprintf("old%03d", i) + Expect(s.CreateChat(ctx, session.ChatRecord{ID: id, Principal: "alice", Title: id, Created: at, Updated: at})).To(Succeed()) + _, err := s.AppendEntry(ctx, id, session.Entry{Role: session.RoleUser, Text: "legacy marmot " + id, At: at}) + Expect(err).NotTo(HaveOccurred()) + } + Expect(db.Exec("DROP TABLE IF EXISTS studio_entry_texts_fts").Error).To(Succeed()) + Expect(db.Exec("DROP TABLE studio_entry_texts").Error).To(Succeed()) + Expect(db.Exec("DROP TABLE studio_entry_text_state").Error).To(Succeed()) +} + +var _ = Describe("the text index of the GORM store on SQLite", func() { + It("uses FTS5 with the trigram tokenizer", func() { + s, err := store.NewGORM(openSQLite(filepath.Join(GinkgoT().TempDir(), "studio.db"))) + Expect(err).NotTo(HaveOccurred()) + Expect(store.TextEngine(s)).To(Equal("fts5")) + }) + + It("indexes the messages of a database from before the index, in the background", func() { + db := openSQLite(filepath.Join(GinkgoT().TempDir(), "studio.db")) + seedLegacy(db, 130) + s, err := store.NewGORM(db) + Expect(err).NotTo(HaveOccurred()) + waitIndexed(s) + res, err := s.(chatsearch.Index).SearchText(context.Background(), chatsearch.Query{Principal: "alice", Text: "marmot", Limit: 200, Now: time.Now()}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Hits).To(HaveLen(130)) + Expect(res.Progress.Indexing).To(BeFalse()) + Expect(res.Progress.Done).To(Equal(130)) + }) + + It("resumes where the backfill stopped and indexes messages written meanwhile once", func() { + db := openSQLite(filepath.Join(GinkgoT().TempDir(), "studio.db")) + seedLegacy(db, 20) + first, err := store.NewGORM(db) + Expect(err).NotTo(HaveOccurred()) + waitIndexed(first) + // A second start finds the work done and changes nothing. + second, err := store.NewGORM(db) + Expect(err).NotTo(HaveOccurred()) + _, err = second.AppendEntry(context.Background(), "old000", session.Entry{Role: session.RoleAssistant, Text: "fresh marmot"}) + Expect(err).NotTo(HaveOccurred()) + waitIndexed(second) + res, err := second.(chatsearch.Index).SearchText(context.Background(), chatsearch.Query{Principal: "alice", Text: "marmot", Limit: 100, Now: time.Now()}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Hits).To(HaveLen(21)) + }) + + It("starts without waiting for the backfill", func() { + db := openSQLite(filepath.Join(GinkgoT().TempDir(), "studio.db")) + seedLegacy(db, 300) + start := time.Now() + s, err := store.NewGORM(db) + Expect(err).NotTo(HaveOccurred()) + Expect(time.Since(start)).To(BeNumerically("<", 3*time.Second)) + waitIndexed(s) + }) + + It("drops the index rows with the chat in one transaction", func() { + db := openSQLite(filepath.Join(GinkgoT().TempDir(), "studio.db")) + s, err := store.NewGORM(db) + Expect(err).NotTo(HaveOccurred()) + ctx := context.Background() + Expect(s.CreateChat(ctx, session.ChatRecord{ID: "c1", Principal: "alice", Title: "t", Created: time.Now(), Updated: time.Now()})).To(Succeed()) + _, err = s.AppendEntry(ctx, "c1", session.Entry{Role: session.RoleUser, Text: "hello"}) + Expect(err).NotTo(HaveOccurred()) + var n int64 + Expect(db.Table("studio_entry_texts").Count(&n).Error).To(Succeed()) + Expect(n).To(Equal(int64(1))) + Expect(s.DeleteChat(ctx, "alice", "c1")).To(Succeed()) + Expect(db.Table("studio_entry_texts").Count(&n).Error).To(Succeed()) + Expect(n).To(BeZero()) + Expect(db.Raw("SELECT count(*) FROM studio_entry_texts_fts WHERE studio_entry_texts_fts MATCH '\"hello\"'").Scan(&n).Error).To(Succeed()) + Expect(n).To(BeZero()) + }) +}) diff --git a/core/services/studio/store/jsonsearch.go b/core/services/studio/store/jsonsearch.go new file mode 100644 index 0000000000..b43b655799 --- /dev/null +++ b/core/services/studio/store/jsonsearch.go @@ -0,0 +1,87 @@ +package store + +import ( + "context" + "sort" + "time" + + "github.com/mudler/LocalAI/core/services/studio/chatsearch" + "github.com/mudler/LocalAI/core/services/studio/session" +) + +// SearchText implements chatsearch.Index. The transcripts are in memory, so it +// scans them, the newest chat first, until chatsearch.ScanBudget is spent. +// Result.Searched and Total say how many chats it read. +func (s *jsonStore) SearchText(ctx context.Context, q chatsearch.Query) (chatsearch.Result, error) { + res := chatsearch.Result{Engine: "memory"} + terms := q.Terms() + if len(terms) == 0 { + return res, nil + } + type listed struct { + slot *chatSlot + updated time.Time + id string + } + var chats []listed + for _, c := range s.slots() { + c.mu.Lock() + if !c.gone && q.ChatMatches(c.Rec) { + chats = append(chats, listed{c, c.Rec.Updated, c.Rec.ID}) + } + c.mu.Unlock() + } + sort.Slice(chats, func(i, j int) bool { + if !chats[i].updated.Equal(chats[j].updated) { + return chats[i].updated.After(chats[j].updated) + } + return chats[i].id > chats[j].id + }) + res.Total = len(chats) + + var ( + cands []chatsearch.Candidate + titles []session.ChatRecord + ) + deadline := time.Now().Add(chatsearch.ScanBudget) + for _, l := range chats { + if err := ctx.Err(); err != nil { + return res, err + } + if time.Now().After(deadline) { + res.Partial = true + break + } + c := l.slot + c.mu.Lock() + if c.gone { + c.mu.Unlock() + continue + } + rec := cloneRec(c.Rec) + rec.HarnessState = nil + if q.Titles && chatsearch.Matches(chatsearch.Fold(rec.Title), terms) { + titles = append(titles, rec) + } + for _, e := range c.entries { + if q.Role != "" && e.Role != q.Role || !q.TimeMatches(e.At) { + continue + } + display, ok := chatsearch.IndexText(e.Role, e.Text, len(e.Attachments) > 0) + if ok && chatsearch.Matches(chatsearch.Fold(display), terms) { + cands = append(cands, chatsearch.Candidate{Chat: rec, Seq: e.Seq, Role: e.Role, Text: display, At: e.At}) + } + } + c.mu.Unlock() + res.Searched++ + } + // The pool is the newest messages, as in an indexed store. + if len(cands) > chatsearch.PoolSize { + sort.SliceStable(cands, func(i, j int) bool { return cands[i].At.After(cands[j].At) }) + cands = cands[:chatsearch.PoolSize] + } + res.Hits = chatsearch.Build(q, cands, titles) + return res, nil +} + +var _ chatsearch.Index = (*jsonStore)(nil) diff --git a/core/services/studio/store/storetest/contract.go b/core/services/studio/store/storetest/contract.go index 8bc848752d..6317e06dbf 100644 --- a/core/services/studio/store/storetest/contract.go +++ b/core/services/studio/store/storetest/contract.go @@ -50,6 +50,7 @@ func texts(ps []store.Pending) []string { func Run(newStore New) { // The loop's own transcript contract. sessiontest.RunStoreSuite(func() session.Store { return newStore() }) + runSearch(newStore) It("skill switches round-trip, survive a chat update and belong to the owner", func() { s := newStore() diff --git a/core/services/studio/store/storetest/search.go b/core/services/studio/store/storetest/search.go new file mode 100644 index 0000000000..be30ab3521 --- /dev/null +++ b/core/services/studio/store/storetest/search.go @@ -0,0 +1,229 @@ +package storetest + +import ( + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/services/studio/chatsearch" + "github.com/mudler/LocalAI/core/services/studio/session" + "github.com/mudler/LocalAI/core/services/studio/store" +) + +// runSearch registers the specs of the text search that every store passes. +func runSearch(newStore New) { + Describe("text search", func() { + var ( + s store.Store + idx chatsearch.Index + ) + BeforeEach(func() { + s = newStore() + var ok bool + idx, ok = s.(chatsearch.Index) + Expect(ok).To(BeTrue(), "the store must implement chatsearch.Index") + }) + + say := func(id, role, text string, at time.Time) int64 { + seq, err := s.AppendEntry(ctx(), id, session.Entry{Role: role, Text: text, At: at}) + Expect(err).NotTo(HaveOccurred()) + return seq + } + search := func(principal, text string, f ...func(*chatsearch.Query)) chatsearch.Result { + q := chatsearch.Query{Principal: principal, Text: text, Titles: true, Now: t0.Add(24 * time.Hour)} + for _, fn := range f { + fn(&q) + } + // A backfill in the background may still run: wait for it. + var res chatsearch.Result + Eventually(func() bool { + var err error + res, err = idx.SearchText(ctx(), q) + Expect(err).NotTo(HaveOccurred()) + return !res.Progress.Indexing + }, "20s", "20ms").Should(BeTrue()) + return res + } + seqs := func(r chatsearch.Result) []string { + var out []string + for _, h := range r.Hits { + out = append(out, fmt.Sprintf("%s/%d", h.Chat.ID, h.Seq)) + } + return out + } + + It("finds the words of user and assistant messages, ignoring case and accents of the script", func() { + chat(s, "c1", "alice") + say("c1", session.RoleUser, "What is the Qdrant port?", t0) + a := say("c1", session.RoleAssistant, "The default port is 6333. Città di Roma", t0.Add(time.Minute)) + say("c1", session.RoleCompacted, "qdrant hidden", t0.Add(2*time.Minute)) + r := search("alice", "QDRANT port") + Expect(seqs(r)).To(Equal([]string{"c1/1"})) + Expect(r.Hits[0].Role).To(Equal(session.RoleUser)) + Expect(r.Hits[0].Snippet).To(ContainSubstring("Qdrant")) + Expect(r.Hits[0].Highlights).NotTo(BeEmpty()) + Expect(r.Hits[0].At.Equal(t0)).To(BeTrue()) + Expect(seqs(search("alice", "città"))).To(Equal([]string{fmt.Sprintf("c1/%d", a)})) + }) + + It("matches every word in one message and a quoted phrase as written", func() { + chat(s, "c1", "alice") + say("c1", session.RoleUser, "the port of qdrant", t0) + say("c1", session.RoleUser, "qdrant only", t0.Add(time.Minute)) + Expect(seqs(search("alice", "qdrant port"))).To(Equal([]string{"c1/1"})) + Expect(search("alice", `"qdrant port"`).Hits).To(BeEmpty()) + Expect(seqs(search("alice", `"port of qdrant"`))).To(Equal([]string{"c1/1"})) + }) + + It("finds words shorter than the trigram size", func() { + chat(s, "c1", "alice") + say("c1", session.RoleUser, "go is fun", t0) + say("c1", session.RoleUser, "nothing", t0.Add(time.Minute)) + Expect(seqs(search("alice", "go"))).To(Equal([]string{"c1/1"})) + Expect(seqs(search("alice", "go fun"))).To(Equal([]string{"c1/1"})) + }) + + It("treats SQL and index syntax in the query as text", func() { + chat(s, "c1", "alice") + say("c1", session.RoleUser, `a 100% sure, a_b, he said ab"c and back\slash`, t0) + say("c1", session.RoleUser, "a 1000 sure aXb abc", t0.Add(time.Minute)) + Expect(seqs(search("alice", "100%"))).To(Equal([]string{"c1/1"})) + Expect(seqs(search("alice", "a_b"))).To(Equal([]string{"c1/1"})) + Expect(seqs(search("alice", `ab"c`))).To(Equal([]string{"c1/1"})) + Expect(seqs(search("alice", `back\slash`))).To(Equal([]string{"c1/1"})) + Expect(search("alice", "NEAR(a b)").Hits).To(BeEmpty()) + Expect(search("alice", "a OR zzz").Hits).To(BeEmpty()) + }) + + It("never shows another user's chats, nor reveals them", func() { + chat(s, "ca", "alice") + chat(s, "cb", "bob") + say("ca", session.RoleUser, "shared word zebra", t0) + say("cb", session.RoleUser, "shared word zebra secret", t0) + Expect(seqs(search("alice", "zebra"))).To(Equal([]string{"ca/1"})) + Expect(seqs(search("bob", "zebra"))).To(Equal([]string{"cb/1"})) + Expect(search("alice", "secret").Hits).To(BeEmpty()) + Expect(search("carol", "zebra").Hits).To(BeEmpty()) + Expect(search("alice", "zebra", func(q *chatsearch.Query) { q.ChatID = "cb" }).Hits).To(BeEmpty()) + }) + + It("leaves out sub-agent chats", func() { + chat(s, "parent", "alice") + Expect(s.CreateChat(ctx(), session.ChatRecord{ID: "sub", Principal: "alice", Title: "sub", Kind: session.KindSubAgent, + ParentID: "parent", Created: t0, Updated: t0})).To(Succeed()) + say("sub", session.RoleAssistant, "findings about zebra", t0) + Expect(search("alice", "zebra").Hits).To(BeEmpty()) + }) + + It("drops a deleted chat from the index at once", func() { + chat(s, "c1", "alice") + say("c1", session.RoleUser, "unique-giraffe-word", t0) + Expect(search("alice", "unique-giraffe-word").Hits).To(HaveLen(1)) + Expect(s.DeleteChat(ctx(), "alice", "c1")).To(Succeed()) + Expect(search("alice", "unique-giraffe-word").Hits).To(BeEmpty()) + // A new chat with the same ID starts clean. + chat(s, "c1", "alice") + say("c1", session.RoleUser, "other words", t0) + Expect(search("alice", "unique-giraffe-word").Hits).To(BeEmpty()) + }) + + It("drops truncated text and indexes the text written in its place", func() { + chat(s, "c1", "alice") + say("c1", session.RoleUser, "question one", t0) + say("c1", session.RoleAssistant, "old answer about pelicans", t0.Add(time.Minute)) + Expect(s.TruncateEntries(ctx(), "alice", "c1", 1)).To(Succeed()) + Expect(search("alice", "pelicans").Hits).To(BeEmpty()) + say("c1", session.RoleAssistant, "new answer about herons", t0.Add(2*time.Minute)) + Expect(seqs(search("alice", "herons"))).To(Equal([]string{"c1/2"})) + Expect(search("alice", "pelicans").Hits).To(BeEmpty()) + }) + + It("indexes a batch of entries", func() { + chat(s, "c1", "alice") + Expect(s.AppendEntries(ctx(), "c1", []session.Entry{ + {Role: session.RoleUser, Text: "first lemur", At: t0}, + {Role: session.RoleAssistant, Text: "second lemur"}, + })).To(Succeed()) + Expect(seqs(search("alice", "lemur"))).To(ConsistOf("c1/1", "c1/2")) + }) + + It("leaves out the list of files of an older message", func() { + chat(s, "c1", "alice") + say("c1", session.RoleUser, "see this\n\nThe user attached these files:\n- /workspace/home/Chats/x/a.txt", t0) + Expect(search("alice", "attached these files").Hits).To(BeEmpty()) + Expect(search("alice", "see this").Hits).To(HaveLen(1)) + }) + + It("matches titles, and returns a title-only chat without a line", func() { + Expect(s.CreateChat(ctx(), session.ChatRecord{ID: "c1", Principal: "alice", Title: "Kubernetes upgrade", Created: t0, Updated: t0})).To(Succeed()) + r := search("alice", "kubernetes") + Expect(r.Hits).To(HaveLen(1)) + Expect(r.Hits[0].InTitle).To(BeTrue()) + Expect(r.Hits[0].Seq).To(BeZero()) + Expect(search("alice", "kubernetes", func(q *chatsearch.Query) { q.Titles = false }).Hits).To(BeEmpty()) + }) + + It("filters by agent, origin, role and time", func() { + Expect(s.CreateChat(ctx(), session.ChatRecord{ID: "web", Principal: "alice", Title: "w", Created: t0, Updated: t0})).To(Succeed()) + Expect(s.CreateChat(ctx(), session.ChatRecord{ID: "ag", Principal: "alice", Title: "a", AgentID: "helper", Created: t0, Updated: t0})).To(Succeed()) + Expect(s.CreateChat(ctx(), session.ChatRecord{ID: "tr", Principal: "alice", Title: "t", Origin: "trigger", OriginID: "t1", Created: t0, Updated: t0})).To(Succeed()) + Expect(s.CreateChat(ctx(), session.ChatRecord{ID: "cn", Principal: "alice", Title: "c", Origin: "connector", OriginID: "b1", Created: t0, Updated: t0})).To(Succeed()) + for _, id := range []string{"web", "ag", "tr", "cn"} { + say(id, session.RoleUser, "walrus question", t0) + say(id, session.RoleAssistant, "walrus answer", t0.Add(48*time.Hour)) + } + with := func(f func(*chatsearch.Query)) []string { + r := search("alice", "walrus", f, func(q *chatsearch.Query) { q.Titles = false }) + var ids []string + for _, h := range r.Hits { + ids = append(ids, fmt.Sprintf("%s/%d", h.Chat.ID, h.Seq)) + } + return ids + } + Expect(with(func(q *chatsearch.Query) { q.AgentID = "helper" })).To(ConsistOf("ag/1", "ag/2")) + Expect(with(func(q *chatsearch.Query) { q.Origin = chatsearch.OriginPerson })).To(ConsistOf("web/1", "web/2", "ag/1", "ag/2")) + Expect(with(func(q *chatsearch.Query) { q.Origin = chatsearch.OriginAutomated })).To(ConsistOf("tr/1", "tr/2", "cn/1", "cn/2")) + Expect(with(func(q *chatsearch.Query) { q.Origin = "connector" })).To(ConsistOf("cn/1", "cn/2")) + Expect(with(func(q *chatsearch.Query) { q.Role = session.RoleAssistant; q.ChatID = "web" })).To(ConsistOf("web/2")) + Expect(with(func(q *chatsearch.Query) { q.Since = t0.Add(24 * time.Hour); q.ChatID = "web" })).To(ConsistOf("web/2")) + Expect(with(func(q *chatsearch.Query) { q.Until = t0.Add(24 * time.Hour); q.ChatID = "web" })).To(ConsistOf("web/1")) + }) + + It("keeps one hit per chat when asked, and ranks the newer chat first on equal text", func() { + chat(s, "old", "alice") + chat(s, "new", "alice") + say("old", session.RoleUser, "ocelot", t0) + say("new", session.RoleUser, "ocelot", t0.Add(10*24*time.Hour)) + say("new", session.RoleAssistant, "another ocelot", t0.Add(10*24*time.Hour)) + r := search("alice", "ocelot", func(q *chatsearch.Query) { q.PerChat = true }) + Expect(r.Hits).To(HaveLen(2)) + Expect(r.Hits[0].Chat.ID).To(Equal("new")) + Expect(r.Hits[0].Matches).To(Equal(2)) + }) + + It("searches every chat, not only the newest few hundred", func() { + chat(s, "oldest", "alice") + say("oldest", session.RoleUser, "the one platypus", t0) + for i := range 520 { + id := fmt.Sprintf("n%03d", i) + Expect(s.CreateChat(ctx(), session.ChatRecord{ID: id, Principal: "alice", Title: id, Created: t0, Updated: t0.Add(time.Duration(i+1) * time.Minute)})).To(Succeed()) + say(id, session.RoleUser, "filler "+strings.Repeat("x", 5), t0.Add(time.Duration(i+1)*time.Minute)) + } + Expect(seqs(search("alice", "platypus"))).To(Equal([]string{"oldest/1"})) + }) + + It("bounds the number of hits and cuts the snippet", func() { + chat(s, "c1", "alice") + long := strings.Repeat("blah ", 100) + "needle" + strings.Repeat(" blah", 100) + for i := range 8 { + say("c1", session.RoleUser, long, t0.Add(time.Duration(i)*time.Minute)) + } + r := search("alice", "needle", func(q *chatsearch.Query) { q.Limit = 3; q.SnippetRunes = 100 }) + Expect(r.Hits).To(HaveLen(3)) + Expect(len([]rune(r.Hits[0].Snippet))).To(BeNumerically("<=", 102)) + }) + }) +} diff --git a/docs/content/features/chats.md b/docs/content/features/chats.md index 95e096ee31..54d3b37192 100644 --- a/docs/content/features/chats.md +++ b/docs/content/features/chats.md @@ -498,7 +498,7 @@ curl -X POST .../api/chats/3f1c.../regenerate curl -X POST .../api/chats/3f1c.../messages/12/edit -d '{"text":"A better question"}' # Copy the chat up to and including line 14 into a new chat. curl -X POST .../api/chats/3f1c.../fork -d '{"from_seq":14,"title":"Other idea"}' -# Find chats by title or by what was said (newest 500 chats). +# Find chats by title or by what was said, in all your chats. curl '.../api/chats/search?q=invoice' # Take a chat out as Markdown (default) or JSON, and bring a JSON file back in. curl '.../api/chats/3f1c.../export?format=json' > chat.json @@ -519,6 +519,17 @@ curl -X POST .../api/chats/import -d "{\"chat\": $(cat chat.json)}" makes a new chat of yours with the model of the file, or the one named in `model`, which you must be allowed to use. +### Search your chats + +`GET /api/chats/search?q=...` searches the titles and the messages (yours and the model's) of all your chats. It does not search the runs of sub-agents, and it never returns a chat of another user. + +- **Words and phrases.** A message matches when it holds every word of `q`, in any order, as part of a word or whole. A phrase in double quotes counts as one word: `q="port of qdrant"`. The search ignores case. +- **Ranking.** The best match comes first: a message with the whole phrase or with whole words ranks above one with word parts, and newer messages rank above older ones. A search ranks the newest 600 matching messages. +- **One hit for each chat.** A hit carries `chat` (with the agent and the origin), `seq` (the line of the message), `role`, `at`, a `snippet` and `highlights`: the parts of the snippet that match, as `start` and `end` offsets in UTF-16 units, so a client can mark them without parsing HTML. Add `group=message` to get every matching line. +- **Filters.** `agent` (a saved agent), `origin` (`person`, `automated`, `trigger`, `connector`, `job` or `page`), `role` (`user` or `assistant`), `since` and `until` (a date `YYYY-MM-DD` or an RFC 3339 time; a date in `until` includes that whole day) and `limit` (default 50, at most 200). +- **After an upgrade.** The server keeps a search index. When it starts with chats from before the index, it adds them in the background and logs the progress; the answer says `indexing: true` until that ends, and older chats can be missing meanwhile. The index uses SQLite FTS5 (trigram), or `pg_trgm` on PostgreSQL; without them the server scans. Deleting a chat, or regenerating or editing from a message, removes the old text from the index in the same step. +- **Chats in files.** A server that keeps chats in files (no database) has no index. It scans the newest chats first for up to two seconds and sets `partial: true` with `searched_chats` and `total_chats` when it stopped early. + ## Images, video and documents Attach files with `multipart/form-data` on `POST /api/chats/{id}/messages` diff --git a/swagger/docs.go b/swagger/docs.go index 10ffb47a20..3cd395b085 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -7535,7 +7535,7 @@ const docTemplate = `{ }, "/api/chats/search": { "get": { - "description": "Finds chats whose title or messages contain the text, ignoring case. The newest chats come first. It reads the newest 500 chats.", + "description": "Finds the messages and titles of your chats that hold every word of the text (a phrase in double quotes counts as one word), ignoring case. It searches all your chats, not only the newest, except the runs of sub-agents. The best match comes first: whole words, the whole phrase and newer messages rank higher. By default each chat appears once, with its best line; group=message returns every matching line. A hit has the line (seq), who wrote it, when, a snippet and the parts of the snippet that match (highlights, UTF-16 offsets). Messages written before the search index existed are added in the background after an upgrade (indexing is true meanwhile). A server that keeps chats in files has no index: it scans for a few seconds and says when it stopped early (partial).", "tags": [ "chats" ], @@ -7543,16 +7543,52 @@ const docTemplate = `{ "parameters": [ { "type": "string", - "description": "text to find", + "description": "text to find: words that must all be in the message", "name": "q", "in": "query", "required": true }, { "type": "integer", - "description": "most chats to return (default 50, at most 200)", + "description": "most hits to return (default 50, at most 200)", "name": "limit", "in": "query" + }, + { + "type": "string", + "description": "only chats of this saved agent", + "name": "agent", + "in": "query" + }, + { + "type": "string", + "description": "only chats of one origin: person, automated, trigger, connector, job or page", + "name": "origin", + "in": "query" + }, + { + "type": "string", + "description": "only lines written by user or assistant", + "name": "role", + "in": "query" + }, + { + "type": "string", + "description": "only lines at or after this date (YYYY-MM-DD) or time (RFC 3339)", + "name": "since", + "in": "query" + }, + { + "type": "string", + "description": "only lines before this time, or on or before this date (YYYY-MM-DD)", + "name": "until", + "in": "query" + }, + { + "type": "string", + "description": "chat (default): one hit per chat; message: every matching line", + "name": "group", + "in": "query" } ], "responses": { @@ -7561,6 +7597,12 @@ const docTemplate = `{ "schema": { "$ref": "#/definitions/schema.ChatsChatSearchResponse" } + }, + "400": { + "description": "The request is not valid", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } } @@ -21327,15 +21369,34 @@ const docTemplate = `{ "schema.ChatsChatHit": { "type": "object", "properties": { + "at": { + "description": "At is when the matching line was written.", + "type": "string" + }, "chat": { "$ref": "#/definitions/schema.ChatsConversation" }, + "highlights": { + "description": "Highlights are the parts of Snippet that match, in order.", + "type": "array", + "items": { + "$ref": "#/definitions/schema.ChatsSearchRange" + } + }, "in_title": { "description": "InTitle is true when the title matches.", "type": "boolean" }, + "matches": { + "description": "Matches is how many lines of the chat match, among the newest matches.", + "type": "integer" + }, + "role": { + "description": "Role is user or assistant: who wrote the matching line.", + "type": "string" + }, "seq": { - "description": "Seq is the first matching transcript line, or 0 when only the title matches.", + "description": "Seq is the matching transcript line, or 0 when only the title matches.", "type": "integer" }, "snippet": { @@ -21347,11 +21408,29 @@ const docTemplate = `{ "schema.ChatsChatSearchResponse": { "type": "object", "properties": { + "engine": { + "description": "Engine is how the server searched: fts5, pg_trgm, scan or memory.", + "type": "string" + }, "hits": { "type": "array", "items": { "$ref": "#/definitions/schema.ChatsChatHit" } + }, + "indexing": { + "description": "Indexing is true while the server still adds earlier chats to its\nsearch index, so older chats may be missing from the hits.", + "type": "boolean" + }, + "partial": { + "description": "Partial is true when a server without an index stopped before it had\nread every chat; SearchedChats of TotalChats were read.", + "type": "boolean" + }, + "searched_chats": { + "type": "integer" + }, + "total_chats": { + "type": "integer" } } }, @@ -24047,6 +24126,17 @@ const docTemplate = `{ } } }, + "schema.ChatsSearchRange": { + "type": "object", + "properties": { + "end": { + "type": "integer" + }, + "start": { + "type": "integer" + } + } + }, "schema.ChatsSearchResult": { "type": "object", "properties": { diff --git a/swagger/swagger.json b/swagger/swagger.json index d85392ea65..14b44bc03b 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -7532,7 +7532,7 @@ }, "/api/chats/search": { "get": { - "description": "Finds chats whose title or messages contain the text, ignoring case. The newest chats come first. It reads the newest 500 chats.", + "description": "Finds the messages and titles of your chats that hold every word of the text (a phrase in double quotes counts as one word), ignoring case. It searches all your chats, not only the newest, except the runs of sub-agents. The best match comes first: whole words, the whole phrase and newer messages rank higher. By default each chat appears once, with its best line; group=message returns every matching line. A hit has the line (seq), who wrote it, when, a snippet and the parts of the snippet that match (highlights, UTF-16 offsets). Messages written before the search index existed are added in the background after an upgrade (indexing is true meanwhile). A server that keeps chats in files has no index: it scans for a few seconds and says when it stopped early (partial).", "tags": [ "chats" ], @@ -7540,16 +7540,52 @@ "parameters": [ { "type": "string", - "description": "text to find", + "description": "text to find: words that must all be in the message", "name": "q", "in": "query", "required": true }, { "type": "integer", - "description": "most chats to return (default 50, at most 200)", + "description": "most hits to return (default 50, at most 200)", "name": "limit", "in": "query" + }, + { + "type": "string", + "description": "only chats of this saved agent", + "name": "agent", + "in": "query" + }, + { + "type": "string", + "description": "only chats of one origin: person, automated, trigger, connector, job or page", + "name": "origin", + "in": "query" + }, + { + "type": "string", + "description": "only lines written by user or assistant", + "name": "role", + "in": "query" + }, + { + "type": "string", + "description": "only lines at or after this date (YYYY-MM-DD) or time (RFC 3339)", + "name": "since", + "in": "query" + }, + { + "type": "string", + "description": "only lines before this time, or on or before this date (YYYY-MM-DD)", + "name": "until", + "in": "query" + }, + { + "type": "string", + "description": "chat (default): one hit per chat; message: every matching line", + "name": "group", + "in": "query" } ], "responses": { @@ -7558,6 +7594,12 @@ "schema": { "$ref": "#/definitions/schema.ChatsChatSearchResponse" } + }, + "400": { + "description": "The request is not valid", + "schema": { + "$ref": "#/definitions/schema.ErrorResponse" + } } } } @@ -21324,15 +21366,34 @@ "schema.ChatsChatHit": { "type": "object", "properties": { + "at": { + "description": "At is when the matching line was written.", + "type": "string" + }, "chat": { "$ref": "#/definitions/schema.ChatsConversation" }, + "highlights": { + "description": "Highlights are the parts of Snippet that match, in order.", + "type": "array", + "items": { + "$ref": "#/definitions/schema.ChatsSearchRange" + } + }, "in_title": { "description": "InTitle is true when the title matches.", "type": "boolean" }, + "matches": { + "description": "Matches is how many lines of the chat match, among the newest matches.", + "type": "integer" + }, + "role": { + "description": "Role is user or assistant: who wrote the matching line.", + "type": "string" + }, "seq": { - "description": "Seq is the first matching transcript line, or 0 when only the title matches.", + "description": "Seq is the matching transcript line, or 0 when only the title matches.", "type": "integer" }, "snippet": { @@ -21344,11 +21405,29 @@ "schema.ChatsChatSearchResponse": { "type": "object", "properties": { + "engine": { + "description": "Engine is how the server searched: fts5, pg_trgm, scan or memory.", + "type": "string" + }, "hits": { "type": "array", "items": { "$ref": "#/definitions/schema.ChatsChatHit" } + }, + "indexing": { + "description": "Indexing is true while the server still adds earlier chats to its\nsearch index, so older chats may be missing from the hits.", + "type": "boolean" + }, + "partial": { + "description": "Partial is true when a server without an index stopped before it had\nread every chat; SearchedChats of TotalChats were read.", + "type": "boolean" + }, + "searched_chats": { + "type": "integer" + }, + "total_chats": { + "type": "integer" } } }, @@ -24044,6 +24123,17 @@ } } }, + "schema.ChatsSearchRange": { + "type": "object", + "properties": { + "end": { + "type": "integer" + }, + "start": { + "type": "integer" + } + } + }, "schema.ChatsSearchResult": { "type": "object", "properties": { diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index a861f64d98..3353616830 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -4496,14 +4496,29 @@ definitions: type: object schema.ChatsChatHit: properties: + at: + description: At is when the matching line was written. + type: string chat: $ref: '#/definitions/schema.ChatsConversation' + highlights: + description: Highlights are the parts of Snippet that match, in order. + items: + $ref: '#/definitions/schema.ChatsSearchRange' + type: array in_title: description: InTitle is true when the title matches. type: boolean + matches: + description: Matches is how many lines of the chat match, among the newest + matches. + type: integer + role: + description: 'Role is user or assistant: who wrote the matching line.' + type: string seq: - description: Seq is the first matching transcript line, or 0 when only the - title matches. + description: Seq is the matching transcript line, or 0 when only the title + matches. type: integer snippet: description: Snippet is the matching text with a little around it. @@ -4511,10 +4526,27 @@ definitions: type: object schema.ChatsChatSearchResponse: properties: + engine: + description: 'Engine is how the server searched: fts5, pg_trgm, scan or memory.' + type: string hits: items: $ref: '#/definitions/schema.ChatsChatHit' type: array + indexing: + description: |- + Indexing is true while the server still adds earlier chats to its + search index, so older chats may be missing from the hits. + type: boolean + partial: + description: |- + Partial is true when a server without an index stopped before it had + read every chat; SearchedChats of TotalChats were read. + type: boolean + searched_chats: + type: integer + total_chats: + type: integer type: object schema.ChatsConnection: properties: @@ -6535,6 +6567,13 @@ definitions: snippet: type: string type: object + schema.ChatsSearchRange: + properties: + end: + type: integer + start: + type: integer + type: object schema.ChatsSearchResult: properties: hits: @@ -18184,23 +18223,61 @@ paths: - chats /api/chats/search: get: - description: Finds chats whose title or messages contain the text, ignoring - case. The newest chats come first. It reads the newest 500 chats. + description: 'Finds the messages and titles of your chats that hold every word + of the text (a phrase in double quotes counts as one word), ignoring case. + It searches all your chats, not only the newest, except the runs of sub-agents. + The best match comes first: whole words, the whole phrase and newer messages + rank higher. By default each chat appears once, with its best line; group=message + returns every matching line. A hit has the line (seq), who wrote it, when, + a snippet and the parts of the snippet that match (highlights, UTF-16 offsets). + Messages written before the search index existed are added in the background + after an upgrade (indexing is true meanwhile). A server that keeps chats in + files has no index: it scans for a few seconds and says when it stopped early + (partial).' parameters: - - description: text to find + - description: 'text to find: words that must all be in the message' in: query name: q required: true type: string - - description: most chats to return (default 50, at most 200) + - description: most hits to return (default 50, at most 200) in: query name: limit type: integer + - description: only chats of this saved agent + in: query + name: agent + type: string + - description: 'only chats of one origin: person, automated, trigger, connector, + job or page' + in: query + name: origin + type: string + - description: only lines written by user or assistant + in: query + name: role + type: string + - description: only lines at or after this date (YYYY-MM-DD) or time (RFC 3339) + in: query + name: since + type: string + - description: only lines before this time, or on or before this date (YYYY-MM-DD) + in: query + name: until + type: string + - description: 'chat (default): one hit per chat; message: every matching line' + in: query + name: group + type: string responses: "200": description: The matching chats schema: $ref: '#/definitions/schema.ChatsChatSearchResponse' + "400": + description: The request is not valid + schema: + $ref: '#/definitions/schema.ErrorResponse' summary: Search your chats. tags: - chats