diff --git a/pkg/ast/ast.go b/pkg/ast/ast.go index f1a7e3263e..f63b989ff0 100644 --- a/pkg/ast/ast.go +++ b/pkg/ast/ast.go @@ -44,6 +44,9 @@ type StringNode struct { Key string Value string Exact bool + // CaseInsensitive marks a case-insensitive restriction; set by the search + // lowering pass, a backend routes it to the field's lowercased form. + CaseInsensitive bool } // BooleanNode represents a bool value diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go new file mode 100644 index 0000000000..184cb9ff52 --- /dev/null +++ b/services/search/pkg/bleve/backend_test.go @@ -0,0 +1,842 @@ +package bleve_test + +import ( + "context" + "fmt" + + bleveSearch "github.com/blevesearch/bleve/v2" + sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" + "github.com/opencloud-eu/reva/v2/pkg/storagespace" + + "github.com/opencloud-eu/opencloud/pkg/log" + searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" + "github.com/opencloud-eu/opencloud/services/search/pkg/content" + bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func hiddenByID(idx bleveSearch.Index, id string) bool { + GinkgoHelper() + + req := bleveSearch.NewSearchRequest(bleveSearch.NewDocIDQuery([]string{id})) + req.Fields = []string{"Hidden"} + + res, err := idx.Search(req) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits).To(HaveLen(1), "no record for %s", id) + + hidden, _ := res.Hits[0].Fields["Hidden"].(bool) + return hidden +} + +var _ = Describe("Bleve", func() { + var ( + eng *bleve.Backend + idx bleveSearch.Index + + doSearch = func(id string, query, path string) (*searchsvc.SearchIndexResponse, error) { + rID, err := storagespace.ParseID(id) + if err != nil { + return nil, err + } + + return eng.Search(context.Background(), &searchsvc.SearchIndexRequest{ + Query: query, + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: rID.StorageId, + SpaceId: rID.SpaceId, + OpaqueId: rID.OpaqueId, + }, + Path: path, + }, + }) + } + + assertDocCount = func(id string, query string, expectedCount int) []*searchmsg.Match { + res, err := doSearch(id, query, "") + + ExpectWithOffset(1, err).ToNot(HaveOccurred()) + ExpectWithOffset(1, len(res.Matches)).To(Equal(expectedCount), "query returned unexpected number of results: "+query) + return res.Matches + } + + rootResource search.Resource + parentResource search.Resource + childResource search.Resource + childResource2 search.Resource + ) + + BeforeEach(func() { + mapping, err := bleve.NewMapping() + Expect(err).ToNot(HaveOccurred()) + + idx, err = bleveSearch.NewMemOnly(mapping) + Expect(err).ToNot(HaveOccurred()) + + eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}) + Expect(err).ToNot(HaveOccurred()) + + rootResource = search.Resource{ + ID: "1$2!2", + RootID: "1$2!2", + Path: ".", + Document: content.Document{}, + } + + parentResource = search.Resource{ + ID: "1$2!3", + ParentID: rootResource.ID, + RootID: rootResource.ID, + Path: "./parent d!r", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER), + Document: content.Document{Name: "parent d!r"}, + } + + childResource = search.Resource{ + ID: "1$2!4", + ParentID: parentResource.ID, + RootID: rootResource.ID, + Path: "./parent d!r/child.pdf", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{Name: "child.pdf"}, + } + + childResource2 = search.Resource{ + ID: "1$2!5", + ParentID: parentResource.ID, + RootID: rootResource.ID, + Path: "./parent d!r/child2.pdf", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{Name: "child2.pdf"}, + } + }) + + Describe("PurgeSpace", func() { + It("takes every record of that space out of the index", func() { + otherSpace := search.Resource{ + ID: "1$9!9", + RootID: "1$9!9", + Path: ".", + Document: content.Document{Name: "other"}, + } + for _, resource := range []search.Resource{rootResource, parentResource, childResource, otherSpace} { + Expect(eng.Upsert(resource.ID, resource)).To(Succeed()) + } + + Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed()) + + count, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(1)), "only the records of that space are gone") + }) + + It("takes a space out that holds more records than one round", func() { + otherSpace := search.Resource{ + ID: "1$9!9", + RootID: "1$9!9", + Path: ".", + Document: content.Document{Name: "other"}, + } + Expect(eng.Upsert(otherSpace.ID, otherSpace)).To(Succeed()) + + for i := range 120 { + resource := search.Resource{ + ID: fmt.Sprintf("%s!file-%d", rootResource.RootID, i), + RootID: rootResource.RootID, + Path: fmt.Sprintf("./file-%d", i), + Document: content.Document{Name: fmt.Sprintf("file-%d", i)}, + } + Expect(eng.Upsert(resource.ID, resource)).To(Succeed()) + } + + Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed()) + + count, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(1)), "only the record of the other space is left") + }) + }) + + Describe("New", func() { + It("returns a new index instance", func() { + b := bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}) + Expect(b).ToNot(BeNil()) + }) + }) + + Describe("Search", func() { + Context("by other fields than filename", func() { + It("finds files by tags", func() { + parentResource.Document.Tags = []string{"foo", "bar"} + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Tags:foo", 1) + assertDocCount(rootResource.ID, "Tags:bar", 1) + assertDocCount(rootResource.ID, "Tags:foo Tags:bar", 1) + assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1) + assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1) + assertDocCount(rootResource.ID, "Tags:baz", 0) + }) + + It("finds files by size", func() { + parentResource.Document.Size = 12345 + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Size:12345", 1) + assertDocCount(rootResource.ID, "Size:>1000", 1) + assertDocCount(rootResource.ID, "Size:<100000", 1) + assertDocCount(rootResource.ID, "Size:12344", 0) + assertDocCount(rootResource.ID, "Size:<1000", 0) + assertDocCount(rootResource.ID, "Size:>100000", 0) + }) + + It("preserves value case for fields not explicitly marked lowercase", func() { + parentResource.Document.Audio = &libregraph.Audio{ + Artist: libregraph.PtrString("Some Artist"), + } + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `audio.artist:"Some Artist"`, 1) + assertDocCount(rootResource.ID, `audio.artist:"some artist"`, 0) + }) + }) + + Context("by filename", func() { + It("finds files with spaces in the filename", func() { + parentResource.Document.Name = "Foo oo.pdf" + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `name:"foo o*"`, 1) + }) + + It("finds files by digits in the filename", func() { + parentResource.Document.Name = "12345.pdf" + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Name:1234*", 1) + }) + + It("filters hidden files", func() { + childResource.Hidden = true + err := eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Hidden:T", 1) + assertDocCount(rootResource.ID, "Hidden:F", 0) + }) + + Context("with a file in the root of the space", func() { + It("scopes the search to the specified space", func() { + parentResource.Document.Name = "foo.pdf" + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Name:foo.pdf", 1) + assertDocCount("9$8!7", "Name:foo.pdf", 0) + }) + }) + + It("limits the search to the specified fields", func() { + parentResource.Document.Name = "bar.pdf" + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Name:bar.pdf", 1) + assertDocCount(rootResource.ID, "Unknown:field", 0) + }) + + It("returns the total number of hits", func() { + parentResource.Document.Name = "bar.pdf" + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + res, err := doSearch(rootResource.ID, "Name:bar*", "") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(1))) + }) + + It("returns all desired fields", func() { + parentResource.Document.Name = "bar.pdf" + parentResource.Type = 3 + parentResource.MimeType = "application/pdf" + + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + matches := assertDocCount(rootResource.ID, fmt.Sprintf("Name:%s", parentResource.Name), 1) + match := matches[0] + Expect(match.Entity.Ref.Path).To(Equal(parentResource.Path)) + Expect(match.Entity.Name).To(Equal(parentResource.Name)) + Expect(match.Entity.Size).To(Equal(parentResource.Size)) + Expect(match.Entity.Type).To(Equal(parentResource.Type)) + Expect(match.Entity.MimeType).To(Equal(parentResource.MimeType)) + Expect(match.Entity.Deleted).To(BeFalse()) + Expect(match.Score > 0).To(BeTrue()) + }) + + It("finds files by name, prefix or substring match", func() { + parentResource.Document.Name = "foo.pdf" + + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + queries := []string{"foo.pdf", "foo*", "*oo.p*"} + for _, query := range queries { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, query, 1) + } + }) + + It("does a case-insensitive search", func() { + parentResource.Document.Name = "foo.pdf" + + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Name:foo*", 1) + assertDocCount(rootResource.ID, "Name:Foo*", 1) + }) + + Context("and an additional file in a subdirectory", func() { + BeforeEach(func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + }) + + It("finds files living deeper in the tree by filename, prefix or substring match", func() { + queries := []string{"child.pdf", "child*", "*ld.*"} + for _, query := range queries { + assertDocCount(rootResource.ID, query, 1) + } + }) + }) + }) + + Context("by path", func() { + BeforeEach(func() { + for _, r := range []search.Resource{parentResource, childResource, childResource2} { + Expect(eng.Upsert(r.ID, r)).To(Succeed()) + } + }) + + It("matches a folder and its descendants", func() { + assertDocCount(rootResource.ID, `path:"./parent d!r"`, 3) + }) + + It("matches a descendant path only itself", func() { + assertDocCount(rootResource.ID, `path:"./parent d!r/child.pdf"`, 1) + }) + + It("matches case-insensitively", func() { + assertDocCount(rootResource.ID, `path:"./PARENT D!R"`, 3) + }) + }) + + Context("Highlights", func() { + + It("highlights only for content searches", func() { + parentResource.Document.Name = "baz.pdf" + parentResource.Document.Content = "foo bar baz" + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + res, err := doSearch(rootResource.ID, "Name:baz*", "") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(1))) + Expect(res.Matches[0].Entity.Highlights).To(Equal("")) + }) + + It("highlights search terms", func() { + parentResource.Document.Name = "baz.pdf" + parentResource.Document.Content = "foo bar baz" + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + res, err := doSearch(rootResource.ID, "Content:bar", "") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(1))) + Expect(res.Matches[0].Entity.Highlights).To(Equal("foo bar baz")) + }) + + }) + + Context("with a file in the root of the space and folder with a file. all of them have the same name", func() { + BeforeEach(func() { + parentResource := search.Resource{ + ID: "1$2!3", + ParentID: rootResource.ID, + RootID: rootResource.ID, + Path: "./doc", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER), + Document: content.Document{Name: "doc"}, + } + + childResource := search.Resource{ + ID: "1$2!4", + ParentID: parentResource.ID, + RootID: rootResource.ID, + Path: "./doc/doc.pdf", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{Name: "doc.pdf"}, + } + + childResource2 := search.Resource{ + ID: "1$2!7", + ParentID: parentResource.ID, + RootID: rootResource.ID, + Path: "./doc/file.pdf", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{Name: "file.pdf"}, + } + + rootChildResource := search.Resource{ + ID: "1$2!5", + ParentID: rootResource.ID, + RootID: rootResource.ID, + Path: "./doc.pdf", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{Name: "doc.pdf"}, + } + + rootChildResource2 := search.Resource{ + ID: "1$2!6", + ParentID: rootResource.ID, + RootID: rootResource.ID, + Path: "./file.pdf", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{Name: "file.pdf"}, + } + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(rootChildResource.ID, rootChildResource) + Expect(err).ToNot(HaveOccurred()) + err = eng.Upsert(rootChildResource2.ID, rootChildResource2) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + err = eng.Upsert(childResource2.ID, childResource2) + Expect(err).ToNot(HaveOccurred()) + }) + It("search *doc* in a root", func() { + res, err := doSearch(rootResource.ID, "Name:*doc*", "") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(3))) + }) + It("search *doc* in a subfolder", func() { + res, err := doSearch(rootResource.ID, "Name:*doc*", "./doc") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(2))) + }) + It("search *file* in a root", func() { + res, err := doSearch(rootResource.ID, "Name:*file*", "") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(2))) + }) + It("search *file* in a subfolder", func() { + res, err := doSearch(rootResource.ID, "Name:*file*", "./doc") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(1))) + }) + }) + + }) + + Describe("Upsert", func() { + It("adds a resourceInfo to the index", func() { + err := eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + count, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(1))) + + query := bleveSearch.NewMatchQuery("child.pdf") + res, err := idx.Search(bleveSearch.NewSearchRequest(query)) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits.Len()).To(Equal(1)) + }) + + It("updates an existing resource in the index", func() { + + err := eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + countA, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(countA).To(Equal(uint64(1))) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + countB, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(countB).To(Equal(uint64(1))) + }) + }) + + Describe("Delete", func() { + It("marks a resource as deleted", func() { + err := eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Name:*child*", 1) + + err = eng.Delete(childResource.ID) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Name:*child*", 0) + }) + + It("marks a child resources as deleted", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) + assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) + + err = eng.Delete(parentResource.ID) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) + assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0) + }) + }) + + Describe("Restore", func() { + It("also marks child resources as restored", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Delete(parentResource.ID) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 0) + assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 0) + + err = eng.Restore(parentResource.ID) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 1) + assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 1) + }) + }) + + Describe("Purge", func() { + It("removes a resource from the index", func() { + err := eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + assertDocCount(rootResource.ID, "Name:child.pdf", 1) + + err = eng.Purge(childResource.ID, false) + + Expect(err).ToNot(HaveOccurred()) + assertDocCount(rootResource.ID, "Name:child.pdf", 0) + }) + It("removes a resource and its children from the index", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) + assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) + + err = eng.Purge(parentResource.ID, false) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) + assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0) + }) + It("removes a resource and ignores its children from the index", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) + + err = eng.Delete(parentResource.ID) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) + + err = eng.Purge(parentResource.ID, true) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) + assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) + }) + }) + + Describe("Move", func() { + It("renames the parent and its child resources", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + parentResource.Path = "newname" + err = eng.Move(parentResource.ID, parentResource.ParentID, "./my/newname") + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, parentResource.Name, 0) + + matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1) + Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3")) + Expect(matches[0].Entity.Ref.Path).To(Equal("./my/newname/child.pdf")) + }) + + DescribeTable("keeps the flag in step with the path", + func(from, target string, hidden bool) { + parentResource.Path = from + parentResource.Hidden = search.IsHidden(from) + childResource.Path = from + "/child.pdf" + childResource.Hidden = parentResource.Hidden + + Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) + Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) + + Expect(eng.Move(parentResource.ID, parentResource.ParentID, target)).To(Succeed()) + + for _, id := range []string{parentResource.ID, childResource.ID} { + Expect(hiddenByID(idx, id)). + To(Equal(hidden), "%s after moving from %s to %s", id, from, target) + } + }, + Entry("into a dot folder", "./parent", "./.trash/parent", true), + Entry("into a plain folder", "./parent", "./archive/parent", false), + Entry("renamed with a leading dot", "./parent", "./.parent", true), + Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false), + Entry("renamed without the leading dot", "./.parent", "./parent", false), + Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true), + ) + + // the trash leaves the path alone, so the flag has to come through untouched + It("carries the flag through the trash and back", func() { + childResource.Path = "./.secret/file.txt" + childResource.Hidden = true + Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) + + Expect(eng.Delete(childResource.ID)).To(Succeed()) + Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after trashing") + + Expect(eng.Restore(childResource.ID)).To(Succeed()) + Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after restoring") + }) + + It("moves the parent and its child resources", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + parentResource.Path = " " + parentResource.ParentID = "1$2!somewhereopaqueid" + + err = eng.Move(parentResource.ID, parentResource.ParentID, "./somewhere/else/newname") + Expect(err).ToNot(HaveOccurred()) + assertDocCount(rootResource.ID, `parent d!r`, 0) + + matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1) + Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3")) + Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname/child.pdf")) + + matches = assertDocCount(rootResource.ID, `newname`, 1) + Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("somewhereopaqueid")) + Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname")) + + }) + + It("keeps case-insensitive search working after a move", func() { + Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) + Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) + + Expect(eng.Move(parentResource.ID, parentResource.ParentID, "./my/NewName")).To(Succeed()) + + // the lowercased siblings are rebuilt at the new path, so a + // case-insensitive query finds the folder under its new name and path, + // including the descendant, and no longer under the old path. + assertDocCount(rootResource.ID, "name:NEWNAME", 1) + assertDocCount(rootResource.ID, `path:"./MY/NEWNAME"`, 2) + assertDocCount(rootResource.ID, `path:"./parent d!r"`, 0) + }) + }) + + Describe("StartBatch", func() { + It("starts a new batch", func() { + b, err := eng.NewBatch(100) + Expect(err).ToNot(HaveOccurred()) + + err = b.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + count, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(0))) + + err = b.Push() + Expect(err).ToNot(HaveOccurred()) + + count, err = idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(1))) + + query := bleveSearch.NewMatchQuery("child.pdf") + res, err := idx.Search(bleveSearch.NewSearchRequest(query)) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits.Len()).To(Equal(1)) + }) + + It("doesn't intertwine different batches", func() { + b, err := eng.NewBatch(100) + Expect(err).ToNot(HaveOccurred()) + + err = b.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + count, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(0))) + + b2, err := eng.NewBatch(100) + Expect(err).ToNot(HaveOccurred()) + + err = b2.Upsert(childResource2.ID, childResource2) + Expect(err).ToNot(HaveOccurred()) + + Expect(b.Push()).To(Succeed()) + count, err = idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(1))) + + Expect(b2.Push()).To(Succeed()) + count, err = idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(2))) + }) + }) + + Describe("File type specific metadata", func() { + + Context("with audio metadata", func() { + BeforeEach(func() { + resource := search.Resource{ + ID: "1$2!7", + ParentID: rootResource.ID, + RootID: rootResource.ID, + Path: "./some_song.mp3", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{ + Name: "some_song.mp3", + MimeType: "audio/mpeg", + Audio: &libregraph.Audio{ + Album: libregraph.PtrString("Some Album"), + AlbumArtist: libregraph.PtrString("Some AlbumArtist"), + Artist: libregraph.PtrString("Some Artist"), + Bitrate: libregraph.PtrInt64(192), + Composers: libregraph.PtrString("Some Composers"), + Copyright: libregraph.PtrString(""), + Disc: libregraph.PtrInt32(2), + DiscCount: libregraph.PtrInt32(5), + Duration: libregraph.PtrInt64(225000), + Genre: libregraph.PtrString("Some Genre"), + HasDrm: libregraph.PtrBool(false), + IsVariableBitrate: libregraph.PtrBool(true), + Title: libregraph.PtrString("Some Title"), + Track: libregraph.PtrInt32(34), + TrackCount: libregraph.PtrInt32(99), + Year: libregraph.PtrInt32(2004), + }, + }, + } + err := eng.Upsert(resource.ID, resource) + Expect(err).ToNot(HaveOccurred()) + }) + + It("returns audio metadata for search", func() { + matches := assertDocCount(rootResource.ID, `*song*`, 1) + audio := matches[0].Entity.Audio + + Expect(audio).ToNot(BeNil()) + + Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album"))) + Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist"))) + Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist"))) + Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192))) + Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers"))) + Expect(audio.Copyright).To(Equal(libregraph.PtrString(""))) + Expect(audio.Disc).To(Equal(libregraph.PtrInt32(2))) + Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5))) + Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225000))) + Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre"))) + Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false))) + Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true))) + Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title"))) + Expect(audio.Track).To(Equal(libregraph.PtrInt32(34))) + Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(99))) + Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) + }) + }) + + Context("with location metadata", func() { + BeforeEach(func() { + resource := search.Resource{ + ID: "1$2!7", + ParentID: rootResource.ID, + RootID: rootResource.ID, + Path: "./team.jpg", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{ + Name: "team.jpg", + MimeType: "image/jpeg", + Location: &libregraph.GeoCoordinates{ + Altitude: libregraph.PtrFloat64(1047.7), + Latitude: libregraph.PtrFloat64(49.48675890884328), + Longitude: libregraph.PtrFloat64(11.103870357204285), + }, + }, + } + err := eng.Upsert(resource.ID, resource) + Expect(err).ToNot(HaveOccurred()) + }) + + It("returns audio metadata for search", func() { + matches := assertDocCount(rootResource.ID, `*team*`, 1) + location := matches[0].Entity.Location + + Expect(location).ToNot(BeNil()) + + Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(1047.7))) + Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328))) + Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285))) + }) + }) + }) +}) diff --git a/services/search/pkg/bleve/index.go b/services/search/pkg/bleve/index.go index 264ad9893b..76a92625ce 100644 --- a/services/search/pkg/bleve/index.go +++ b/services/search/pkg/bleve/index.go @@ -10,10 +10,8 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" - regexpCharFilter "github.com/blevesearch/bleve/v2/analysis/char/regexp" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" "github.com/blevesearch/bleve/v2/analysis/token/porter" - "github.com/blevesearch/bleve/v2/analysis/tokenizer/single" "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" "github.com/blevesearch/bleve/v2/mapping" storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -60,44 +58,6 @@ func NewMapping() (mapping.IndexMapping, error) { indexMapping := bleve.NewIndexMapping() indexMapping.DefaultAnalyzer = keyword.Name indexMapping.DefaultMapping = docMapping - err = indexMapping.AddCustomCharFilter("dotToSpace", - map[string]any{ - "type": regexpCharFilter.Name, - "regexp": `\.`, - "replace": " ", - }, - ) - if err != nil { - return nil, err - } - - err = indexMapping.AddCustomAnalyzer("lowercaseWords", - map[string]any{ - "type": custom.Name, - "char_filters": []string{"dotToSpace"}, - "tokenizer": unicode.Name, - "token_filters": []string{ - lowercase.Name, - }, - }, - ) - if err != nil { - return nil, err - } - - err = indexMapping.AddCustomAnalyzer("lowercaseKeyword", - map[string]any{ - "type": custom.Name, - "tokenizer": single.Name, - "token_filters": []string{ - lowercase.Name, - }, - }, - ) - if err != nil { - return nil, err - } - err = indexMapping.AddCustomAnalyzer("fulltext", map[string]any{ "type": custom.Name, diff --git a/services/search/pkg/mapping/bleve.go b/services/search/pkg/mapping/bleve.go index f357e4f305..70e1f0949e 100644 --- a/services/search/pkg/mapping/bleve.go +++ b/services/search/pkg/mapping/bleve.go @@ -12,10 +12,8 @@ import ( // struct via reflection. Field names come from json tags; overrides are // keyed by those names (or dotted paths for nested fields). // -// The returned mapping references analyzer names (Analyzer field on the -// FieldOpts, plus "fulltext" / "path_hierarchy" for the corresponding Types); -// the caller is responsible for registering those analyzers on the enclosing -// IndexMapping. +// The returned mapping references the "fulltext" analyzer for Fulltext fields; +// the caller registers it on the enclosing IndexMapping. func BleveBuildMapping(t reflect.Type, overrides map[string]FieldOpts) (*bleveMapping.DocumentMapping, error) { return buildBleveDocMapping(t, overrides, "") } @@ -61,6 +59,16 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix return nil } + if fieldType == TypeKeyword || fieldType == TypePath { + // bleve has no path tokenizer, so a path is a plain keyword here. + base := bleveKeywordMapping(fieldType, opts) + doc.AddFieldMappingsAt(fi.Name, base) + if opts.caseInsensitive() { + doc.AddFieldMappingsAt(fi.Name+LowercaseSuffix, lowercaseSibling(base)) + } + return nil + } + fm, err := bleveFieldMapping(fieldType, opts) if err != nil { return fmt.Errorf("mapping: field %q: %w", key, err) @@ -71,26 +79,46 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix return doc, err } +// bleveKeywordMapping is a case-preserving keyword field; path fields stay out +// of _all by default. +func bleveKeywordMapping(fieldType string, opts FieldOpts) *bleveMapping.FieldMapping { + fm := bleve.NewKeywordFieldMapping() + switch { + case opts.IncludeInAll != nil: + fm.IncludeInAll = *opts.IncludeInAll + case fieldType == TypePath: + fm.IncludeInAll = false + } + return fm +} + +// lowercaseSibling derives the lowercased shadow of a keyword/path field from its +// base mapping: used only for case-insensitive matching, so indexed but never +// stored, kept out of _all, and without doc values, since the case-preserved base +// field is what we return and aggregate on. +func lowercaseSibling(base *bleveMapping.FieldMapping) *bleveMapping.FieldMapping { + fm := *base + fm.Store = false + fm.IncludeInAll = false + fm.DocValues = false + return &fm +} + func bleveFieldMapping(fieldType string, opts FieldOpts) (*bleveMapping.FieldMapping, error) { switch fieldType { case TypeWildcard: // bleve has no wildcard type; fall back to keyword-ish text. fieldType = TypeKeyword fallthrough - case TypeKeyword, TypeFulltext, TypePath: + case TypeKeyword, TypeFulltext: fm := bleve.NewTextFieldMapping() - switch { - case opts.Analyzer != "": - fm.Analyzer = opts.Analyzer - case fieldType == TypeFulltext: + if fieldType == TypeFulltext { fm.Analyzer = "fulltext" - case fieldType == TypePath: - fm.Analyzer = "path_hierarchy" } switch { case opts.IncludeInAll != nil: fm.IncludeInAll = *opts.IncludeInAll - case fieldType == TypeFulltext, fieldType == TypePath: + case fieldType == TypeFulltext: fm.IncludeInAll = false } return fm, nil diff --git a/services/search/pkg/mapping/bleve_test.go b/services/search/pkg/mapping/bleve_test.go index a9cd81954f..4fcb09f7ab 100644 --- a/services/search/pkg/mapping/bleve_test.go +++ b/services/search/pkg/mapping/bleve_test.go @@ -65,21 +65,31 @@ var _ = Describe("BleveBuildMapping", func() { }) It("applies field overrides", func() { - includeInAllFalse := false + True, False := true, false dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{ - "Name": {Analyzer: "lowercaseKeyword"}, + "Name": {CaseInsensitive: &True}, "Content": {Type: TypeFulltext}, - "Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &includeInAllFalse}, + "Tags": {CaseInsensitive: &True, IncludeInAll: &False}, }) Expect(err).ToNot(HaveOccurred()) - nameField := dm.Properties["Name"].Fields[0] - Expect(nameField.Analyzer).To(Equal("lowercaseKeyword"), "Name analyzer") - Expect(nameField.IncludeInAll).To(BeTrue(), "Name IncludeInAll should stay default-true when not overridden") + // Name: case-preserved base keyword + lowercased sibling. + Expect(dm.Properties["Name"]).ToNot(BeNil(), "Name base field") + Expect(dm.Properties["Name"].Fields[0].Analyzer).To(Equal("keyword"), "Name base is a keyword") + Expect(dm.Properties["Name"].Fields[0].Store).To(BeTrue(), "Name base is stored (returned)") + Expect(dm.Properties["Name_lowercase"]).ToNot(BeNil(), "Name_lowercase sibling") + // The sibling is a search-only shadow: indexed but never stored, kept out + // of _all, no doc values (the base is what we return). + sibling := dm.Properties["Name_lowercase"].Fields[0] + Expect(sibling.Index).To(BeTrue(), "Name_lowercase is indexed") + Expect(sibling.Store).To(BeFalse(), "Name_lowercase is not stored") + Expect(sibling.IncludeInAll).To(BeFalse(), "Name_lowercase is out of _all") + Expect(sibling.DocValues).To(BeFalse(), "Name_lowercase has no doc values") contentField := dm.Properties["Content"].Fields[0] Expect(contentField.Analyzer).To(Equal("fulltext"), "Content analyzer") Expect(contentField.IncludeInAll).To(BeFalse(), "Content IncludeInAll should default to false for fulltext type") - tagsField := dm.Properties["Tags"].Fields[0] - Expect(tagsField.IncludeInAll).To(BeFalse(), "Tags IncludeInAll should honor the explicit false override") + // Tags: base + lowercased sibling, both honoring the IncludeInAll override. + Expect(dm.Properties["Tags"].Fields[0].IncludeInAll).To(BeFalse(), "Tags base IncludeInAll honored") + Expect(dm.Properties["Tags_lowercase"].Fields[0].IncludeInAll).To(BeFalse(), "Tags sibling IncludeInAll honored") }) It("builds an object sub-document plus a geopoint sibling", func() { diff --git a/services/search/pkg/mapping/casing.go b/services/search/pkg/mapping/casing.go new file mode 100644 index 0000000000..3ecceda993 --- /dev/null +++ b/services/search/pkg/mapping/casing.go @@ -0,0 +1,57 @@ +package mapping + +import "strings" + +func addLowercaseSiblings(m map[string]any, overrides map[string]FieldOpts) { + for key, opts := range overrides { + if !opts.caseInsensitive() || !isCasedType(opts) { + continue + } + parent, leaf, ok := resolveLeaf(m, key) + if !ok { + continue + } + addLowercaseSibling(parent, leaf) + } +} + +func isCasedType(opts FieldOpts) bool { + return opts.Type == "" || opts.Type == TypeKeyword || opts.Type == TypePath +} + +func resolveLeaf(m map[string]any, dottedPath string) (map[string]any, string, bool) { + parts := strings.Split(dottedPath, ".") + parent := m + for _, p := range parts[:len(parts)-1] { + next, ok := parent[p].(map[string]any) + if !ok { + return nil, "", false + } + parent = next + } + return parent, parts[len(parts)-1], true +} + +// addLowercaseSibling writes a _lowercase sibling; no-op for non-strings. +func addLowercaseSibling(parent map[string]any, leaf string) { + switch v := parent[leaf].(type) { + case string: + parent[leaf+LowercaseSuffix] = strings.ToLower(v) + case []any: + out := make([]any, 0, len(v)) + for _, item := range v { + if s, ok := item.(string); ok { + out = append(out, strings.ToLower(s)) + } + } + if len(out) > 0 { + parent[leaf+LowercaseSuffix] = out + } + case []string: + out := make([]string, len(v)) + for i, s := range v { + out[i] = strings.ToLower(s) + } + parent[leaf+LowercaseSuffix] = out + } +} diff --git a/services/search/pkg/mapping/casing_test.go b/services/search/pkg/mapping/casing_test.go new file mode 100644 index 0000000000..d846158fb8 --- /dev/null +++ b/services/search/pkg/mapping/casing_test.go @@ -0,0 +1,41 @@ +package mapping + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("PrepareForIndex casing", func() { + It("adds lowercased siblings for CaseInsensitive keyword and path fields", func() { + True := true + type doc struct { + Name string `json:"Name"` + Path string `json:"Path"` + Tags []string `json:"Tags"` + } + d := doc{Name: "Report FINAL", Path: "/Foo/Bar", Tags: []string{"Work", "Urgent"}} + m, err := PrepareForIndex(d, map[string]FieldOpts{ + "Name": {CaseInsensitive: &True}, + "Path": {Type: TypePath, CaseInsensitive: &True}, + "Tags": {CaseInsensitive: &True}, + }) + Expect(err).ToNot(HaveOccurred()) + + // Originals stay for the case-preserved base fields and the cascade. + Expect(m["Name"]).To(Equal("Report FINAL")) + Expect(m["Path"]).To(Equal("/Foo/Bar")) + + Expect(m["Name_lowercase"]).To(Equal("report final")) + Expect(m["Path_lowercase"]).To(Equal("/foo/bar")) + Expect(m["Tags_lowercase"]).To(Equal([]any{"work", "urgent"})) + }) + + It("writes no sibling without CaseInsensitive", func() { + type doc struct { + ID string `json:"ID"` + } + m, err := PrepareForIndex(doc{ID: "ABC"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(m).ToNot(HaveKey("ID" + LowercaseSuffix)) + }) +}) diff --git a/services/search/pkg/mapping/opensearch.go b/services/search/pkg/mapping/opensearch.go index abeeb96d65..6b6b243ec7 100644 --- a/services/search/pkg/mapping/opensearch.go +++ b/services/search/pkg/mapping/opensearch.go @@ -56,7 +56,20 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p return nil } - fm, err := openSearchFieldMapping(fieldType, opts, fi.GoField.Type) + if fieldType == TypeKeyword || fieldType == TypePath { + // path_hierarchy is case-preserving here; casing lives in the value. + m := map[string]any{"type": "keyword"} + if fieldType == TypePath { + m = map[string]any{"type": "text", "analyzer": "path_hierarchy"} + } + props[fi.Name] = m + if opts.caseInsensitive() { + props[fi.Name+LowercaseSuffix] = m + } + return nil + } + + fm, err := openSearchFieldMapping(fieldType, fi.GoField.Type) if err != nil { return fmt.Errorf("mapping: field %q: %w", key, err) } @@ -66,32 +79,15 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p return props, err } -func openSearchFieldMapping(fieldType string, opts FieldOpts, goType reflect.Type) (map[string]any, error) { +// openSearchFieldMapping handles the non-keyword/path types; keyword and path +// are emitted (with their cased forms) by buildOpenSearchProperties directly. +func openSearchFieldMapping(fieldType string, goType reflect.Type) (map[string]any, error) { switch fieldType { - case TypeKeyword: - m := map[string]any{"type": "keyword"} - if opts.Analyzer != "" { - m["type"] = "text" - m["analyzer"] = opts.Analyzer - } - return m, nil case TypeFulltext: - m := map[string]any{ + return map[string]any{ "type": "text", "term_vector": "with_positions_offsets", - } - if opts.Analyzer != "" { - m["analyzer"] = opts.Analyzer - } - return m, nil - case TypePath: - m := map[string]any{"type": "text"} - if opts.Analyzer != "" { - m["analyzer"] = opts.Analyzer - } else { - m["analyzer"] = "path_hierarchy" - } - return m, nil + }, nil case TypeWildcard: // OpenSearch stores wildcard fields with doc_values=false by // default, so emit it explicitly to keep local and remote diff --git a/services/search/pkg/mapping/opensearch_test.go b/services/search/pkg/mapping/opensearch_test.go index 3ff80f20e3..b64520cb73 100644 --- a/services/search/pkg/mapping/opensearch_test.go +++ b/services/search/pkg/mapping/opensearch_test.go @@ -78,6 +78,7 @@ var _ = Describe("OpenSearchBuildMapping", func() { }) It("applies field overrides", func() { + True := true type doc struct { Name string `json:"Name"` Content string `json:"Content"` @@ -85,23 +86,23 @@ var _ = Describe("OpenSearchBuildMapping", func() { MimeType string `json:"MimeType"` } props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{ - "Name": {Analyzer: "lowercaseKeyword"}, + "Name": {CaseInsensitive: &True}, "Content": {Type: TypeFulltext}, - "Path": {Type: TypePath}, + "Path": {Type: TypePath, CaseInsensitive: &True}, "MimeType": {Type: TypeWildcard}, }) Expect(err).ToNot(HaveOccurred()) - name := props["Name"].(map[string]any) - Expect(name["type"]).To(Equal("text"), "Name: %#v", name) - Expect(name["analyzer"]).To(Equal("lowercaseKeyword"), "Name: %#v", name) + // Name: case-preserved keyword base + lowercased keyword sibling. + Expect(props["Name"]).To(Equal(map[string]any{"type": "keyword"})) + Expect(props["Name_lowercase"]).To(Equal(map[string]any{"type": "keyword"})) content := props["Content"].(map[string]any) Expect(content["type"]).To(Equal("text"), "Content: %#v", content) Expect(content["term_vector"]).To(Equal("with_positions_offsets"), "Content: %#v", content) _, ok := content["analyzer"] Expect(ok).To(BeFalse(), "Content should leave analyzer unset (use OpenSearch default)") - path := props["Path"].(map[string]any) - Expect(path["type"]).To(Equal("text"), "Path: %#v", path) - Expect(path["analyzer"]).To(Equal("path_hierarchy"), "Path: %#v", path) + // Path: path_hierarchy base + lowercased sibling, both case-preserving. + Expect(props["Path"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"})) + Expect(props["Path_lowercase"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"})) mime := props["MimeType"].(map[string]any) Expect(mime["type"]).To(Equal("wildcard"), "MimeType: %#v", mime) }) diff --git a/services/search/pkg/mapping/opts.go b/services/search/pkg/mapping/opts.go index 3daa258a62..9ae7d295a0 100644 --- a/services/search/pkg/mapping/opts.go +++ b/services/search/pkg/mapping/opts.go @@ -17,6 +17,9 @@ const ( TypeGeopoint = "geopoint" ) +// LowercaseSuffix names the lowercased sibling of a keyword/path field. +const LowercaseSuffix = "_lowercase" + // FieldOpts overrides the default type inference for a struct field. Keys in // the override map are json-tag names (e.g. "Name", "location", "audio.artist"), // not Go field names. @@ -24,12 +27,14 @@ type FieldOpts struct { // Type is one of the Type* constants. Empty means "infer from Go type". Type string - // Analyzer is the name of a custom analyzer registered on the bleve - // IndexMapping (e.g. "lowercaseKeyword", "fulltext"). For OpenSearch it - // becomes the analyzer attribute on the field. - Analyzer string + // CaseInsensitive additionally indexes a lowercased _lowercase sibling + // for case-insensitive search; the case-preserved base is always indexed. + // Nil/false means off. Keyword/path only. + CaseInsensitive *bool // IncludeInAll controls bleve's _all field inclusion. Nil means "use the // bleve default for this field type". Has no effect on OpenSearch. IncludeInAll *bool } + +func (o FieldOpts) caseInsensitive() bool { return o.CaseInsensitive != nil && *o.CaseInsensitive } diff --git a/services/search/pkg/mapping/serialize.go b/services/search/pkg/mapping/serialize.go index 027d10da71..ec5420784a 100644 --- a/services/search/pkg/mapping/serialize.go +++ b/services/search/pkg/mapping/serialize.go @@ -19,5 +19,6 @@ func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, err return out, nil } addGeopointSiblings(out, overrides) + addLowercaseSiblings(out, overrides) return out, nil } diff --git a/services/search/pkg/mapping/validate_test.go b/services/search/pkg/mapping/validate_test.go index 278bb3820f..3b15f39caf 100644 --- a/services/search/pkg/mapping/validate_test.go +++ b/services/search/pkg/mapping/validate_test.go @@ -23,9 +23,9 @@ type sample struct { var _ = Describe("Validate", func() { It("accepts known override keys", func() { err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{ - "Name": {Analyzer: "lowercaseKeyword"}, + "Name": {}, "audio": {Type: TypeObject}, - "audio.artist": {Analyzer: "lowercaseKeyword"}, + "audio.artist": {}, "location": {Type: TypeGeopoint}, }) Expect(err).ToNot(HaveOccurred()) diff --git a/services/search/pkg/opensearch/batch.go b/services/search/pkg/opensearch/batch.go index cd1a89e386..1dc0bdf8ca 100644 --- a/services/search/pkg/opensearch/batch.go +++ b/services/search/pkg/opensearch/batch.go @@ -64,27 +64,45 @@ func (b *Batch) Upsert(id string, r search.Resource) error { }) } -func (b *Batch) Move(id string, parentID string, targetPath string) error { +func (b *Batch) Move(id string, parentID string, location string) error { return b.withSizeLimit(func() error { op := func() error { return updateSelfAndDescendants(context.Background(), b.client, b.index, id, func(rootResource search.Resource) *osu.BodyParamScript { + newPath := utils.MakeRelativePath(location) + newName := path.Base(newPath) return &osu.BodyParamScript{ - Source: ` - if (ctx._source.ID == params.id ) { ctx._source.Name = params.newName; ctx._source.ParentID = params.parentID; } - ctx._source.Path = ctx._source.Path.replace(params.oldPath, params.newPath); - boolean hidden = false; - for (String name : ctx._source.Path.splitOnToken('/')) { - if (!name.equals('.') && !name.equals('..') && name.startsWith('.')) { hidden = true; break; } - } - ctx._source.Hidden = hidden; - `, + // Keep the case-preserved base fields and their lowercased + // search siblings in sync: swap the moved prefix in both. The + // lowercased new values come from Go's strings.ToLower via + // params, so the sibling stays byte-identical to what + // PrepareForIndex writes on upsert (painless toLowerCase would + // lowercase differently than Go). + Source: fmt.Sprintf(` + if (ctx._source.ID == params.id) { + ctx._source.Name = params.newName; + ctx._source.ParentID = params.parentID; + if (ctx._source.Name%[1]s != null) { ctx._source.Name%[1]s = params.newNameLower; } + } + ctx._source.Path = ctx._source.Path.replace(params.oldPath, params.newPath); + if (ctx._source.Path%[1]s != null) { + ctx._source.Path%[1]s = ctx._source.Path%[1]s.replace(params.oldPathLower, params.newPathLower); + } + boolean hidden = false; + for (String name : ctx._source.Path.splitOnToken('/')) { + if (!name.equals('.') && !name.equals('..') && name.startsWith('.')) { hidden = true; break; } + } + ctx._source.Hidden = hidden; + `, mapping.LowercaseSuffix), Lang: "painless", Params: map[string]any{ - "id": id, - "parentID": parentID, - "oldPath": rootResource.Path, - "newPath": utils.MakeRelativePath(targetPath), - "newName": path.Base(utils.MakeRelativePath(targetPath)), + "id": id, + "parentID": parentID, + "oldPath": rootResource.Path, + "newPath": newPath, + "newName": newName, + "oldPathLower": strings.ToLower(rootResource.Path), + "newPathLower": strings.ToLower(newPath), + "newNameLower": strings.ToLower(newName), }, } }) diff --git a/services/search/pkg/opensearch/index.go b/services/search/pkg/opensearch/index.go index 120ed6cea6..93d454e53b 100644 --- a/services/search/pkg/opensearch/index.go +++ b/services/search/pkg/opensearch/index.go @@ -61,7 +61,6 @@ func buildResourceMapping() ([]byte, error) { resourceType := reflect.TypeFor[search.Resource]() overrides := maps.Clone(search.Resource{}.SearchFieldOverrides()) overrides["MimeType"] = searchmapping.FieldOpts{Type: searchmapping.TypeWildcard} - overrides["Path"] = searchmapping.FieldOpts{Type: searchmapping.TypePath} if err := searchmapping.Validate(resourceType, overrides); err != nil { return nil, err } @@ -75,16 +74,11 @@ func buildResourceMapping() ([]byte, error) { "number_of_shards": "1", "number_of_replicas": "1", "analysis": map[string]any{ + // path_hierarchy is case-preserving; casing lives in the value. "analyzer": map[string]any{ "path_hierarchy": map[string]any{ "type": "custom", "tokenizer": "path_hierarchy", - "filter": []string{"lowercase"}, - }, - "lowercaseKeyword": map[string]any{ - "type": "custom", - "tokenizer": "keyword", - "filter": []string{"lowercase"}, }, }, "tokenizer": map[string]any{ diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand.go b/services/search/pkg/opensearch/internal/convert/kql_expand.go deleted file mode 100644 index 8d382c8745..0000000000 --- a/services/search/pkg/opensearch/internal/convert/kql_expand.go +++ /dev/null @@ -1,25 +0,0 @@ -package convert - -import ( - "strings" - - "github.com/opencloud-eu/opencloud/pkg/ast" - "github.com/opencloud-eu/opencloud/services/search/pkg/search" -) - -// LowerValues folds restriction values for fields whose index analyzer -// lowercases (search.LowercaseValueFields, shared with bleve); case-preserved -// fields keep their casing. Runs after query.Normalize, so keys are resolved. -func LowerValues(nodes []ast.Node) []ast.Node { - for _, n := range nodes { - switch node := n.(type) { - case *ast.StringNode: - if _, ok := search.LowercaseValueFields()[node.Key]; ok { - node.Value = strings.ToLower(node.Value) - } - case *ast.GroupNode: - LowerValues(node.Nodes) - } - } - return nodes -} diff --git a/services/search/pkg/opensearch/internal/convert/kql_query.go b/services/search/pkg/opensearch/internal/convert/kql_query.go index 18687dda74..711a67b4b4 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_query.go +++ b/services/search/pkg/opensearch/internal/convert/kql_query.go @@ -18,11 +18,10 @@ func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) { return nil, err } - // shared lowering (field resolution + media-type), then value lowercasing. + // shared lowering: field resolution, media-type expansion, value lowercasing. kqlAst = query.Normalize(kqlAst, query.ResolveField) - kqlNodes := LowerValues(kqlAst.Nodes) - builder, err := TranspileKQLToOpenSearch(kqlNodes) + builder, err := TranspileKQLToOpenSearch(kqlAst.Nodes) if err != nil { return nil, fmt.Errorf("failed to compile query: %w", err) } diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index e3fb25a807..8a060a69d8 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -4,12 +4,12 @@ import ( "errors" "fmt" "slices" - "strconv" "strings" "time" "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/pkg/kql" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" ) @@ -99,7 +99,28 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { case *ast.BooleanNode: return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil case *ast.StringNode: - return stringNodeQuery(node), nil + field, value := node.Key, node.Value + if node.CaseInsensitive { + field += mapping.LowercaseSuffix + value = strings.ToLower(value) + } + + isWildcard := strings.Contains(value, "*") + if isWildcard { + return osu.NewWildcardQuery(field).Value(value), nil + } + + totalTerms := strings.Split(value, " ") + isSingleTerm := len(totalTerms) == 1 + isMultiTerm := len(totalTerms) >= 1 + switch { + case isSingleTerm: + return osu.NewTermQuery[string](field).Value(value), nil + case isMultiTerm: + return osu.NewMatchPhraseQuery(field).Query(value), nil + } + + return nil, fmt.Errorf("unsupported string node value: %s", value) case *ast.DateTimeNode: return dateTimeNodeQuery(node) case *ast.NumberNode: @@ -116,67 +137,26 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { return nil, fmt.Errorf("%w: %T", ErrUnsupportedNodeType, node) } -// stringNodeQuery picks the query a string node turns into. -func stringNodeQuery(node *ast.StringNode) osu.Builder { - isWildcard := strings.ContainsAny(node.Value, "*?") - - switch { - // Name: "*oo-bar", "*oo ba*", "*OO*" - // Title: "*rterly rep*" - // Tags: "*spaced tag*" - case isWildcard && slices.Contains([]string{"Name", "Title"}, node.Key): - patterns := []osu.Builder{wildcardOn(node.Key+".wildcard", node.Value)} - if !strings.HasSuffix(node.Value, "*") { - patterns = append(patterns, wildcardOn(node.Key+".wildcard", node.Value+".*")) - } - - return osu.NewBoolQuery(). - Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). - Should(patterns...) - // Tags: "*foo*", "*paced ta*" - case isWildcard && node.Key == "Tags": - return wildcardOn(node.Key+".wildcard", node.Value) - // Path: "./foo*", MimeType: "*plain" - case isWildcard: - return osu.NewWildcardQuery(node.Key).Value(node.Value) - // Name: =new, Title: ="quarterly report" - case node.Exact && slices.Contains([]string{"Name", "Title"}, node.Key): - return osu.NewTermQuery[string](node.Key + ".wildcard"). - Value(node.Value). - Params(&osu.TermQueryParams{CaseInsensitive: true}) - // Tags: "foo-bar", "spaced tag", "FOO-BAR" - case node.Key == "Tags": - return osu.NewTermQuery[string](node.Key + ".wildcard"). - Value(node.Value). - Params(&osu.TermQueryParams{CaseInsensitive: true}) - // Name: "foo-bar", "foo bar" - // Title: "quarterly report" - // Content: "foo bar" - case slices.Contains([]string{"Name", "Title", "Content"}, node.Key): - return osu.NewMatchPhraseQuery(node.Key).Query(node.Value) - // Size: "42", Type: "1" - case slices.Contains([]string{"Size", "Type"}, node.Key): - number, err := strconv.ParseInt(node.Value, 10, 64) - if err != nil { - return osu.NewMatchNoneQuery() - } - - return osu.NewTermQuery[int64](node.Key).Value(number) - // Path: "./foo bar/", the hierarchy tokens carry no trailing slash - case node.Key == "Path": - return osu.NewTermQuery[string](node.Key).Value(strings.TrimSuffix(node.Value, "/")) - // Hidden: "TRUE" arrives lowered, anything that is no bool matches nothing - case node.Key == "Hidden": - value, err := strconv.ParseBool(node.Value) - if err != nil { - return osu.NewMatchNoneQuery() - } - - return osu.NewTermQuery[bool](node.Key).Value(value) - // MimeType: "text/plain" - default: - return osu.NewTermQuery[string](node.Key).Value(node.Value) +// dateTimeNodeQuery turns a date time node into a range query. +func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) { + if node.Operator == nil { + return nil, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType) } + + query := osu.NewRangeQuery[time.Time](node.Key) + + switch node.Operator.Value { + case ">": + return query.Gt(node.Value), nil + case ">=": + return query.Gte(node.Value), nil + case "<": + return query.Lt(node.Value), nil + case "<=": + return query.Lte(node.Value), nil + } + + return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType) } func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) { @@ -203,31 +183,3 @@ func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) { return nil, fmt.Errorf("unsupported operator %s for number node: %w", node.Operator.Value, ErrUnsupportedNodeType) } - -func wildcardOn(field, value string) osu.Builder { - return osu.NewWildcardQuery(field). - Value(value). - Params(&osu.WildcardQueryParams{CaseInsensitive: true}) -} - -// dateTimeNodeQuery turns a date time node into a range query. -func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) { - if node.Operator == nil { - return nil, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType) - } - - query := osu.NewRangeQuery[time.Time](node.Key) - - switch node.Operator.Value { - case ">": - return query.Gt(node.Value), nil - case ">=": - return query.Gte(node.Value), nil - case "<": - return query.Lt(node.Value), nil - case "<=": - return query.Lte(node.Value), nil - } - - return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType) -} diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go index 9e03fd23df..5f29306848 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go @@ -16,13 +16,31 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{ // kql to os dsl - type tests { - Name: "match phrase query - string node on an analyzed field", + Name: "term query - string node", Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "Name", Value: "openCloud"}, }, }, - Want: osu.NewMatchPhraseQuery("Name").Query("openCloud"), + Want: osu.NewTermQuery[string]("Name").Value("openCloud"), + }, + { + Name: "case-insensitive term routes to the lowercased sibling", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "openCloud", CaseInsensitive: true}, + }, + }, + Want: osu.NewTermQuery[string]("Name_lowercase").Value("opencloud"), + }, + { + Name: "case-insensitive wildcard routes to the lowercased sibling", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "Open*", CaseInsensitive: true}, + }, + }, + Want: osu.NewWildcardQuery("Name_lowercase").Value("open*"), }, { Name: "term query - boolean node - true", @@ -58,16 +76,10 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { &ast.StringNode{Key: "Name", Value: "open*"}, }, }, - Want: osu.NewBoolQuery(). - Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). - Should( - osu.NewWildcardQuery("Name.wildcard"). - Value("open*"). - Params(&osu.WildcardQueryParams{CaseInsensitive: true}), - ), + Want: osu.NewWildcardQuery("Name").Value("open*"), }, { - Name: "wildcard query - string node without an unanalyzed sub field", + Name: "wildcard query - fulltext field", Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "Content", Value: "open*"}, @@ -142,8 +154,8 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, }, Want: osu.NewBoolQuery().Must( - osu.NewMatchPhraseQuery("Name").Query("a"), - osu.NewMatchPhraseQuery("Name").Query("b"), + osu.NewTermQuery[string]("Name").Value("a"), + osu.NewTermQuery[string]("Name").Value("b"), ), }, { @@ -155,7 +167,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }}, }, }, - Want: osu.NewMatchPhraseQuery("Name").Query("any"), + Want: osu.NewTermQuery[string]("Name").Value("any"), }, { Name: "range query >", @@ -217,7 +229,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { &ast.StringNode{Key: "Name", Value: "openCloud"}, }, }, - Want: osu.NewMatchPhraseQuery("Name").Query("openCloud"), + Want: osu.NewTermQuery[string]("Name").Value("openCloud"), }, { Name: "[* *]", @@ -229,7 +241,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewBoolQuery(). Must( - osu.NewMatchPhraseQuery("Name").Query("openCloud"), + osu.NewTermQuery[string]("Name").Value("openCloud"), osu.NewTermQuery[string]("age").Value("32"), ), }, @@ -244,7 +256,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewBoolQuery(). Must( - osu.NewMatchPhraseQuery("Name").Query("openCloud"), + osu.NewTermQuery[string]("Name").Value("openCloud"), osu.NewTermQuery[string]("age").Value("32"), ), }, @@ -260,7 +272,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { Want: osu.NewBoolQuery(). Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). Should( - osu.NewMatchPhraseQuery("Name").Query("openCloud"), + osu.NewTermQuery[string]("Name").Value("openCloud"), osu.NewTermQuery[string]("age").Value("32"), ), }, @@ -288,7 +300,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewBoolQuery(). Must( - osu.NewMatchPhraseQuery("Name").Query("openCloud"), + osu.NewTermQuery[string]("Name").Value("openCloud"), ). MustNot( osu.NewTermQuery[string]("age").Value("32"), @@ -308,7 +320,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { Want: osu.NewBoolQuery(). Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). Should( - osu.NewMatchPhraseQuery("Name").Query("openCloud"), + osu.NewTermQuery[string]("Name").Value("openCloud"), osu.NewTermQuery[string]("age").Value("32"), osu.NewTermQuery[string]("age").Value("44"), ), diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index c5b32835b0..51e9ab4f0d 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -12,31 +12,10 @@ import ( bleveQuery "github.com/blevesearch/bleve/v2/search/query" "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/pkg/kql" - "github.com/opencloud-eu/opencloud/services/search/pkg/search" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" + searchQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query" ) -// lowercaseFields holds the fields whose query-side value is pre-lowercased so -// it matches the index-time lowercasing analyzer. Shared with the OpenSearch -// backend via search.LowercaseValueFields; every other field keeps its casing. -var lowercaseFields = search.LowercaseValueFields() - -var _fields = map[string]string{ - "rootid": "RootID", - "path": "Path", - "id": "ID", - "name": "Name", - "size": "Size", - "mtime": "Mtime", - "mediatype": "MimeType", - "type": "Type", - "tag": "Tags", - "tags": "Tags", - "content": "Content", - "title": "Title", - "hidden": "Hidden", - "favorite": "Favorites", -} - // The following quoted string enumerates the characters which may be escaped: "+-=&|>\<\!\(\)\{\}\[\]\^\"\~\:\ `), }), wantErr: false, }, diff --git a/services/search/pkg/query/normalize.go b/services/search/pkg/query/normalize.go index 78a4a55fe9..064a78e2e2 100644 --- a/services/search/pkg/query/normalize.go +++ b/services/search/pkg/query/normalize.go @@ -2,6 +2,7 @@ package query import ( "reflect" + "strings" "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/services/search/pkg/query/mimetype" @@ -31,10 +32,14 @@ func normalizeNodes(nodes []ast.Node, resolve func(string) string, defaultKey st switch node := n.(type) { case *ast.StringNode: node.Key = resolveKey(node.Key) + if FieldValueIsNormalized(node.Key) { + node.Value = strings.ToLower(node.Value) + } if exp := mimetype.Expand(node.Key, node.Value); exp != nil { out = append(out, normalizeNodes(exp, resolve, defaultKey)...) continue } + node.CaseInsensitive = FieldIsCaseInsensitive(node.Key) out = append(out, node) case *ast.DateTimeNode: node.Key = resolveKey(node.Key) diff --git a/services/search/pkg/query/normalize_test.go b/services/search/pkg/query/normalize_test.go index 66ba3243f0..df7568f0c0 100644 --- a/services/search/pkg/query/normalize_test.go +++ b/services/search/pkg/query/normalize_test.go @@ -20,13 +20,24 @@ func norm(nodes ...ast.Node) []ast.Node { func TestResolveField(t *testing.T) { require.Equal(t, "Name", query.ResolveField("")) // empty -> free-text default - require.Equal(t, "Name", query.ResolveField("NAME")) // case-insensitive + require.Equal(t, "Name", query.ResolveField("NAME")) // canonical, case-insensitive key match require.Equal(t, "Tags", query.ResolveField("tag")) // singular alias - require.Equal(t, "MimeType", query.ResolveField("mimetype")) // real field, case-insensitive - require.Equal(t, "photo.cameraMake", query.ResolveField("photo.CAMERAMAKE")) // facet, case-insensitive + require.Equal(t, "MimeType", query.ResolveField("mimetype")) // real field + require.Equal(t, "photo.cameraMake", query.ResolveField("photo.CAMERAMAKE")) // facet, case-insensitive key match require.Equal(t, "unknown.field", query.ResolveField("unknown.field")) // unknown key: unchanged, becomes a dead query } +func TestFieldIsCaseInsensitive(t *testing.T) { + // The four CaseInsensitive override fields (resolved canonical names). + for _, f := range []string{"Name", "Path", "Tags", "Favorites"} { + require.True(t, query.FieldIsCaseInsensitive(f), f) + } + // Case-preserved / non-keyword fields are not. + for _, f := range []string{"MimeType", "ID", "Content", "unknown"} { + require.False(t, query.FieldIsCaseInsensitive(f), f) + } +} + func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) { got := norm( &ast.StringNode{Key: "", Value: "free"}, @@ -40,9 +51,9 @@ func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) { ast.NumberNode{Key: "size", Value: 100}, ) require.Equal(t, []ast.Node{ - &ast.StringNode{Key: "Name", Value: "free"}, + &ast.StringNode{Key: "Name", Value: "free", CaseInsensitive: true}, &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Tags", Value: "x"}, + &ast.StringNode{Key: "Tags", Value: "x", CaseInsensitive: true}, &ast.OperatorNode{Value: "AND"}, &ast.StringNode{Key: "photo.cameraMake", Value: "canon"}, &ast.OperatorNode{Value: "AND"}, @@ -71,11 +82,11 @@ func TestNormalize_GroupKeyDefaulting(t *testing.T) { &ast.GroupNode{Key: "author", Nodes: []ast.Node{ &ast.StringNode{Key: "author", Value: "b"}, &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "Name", Value: "d"}, + &ast.StringNode{Key: "Name", Value: "d", CaseInsensitive: true}, }}, &ast.OperatorNode{Value: "AND"}, &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "e"}, + &ast.StringNode{Key: "Name", Value: "e", CaseInsensitive: true}, }}, }, got) } @@ -87,7 +98,7 @@ func TestNormalize_ConvertsValueNodesToPointers(t *testing.T) { ast.DateTimeNode{Key: "mtime"}, ) require.Equal(t, []ast.Node{ - &ast.StringNode{Key: "Name", Value: "x"}, + &ast.StringNode{Key: "Name", Value: "x", CaseInsensitive: true}, &ast.OperatorNode{Value: "AND"}, &ast.DateTimeNode{Key: "Mtime"}, }, got) diff --git a/services/search/pkg/query/resolver.go b/services/search/pkg/query/resolver.go index 5a4127a817..edc6e15951 100644 --- a/services/search/pkg/query/resolver.go +++ b/services/search/pkg/query/resolver.go @@ -15,27 +15,72 @@ var aliases = map[string]string{ "favorite": "Favorites", } -// fieldIndex maps a lowercased KQL key to the real field name: derived once from -// the resource struct, overlaid with the explicit aliases. +// fieldIndex maps a lowercased KQL key to its canonical field name ("" is the +// bare-search default). var fieldIndex = sync.OnceValue(func() map[string]string { - idx := mapping.FieldNameIndex( - reflect.TypeFor[search.Resource](), - search.Resource{}.SearchFieldOverrides(), - ) + idx := mapping.FieldNameIndex(reflect.TypeFor[search.Resource](), search.Resource{}.SearchFieldOverrides()) for k, v := range aliases { idx[k] = v } + idx[""] = idx["name"] return idx }) -// ResolveField maps a KQL key to the index field name: empty -> Name, a known -// key (case-insensitive) -> its field, anything else unchanged. -func ResolveField(name string) string { - if name == "" { - return "Name" +// caseInsensitiveFields are the fields searched case-insensitively by default, +// derived from the CaseInsensitive overrides. +var caseInsensitiveFields = sync.OnceValue(func() map[string]struct{} { + out := map[string]struct{}{} + for field, opts := range (search.Resource{}).SearchFieldOverrides() { + if opts.CaseInsensitive != nil && *opts.CaseInsensitive { + out[field] = struct{}{} + } } + return out +}) + +// pathFields are hierarchical path fields (TypePath), derived from the overrides. +var pathFields = sync.OnceValue(func() map[string]struct{} { + out := map[string]struct{}{} + for field, opts := range (search.Resource{}).SearchFieldOverrides() { + if opts.Type == mapping.TypePath { + out[field] = struct{}{} + } + } + return out +}) + +// ResolveField maps a KQL key to its canonical field name; unknown keys pass through. +func ResolveField(name string) string { if v, ok := fieldIndex()[strings.ToLower(name)]; ok { return v } return name } + +// normalizedValueFields have their stored values normalized to lowercase at +// index time, so query values fold to match even though the fields themselves +// are case-preserved keywords. +var normalizedValueFields = map[string]struct{}{ + "MimeType": {}, + "Type": {}, + "Hidden": {}, +} + +// FieldValueIsNormalized reports whether a field's stored values are +// normalized lowercase. +func FieldValueIsNormalized(field string) bool { + _, ok := normalizedValueFields[field] + return ok +} + +// FieldIsCaseInsensitive reports whether a field's default search is case-insensitive. +func FieldIsCaseInsensitive(field string) bool { + _, ok := caseInsensitiveFields()[field] + return ok +} + +// FieldIsPath reports whether a field is a hierarchical path field. +func FieldIsPath(field string) bool { + _, ok := pathFields()[field] + return ok +} diff --git a/services/search/pkg/search/search.go b/services/search/pkg/search/search.go index fb4ebef31e..73b07cae01 100644 --- a/services/search/pkg/search/search.go +++ b/services/search/pkg/search/search.go @@ -72,12 +72,13 @@ type Resource struct { // resourceFieldOverrides is built once (it never changes) and reused on hot // paths instead of reallocating per call. var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts { - excludeFromAll := false + True, False := true, false return map[string]mapping.FieldOpts{ - "Name": {Analyzer: "lowercaseKeyword"}, + "Name": {CaseInsensitive: &True}, + "Path": {Type: mapping.TypePath, CaseInsensitive: &True}, "Content": {Type: mapping.TypeFulltext}, - "Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll}, - "Favorites": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll}, + "Tags": {CaseInsensitive: &True, IncludeInAll: &False}, + "Favorites": {CaseInsensitive: &True, IncludeInAll: &False}, "location": {Type: mapping.TypeGeopoint}, } }) @@ -89,32 +90,6 @@ func (Resource) SearchFieldOverrides() map[string]mapping.FieldOpts { return resourceFieldOverrides() } -// lowercaseValueFields is the set of index field names whose query values must -// be lowercased to match their index-time lowercasing analyzer (lowercaseKeyword -// or the fulltext type). Built once from the field overrides. -var lowercaseValueFields = sync.OnceValue(func() map[string]struct{} { - out := map[string]struct{}{} - for key, opts := range resourceFieldOverrides() { - if opts.Analyzer == "lowercaseKeyword" || opts.Type == mapping.TypeFulltext { - out[key] = struct{}{} - } - } - // stored values are normalized lowercase, so query values must fold too - // even though the index fields preserve case - for _, key := range []string{"MimeType", "Type", "Hidden"} { - out[key] = struct{}{} - } - return out -}) - -// LowercaseValueFields returns the set of index field names whose query values -// must be lowercased so query-side matching lines up with the index-time -// analyzer. Both search backends use it, so value casing stays consistent; every -// other (case-preserved) field keeps its original case. Read-only, do not mutate. -func LowercaseValueFields() map[string]struct{} { - return lowercaseValueFields() -} - // ResolveReference makes sure the path is relative to the space root func ResolveReference(ctx context.Context, ref *provider.Reference, ri *provider.ResourceInfo, gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) (*provider.Reference, error) { if ref.GetResourceId().GetOpaqueId() == ref.GetResourceId().GetSpaceId() {