From 545aa114e89b1c17bc4476f80e6088f2bf026f0f Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Fri, 28 Aug 2026 19:38:01 +0200 Subject: [PATCH] test(search): let the parity suite replace the per-engine backend suites Everything the bleve and OpenSearch backend suites checked now runs against both engines in services/search/pkg/parity. The two cases without a twin there join it (a facet value keeps its case, batches stay apart), the one thing only OpenSearch can do, refuse an unhealthy cluster, stays in its package. --- services/search/pkg/bleve/backend_test.go | 808 ------------------ .../search/pkg/opensearch/backend_test.go | 662 +------------- services/search/pkg/parity/README.md | 5 + .../search/pkg/parity/lifecycle_batch_test.go | 29 + .../search/pkg/parity/query_fields_test.go | 6 + 5 files changed, 42 insertions(+), 1468 deletions(-) delete mode 100644 services/search/pkg/bleve/backend_test.go diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go deleted file mode 100644 index b9feef8e8d..0000000000 --- a/services/search/pkg/bleve/backend_test.go +++ /dev/null @@ -1,808 +0,0 @@ -package bleve_test - -import ( - "context" - "fmt" - - bleveSearch "github.com/blevesearch/bleve/v2" - sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - libregraph "github.com/opencloud-eu/libre-graph-api-go" - "github.com/opencloud-eu/reva/v2/pkg/storagespace" - - "github.com/opencloud-eu/opencloud/pkg/log" - searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" - searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" - "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" - "github.com/opencloud-eu/opencloud/services/search/pkg/content" - bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve" - "github.com/opencloud-eu/opencloud/services/search/pkg/search" -) - -func hiddenByID(idx bleveSearch.Index, id string) bool { - GinkgoHelper() - - req := bleveSearch.NewSearchRequest(bleveSearch.NewDocIDQuery([]string{id})) - req.Fields = []string{"Hidden"} - - res, err := idx.Search(req) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Hits).To(HaveLen(1), "no record for %s", id) - - hidden, _ := res.Hits[0].Fields["Hidden"].(bool) - return hidden -} - -var _ = Describe("Bleve", func() { - var ( - eng *bleve.Backend - idx bleveSearch.Index - - doSearch = func(id string, query, path string) (*searchsvc.SearchIndexResponse, error) { - rID, err := storagespace.ParseID(id) - if err != nil { - return nil, err - } - - return eng.Search(context.Background(), &searchsvc.SearchIndexRequest{ - Query: query, - Ref: &searchmsg.Reference{ - ResourceId: &searchmsg.ResourceID{ - StorageId: rID.StorageId, - SpaceId: rID.SpaceId, - OpaqueId: rID.OpaqueId, - }, - Path: path, - }, - }) - } - - assertDocCount = func(id string, query string, expectedCount int) []*searchmsg.Match { - res, err := doSearch(id, query, "") - - ExpectWithOffset(1, err).ToNot(HaveOccurred()) - ExpectWithOffset(1, len(res.Matches)).To(Equal(expectedCount), "query returned unexpected number of results: "+query) - return res.Matches - } - - rootResource search.Resource - parentResource search.Resource - childResource search.Resource - childResource2 search.Resource - ) - - BeforeEach(func() { - mapping, err := bleve.NewMapping() - Expect(err).ToNot(HaveOccurred()) - - idx, err = bleveSearch.NewMemOnly(mapping) - Expect(err).ToNot(HaveOccurred()) - - eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}) - Expect(err).ToNot(HaveOccurred()) - - rootResource = search.Resource{ - ID: "1$2!2", - RootID: "1$2!2", - Path: ".", - Document: content.Document{}, - } - - parentResource = search.Resource{ - ID: "1$2!3", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./parent d!r", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER), - Document: content.Document{Name: "parent d!r"}, - } - - childResource = search.Resource{ - ID: "1$2!4", - ParentID: parentResource.ID, - RootID: rootResource.ID, - Path: "./parent d!r/child.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "child.pdf"}, - } - - childResource2 = search.Resource{ - ID: "1$2!5", - ParentID: parentResource.ID, - RootID: rootResource.ID, - Path: "./parent d!r/child2.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "child2.pdf"}, - } - }) - - Describe("PurgeSpace", func() { - It("takes every record of that space out of the index", func() { - otherSpace := search.Resource{ - ID: "1$9!9", - RootID: "1$9!9", - Path: ".", - Document: content.Document{Name: "other"}, - } - for _, resource := range []search.Resource{rootResource, parentResource, childResource, otherSpace} { - Expect(eng.Upsert(resource.ID, resource)).To(Succeed()) - } - - Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1)), "only the records of that space are gone") - }) - - It("takes a space out that holds more records than one round", func() { - otherSpace := search.Resource{ - ID: "1$9!9", - RootID: "1$9!9", - Path: ".", - Document: content.Document{Name: "other"}, - } - Expect(eng.Upsert(otherSpace.ID, otherSpace)).To(Succeed()) - - for i := range 120 { - resource := search.Resource{ - ID: fmt.Sprintf("%s!file-%d", rootResource.RootID, i), - RootID: rootResource.RootID, - Path: fmt.Sprintf("./file-%d", i), - Document: content.Document{Name: fmt.Sprintf("file-%d", i)}, - } - Expect(eng.Upsert(resource.ID, resource)).To(Succeed()) - } - - Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1)), "only the record of the other space is left") - }) - }) - - Describe("New", func() { - It("returns a new index instance", func() { - b := bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}) - Expect(b).ToNot(BeNil()) - }) - }) - - Describe("Search", func() { - Context("by other fields than filename", func() { - It("finds files by tags", func() { - parentResource.Document.Tags = []string{"foo", "bar"} - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Tags:foo", 1) - assertDocCount(rootResource.ID, "Tags:bar", 1) - assertDocCount(rootResource.ID, "Tags:foo Tags:bar", 1) - assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1) - assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1) - assertDocCount(rootResource.ID, "Tags:baz", 0) - }) - - It("finds files by size", func() { - parentResource.Document.Size = 12345 - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Size:12345", 1) - assertDocCount(rootResource.ID, "Size:>1000", 1) - assertDocCount(rootResource.ID, "Size:<100000", 1) - assertDocCount(rootResource.ID, "Size:12344", 0) - assertDocCount(rootResource.ID, "Size:<1000", 0) - assertDocCount(rootResource.ID, "Size:>100000", 0) - }) - - It("preserves value case for fields not explicitly marked lowercase", func() { - parentResource.Document.Audio = &libregraph.Audio{ - Artist: libregraph.PtrString("Some Artist"), - } - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `audio.artist:"Some Artist"`, 1) - assertDocCount(rootResource.ID, `audio.artist:"some artist"`, 0) - }) - }) - - Context("by filename", func() { - It("finds files with spaces in the filename", func() { - parentResource.Document.Name = "Foo oo.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `name:"foo o*"`, 1) - }) - - It("finds files by digits in the filename", func() { - parentResource.Document.Name = "12345.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:1234*", 1) - }) - - It("filters hidden files", func() { - childResource.Hidden = true - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Hidden:T", 1) - assertDocCount(rootResource.ID, "Hidden:F", 0) - }) - - Context("with a file in the root of the space", func() { - It("scopes the search to the specified space", func() { - parentResource.Document.Name = "foo.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:foo.pdf", 1) - assertDocCount("9$8!7", "Name:foo.pdf", 0) - }) - }) - - It("limits the search to the specified fields", func() { - parentResource.Document.Name = "bar.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:bar.pdf", 1) - assertDocCount(rootResource.ID, "Unknown:field", 0) - }) - - It("returns the total number of hits", func() { - parentResource.Document.Name = "bar.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - res, err := doSearch(rootResource.ID, "Name:bar*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - }) - - It("returns all desired fields", func() { - parentResource.Document.Name = "bar.pdf" - parentResource.Type = 3 - parentResource.MimeType = "application/pdf" - - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - matches := assertDocCount(rootResource.ID, fmt.Sprintf("Name:%s", parentResource.Name), 1) - match := matches[0] - Expect(match.Entity.Ref.Path).To(Equal(parentResource.Path)) - Expect(match.Entity.Name).To(Equal(parentResource.Name)) - Expect(match.Entity.Size).To(Equal(parentResource.Size)) - Expect(match.Entity.Type).To(Equal(parentResource.Type)) - Expect(match.Entity.MimeType).To(Equal(parentResource.MimeType)) - Expect(match.Entity.Deleted).To(BeFalse()) - Expect(match.Score > 0).To(BeTrue()) - }) - - It("finds files by name, prefix or substring match", func() { - parentResource.Document.Name = "foo.pdf" - - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - queries := []string{"foo.pdf", "foo*", "*oo.p*"} - for _, query := range queries { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, query, 1) - } - }) - - It("does a case-insensitive search", func() { - parentResource.Document.Name = "foo.pdf" - - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:foo*", 1) - assertDocCount(rootResource.ID, "Name:Foo*", 1) - }) - - Context("and an additional file in a subdirectory", func() { - BeforeEach(func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - }) - - It("finds files living deeper in the tree by filename, prefix or substring match", func() { - queries := []string{"child.pdf", "child*", "*ld.*"} - for _, query := range queries { - assertDocCount(rootResource.ID, query, 1) - } - }) - }) - }) - - Context("Highlights", func() { - - It("highlights only for content searches", func() { - parentResource.Document.Name = "baz.pdf" - parentResource.Document.Content = "foo bar baz" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - res, err := doSearch(rootResource.ID, "Name:baz*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - Expect(res.Matches[0].Entity.Highlights).To(Equal("")) - }) - - It("highlights search terms", func() { - parentResource.Document.Name = "baz.pdf" - parentResource.Document.Content = "foo bar baz" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - res, err := doSearch(rootResource.ID, "Content:bar", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - Expect(res.Matches[0].Entity.Highlights).To(Equal("foo bar baz")) - }) - - }) - - Context("with a file in the root of the space and folder with a file. all of them have the same name", func() { - BeforeEach(func() { - parentResource := search.Resource{ - ID: "1$2!3", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./doc", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER), - Document: content.Document{Name: "doc"}, - } - - childResource := search.Resource{ - ID: "1$2!4", - ParentID: parentResource.ID, - RootID: rootResource.ID, - Path: "./doc/doc.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "doc.pdf"}, - } - - childResource2 := search.Resource{ - ID: "1$2!7", - ParentID: parentResource.ID, - RootID: rootResource.ID, - Path: "./doc/file.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "file.pdf"}, - } - - rootChildResource := search.Resource{ - ID: "1$2!5", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./doc.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "doc.pdf"}, - } - - rootChildResource2 := search.Resource{ - ID: "1$2!6", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./file.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "file.pdf"}, - } - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(rootChildResource.ID, rootChildResource) - Expect(err).ToNot(HaveOccurred()) - err = eng.Upsert(rootChildResource2.ID, rootChildResource2) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - err = eng.Upsert(childResource2.ID, childResource2) - Expect(err).ToNot(HaveOccurred()) - }) - It("search *doc* in a root", func() { - res, err := doSearch(rootResource.ID, "Name:*doc*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(3))) - }) - It("search *doc* in a subfolder", func() { - res, err := doSearch(rootResource.ID, "Name:*doc*", "./doc") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(2))) - }) - It("search *file* in a root", func() { - res, err := doSearch(rootResource.ID, "Name:*file*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(2))) - }) - It("search *file* in a subfolder", func() { - res, err := doSearch(rootResource.ID, "Name:*file*", "./doc") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - }) - }) - - }) - - Describe("Upsert", func() { - It("adds a resourceInfo to the index", func() { - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - query := bleveSearch.NewMatchQuery("child.pdf") - res, err := idx.Search(bleveSearch.NewSearchRequest(query)) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Hits.Len()).To(Equal(1)) - }) - - It("updates an existing resource in the index", func() { - - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - countA, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(countA).To(Equal(uint64(1))) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - countB, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(countB).To(Equal(uint64(1))) - }) - }) - - Describe("Delete", func() { - It("marks a resource as deleted", func() { - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:*child*", 1) - - err = eng.Delete(childResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:*child*", 0) - }) - - It("marks a child resources as deleted", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - - err = eng.Delete(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0) - }) - }) - - Describe("Restore", func() { - It("also marks child resources as restored", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Delete(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 0) - - err = eng.Restore(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 1) - assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 1) - }) - }) - - Describe("Purge", func() { - It("removes a resource from the index", func() { - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - assertDocCount(rootResource.ID, "Name:child.pdf", 1) - - err = eng.Purge(childResource.ID, false) - - Expect(err).ToNot(HaveOccurred()) - assertDocCount(rootResource.ID, "Name:child.pdf", 0) - }) - It("removes a resource and its children from the index", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - - err = eng.Purge(parentResource.ID, false) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0) - }) - It("removes a resource and ignores its children from the index", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) - - err = eng.Delete(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - - err = eng.Purge(parentResource.ID, true) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - }) - }) - - Describe("Move", func() { - It("renames the parent and its child resources", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - parentResource.Path = "newname" - err = eng.Move(parentResource.ID, parentResource.ParentID, "./my/newname") - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, parentResource.Name, 0) - - matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1) - Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3")) - Expect(matches[0].Entity.Ref.Path).To(Equal("./my/newname/child.pdf")) - }) - - DescribeTable("keeps the flag in step with the path", - func(from, target string, hidden bool) { - parentResource.Path = from - parentResource.Hidden = search.IsHidden(from) - childResource.Path = from + "/child.pdf" - childResource.Hidden = parentResource.Hidden - - Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) - Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) - - Expect(eng.Move(parentResource.ID, parentResource.ParentID, target)).To(Succeed()) - - for _, id := range []string{parentResource.ID, childResource.ID} { - Expect(hiddenByID(idx, id)). - To(Equal(hidden), "%s after moving from %s to %s", id, from, target) - } - }, - Entry("into a dot folder", "./parent", "./.trash/parent", true), - Entry("into a plain folder", "./parent", "./archive/parent", false), - Entry("renamed with a leading dot", "./parent", "./.parent", true), - Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false), - Entry("renamed without the leading dot", "./.parent", "./parent", false), - Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true), - ) - - // the trash leaves the path alone, so the flag has to come through untouched - It("carries the flag through the trash and back", func() { - childResource.Path = "./.secret/file.txt" - childResource.Hidden = true - Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) - - Expect(eng.Delete(childResource.ID)).To(Succeed()) - Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after trashing") - - Expect(eng.Restore(childResource.ID)).To(Succeed()) - Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after restoring") - }) - - It("moves the parent and its child resources", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - parentResource.Path = " " - parentResource.ParentID = "1$2!somewhereopaqueid" - - err = eng.Move(parentResource.ID, parentResource.ParentID, "./somewhere/else/newname") - Expect(err).ToNot(HaveOccurred()) - assertDocCount(rootResource.ID, `parent d!r`, 0) - - matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1) - Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3")) - Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname/child.pdf")) - - matches = assertDocCount(rootResource.ID, `newname`, 1) - Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("somewhereopaqueid")) - Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname")) - - }) - }) - - Describe("StartBatch", func() { - It("starts a new batch", func() { - b, err := eng.NewBatch(100) - Expect(err).ToNot(HaveOccurred()) - - err = b.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(0))) - - err = b.Push() - Expect(err).ToNot(HaveOccurred()) - - count, err = idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - query := bleveSearch.NewMatchQuery("child.pdf") - res, err := idx.Search(bleveSearch.NewSearchRequest(query)) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Hits.Len()).To(Equal(1)) - }) - - It("doesn't intertwine different batches", func() { - b, err := eng.NewBatch(100) - Expect(err).ToNot(HaveOccurred()) - - err = b.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(0))) - - b2, err := eng.NewBatch(100) - Expect(err).ToNot(HaveOccurred()) - - err = b2.Upsert(childResource2.ID, childResource2) - Expect(err).ToNot(HaveOccurred()) - - Expect(b.Push()).To(Succeed()) - count, err = idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - Expect(b2.Push()).To(Succeed()) - count, err = idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(2))) - }) - }) - - Describe("File type specific metadata", func() { - - Context("with audio metadata", func() { - BeforeEach(func() { - resource := search.Resource{ - ID: "1$2!7", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./some_song.mp3", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{ - Name: "some_song.mp3", - MimeType: "audio/mpeg", - Audio: &libregraph.Audio{ - Album: libregraph.PtrString("Some Album"), - AlbumArtist: libregraph.PtrString("Some AlbumArtist"), - Artist: libregraph.PtrString("Some Artist"), - Bitrate: libregraph.PtrInt64(192), - Composers: libregraph.PtrString("Some Composers"), - Copyright: libregraph.PtrString(""), - Disc: libregraph.PtrInt32(2), - DiscCount: libregraph.PtrInt32(5), - Duration: libregraph.PtrInt64(225000), - Genre: libregraph.PtrString("Some Genre"), - HasDrm: libregraph.PtrBool(false), - IsVariableBitrate: libregraph.PtrBool(true), - Title: libregraph.PtrString("Some Title"), - Track: libregraph.PtrInt32(34), - TrackCount: libregraph.PtrInt32(99), - Year: libregraph.PtrInt32(2004), - }, - }, - } - err := eng.Upsert(resource.ID, resource) - Expect(err).ToNot(HaveOccurred()) - }) - - It("returns audio metadata for search", func() { - matches := assertDocCount(rootResource.ID, `*song*`, 1) - audio := matches[0].Entity.Audio - - Expect(audio).ToNot(BeNil()) - - Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album"))) - Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist"))) - Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist"))) - Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192))) - Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers"))) - Expect(audio.Copyright).To(Equal(libregraph.PtrString(""))) - Expect(audio.Disc).To(Equal(libregraph.PtrInt32(2))) - Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5))) - Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225000))) - Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre"))) - Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false))) - Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true))) - Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title"))) - Expect(audio.Track).To(Equal(libregraph.PtrInt32(34))) - Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(99))) - Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) - }) - }) - - Context("with location metadata", func() { - BeforeEach(func() { - resource := search.Resource{ - ID: "1$2!7", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./team.jpg", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{ - Name: "team.jpg", - MimeType: "image/jpeg", - Location: &libregraph.GeoCoordinates{ - Altitude: libregraph.PtrFloat64(1047.7), - Latitude: libregraph.PtrFloat64(49.48675890884328), - Longitude: libregraph.PtrFloat64(11.103870357204285), - }, - }, - } - err := eng.Upsert(resource.ID, resource) - Expect(err).ToNot(HaveOccurred()) - }) - - It("returns audio metadata for search", func() { - matches := assertDocCount(rootResource.ID, `*team*`, 1) - location := matches[0].Entity.Location - - Expect(location).ToNot(BeNil()) - - Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(1047.7))) - Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328))) - Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285))) - }) - }) - }) -}) diff --git a/services/search/pkg/opensearch/backend_test.go b/services/search/pkg/opensearch/backend_test.go index 850b9a7bfa..71ef3860fc 100644 --- a/services/search/pkg/opensearch/backend_test.go +++ b/services/search/pkg/opensearch/backend_test.go @@ -1,9 +1,6 @@ package opensearch_test import ( - "context" - "fmt" - "strings" "testing" . "github.com/onsi/ginkgo/v2" @@ -11,12 +8,7 @@ import ( opensearchgo "github.com/opensearch-project/opensearch-go/v4" opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - - searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" - "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) func TestOpenSearchBackend(t *testing.T) { @@ -24,56 +16,8 @@ func TestOpenSearchBackend(t *testing.T) { RunSpecs(t, "OpenSearch Backend Suite") } -func deleteIndexOnCleanup(tc *opensearchtest.TestClient, indexName string) { - DeferCleanup(func() { - Expect(tc.IndicesDelete(context.Background(), []string{indexName})).To(Succeed()) - }) -} - -func resourceByID(tc *opensearchtest.TestClient, index, id string) search.Resource { - GinkgoHelper() - - body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "query": map[string]any{ - "ids": map[string]any{ - "values": []string{id}, - }, - }, - }) - - resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(index, strings.NewReader(body)).Hits) - Expect(resources).To(HaveLen(1)) - return resources[0] -} - -// otherRoot returns a copy of the given resource that lives in a different root (space) -// while keeping the same path, so it can be used to assert that cross-root updates do -// not affect identically-named resources in other roots. -func otherRoot(r search.Resource) search.Resource { - r.ID = "2$2!3" - r.RootID = "2$2!1" - r.ParentID = "2$2!2" - return r -} - -func newBackend(indexName string, resources ...search.Resource) (*opensearch.Backend, *opensearchtest.TestClient) { - GinkgoHelper() - - tc := opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - - backend, err := opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - - for _, r := range resources { - tc.Require.DocumentCreate(indexName, r.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), r))) - } - tc.Require.IndicesCount([]string{indexName}, nil, len(resources)) - - return backend, tc -} - +// what the engine does with its index is covered for both engines by +// services/search/pkg/parity; this is the one thing only OpenSearch can do var _ = Describe("Backend", func() { Describe("NewBackend", func() { It("fails to create if the cluster is not healthy", func() { @@ -89,606 +33,4 @@ var _ = Describe("Backend", func() { Expect(err).To(MatchError(opensearch.ErrUnhealthyCluster)) }) }) - - Describe("Search", func() { - const indexName = "opencloud-test-engine-search" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - document search.Resource - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - - document = opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - }) - - It("performs the most simple search", func() { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ - Query: fmt.Sprintf(`"%s"`, document.Name), - }) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(HaveLen(1)) - Expect(resp.TotalMatches).To(Equal(int32(1))) - Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID)) - }) - - It("ignores files that are marked as deleted", func() { - deletedDocument := opensearchtest.Testdata.Resources.File - deletedDocument.ID = "1$2!4" - deletedDocument.Deleted = true - - tc.Require.DocumentCreate(indexName, deletedDocument.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), deletedDocument))) - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ - Query: fmt.Sprintf(`"%s"`, document.Name), - }) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(HaveLen(1)) - Expect(resp.TotalMatches).To(Equal(int32(1))) - Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID)) - }) - }) - - Describe("Upsert", func() { - const indexName = "opencloud-test-engine-upsert" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("upserts a full document", func() { - document := opensearchtest.Testdata.Resources.File - Expect(backend.Upsert(document.ID, document)).To(Succeed()) - - tc.Require.IndicesCount([]string{indexName}, nil, 1) - }) - }) - - Describe("Move", func() { - const indexName = "opencloud-test-engine-move" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("moves the document to a new path", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "query": map[string]any{ - "ids": map[string]any{ - "values": []string{document.ID}, - }, - }, - }) - - resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits) - Expect(resources).To(HaveLen(1)) - Expect(resources[0].Path).To(Equal(document.Path)) - - document.Path = "./new/path/to/resource" - Expect(backend.Move(document.ID, document.ParentID, document.Path)).To(Succeed()) - - resources = opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits) - Expect(resources).To(HaveLen(1)) - Expect(resources[0].Path).To(Equal(document.Path)) - }) - }) - - Describe("WriteVisibility", func() { - const indexName = "opencloud-test-engine-write-visibility" - - It("deletes a record that was just written", func() { - document := opensearchtest.Testdata.Resources.File - document.ID = "1$1!95" - document.Name = "textfile.txt" - document.Path = "./textfile.txt" - - backend, tc := newBackend(indexName) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Upsert(document.ID, document)).To(Succeed()) - Expect(backend.Delete(document.ID)).To(Succeed()) - - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ - Query: fmt.Sprintf(`name:"%s"`, document.Name), - }) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(BeEmpty()) - }) - }) - - Describe("Delete", func() { - const indexName = "opencloud-test-engine-delete" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("marks the document as deleted", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "query": map[string]any{ - "term": map[string]any{ - "Deleted": map[string]any{ - "value": true, - }, - }, - }, - }) - - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0) - - Expect(backend.Delete(document.ID)).To(Succeed()) - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1) - }) - }) - - Describe("Restore", func() { - const indexName = "opencloud-test-engine-restore" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("marks the document as not deleted", func() { - document := opensearchtest.Testdata.Resources.File - document.Deleted = true - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "query": map[string]any{ - "term": map[string]any{ - "Deleted": map[string]any{ - "value": true, - }, - }, - }, - }) - - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1) - - Expect(backend.Restore(document.ID)).To(Succeed()) - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0) - }) - }) - - Describe("Purge", func() { - const indexName = "opencloud-test-engine-purge" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("purges a full document", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - Expect(backend.Purge(document.ID, false)).To(Succeed()) - - tc.Require.IndicesCount([]string{indexName}, nil, 0) - }) - - It("purges resource trees", func() { - resourceFolder := opensearchtest.Testdata.Resources.Folder - tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder))) - - resourceFile := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile))) - - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - Expect(backend.Purge(resourceFolder.ID, false)).To(Succeed()) - - tc.Require.IndicesCount([]string{indexName}, nil, 0) - }) - - It("purges resource trees and ignores undeleted resources", func() { - resourceFolder := opensearchtest.Testdata.Resources.Folder - tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder))) - - resourceFile := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile))) - - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - Expect(backend.Delete(resourceFile.ID)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - Expect(backend.Purge(resourceFolder.ID, true)).To(Succeed()) - - tc.Require.IndicesCount([]string{indexName}, nil, 1) - }) - }) - - Describe("PurgeSpace", func() { - const indexName = "opencloud-test-engine-purge-space" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("takes every record of that space out of the index", func() { - gone := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, gone.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), gone))) - - stays := opensearchtest.Testdata.Resources.File - stays.ID = "1$2!3" - stays.RootID = "1$2!2" - tc.Require.DocumentCreate(indexName, stays.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), stays))) - - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - Expect(backend.PurgeSpace(gone.RootID)).To(Succeed()) - - tc.Require.IndicesRefresh([]string{indexName}, nil) - left := opensearchtest.SearchHitsMustBeConverted[search.Resource]( - GinkgoTB(), - tc.Require.Search(indexName, strings.NewReader(`{"query":{"match_all":{}}}`)).Hits, - ) - Expect(left).To(HaveLen(1), "only the records of that space are gone") - Expect(left[0].ID).To(Equal(stays.ID)) - }) - }) - - Describe("Hidden", func() { - const indexName = "opencloud-test-engine-hidden" - - DescribeTable("keeps the flag in step with the path", - func(from, target string, hidden bool) { - folder := opensearchtest.Testdata.Resources.Folder - folder.ID = "1$1!30" - folder.Name = "parent" - folder.Path = from - folder.Hidden = search.IsHidden(from) - - child := opensearchtest.Testdata.Resources.File - child.ID = "1$1!31" - child.Name = "child.txt" - child.Path = from + "/child.txt" - child.ParentID = folder.ID - child.Hidden = folder.Hidden - - backend, tc := newBackend(indexName, folder, child) - deleteIndexOnCleanup(tc, indexName) - tc.Require.IndicesRefresh([]string{indexName}, nil) - - Expect(backend.Move(folder.ID, folder.ParentID, target)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - - for _, id := range []string{folder.ID, child.ID} { - Expect(resourceByID(tc, indexName, id).Hidden). - To(Equal(hidden), "%s after moving from %s to %s", id, from, target) - } - }, - Entry("into a dot folder", "./parent", "./.trash/parent", true), - Entry("into a plain folder", "./parent", "./archive/parent", false), - Entry("renamed with a leading dot", "./parent", "./.parent", true), - Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false), - Entry("renamed without the leading dot", "./.parent", "./parent", false), - Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true), - ) - - It("carries the flag through the trash and back", func() { - hidden := opensearchtest.Testdata.Resources.File - hidden.ID = "1$1!32" - hidden.Path = "./.secret/file.txt" - hidden.Hidden = true - - backend, tc := newBackend(indexName, hidden) - deleteIndexOnCleanup(tc, indexName) - tc.Require.IndicesRefresh([]string{indexName}, nil) - - Expect(backend.Delete(hidden.ID)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - Expect(resourceByID(tc, indexName, hidden.ID).Hidden).To(BeTrue(), "after trashing") - - Expect(backend.Restore(hidden.ID)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - Expect(resourceByID(tc, indexName, hidden.ID).Hidden).To(BeTrue(), "after restoring") - }) - }) - - Describe("DocCount", func() { - const indexName = "opencloud-test-engine-doc-count" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("ignores deleted documents", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - count, err := backend.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - tc.Require.Update(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "doc": map[string]any{ - "Deleted": true, - }, - }))) - - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - count, err = backend.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(0))) - }) - }) - - // The following specs ensure that updates which affect a resource and its descendants - // (Delete, Restore, Move) are scoped to the root (space) of the target resource. Two - // resources living in different roots may share the exact same path, so matching by - // path alone would incorrectly update the wrong resource. - Describe("updateSelfAndDescendants root scope", func() { - It("deletes only the resource in the target root", func() { - const indexName = "opencloud-test-engine-root-scope-delete" - - target := opensearchtest.Testdata.Resources.File - other := otherRoot(target) - - backend, tc := newBackend(indexName, target, other) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Delete(target.ID)).To(Succeed()) - - Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeTrue(), "target resource should be marked as deleted") - Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeFalse(), "resource in a different root must not be affected") - }) - - It("restores only the resource in the target root", func() { - const indexName = "opencloud-test-engine-root-scope-restore" - - target := opensearchtest.Testdata.Resources.File - target.Deleted = true - other := otherRoot(target) - - backend, tc := newBackend(indexName, target, other) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Restore(target.ID)).To(Succeed()) - - Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeFalse(), "target resource should be restored") - Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeTrue(), "resource in a different root must not be affected") - }) - - It("moves only the resource in the target root", func() { - const indexName = "opencloud-test-engine-root-scope-move" - - target := opensearchtest.Testdata.Resources.File - other := otherRoot(target) - - backend, tc := newBackend(indexName, target, other) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Move(target.ID, target.ParentID, "./new/path/to/resource")).To(Succeed()) - - Expect(resourceByID(tc, indexName, target.ID).Path).To(Equal("./new/path/to/resource"), "target resource should be moved") - Expect(resourceByID(tc, indexName, other.ID).Path).To(Equal(other.Path), "resource in a different root must not be moved") - }) - }) - - Describe("SearchInAnalyzedFields", func() { - const indexName = "opencloud-test-engine-search-analyzed-fields" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - dashed := opensearchtest.Testdata.Resources.Folder - dashed.ID = "1$1!10" - dashed.Name = "new-folder" - dashed.Path = "./new-folder" - dashed.Title = "quarterly report" - - plain := opensearchtest.Testdata.Resources.Folder - plain.ID = "1$1!11" - plain.Name = "documents" - plain.Path = "./documents" - plain.Title = "notes" - - spaced := opensearchtest.Testdata.Resources.Folder - spaced.ID = "1$1!12" - spaced.Name = "foo bar" - spaced.Path = "./foo bar" - spaced.Title = "spaced out" - - backend, tc = newBackend(indexName, dashed, plain, spaced) - deleteIndexOnCleanup(tc, indexName) - tc.Require.IndicesRefresh([]string{indexName}, nil) - }) - - DescribeTable("finds what the analyzer made of the value", - func(query string, want []string) { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query}) - Expect(err).ToNot(HaveOccurred()) - - names := make([]string, 0, len(resp.Matches)) - for _, match := range resp.Matches { - names = append(names, match.Entity.Name) - } - Expect(names).To(ConsistOf(want)) - }, - Entry("the full name with the dash", "new-folder", []string{"new-folder"}), - Entry("one token of it", "new", []string{"new-folder"}), - Entry("a name without a dash", "documents", []string{"documents"}), - Entry("a wildcard", "*folder*", []string{"new-folder"}), - // the shape the web client sends for every name search - Entry("a wildcard around the whole dashed name", `name:"*new-folder*"`, []string{"new-folder"}), - Entry("a wildcard spanning the dash", `name:"*w-fol*"`, []string{"new-folder"}), - Entry("a wildcard in a different case", `name:"*NEW-FOLDER*"`, []string{"new-folder"}), - Entry("a wildcard spanning a space", `name:"*oo ba*"`, []string{"foo bar"}), - Entry("a wildcard around a name with a space", `name:"*foo bar*"`, []string{"foo bar"}), - Entry("a name with a space", `name:"foo bar"`, []string{"foo bar"}), - Entry("a title of two words", `Title:"quarterly report"`, []string{"new-folder"}), - Entry("one token of a title", "Title:quarterly", []string{"new-folder"}), - ) - }) - - Describe("SearchByTag", func() { - const indexName = "opencloud-test-engine-search-by-tag" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tagged := opensearchtest.Testdata.Resources.Folder - tagged.ID = "1$1!20" - tagged.Name = "tagged" - tagged.Path = "./tagged" - tagged.Tags = []string{"foo-bar"} - - other := opensearchtest.Testdata.Resources.Folder - other.ID = "1$1!21" - other.Name = "other" - other.Path = "./other" - other.Tags = []string{"foo"} - - backend, tc = newBackend(indexName, tagged, other) - deleteIndexOnCleanup(tc, indexName) - tc.Require.IndicesRefresh([]string{indexName}, nil) - }) - - // a tag is one label, not prose, so it matches as a whole or not at all - DescribeTable("matches a tag as a whole", - func(query string, want []string) { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query}) - Expect(err).ToNot(HaveOccurred()) - - names := make([]string, 0, len(resp.Matches)) - for _, match := range resp.Matches { - names = append(names, match.Entity.Name) - } - Expect(names).To(ConsistOf(want)) - }, - Entry("the whole tag", `tag:("foo-bar")`, []string{"tagged"}), - Entry("a token of a tag does not match it", `tag:("foo")`, []string{"other"}), - Entry("a tag in a different case", `tag:("FOO-BAR")`, []string{"tagged"}), - Entry("a wildcard reaches both", `tag:("*foo*")`, []string{"tagged", "other"}), - ) - }) - - Describe("SearchWithAnInvalidQuery", func() { - const indexName = "opencloud-test-engine-search-invalid-query" - - It("answers with a bad request", func() { - backend, tc := newBackend(indexName) - deleteIndexOnCleanup(tc, indexName) - - _, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: "AND mediatype:document"}) - Expect(err).To(HaveOccurred()) - Expect(err).To(BeAssignableToTypeOf(errtypes.BadRequest(""))) - Expect(err.Error()).To(Equal(`error: bad request: the expression can't begin from a binary operator: 'AND'`)) - }) - }) }) diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index ad880695b4..a8acafba94 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -206,6 +206,7 @@ Fixtures: - `plain.txt`, ID = 1$1!plain.txt - `box`, ID = 1$1!box, folder - `boxed.txt`, ID = 1$1!boxed.txt, Path = ./box/boxed.txt +- `song.mp3`, ID = 1$1!song.mp3, MimeType = audio/mpeg | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| @@ -221,6 +222,8 @@ Fixtures: | FIELDS-10 | `hidden:TRUE` | hidden.txt | no match | error | ❌ known | | FIELDS-11 | `id:"1$1!AB-23"` | cased.txt | cased.txt | no match | ❌ known | | FIELDS-12 | `id:"1$1!ab-23"` | no match | no match | no match | ✅ | +| FIELDS-13 | `audio.artist:"Some Artist"` | song.mp3 | song.mp3 | no match | ❌ known | +| FIELDS-14 | `audio.artist:"some artist"` | no match | no match | no match | ✅ | ### deleted @@ -582,6 +585,8 @@ Fixtures: | BATCH-03 | takes a resource out the same way a delete does, then `name:"*child*"` | no match | no match | no match | ✅ | | BATCH-04 | moves a resource the same way a move does, then `path:"./my/newname/child.pdf"` | child.pdf | child.pdf | child.pdf | ✅ | | BATCH-04 | moves a resource the same way a move does, then `path:"./parent/child.pdf"` | no match | no match | no match | ✅ | +| BATCH-05 | keeps what another batch holds out of its push, then `name:"*added*"` | added.pdf | added.pdf | added.pdf | ✅ | +| BATCH-05 | keeps what another batch holds out of its push, then `name:"*other*"` | no match | no match | no match | ✅ | ## Response diff --git a/services/search/pkg/parity/lifecycle_batch_test.go b/services/search/pkg/parity/lifecycle_batch_test.go index 5234e7758d..eb718714db 100644 --- a/services/search/pkg/parity/lifecycle_batch_test.go +++ b/services/search/pkg/parity/lifecycle_batch_test.go @@ -9,6 +9,7 @@ func batchLifecycle() lifecycleGroup { parent, child := fixtureTree() added := fixtureDoc("added.pdf", withID("1$1!7"), withParent(parent.ID), withPath("./parent/added.pdf")) + other := fixtureDoc("other.pdf", withID("1$1!8"), withParent(parent.ID), withPath("./parent/other.pdf")) return lifecycleGroup{ name: "batch", @@ -78,6 +79,34 @@ func batchLifecycle() lifecycleGroup { {`path:"./parent/child.pdf"`, nil}, }, }, + { + id: 5, title: "keeps what another batch holds out of its push", + do: func(e search.Engine) error { + first, err := e.NewBatch(100) + if err != nil { + return err + } + + second, err := e.NewBatch(100) + if err != nil { + return err + } + + if err := first.Upsert(added.ID, added); err != nil { + return err + } + + if err := second.Upsert(other.ID, other); err != nil { + return err + } + + return first.Push() + }, + expect: []expectation{ + {`name:"*added*"`, []string{"added.pdf"}}, + {`name:"*other*"`, nil}, + }, + }, }, } } diff --git a/services/search/pkg/parity/query_fields_test.go b/services/search/pkg/parity/query_fields_test.go index 265a7aecd5..a643f0cfae 100644 --- a/services/search/pkg/parity/query_fields_test.go +++ b/services/search/pkg/parity/query_fields_test.go @@ -1,6 +1,8 @@ package parity import ( + libregraph "github.com/opencloud-eu/libre-graph-api-go" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) @@ -16,6 +18,7 @@ func fieldsGroup() queryGroup { fixtureDoc("plain.txt"), fixtureFolder("box"), fixtureDoc("boxed.txt", withParent("1$1!box"), withPath("./box/boxed.txt")), + fixtureDoc("song.mp3", withMime("audio/mpeg"), withAudio(&libregraph.Audio{Artist: libregraph.PtrString("Some Artist")})), }, cases: []queryCase{ {id: 1, query: `size:42`, want: []string{"small.txt"}}, @@ -30,6 +33,9 @@ func fieldsGroup() queryGroup { {id: 10, query: `hidden:TRUE`, want: []string{"hidden.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, {id: 11, query: `id:"1$1!AB-23"`, want: []string{"cased.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, {id: 12, query: `id:"1$1!ab-23"`}, + // a facet value keeps its case, the field is not marked lowercase + {id: 13, query: `audio.artist:"Some Artist"`, want: []string{"song.mp3"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 14, query: `audio.artist:"some artist"`}, }, } }