diff --git a/.codacy.yml b/.codacy.yml index 959f2f3cea..f5eba526d9 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -18,6 +18,8 @@ exclude_paths: - 'deployments/**' - "release-config.ts" - 'tests/acceptance/expected-failures-*.md' + # written by the search engine parity suite, table rows are as wide as they are + - 'services/search/pkg/parity/README.md' - 'tests/acceptance/bootstrap/**' - 'tests/acceptance/TestHelpers/**' - 'tests/acceptance/scripts/run.sh' diff --git a/services/search/pkg/opensearch/internal/test/helper.go b/services/search/internal/opensearchtest/helper.go similarity index 100% rename from services/search/pkg/opensearch/internal/test/helper.go rename to services/search/internal/opensearchtest/helper.go diff --git a/services/search/pkg/opensearch/internal/test/os.go b/services/search/internal/opensearchtest/os.go similarity index 100% rename from services/search/pkg/opensearch/internal/test/os.go rename to services/search/internal/opensearchtest/os.go diff --git a/services/search/pkg/opensearch/internal/test/suite.go b/services/search/internal/opensearchtest/suite.go similarity index 66% rename from services/search/pkg/opensearch/internal/test/suite.go rename to services/search/internal/opensearchtest/suite.go index e565a399e7..433907292a 100644 --- a/services/search/pkg/opensearch/internal/test/suite.go +++ b/services/search/internal/opensearchtest/suite.go @@ -3,6 +3,7 @@ package opensearchtest import ( "context" "fmt" + "net/http" "os" "slices" "strings" @@ -18,7 +19,9 @@ import ( ) const ( - openSearchImage = "opensearchproject/opensearch:2" + openSearchImage = "opensearchproject/opensearch:2" + openSearchPort = "9200" + openSearchStartupTimeout = 3 * time.Minute ) func SetupTests(ctx context.Context) (*config.Config, func(), error) { @@ -62,6 +65,14 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func return func() {}, nil } + keepContainer := os.Getenv("KEEP_TEST_CONTAINER") == "true" + if keepContainer { + // the reaper would take the kept container down with the session + if err := os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true"); err != nil { + return nil, fmt.Errorf("failed to disable the testcontainers reaper: %w", err) + } + } + containerName := fmt.Sprintf("opencloud/test/%s", openSearchImage) containerName = strings.Replace(containerName, "/", "__", -1) containerName = strings.Replace(containerName, ":", "_", -1) @@ -72,9 +83,21 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func opensearch.WithPassword(cfg.Engine.OpenSearch.Client.Password), testcontainers.WithName(containerName), testcontainers.WithReuseByName(containerName), + // test indexes are tiny; don't let a full host disk trip the flood-stage + // create-index / read-only blocks mid-run + testcontainers.WithEnv(map[string]string{ + "cluster.routing.allocation.disk.threshold_enabled": "false", + }), + // a health probe answers at once on a reused container, a log wait + // would sit out the log timeout on it; a cold boot takes well over the + // previous 5s testcontainers.WithWaitStrategy( - wait.ForLog("ML configuration initialized successfully"). - WithStartupTimeout(5*time.Second), + wait.ForHTTP("/_cluster/health?wait_for_status=yellow&timeout=1s"). + WithPort(openSearchPort). + WithTLS(false). + WithBasicAuth(cfg.Engine.OpenSearch.Client.Username, cfg.Engine.OpenSearch.Client.Password). + WithStatusCodeMatcher(func(status int) bool { return status == http.StatusOK }). + WithStartupTimeout(openSearchStartupTimeout), ), ) if err != nil { @@ -91,6 +114,13 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func cfg.Engine.OpenSearch.Client.Addresses = []string{address} return func() { + // KEEP_TEST_CONTAINER=true leaves the container up for the next run, + // which picks it up again by name instead of booting a fresh one + if keepContainer { + _, _ = fmt.Fprintf(os.Stderr, "keeping OpenSearch container %s\n", containerName) + return + } + err := container.Terminate(ctx) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "failed to terminate OpenSearch container: %v\n", err) diff --git a/services/search/pkg/opensearch/internal/test/test.go b/services/search/internal/opensearchtest/test.go similarity index 100% rename from services/search/pkg/opensearch/internal/test/test.go rename to services/search/internal/opensearchtest/test.go diff --git a/services/search/pkg/opensearch/internal/test/testdata.go b/services/search/internal/opensearchtest/testdata.go similarity index 100% rename from services/search/pkg/opensearch/internal/test/testdata.go rename to services/search/internal/opensearchtest/testdata.go diff --git a/services/search/pkg/opensearch/internal/test/testdata/resource_file.json b/services/search/internal/opensearchtest/testdata/resource_file.json similarity index 100% rename from services/search/pkg/opensearch/internal/test/testdata/resource_file.json rename to services/search/internal/opensearchtest/testdata/resource_file.json diff --git a/services/search/pkg/opensearch/internal/test/testdata/resource_folder.json b/services/search/internal/opensearchtest/testdata/resource_folder.json similarity index 100% rename from services/search/pkg/opensearch/internal/test/testdata/resource_folder.json rename to services/search/internal/opensearchtest/testdata/resource_folder.json diff --git a/services/search/pkg/opensearch/internal/test/testdata/resource_root.json b/services/search/internal/opensearchtest/testdata/resource_root.json similarity index 100% rename from services/search/pkg/opensearch/internal/test/testdata/resource_root.json rename to services/search/internal/opensearchtest/testdata/resource_root.json diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go deleted file mode 100644 index b9feef8e8d..0000000000 --- a/services/search/pkg/bleve/backend_test.go +++ /dev/null @@ -1,808 +0,0 @@ -package bleve_test - -import ( - "context" - "fmt" - - bleveSearch "github.com/blevesearch/bleve/v2" - sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - libregraph "github.com/opencloud-eu/libre-graph-api-go" - "github.com/opencloud-eu/reva/v2/pkg/storagespace" - - "github.com/opencloud-eu/opencloud/pkg/log" - searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" - searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" - "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" - "github.com/opencloud-eu/opencloud/services/search/pkg/content" - bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve" - "github.com/opencloud-eu/opencloud/services/search/pkg/search" -) - -func hiddenByID(idx bleveSearch.Index, id string) bool { - GinkgoHelper() - - req := bleveSearch.NewSearchRequest(bleveSearch.NewDocIDQuery([]string{id})) - req.Fields = []string{"Hidden"} - - res, err := idx.Search(req) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Hits).To(HaveLen(1), "no record for %s", id) - - hidden, _ := res.Hits[0].Fields["Hidden"].(bool) - return hidden -} - -var _ = Describe("Bleve", func() { - var ( - eng *bleve.Backend - idx bleveSearch.Index - - doSearch = func(id string, query, path string) (*searchsvc.SearchIndexResponse, error) { - rID, err := storagespace.ParseID(id) - if err != nil { - return nil, err - } - - return eng.Search(context.Background(), &searchsvc.SearchIndexRequest{ - Query: query, - Ref: &searchmsg.Reference{ - ResourceId: &searchmsg.ResourceID{ - StorageId: rID.StorageId, - SpaceId: rID.SpaceId, - OpaqueId: rID.OpaqueId, - }, - Path: path, - }, - }) - } - - assertDocCount = func(id string, query string, expectedCount int) []*searchmsg.Match { - res, err := doSearch(id, query, "") - - ExpectWithOffset(1, err).ToNot(HaveOccurred()) - ExpectWithOffset(1, len(res.Matches)).To(Equal(expectedCount), "query returned unexpected number of results: "+query) - return res.Matches - } - - rootResource search.Resource - parentResource search.Resource - childResource search.Resource - childResource2 search.Resource - ) - - BeforeEach(func() { - mapping, err := bleve.NewMapping() - Expect(err).ToNot(HaveOccurred()) - - idx, err = bleveSearch.NewMemOnly(mapping) - Expect(err).ToNot(HaveOccurred()) - - eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}) - Expect(err).ToNot(HaveOccurred()) - - rootResource = search.Resource{ - ID: "1$2!2", - RootID: "1$2!2", - Path: ".", - Document: content.Document{}, - } - - parentResource = search.Resource{ - ID: "1$2!3", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./parent d!r", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER), - Document: content.Document{Name: "parent d!r"}, - } - - childResource = search.Resource{ - ID: "1$2!4", - ParentID: parentResource.ID, - RootID: rootResource.ID, - Path: "./parent d!r/child.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "child.pdf"}, - } - - childResource2 = search.Resource{ - ID: "1$2!5", - ParentID: parentResource.ID, - RootID: rootResource.ID, - Path: "./parent d!r/child2.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "child2.pdf"}, - } - }) - - Describe("PurgeSpace", func() { - It("takes every record of that space out of the index", func() { - otherSpace := search.Resource{ - ID: "1$9!9", - RootID: "1$9!9", - Path: ".", - Document: content.Document{Name: "other"}, - } - for _, resource := range []search.Resource{rootResource, parentResource, childResource, otherSpace} { - Expect(eng.Upsert(resource.ID, resource)).To(Succeed()) - } - - Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1)), "only the records of that space are gone") - }) - - It("takes a space out that holds more records than one round", func() { - otherSpace := search.Resource{ - ID: "1$9!9", - RootID: "1$9!9", - Path: ".", - Document: content.Document{Name: "other"}, - } - Expect(eng.Upsert(otherSpace.ID, otherSpace)).To(Succeed()) - - for i := range 120 { - resource := search.Resource{ - ID: fmt.Sprintf("%s!file-%d", rootResource.RootID, i), - RootID: rootResource.RootID, - Path: fmt.Sprintf("./file-%d", i), - Document: content.Document{Name: fmt.Sprintf("file-%d", i)}, - } - Expect(eng.Upsert(resource.ID, resource)).To(Succeed()) - } - - Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1)), "only the record of the other space is left") - }) - }) - - Describe("New", func() { - It("returns a new index instance", func() { - b := bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}) - Expect(b).ToNot(BeNil()) - }) - }) - - Describe("Search", func() { - Context("by other fields than filename", func() { - It("finds files by tags", func() { - parentResource.Document.Tags = []string{"foo", "bar"} - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Tags:foo", 1) - assertDocCount(rootResource.ID, "Tags:bar", 1) - assertDocCount(rootResource.ID, "Tags:foo Tags:bar", 1) - assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1) - assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1) - assertDocCount(rootResource.ID, "Tags:baz", 0) - }) - - It("finds files by size", func() { - parentResource.Document.Size = 12345 - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Size:12345", 1) - assertDocCount(rootResource.ID, "Size:>1000", 1) - assertDocCount(rootResource.ID, "Size:<100000", 1) - assertDocCount(rootResource.ID, "Size:12344", 0) - assertDocCount(rootResource.ID, "Size:<1000", 0) - assertDocCount(rootResource.ID, "Size:>100000", 0) - }) - - It("preserves value case for fields not explicitly marked lowercase", func() { - parentResource.Document.Audio = &libregraph.Audio{ - Artist: libregraph.PtrString("Some Artist"), - } - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `audio.artist:"Some Artist"`, 1) - assertDocCount(rootResource.ID, `audio.artist:"some artist"`, 0) - }) - }) - - Context("by filename", func() { - It("finds files with spaces in the filename", func() { - parentResource.Document.Name = "Foo oo.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `name:"foo o*"`, 1) - }) - - It("finds files by digits in the filename", func() { - parentResource.Document.Name = "12345.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:1234*", 1) - }) - - It("filters hidden files", func() { - childResource.Hidden = true - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Hidden:T", 1) - assertDocCount(rootResource.ID, "Hidden:F", 0) - }) - - Context("with a file in the root of the space", func() { - It("scopes the search to the specified space", func() { - parentResource.Document.Name = "foo.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:foo.pdf", 1) - assertDocCount("9$8!7", "Name:foo.pdf", 0) - }) - }) - - It("limits the search to the specified fields", func() { - parentResource.Document.Name = "bar.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:bar.pdf", 1) - assertDocCount(rootResource.ID, "Unknown:field", 0) - }) - - It("returns the total number of hits", func() { - parentResource.Document.Name = "bar.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - res, err := doSearch(rootResource.ID, "Name:bar*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - }) - - It("returns all desired fields", func() { - parentResource.Document.Name = "bar.pdf" - parentResource.Type = 3 - parentResource.MimeType = "application/pdf" - - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - matches := assertDocCount(rootResource.ID, fmt.Sprintf("Name:%s", parentResource.Name), 1) - match := matches[0] - Expect(match.Entity.Ref.Path).To(Equal(parentResource.Path)) - Expect(match.Entity.Name).To(Equal(parentResource.Name)) - Expect(match.Entity.Size).To(Equal(parentResource.Size)) - Expect(match.Entity.Type).To(Equal(parentResource.Type)) - Expect(match.Entity.MimeType).To(Equal(parentResource.MimeType)) - Expect(match.Entity.Deleted).To(BeFalse()) - Expect(match.Score > 0).To(BeTrue()) - }) - - It("finds files by name, prefix or substring match", func() { - parentResource.Document.Name = "foo.pdf" - - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - queries := []string{"foo.pdf", "foo*", "*oo.p*"} - for _, query := range queries { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, query, 1) - } - }) - - It("does a case-insensitive search", func() { - parentResource.Document.Name = "foo.pdf" - - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:foo*", 1) - assertDocCount(rootResource.ID, "Name:Foo*", 1) - }) - - Context("and an additional file in a subdirectory", func() { - BeforeEach(func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - }) - - It("finds files living deeper in the tree by filename, prefix or substring match", func() { - queries := []string{"child.pdf", "child*", "*ld.*"} - for _, query := range queries { - assertDocCount(rootResource.ID, query, 1) - } - }) - }) - }) - - Context("Highlights", func() { - - It("highlights only for content searches", func() { - parentResource.Document.Name = "baz.pdf" - parentResource.Document.Content = "foo bar baz" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - res, err := doSearch(rootResource.ID, "Name:baz*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - Expect(res.Matches[0].Entity.Highlights).To(Equal("")) - }) - - It("highlights search terms", func() { - parentResource.Document.Name = "baz.pdf" - parentResource.Document.Content = "foo bar baz" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - res, err := doSearch(rootResource.ID, "Content:bar", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - Expect(res.Matches[0].Entity.Highlights).To(Equal("foo bar baz")) - }) - - }) - - Context("with a file in the root of the space and folder with a file. all of them have the same name", func() { - BeforeEach(func() { - parentResource := search.Resource{ - ID: "1$2!3", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./doc", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER), - Document: content.Document{Name: "doc"}, - } - - childResource := search.Resource{ - ID: "1$2!4", - ParentID: parentResource.ID, - RootID: rootResource.ID, - Path: "./doc/doc.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "doc.pdf"}, - } - - childResource2 := search.Resource{ - ID: "1$2!7", - ParentID: parentResource.ID, - RootID: rootResource.ID, - Path: "./doc/file.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "file.pdf"}, - } - - rootChildResource := search.Resource{ - ID: "1$2!5", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./doc.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "doc.pdf"}, - } - - rootChildResource2 := search.Resource{ - ID: "1$2!6", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./file.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "file.pdf"}, - } - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(rootChildResource.ID, rootChildResource) - Expect(err).ToNot(HaveOccurred()) - err = eng.Upsert(rootChildResource2.ID, rootChildResource2) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - err = eng.Upsert(childResource2.ID, childResource2) - Expect(err).ToNot(HaveOccurred()) - }) - It("search *doc* in a root", func() { - res, err := doSearch(rootResource.ID, "Name:*doc*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(3))) - }) - It("search *doc* in a subfolder", func() { - res, err := doSearch(rootResource.ID, "Name:*doc*", "./doc") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(2))) - }) - It("search *file* in a root", func() { - res, err := doSearch(rootResource.ID, "Name:*file*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(2))) - }) - It("search *file* in a subfolder", func() { - res, err := doSearch(rootResource.ID, "Name:*file*", "./doc") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - }) - }) - - }) - - Describe("Upsert", func() { - It("adds a resourceInfo to the index", func() { - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - query := bleveSearch.NewMatchQuery("child.pdf") - res, err := idx.Search(bleveSearch.NewSearchRequest(query)) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Hits.Len()).To(Equal(1)) - }) - - It("updates an existing resource in the index", func() { - - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - countA, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(countA).To(Equal(uint64(1))) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - countB, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(countB).To(Equal(uint64(1))) - }) - }) - - Describe("Delete", func() { - It("marks a resource as deleted", func() { - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:*child*", 1) - - err = eng.Delete(childResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:*child*", 0) - }) - - It("marks a child resources as deleted", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - - err = eng.Delete(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0) - }) - }) - - Describe("Restore", func() { - It("also marks child resources as restored", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Delete(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 0) - - err = eng.Restore(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 1) - assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 1) - }) - }) - - Describe("Purge", func() { - It("removes a resource from the index", func() { - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - assertDocCount(rootResource.ID, "Name:child.pdf", 1) - - err = eng.Purge(childResource.ID, false) - - Expect(err).ToNot(HaveOccurred()) - assertDocCount(rootResource.ID, "Name:child.pdf", 0) - }) - It("removes a resource and its children from the index", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - - err = eng.Purge(parentResource.ID, false) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0) - }) - It("removes a resource and ignores its children from the index", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) - - err = eng.Delete(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - - err = eng.Purge(parentResource.ID, true) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - }) - }) - - Describe("Move", func() { - It("renames the parent and its child resources", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - parentResource.Path = "newname" - err = eng.Move(parentResource.ID, parentResource.ParentID, "./my/newname") - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, parentResource.Name, 0) - - matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1) - Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3")) - Expect(matches[0].Entity.Ref.Path).To(Equal("./my/newname/child.pdf")) - }) - - DescribeTable("keeps the flag in step with the path", - func(from, target string, hidden bool) { - parentResource.Path = from - parentResource.Hidden = search.IsHidden(from) - childResource.Path = from + "/child.pdf" - childResource.Hidden = parentResource.Hidden - - Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) - Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) - - Expect(eng.Move(parentResource.ID, parentResource.ParentID, target)).To(Succeed()) - - for _, id := range []string{parentResource.ID, childResource.ID} { - Expect(hiddenByID(idx, id)). - To(Equal(hidden), "%s after moving from %s to %s", id, from, target) - } - }, - Entry("into a dot folder", "./parent", "./.trash/parent", true), - Entry("into a plain folder", "./parent", "./archive/parent", false), - Entry("renamed with a leading dot", "./parent", "./.parent", true), - Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false), - Entry("renamed without the leading dot", "./.parent", "./parent", false), - Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true), - ) - - // the trash leaves the path alone, so the flag has to come through untouched - It("carries the flag through the trash and back", func() { - childResource.Path = "./.secret/file.txt" - childResource.Hidden = true - Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) - - Expect(eng.Delete(childResource.ID)).To(Succeed()) - Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after trashing") - - Expect(eng.Restore(childResource.ID)).To(Succeed()) - Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after restoring") - }) - - It("moves the parent and its child resources", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - parentResource.Path = " " - parentResource.ParentID = "1$2!somewhereopaqueid" - - err = eng.Move(parentResource.ID, parentResource.ParentID, "./somewhere/else/newname") - Expect(err).ToNot(HaveOccurred()) - assertDocCount(rootResource.ID, `parent d!r`, 0) - - matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1) - Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3")) - Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname/child.pdf")) - - matches = assertDocCount(rootResource.ID, `newname`, 1) - Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("somewhereopaqueid")) - Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname")) - - }) - }) - - Describe("StartBatch", func() { - It("starts a new batch", func() { - b, err := eng.NewBatch(100) - Expect(err).ToNot(HaveOccurred()) - - err = b.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(0))) - - err = b.Push() - Expect(err).ToNot(HaveOccurred()) - - count, err = idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - query := bleveSearch.NewMatchQuery("child.pdf") - res, err := idx.Search(bleveSearch.NewSearchRequest(query)) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Hits.Len()).To(Equal(1)) - }) - - It("doesn't intertwine different batches", func() { - b, err := eng.NewBatch(100) - Expect(err).ToNot(HaveOccurred()) - - err = b.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(0))) - - b2, err := eng.NewBatch(100) - Expect(err).ToNot(HaveOccurred()) - - err = b2.Upsert(childResource2.ID, childResource2) - Expect(err).ToNot(HaveOccurred()) - - Expect(b.Push()).To(Succeed()) - count, err = idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - Expect(b2.Push()).To(Succeed()) - count, err = idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(2))) - }) - }) - - Describe("File type specific metadata", func() { - - Context("with audio metadata", func() { - BeforeEach(func() { - resource := search.Resource{ - ID: "1$2!7", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./some_song.mp3", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{ - Name: "some_song.mp3", - MimeType: "audio/mpeg", - Audio: &libregraph.Audio{ - Album: libregraph.PtrString("Some Album"), - AlbumArtist: libregraph.PtrString("Some AlbumArtist"), - Artist: libregraph.PtrString("Some Artist"), - Bitrate: libregraph.PtrInt64(192), - Composers: libregraph.PtrString("Some Composers"), - Copyright: libregraph.PtrString(""), - Disc: libregraph.PtrInt32(2), - DiscCount: libregraph.PtrInt32(5), - Duration: libregraph.PtrInt64(225000), - Genre: libregraph.PtrString("Some Genre"), - HasDrm: libregraph.PtrBool(false), - IsVariableBitrate: libregraph.PtrBool(true), - Title: libregraph.PtrString("Some Title"), - Track: libregraph.PtrInt32(34), - TrackCount: libregraph.PtrInt32(99), - Year: libregraph.PtrInt32(2004), - }, - }, - } - err := eng.Upsert(resource.ID, resource) - Expect(err).ToNot(HaveOccurred()) - }) - - It("returns audio metadata for search", func() { - matches := assertDocCount(rootResource.ID, `*song*`, 1) - audio := matches[0].Entity.Audio - - Expect(audio).ToNot(BeNil()) - - Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album"))) - Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist"))) - Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist"))) - Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192))) - Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers"))) - Expect(audio.Copyright).To(Equal(libregraph.PtrString(""))) - Expect(audio.Disc).To(Equal(libregraph.PtrInt32(2))) - Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5))) - Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225000))) - Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre"))) - Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false))) - Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true))) - Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title"))) - Expect(audio.Track).To(Equal(libregraph.PtrInt32(34))) - Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(99))) - Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) - }) - }) - - Context("with location metadata", func() { - BeforeEach(func() { - resource := search.Resource{ - ID: "1$2!7", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./team.jpg", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{ - Name: "team.jpg", - MimeType: "image/jpeg", - Location: &libregraph.GeoCoordinates{ - Altitude: libregraph.PtrFloat64(1047.7), - Latitude: libregraph.PtrFloat64(49.48675890884328), - Longitude: libregraph.PtrFloat64(11.103870357204285), - }, - }, - } - err := eng.Upsert(resource.ID, resource) - Expect(err).ToNot(HaveOccurred()) - }) - - It("returns audio metadata for search", func() { - matches := assertDocCount(rootResource.ID, `*team*`, 1) - location := matches[0].Entity.Location - - Expect(location).ToNot(BeNil()) - - Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(1047.7))) - Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328))) - Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285))) - }) - }) - }) -}) diff --git a/services/search/pkg/opensearch/backend_test.go b/services/search/pkg/opensearch/backend_test.go index 29859f751a..71ef3860fc 100644 --- a/services/search/pkg/opensearch/backend_test.go +++ b/services/search/pkg/opensearch/backend_test.go @@ -1,9 +1,6 @@ package opensearch_test import ( - "context" - "fmt" - "strings" "testing" . "github.com/onsi/ginkgo/v2" @@ -11,12 +8,7 @@ import ( opensearchgo "github.com/opensearch-project/opensearch-go/v4" opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - - searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" - "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) func TestOpenSearchBackend(t *testing.T) { @@ -24,56 +16,8 @@ func TestOpenSearchBackend(t *testing.T) { RunSpecs(t, "OpenSearch Backend Suite") } -func deleteIndexOnCleanup(tc *opensearchtest.TestClient, indexName string) { - DeferCleanup(func() { - Expect(tc.IndicesDelete(context.Background(), []string{indexName})).To(Succeed()) - }) -} - -func resourceByID(tc *opensearchtest.TestClient, index, id string) search.Resource { - GinkgoHelper() - - body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "query": map[string]any{ - "ids": map[string]any{ - "values": []string{id}, - }, - }, - }) - - resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(index, strings.NewReader(body)).Hits) - Expect(resources).To(HaveLen(1)) - return resources[0] -} - -// otherRoot returns a copy of the given resource that lives in a different root (space) -// while keeping the same path, so it can be used to assert that cross-root updates do -// not affect identically-named resources in other roots. -func otherRoot(r search.Resource) search.Resource { - r.ID = "2$2!3" - r.RootID = "2$2!1" - r.ParentID = "2$2!2" - return r -} - -func newBackend(indexName string, resources ...search.Resource) (*opensearch.Backend, *opensearchtest.TestClient) { - GinkgoHelper() - - tc := opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - - backend, err := opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - - for _, r := range resources { - tc.Require.DocumentCreate(indexName, r.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), r))) - } - tc.Require.IndicesCount([]string{indexName}, nil, len(resources)) - - return backend, tc -} - +// what the engine does with its index is covered for both engines by +// services/search/pkg/parity; this is the one thing only OpenSearch can do var _ = Describe("Backend", func() { Describe("NewBackend", func() { It("fails to create if the cluster is not healthy", func() { @@ -89,606 +33,4 @@ var _ = Describe("Backend", func() { Expect(err).To(MatchError(opensearch.ErrUnhealthyCluster)) }) }) - - Describe("Search", func() { - const indexName = "opencloud-test-engine-search" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - document search.Resource - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - - document = opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - }) - - It("performs the most simple search", func() { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ - Query: fmt.Sprintf(`"%s"`, document.Name), - }) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(HaveLen(1)) - Expect(resp.TotalMatches).To(Equal(int32(1))) - Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID)) - }) - - It("ignores files that are marked as deleted", func() { - deletedDocument := opensearchtest.Testdata.Resources.File - deletedDocument.ID = "1$2!4" - deletedDocument.Deleted = true - - tc.Require.DocumentCreate(indexName, deletedDocument.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), deletedDocument))) - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ - Query: fmt.Sprintf(`"%s"`, document.Name), - }) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(HaveLen(1)) - Expect(resp.TotalMatches).To(Equal(int32(1))) - Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID)) - }) - }) - - Describe("Upsert", func() { - const indexName = "opencloud-test-engine-upsert" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("upserts a full document", func() { - document := opensearchtest.Testdata.Resources.File - Expect(backend.Upsert(document.ID, document)).To(Succeed()) - - tc.Require.IndicesCount([]string{indexName}, nil, 1) - }) - }) - - Describe("Move", func() { - const indexName = "opencloud-test-engine-move" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("moves the document to a new path", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "query": map[string]any{ - "ids": map[string]any{ - "values": []string{document.ID}, - }, - }, - }) - - resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits) - Expect(resources).To(HaveLen(1)) - Expect(resources[0].Path).To(Equal(document.Path)) - - document.Path = "./new/path/to/resource" - Expect(backend.Move(document.ID, document.ParentID, document.Path)).To(Succeed()) - - resources = opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits) - Expect(resources).To(HaveLen(1)) - Expect(resources[0].Path).To(Equal(document.Path)) - }) - }) - - Describe("WriteVisibility", func() { - const indexName = "opencloud-test-engine-write-visibility" - - It("deletes a record that was just written", func() { - document := opensearchtest.Testdata.Resources.File - document.ID = "1$1!95" - document.Name = "textfile.txt" - document.Path = "./textfile.txt" - - backend, tc := newBackend(indexName) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Upsert(document.ID, document)).To(Succeed()) - Expect(backend.Delete(document.ID)).To(Succeed()) - - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ - Query: fmt.Sprintf(`name:"%s"`, document.Name), - }) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(BeEmpty()) - }) - }) - - Describe("Delete", func() { - const indexName = "opencloud-test-engine-delete" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("marks the document as deleted", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "query": map[string]any{ - "term": map[string]any{ - "Deleted": map[string]any{ - "value": true, - }, - }, - }, - }) - - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0) - - Expect(backend.Delete(document.ID)).To(Succeed()) - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1) - }) - }) - - Describe("Restore", func() { - const indexName = "opencloud-test-engine-restore" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("marks the document as not deleted", func() { - document := opensearchtest.Testdata.Resources.File - document.Deleted = true - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "query": map[string]any{ - "term": map[string]any{ - "Deleted": map[string]any{ - "value": true, - }, - }, - }, - }) - - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1) - - Expect(backend.Restore(document.ID)).To(Succeed()) - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0) - }) - }) - - Describe("Purge", func() { - const indexName = "opencloud-test-engine-purge" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("purges a full document", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - Expect(backend.Purge(document.ID, false)).To(Succeed()) - - tc.Require.IndicesCount([]string{indexName}, nil, 0) - }) - - It("purges resource trees", func() { - resourceFolder := opensearchtest.Testdata.Resources.Folder - tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder))) - - resourceFile := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile))) - - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - Expect(backend.Purge(resourceFolder.ID, false)).To(Succeed()) - - tc.Require.IndicesCount([]string{indexName}, nil, 0) - }) - - It("purges resource trees and ignores undeleted resources", func() { - resourceFolder := opensearchtest.Testdata.Resources.Folder - tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder))) - - resourceFile := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile))) - - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - Expect(backend.Delete(resourceFile.ID)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - Expect(backend.Purge(resourceFolder.ID, true)).To(Succeed()) - - tc.Require.IndicesCount([]string{indexName}, nil, 1) - }) - }) - - Describe("PurgeSpace", func() { - const indexName = "opencloud-test-engine-purge-space" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("takes every record of that space out of the index", func() { - gone := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, gone.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), gone))) - - stays := opensearchtest.Testdata.Resources.File - stays.ID = "1$2!3" - stays.RootID = "1$2!2" - tc.Require.DocumentCreate(indexName, stays.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), stays))) - - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - Expect(backend.PurgeSpace(gone.RootID)).To(Succeed()) - - tc.Require.IndicesRefresh([]string{indexName}, nil) - left := opensearchtest.SearchHitsMustBeConverted[search.Resource]( - GinkgoTB(), - tc.Require.Search(indexName, strings.NewReader(`{"query":{"match_all":{}}}`)).Hits, - ) - Expect(left).To(HaveLen(1), "only the records of that space are gone") - Expect(left[0].ID).To(Equal(stays.ID)) - }) - }) - - Describe("Hidden", func() { - const indexName = "opencloud-test-engine-hidden" - - DescribeTable("keeps the flag in step with the path", - func(from, target string, hidden bool) { - folder := opensearchtest.Testdata.Resources.Folder - folder.ID = "1$1!30" - folder.Name = "parent" - folder.Path = from - folder.Hidden = search.IsHidden(from) - - child := opensearchtest.Testdata.Resources.File - child.ID = "1$1!31" - child.Name = "child.txt" - child.Path = from + "/child.txt" - child.ParentID = folder.ID - child.Hidden = folder.Hidden - - backend, tc := newBackend(indexName, folder, child) - deleteIndexOnCleanup(tc, indexName) - tc.Require.IndicesRefresh([]string{indexName}, nil) - - Expect(backend.Move(folder.ID, folder.ParentID, target)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - - for _, id := range []string{folder.ID, child.ID} { - Expect(resourceByID(tc, indexName, id).Hidden). - To(Equal(hidden), "%s after moving from %s to %s", id, from, target) - } - }, - Entry("into a dot folder", "./parent", "./.trash/parent", true), - Entry("into a plain folder", "./parent", "./archive/parent", false), - Entry("renamed with a leading dot", "./parent", "./.parent", true), - Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false), - Entry("renamed without the leading dot", "./.parent", "./parent", false), - Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true), - ) - - It("carries the flag through the trash and back", func() { - hidden := opensearchtest.Testdata.Resources.File - hidden.ID = "1$1!32" - hidden.Path = "./.secret/file.txt" - hidden.Hidden = true - - backend, tc := newBackend(indexName, hidden) - deleteIndexOnCleanup(tc, indexName) - tc.Require.IndicesRefresh([]string{indexName}, nil) - - Expect(backend.Delete(hidden.ID)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - Expect(resourceByID(tc, indexName, hidden.ID).Hidden).To(BeTrue(), "after trashing") - - Expect(backend.Restore(hidden.ID)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - Expect(resourceByID(tc, indexName, hidden.ID).Hidden).To(BeTrue(), "after restoring") - }) - }) - - Describe("DocCount", func() { - const indexName = "opencloud-test-engine-doc-count" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("ignores deleted documents", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - count, err := backend.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - tc.Require.Update(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "doc": map[string]any{ - "Deleted": true, - }, - }))) - - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - count, err = backend.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(0))) - }) - }) - - // The following specs ensure that updates which affect a resource and its descendants - // (Delete, Restore, Move) are scoped to the root (space) of the target resource. Two - // resources living in different roots may share the exact same path, so matching by - // path alone would incorrectly update the wrong resource. - Describe("updateSelfAndDescendants root scope", func() { - It("deletes only the resource in the target root", func() { - const indexName = "opencloud-test-engine-root-scope-delete" - - target := opensearchtest.Testdata.Resources.File - other := otherRoot(target) - - backend, tc := newBackend(indexName, target, other) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Delete(target.ID)).To(Succeed()) - - Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeTrue(), "target resource should be marked as deleted") - Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeFalse(), "resource in a different root must not be affected") - }) - - It("restores only the resource in the target root", func() { - const indexName = "opencloud-test-engine-root-scope-restore" - - target := opensearchtest.Testdata.Resources.File - target.Deleted = true - other := otherRoot(target) - - backend, tc := newBackend(indexName, target, other) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Restore(target.ID)).To(Succeed()) - - Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeFalse(), "target resource should be restored") - Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeTrue(), "resource in a different root must not be affected") - }) - - It("moves only the resource in the target root", func() { - const indexName = "opencloud-test-engine-root-scope-move" - - target := opensearchtest.Testdata.Resources.File - other := otherRoot(target) - - backend, tc := newBackend(indexName, target, other) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Move(target.ID, target.ParentID, "./new/path/to/resource")).To(Succeed()) - - Expect(resourceByID(tc, indexName, target.ID).Path).To(Equal("./new/path/to/resource"), "target resource should be moved") - Expect(resourceByID(tc, indexName, other.ID).Path).To(Equal(other.Path), "resource in a different root must not be moved") - }) - }) - - Describe("SearchInAnalyzedFields", func() { - const indexName = "opencloud-test-engine-search-analyzed-fields" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - dashed := opensearchtest.Testdata.Resources.Folder - dashed.ID = "1$1!10" - dashed.Name = "new-folder" - dashed.Path = "./new-folder" - dashed.Title = "quarterly report" - - plain := opensearchtest.Testdata.Resources.Folder - plain.ID = "1$1!11" - plain.Name = "documents" - plain.Path = "./documents" - plain.Title = "notes" - - spaced := opensearchtest.Testdata.Resources.Folder - spaced.ID = "1$1!12" - spaced.Name = "foo bar" - spaced.Path = "./foo bar" - spaced.Title = "spaced out" - - backend, tc = newBackend(indexName, dashed, plain, spaced) - deleteIndexOnCleanup(tc, indexName) - tc.Require.IndicesRefresh([]string{indexName}, nil) - }) - - DescribeTable("finds what the analyzer made of the value", - func(query string, want []string) { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query}) - Expect(err).ToNot(HaveOccurred()) - - names := make([]string, 0, len(resp.Matches)) - for _, match := range resp.Matches { - names = append(names, match.Entity.Name) - } - Expect(names).To(ConsistOf(want)) - }, - Entry("the full name with the dash", "new-folder", []string{"new-folder"}), - Entry("one token of it", "new", []string{"new-folder"}), - Entry("a name without a dash", "documents", []string{"documents"}), - Entry("a wildcard", "*folder*", []string{"new-folder"}), - // the shape the web client sends for every name search - Entry("a wildcard around the whole dashed name", `name:"*new-folder*"`, []string{"new-folder"}), - Entry("a wildcard spanning the dash", `name:"*w-fol*"`, []string{"new-folder"}), - Entry("a wildcard in a different case", `name:"*NEW-FOLDER*"`, []string{"new-folder"}), - Entry("a wildcard spanning a space", `name:"*oo ba*"`, []string{"foo bar"}), - Entry("a wildcard around a name with a space", `name:"*foo bar*"`, []string{"foo bar"}), - Entry("a name with a space", `name:"foo bar"`, []string{"foo bar"}), - Entry("a title of two words", `Title:"quarterly report"`, []string{"new-folder"}), - Entry("one token of a title", "Title:quarterly", []string{"new-folder"}), - ) - }) - - Describe("SearchByTag", func() { - const indexName = "opencloud-test-engine-search-by-tag" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tagged := opensearchtest.Testdata.Resources.Folder - tagged.ID = "1$1!20" - tagged.Name = "tagged" - tagged.Path = "./tagged" - tagged.Tags = []string{"foo-bar"} - - other := opensearchtest.Testdata.Resources.Folder - other.ID = "1$1!21" - other.Name = "other" - other.Path = "./other" - other.Tags = []string{"foo"} - - backend, tc = newBackend(indexName, tagged, other) - deleteIndexOnCleanup(tc, indexName) - tc.Require.IndicesRefresh([]string{indexName}, nil) - }) - - // a tag is one label, not prose, so it matches as a whole or not at all - DescribeTable("matches a tag as a whole", - func(query string, want []string) { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query}) - Expect(err).ToNot(HaveOccurred()) - - names := make([]string, 0, len(resp.Matches)) - for _, match := range resp.Matches { - names = append(names, match.Entity.Name) - } - Expect(names).To(ConsistOf(want)) - }, - Entry("the whole tag", `tag:("foo-bar")`, []string{"tagged"}), - Entry("a token of a tag does not match it", `tag:("foo")`, []string{"other"}), - Entry("a tag in a different case", `tag:("FOO-BAR")`, []string{"tagged"}), - Entry("a wildcard reaches both", `tag:("*foo*")`, []string{"tagged", "other"}), - ) - }) - - Describe("SearchWithAnInvalidQuery", func() { - const indexName = "opencloud-test-engine-search-invalid-query" - - It("answers with a bad request", func() { - backend, tc := newBackend(indexName) - deleteIndexOnCleanup(tc, indexName) - - _, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: "AND mediatype:document"}) - Expect(err).To(HaveOccurred()) - Expect(err).To(BeAssignableToTypeOf(errtypes.BadRequest(""))) - Expect(err.Error()).To(Equal(`error: bad request: the expression can't begin from a binary operator: 'AND'`)) - }) - }) }) diff --git a/services/search/pkg/opensearch/index_test.go b/services/search/pkg/opensearch/index_test.go index f4b5451a9d..6a813572a2 100644 --- a/services/search/pkg/opensearch/index_test.go +++ b/services/search/pkg/opensearch/index_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/require" "github.com/tidwall/sjson" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestIndexManager(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go index 49218d5aa5..1b3cb0b914 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go @@ -9,7 +9,7 @@ import ( "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" "github.com/opencloud-eu/opencloud/pkg/ast" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" ) func TestExpandKQLAST(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go index 9dbf7f2b9a..1b627c8133 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go @@ -7,9 +7,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestTranspileKQLToOpenSearch(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/convert/opensearch_test.go b/services/search/pkg/opensearch/internal/convert/opensearch_test.go index bf80b7058f..d2fba09c70 100644 --- a/services/search/pkg/opensearch/internal/convert/opensearch_test.go +++ b/services/search/pkg/opensearch/internal/convert/opensearch_test.go @@ -10,8 +10,8 @@ import ( "github.com/opencloud-eu/opencloud/pkg/conversions" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) diff --git a/services/search/pkg/opensearch/internal/osu/query_bool_test.go b/services/search/pkg/opensearch/internal/osu/query_bool_test.go index 3f33d4c1f4..d101cc5b7e 100644 --- a/services/search/pkg/opensearch/internal/osu/query_bool_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_bool_test.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/assert" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestBoolQuery(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase_test.go b/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase_test.go index 2fa712d039..b2c4b36a19 100644 --- a/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase_test.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/assert" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestNewMatchPhraseQuery(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/osu/query_term_level_ids_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_ids_test.go index 1a15777649..ecfca2fff4 100644 --- a/services/search/pkg/opensearch/internal/osu/query_term_level_ids_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_ids_test.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/assert" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestIDsQuery(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/osu/query_term_level_range_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_range_test.go index 0bb65fca73..43b6241768 100644 --- a/services/search/pkg/opensearch/internal/osu/query_term_level_range_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_range_test.go @@ -8,8 +8,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestRangeQuery(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/osu/query_term_level_term_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_term_test.go index c4e9e8f325..e4244bf0a8 100644 --- a/services/search/pkg/opensearch/internal/osu/query_term_level_term_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_term_test.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/assert" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestTermQuery(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard_test.go index afc810e36d..e1a5f37088 100644 --- a/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard_test.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/assert" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestWildcardQuery(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/osu/request_test.go b/services/search/pkg/opensearch/internal/osu/request_test.go index 535f05bd82..f8e0010258 100644 --- a/services/search/pkg/opensearch/internal/osu/request_test.go +++ b/services/search/pkg/opensearch/internal/osu/request_test.go @@ -8,8 +8,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestRequestBody(t *testing.T) { diff --git a/services/search/pkg/opensearch/opensearch_test.go b/services/search/pkg/opensearch/opensearch_test.go index 6278ce6d66..cd5fb6fdbc 100644 --- a/services/search/pkg/opensearch/opensearch_test.go +++ b/services/search/pkg/opensearch/opensearch_test.go @@ -6,8 +6,8 @@ import ( "os" "testing" + opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/config" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) var defaultConfig *config.Config diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md new file mode 100644 index 0000000000..a8acafba94 --- /dev/null +++ b/services/search/pkg/parity/README.md @@ -0,0 +1,631 @@ +# Engine parity + +Written by the parity suite (`go test ./services/search/pkg/parity/`), do not edit. +Every case runs against bleve and OpenSearch. `same?` is ✅ when both answer as +expected, `❌ known` when an engine's divergence is documented in the case +(`engineOverrides`), `❌` when it is not. + +## Queries + +### name + +Fixtures: + +- `new-folder`, folder +- `quarterly notes.txt` +- `Report.txt` +- `Übung.txt` +- `a+b.txt` +- `c(d).txt` +- `e&f.txt` +- `v1.2.3.txt` +- `foo bar.txt` +- `aaaaaaaaaa...edle.txt` + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| NAME-01 | `new` | new-folder | no match | new-folder | ❌ known | +| NAME-02 | `quarterly` | quarterly notes.txt | no match | quarterly notes.txt | ❌ known | +| NAME-03 | `report` | Report.txt | no match | no match | ❌ known | +| NAME-04 | `name:"*new-folder*"` | new-folder | new-folder | new-folder | ✅ | +| NAME-05 | `name:"*w-fol*"` | new-folder | new-folder | new-folder | ✅ | +| NAME-06 | `name:"*oo ba*"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | +| NAME-07 | `name:"*REPORT*"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-08 | `name:"*übung*"` | Übung.txt | Übung.txt | no match | ❌ known | +| NAME-09 | `name:"*ÜBUNG*"` | Übung.txt | Übung.txt | no match | ❌ known | +| NAME-10 | `name:"*a+b*"` | a+b.txt | a+b.txt | a+b.txt | ✅ | +| NAME-11 | `name:"*c(d)*"` | c(d).txt | c(d).txt | c(d).txt | ✅ | +| NAME-12 | `name:"*e&f*"` | e&f.txt | e&f.txt | e&f.txt | ✅ | +| NAME-13 | `name:"*v1.2*"` | v1.2.3.txt | v1.2.3.txt | v1.2.3.txt | ✅ | +| NAME-14 | `new-folder` | new-folder | new-folder | new-folder | ✅ | +| NAME-15 | `*folder*` | new-folder | new-folder | new-folder | ✅ | +| NAME-16 | `name:"*foo bar*"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | +| NAME-17 | `name:"foo bar.txt"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | +| NAME-18 | `name:"*needle*"` | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | no match | ❌ known | +| NAME-19 | `name:"report*"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-20 | `name:"*report"` | Report.txt | no match | no match | ❌ known | +| NAME-21 | `name:"Rep*rt.txt"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-22 | `Name:"*report*"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-23 | `NAME:"*report*"` | Report.txt | Report.txt | no match | ❌ known | +| NAME-24 | `name:Rep?rt.txt` | Report.txt | Report.txt | no match | ❌ known | +| NAME-25 | `name:"*eport"` | Report.txt | no match | no match | ❌ known | +| NAME-26 | `name:"repor*"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-27 | `REPORT` | Report.txt | no match | no match | ❌ known | +| NAME-28 | `name:REPORT` | Report.txt | no match | no match | ❌ known | +| NAME-29 | `name:"REPORT.TXT"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-30 | `name:"FOO BAR.TXT"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | +| NAME-31 | `name:"ÜBUNG.TXT"` | Übung.txt | Übung.txt | Übung.txt | ✅ | +| NAME-32 | `name:"folder*"` | no match | no match | no match | ✅ | +| NAME-33 | `name:"*new"` | no match | no match | no match | ✅ | +| NAME-34 | `name:new` | new-folder | no match | new-folder | ❌ known | +| NAME-35 | `name:"new"` | no match | no match | new-folder | ❌ known | +| NAME-36 | `name:"new-folder"` | new-folder | new-folder | new-folder | ✅ | +| NAME-37 | `name:"new-*"` | new-folder | new-folder | new-folder | ✅ | +| NAME-38 | `name:"new*"` | new-folder | new-folder | new-folder | ✅ | +| NAME-39 | `name:new-*` | new-folder | new-folder | new-folder | ✅ | +| NAME-40 | `name:"*-folder"` | new-folder | new-folder | new-folder | ✅ | +| NAME-41 | `name:"Rep?rt.txt"` | Report.txt | Report.txt | no match | ❌ known | + +### extension + +Fixtures: + +- `report.txt` +- `notes.md`, MimeType = text/markdown +- `archive`, folder + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| EXTENSION-01 | `txt` | report.txt | no match | no match | ❌ known | +| EXTENSION-02 | `md` | notes.md | no match | no match | ❌ known | +| EXTENSION-03 | `name:"*.txt"` | report.txt | report.txt | report.txt | ✅ | +| EXTENSION-04 | `report` | report.txt | no match | no match | ❌ known | + +### tags + +Fixtures: + +- `invoice.txt`, Tags = foo-bar +- `memo.txt`, Tags = foo +- `spaced.txt`, Tags = spaced tag +- `project`, folder, Tags = work +- `draft.txt`, Path = ./project/draft.txt +- `longtag.txt`, Tags = zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzneedle + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| TAGS-01 | `name:"*foo-bar*"` | no match | no match | no match | ✅ | +| TAGS-02 | `tag:("foo-bar")` | invoice.txt | invoice.txt | invoice.txt | ✅ | +| TAGS-03 | `tag:("foo")` | memo.txt | memo.txt | memo.txt | ✅ | +| TAGS-04 | `tag:("FOO-BAR")` | invoice.txt | invoice.txt | invoice.txt | ✅ | +| TAGS-05 | `tag:("*foo*")` | invoice.txt, memo.txt | invoice.txt, memo.txt | invoice.txt, memo.txt | ✅ | +| TAGS-06 | `tag:("spaced tag")` | spaced.txt | spaced.txt | spaced.txt | ✅ | +| TAGS-07 | `tag:("*paced ta*")` | spaced.txt | spaced.txt | spaced.txt | ✅ | +| TAGS-08 | `tag:("work")` | project | project | project | ✅ | +| TAGS-09 | `tag:("zzzzzzzzzzzzzzzzzzzzzzzzzz...zzzzzzzzzzzzzzzzneedle")` | longtag.txt | longtag.txt | no match | ❌ known | + +### title + +Fixtures: + +- `q1.html`, MimeType = text/html, Title = "quarterly report" + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| TITLE-01 | `Title:"quarterly report"` | q1.html | q1.html | q1.html | ✅ | +| TITLE-02 | `Title:quarterly` | q1.html | no match | q1.html | ❌ known | +| TITLE-03 | `Title:QUARTERLY` | q1.html | no match | q1.html | ❌ known | +| TITLE-04 | `Title:quarterl*` | q1.html | q1.html | q1.html | ✅ | +| TITLE-05 | `Title:"*ly rep*"` | q1.html | q1.html | q1.html | ✅ | +| TITLE-06 | `title:quarterly` | q1.html | no match | no match | ❌ known | +| TITLE-07 | `Title:"QUARTERLY REPORT"` | q1.html | no match | q1.html | ❌ known | + +### content + +Fixtures: + +- `monthly.txt`, Content = "the monthly reports are due" +- `links.txt`, Content = "see https://opencloud.example.com/help or write to alan@example.org" + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| CONTENT-01 | `Content:report` | no match | monthly.txt | no match | ❌ known | +| CONTENT-02 | `Content:REPORTS` | monthly.txt | monthly.txt | monthly.txt | ✅ | +| CONTENT-03 | `Content:"monthly reports"` | monthly.txt | monthly.txt | monthly.txt | ✅ | +| CONTENT-04 | `Content:"reports monthly"` | no match | monthly.txt | no match | ❌ known | +| CONTENT-05 | `Content:report*` | monthly.txt | monthly.txt | monthly.txt | ✅ | +| CONTENT-06 | `Content:*eport*` | monthly.txt | monthly.txt | monthly.txt | ✅ | +| CONTENT-07 | `Content:month*` | monthly.txt | monthly.txt | monthly.txt | ✅ | +| CONTENT-08 | `Content:"https://opencloud.example.com/help"` | links.txt | links.txt | links.txt | ✅ | +| CONTENT-09 | `Content:"alan@example.org"` | links.txt | links.txt | links.txt | ✅ | +| CONTENT-10 | `Content:opencloud` | links.txt | no match | no match | ❌ known | + +### favorites + +Fixtures: + +- `starred.txt`, Favorites = A1B2-Upper +- `plain.txt` +- `keepsakes`, folder, Favorites = A1B2-Upper +- `photo.jpg`, MimeType = image/jpeg, Path = ./keepsakes/photo.jpg + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| FAVORITES-01 | `Favorites:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | no match | ❌ known | +| FAVORITES-02 | `favorite:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | no match | ❌ known | +| FAVORITES-03 | `Favorites:"somebody-else"` | no match | no match | no match | ✅ | + +### mediatype + +Fixtures: + +- `notes.md`, MimeType = text/markdown +- `photo.jpg`, MimeType = image/jpeg +- `albums`, folder +- `drafts`, folder + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| MEDIATYPE-01 | `mediatype:text/markdown` | notes.md | notes.md | notes.md | ✅ | +| MEDIATYPE-02 | `mediatype:TEXT/MARKDOWN` | notes.md | no match | notes.md | ❌ known | +| MEDIATYPE-03 | `mediatype:image/jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ | +| MEDIATYPE-04 | `mediatype:*jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ | +| MEDIATYPE-05 | `mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ | +| MEDIATYPE-06 | `mediatype:folder` | albums, drafts | albums, drafts | albums, drafts | ✅ | + +### path + +Fixtures: + +- `parent`, folder +- `child.jpg`, MimeType = image/jpeg, Path = ./parent/child.jpg +- `docs-lower`, folder, Path = ./documents +- `docs-upper`, folder, Path = ./DOCUMENTS +- `docs-mixed`, folder, Path = ./Documents + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| PATH-01 | `path:"./parent"` | child.jpg, parent | parent | child.jpg, parent | ❌ known | +| PATH-02 | `path:"./parent/child.jpg"` | child.jpg | child.jpg | child.jpg | ✅ | +| PATH-03 | `path:"./Parent"` | no match | no match | child.jpg, parent | ❌ known | +| PATH-04 | `path:"*child*"` | child.jpg | child.jpg | child.jpg | ✅ | +| PATH-05 | `path:"./documents"` | docs-lower | docs-lower | docs-lower, docs-mixed, docs-upper | ❌ known | +| PATH-06 | `path:"./DOCUMENTS"` | docs-upper | docs-upper | docs-lower, docs-mixed, docs-upper | ❌ known | +| PATH-07 | `path:"./Documents"` | docs-mixed | docs-mixed | docs-lower, docs-mixed, docs-upper | ❌ known | +| PATH-08 | `path:"./parent/"` | child.jpg, parent | no match | no match | ❌ known | + +### fields + +Fixtures: + +- `small.txt`, ID = 1$1!small.txt, Size = 42 +- `old.txt`, ID = 1$1!old.txt, Mtime = 2020-01-01T00:00:00Z +- `known.txt`, ID = 1$1!23 +- `cased.txt`, ID = 1$1!AB-23 +- `hidden.txt`, ID = 1$1!hidden.txt, hidden +- `plain.txt`, ID = 1$1!plain.txt +- `box`, ID = 1$1!box, folder +- `boxed.txt`, ID = 1$1!boxed.txt, Path = ./box/boxed.txt +- `song.mp3`, ID = 1$1!song.mp3, MimeType = audio/mpeg + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| FIELDS-01 | `size:42` | small.txt | small.txt | small.txt | ✅ | +| FIELDS-02 | `mtime<"2021-01-01T00:00:00Z"` | old.txt | old.txt | old.txt | ✅ | +| FIELDS-03 | `id:"1$1!23"` | known.txt | known.txt | known.txt | ✅ | +| FIELDS-04 | `hidden:true` | hidden.txt | no match | hidden.txt | ❌ known | +| FIELDS-05 | `type:file` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | no match | error | ❌ known | +| FIELDS-06 | `type:folder` | box | no match | error | ❌ known | +| FIELDS-07 | `unknown:field` | no match | no match | no match | ✅ | +| FIELDS-08 | `type:File` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | no match | error | ❌ known | +| FIELDS-09 | `type:FOLDER` | box | no match | error | ❌ known | +| FIELDS-10 | `hidden:TRUE` | hidden.txt | no match | error | ❌ known | +| FIELDS-11 | `id:"1$1!AB-23"` | cased.txt | cased.txt | no match | ❌ known | +| FIELDS-12 | `id:"1$1!ab-23"` | no match | no match | no match | ✅ | +| FIELDS-13 | `audio.artist:"Some Artist"` | song.mp3 | song.mp3 | no match | ❌ known | +| FIELDS-14 | `audio.artist:"some artist"` | no match | no match | no match | ✅ | + +### deleted + +Fixtures: + +- `trashed.txt`, deleted +- `kept.txt` +- `bin`, folder, deleted +- `receipt.txt`, Path = ./bin/receipt.txt, deleted +- `shelf`, folder +- `book.txt`, Path = ./shelf/book.txt + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| DELETED-01 | `name:"*trashed*"` | no match | no match | no match | ✅ | +| DELETED-02 | `name:"*.txt"` | book.txt, kept.txt | book.txt, kept.txt | book.txt, kept.txt | ✅ | +| DELETED-03 | `name:"*receipt*"` | no match | no match | no match | ✅ | +| DELETED-04 | `path:"./bin"` | no match | no match | no match | ✅ | +| DELETED-05 | `path:"./shelf"` | book.txt, shelf | shelf | book.txt, shelf | ❌ known | + +### visibility + +Fixtures: + +- `visible.txt` +- `dotfile.txt`, hidden +- `.private`, folder, hidden +- `secret.txt`, Path = ./.private/secret.txt, hidden + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| VISIBILITY-01 | `hidden:true` | .private, dotfile.txt, secret.txt | no match | .private, dotfile.txt, secret.txt | ❌ known | +| VISIBILITY-02 | `hidden:TRUE` | .private, dotfile.txt, secret.txt | no match | error | ❌ known | +| VISIBILITY-03 | `hidden:false` | visible.txt | no match | visible.txt | ❌ known | +| VISIBILITY-04 | `name:"*secret*"` | secret.txt | secret.txt | secret.txt | ✅ | +| VISIBILITY-05 | `path:"./.private"` | .private, secret.txt | .private | .private, secret.txt | ❌ known | +| VISIBILITY-06 | `hidden:banana` | no match | no match | error | ❌ known | +| VISIBILITY-07 | `hidden:"true"` | .private, dotfile.txt, secret.txt | no match | .private, dotfile.txt, secret.txt | ❌ known | + +### boolean + +Fixtures: + +- `alpha.txt`, Tags = red +- `beta.txt`, Tags = blue +- `gamma.md`, MimeType = text/markdown + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| BOOLEAN-01 | `name:"*alpha*" AND name:"*txt*"` | alpha.txt | alpha.txt | alpha.txt | ✅ | +| BOOLEAN-02 | `name:"*alpha*" OR name:"*beta*"` | alpha.txt, beta.txt | alpha.txt, beta.txt | alpha.txt, beta.txt | ✅ | +| BOOLEAN-03 | `name:"*a*" AND NOT name:"*alpha*"` | beta.txt, gamma.md | beta.txt, gamma.md | beta.txt, gamma.md | ✅ | +| BOOLEAN-04 | `name:"*a*" AND tag:("red")` | alpha.txt | alpha.txt | alpha.txt | ✅ | +| BOOLEAN-05 | `(name:"*alpha*" OR name:"*beta*") AND mediatype:text/plain` | alpha.txt, beta.txt | alpha.txt, beta.txt | alpha.txt, beta.txt | ✅ | +| BOOLEAN-06 | `name:"*a*" AND mediatype:text/markdown` | gamma.md | gamma.md | gamma.md | ✅ | + +### samename + +Fixtures: + +- `doc`, folder +- `doc.pdf` +- `file.pdf` +- `doc.pdf`, Path = ./doc/doc.pdf +- `file.pdf`, Path = ./doc/file.pdf + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| SAMENAME-01 | `name:"*doc*"` | 3 items | doc, doc.pdf, doc.pdf | doc, doc.pdf, doc.pdf | ✅ | +| SAMENAME-02 | `name:"*doc*"` in 1$1!1 under ./doc | 2 items | doc, doc.pdf | doc, doc.pdf | ✅ | +| SAMENAME-03 | `name:"*file*"` | 2 items | file.pdf, file.pdf | file.pdf, file.pdf | ✅ | +| SAMENAME-04 | `name:"*file*"` in 1$1!1 under ./doc | 1 items | file.pdf | file.pdf | ✅ | + +### stress + +Fixtures: + +- `quarterly report.docx`, MimeType = application/vnd.openxmlformats-officedocument.wordprocessingml.document, Tags = final, Size = 2000 +- `draft report.txt`, Tags = draft, Size = 50, Mtime = 2020-01-01T00:00:00Z +- `photo.jpg`, MimeType = image/jpeg, Tags = final +- `notes.md`, MimeType = text/markdown, hidden +- `archive`, folder + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| STRESS-01 | `name:"*report*" AND mediatype:document` | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | ✅ | +| STRESS-02 | `name:"*report*" AND NOT tag:("draft")` | quarterly report.docx | draft report.txt, quarterly report.docx | quarterly report.docx | ❌ known | +| STRESS-03 | `(tag:("final") OR tag:("draft")) AND mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ | +| STRESS-04 | `mediatype:document AND mtime>"2021-01-01T00:00:00Z"` | notes.md, quarterly report.docx | notes.md, quarterly report.docx | notes.md, quarterly report.docx | ✅ | +| STRESS-05 | `name:"*report*" AND size>100` | quarterly report.docx | no match | no match | ❌ known | +| STRESS-06 | `tag:("final") AND NOT mediatype:folder` | photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | ✅ | +| STRESS-07 | `hidden:true AND name:"*notes*"` | notes.md | no match | notes.md | ❌ known | +| STRESS-08 | `name:quarterly report` | quarterly report.docx | no match | no match | ❌ known | +| STRESS-09 | `name:"quarterly report"` | no match | no match | no match | ✅ | +| STRESS-10 | `name:"quarterly report.docx"` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | +| STRESS-11 | `NOT tag:("draft")` | archive, notes.md, photo.jpg, quarterly report.docx | archive, draft report.txt, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | ❌ known | +| STRESS-12 | `tag:("final") OR hidden:true` | notes.md, photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | ❌ known | +| STRESS-13 | `(name:"*report*" OR name:"*notes..."draft") OR hidden:true)` | quarterly report.docx | notes.md, quarterly report.docx | quarterly report.docx | ❌ known | +| STRESS-14 | `mediatype:image OR (mediatype:document AND tag:("draft"))` | draft report.txt, photo.jpg | draft report.txt, photo.jpg | draft report.txt, photo.jpg | ✅ | +| STRESS-15 | `NOT (mediatype:folder OR hidden:true)` | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, notes.md, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | ❌ known | +| STRESS-16 | `name:"*report*" AND (size>100 OR tag:("draft"))` | draft report.txt, quarterly report.docx | draft report.txt | draft report.txt | ❌ known | + +### everything + +Fixtures: + +- `alpha.txt` +- `beta.txt` +- `box`, folder + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| EVERYTHING-01 | `*` | alpha.txt, beta.txt, box | alpha.txt, beta.txt, box | alpha.txt, beta.txt, box | ✅ | +| EVERYTHING-02 | `name:"*"` | alpha.txt, beta.txt, box | alpha.txt, beta.txt, box | alpha.txt, beta.txt, box | ✅ | +| EVERYTHING-03 | `*` with a limit of 2 | 2 items | alpha.txt, beta.txt | alpha.txt, beta.txt | ✅ | +| EVERYTHING-04 | `*` with a limit of -1 | 3 items | alpha.txt, beta.txt, box | alpha.txt, beta.txt, box | ✅ | + +### range + +Fixtures: + +- `small.txt`, Size = 50 +- `big.txt`, Size = 500 +- `ancient.txt`, Size = 10, Mtime = 2020-01-01T00:00:00Z + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| RANGE-01 | `size>100` | big.txt | no match | no match | ❌ known | +| RANGE-02 | `size<100` | ancient.txt, small.txt | no match | no match | ❌ known | +| RANGE-03 | `mtime>"2021-01-01T00:00:00Z"` | big.txt, small.txt | big.txt, small.txt | big.txt, small.txt | ✅ | +| RANGE-04 | `mtime<"2021-01-01T00:00:00Z"` | ancient.txt | ancient.txt | ancient.txt | ✅ | +| RANGE-05 | `Mtime:"today"` | big.txt, small.txt | big.txt, small.txt | big.txt, small.txt | ✅ | +| RANGE-06 | `Mtime:"yesterday"` | no match | no match | no match | ✅ | +| RANGE-07 | `mtime>2021` | no match | no match | no match | ✅ | +| RANGE-08 | `name>100` | no match | no match | no match | ✅ | + +### scope + +Fixtures: + +- `parent`, folder +- `child.pdf`, Path = ./parent/child.pdf +- `outside.txt` +- `elsewhere.txt` + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| SCOPE-01 | `*` | child.pdf, elsewhere.txt, outside.txt, parent | child.pdf, elsewhere.txt, outside.txt, parent | child.pdf, elsewhere.txt, outside.txt, parent | ✅ | +| SCOPE-02 | `*` in 1$1!1 | child.pdf, outside.txt, parent | child.pdf, outside.txt, parent | child.pdf, outside.txt, parent | ✅ | +| SCOPE-03 | `*` in 1$1!1 under ./parent | child.pdf, parent | child.pdf, parent | child.pdf, parent | ✅ | +| SCOPE-04 | `*` in 1$1!1 under ./parent/child.pdf | child.pdf | child.pdf | child.pdf | ✅ | +| SCOPE-05 | `name:"*elsewhere*"` in 1$1!1 | no match | no match | no match | ✅ | + +### invalid + +Fixtures: + +- `alpha.txt` + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| INVALID-01 | `AND mediatype:document` | bad request | bad request | bad request | ✅ | +| INVALID-02 | `mediatype:document AND` | alpha.txt | alpha.txt | alpha.txt | ✅ | + +## Operations + +### delete + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| DELETE-01 | takes the resource out of the results, then `name:"*parent*"` | parent | parent | parent | ✅ | +| DELETE-01 | takes the resource out of the results, then `name:"*child*"` | no match | no match | no match | ✅ | +| DELETE-02 | takes the descendants along, then `name:"*parent*"` | no match | no match | no match | ✅ | +| DELETE-02 | takes the descendants along, then `name:"*child*"` | no match | no match | no match | ✅ | +| DELETE-03 | leaves the resource in the index, then `DocCount()` | 2 | 2 | 1 | ❌ known | +| DELETE-04 | takes a resource out that was just written, then `name:"*fresh*"` | no match | no match | no match | ✅ | + +### restore + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf +- `file.txt`, ID = 1$1!5, Path = ./.secret/file.txt, hidden + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| RESTORE-01 | brings the descendants back, then `name:"*parent*"` | parent | parent | parent | ✅ | +| RESTORE-01 | brings the descendants back, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ | +| RESTORE-02 | leaves the hidden flag alone, then `hidden:true` | file.txt | no match | file.txt | ❌ known | + +### purge + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| PURGE-01 | removes one resource, then `name:"*parent*"` | parent | parent | parent | ✅ | +| PURGE-01 | removes one resource, then `name:"*child*"` | no match | no match | no match | ✅ | +| PURGE-01 | removes one resource, then `DocCount()` | 1 | 1 | 1 | ✅ | +| PURGE-02 | removes the tree, then `name:"*parent*"` | no match | no match | no match | ✅ | +| PURGE-02 | removes the tree, then `name:"*child*"` | no match | no match | no match | ✅ | +| PURGE-02 | removes the tree, then `DocCount()` | 0 | 0 | 0 | ✅ | +| PURGE-03 | takes only the deleted ones when it is told to, then `name:"*parent*"` | parent | parent | parent | ✅ | +| PURGE-03 | takes only the deleted ones when it is told to, then `name:"*child*"` | no match | no match | no match | ✅ | + +### purgespace + +Fixtures: + +- `inSpace.txt`, ID = 1$1!4 +- `elsewhere.txt`, ID = 2$2!4 +- `bulk-0.txt`, ID = 1$1!bulk-0.txt +- `bulk-1.txt`, ID = 1$1!bulk-1.txt +- `bulk-2.txt`, ID = 1$1!bulk-2.txt +- `bulk-3.txt`, ID = 1$1!bulk-3.txt +- `bulk-4.txt`, ID = 1$1!bulk-4.txt +- `bulk-5.txt`, ID = 1$1!bulk-5.txt +- `bulk-6.txt`, ID = 1$1!bulk-6.txt +- `bulk-7.txt`, ID = 1$1!bulk-7.txt +- ... and 52 more of the same + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| PURGESPACE-01 | leaves the other space alone, then `name:"*inSpace*"` | no match | no match | no match | ✅ | +| PURGESPACE-01 | leaves the other space alone, then `name:"*elsewhere*"` | elsewhere.txt | elsewhere.txt | elsewhere.txt | ✅ | +| PURGESPACE-02 | takes a space out that holds more than one round, then `name:"*bulk*"` | no match | no match | no match | ✅ | +| PURGESPACE-02 | takes a space out that holds more than one round, then `name:"*elsewhere*"` | elsewhere.txt | elsewhere.txt | elsewhere.txt | ✅ | +| PURGESPACE-02 | takes a space out that holds more than one round, then `DocCount()` | 1 | 1 | 1 | ✅ | + +### move + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| MOVE-01 | carries the descendants to the new path, then `path:"./my/newname/child.pdf"` | child.pdf | child.pdf | child.pdf | ✅ | +| MOVE-01 | carries the descendants to the new path, then `path:"./parent/child.pdf"` | no match | no match | no match | ✅ | +| MOVE-02 | through the trash and back leaves the flag behind, then `hidden:true` | no match | no match | no match | ✅ | + +### rootscope + +Fixtures: + +- `target.txt`, ID = 1$1!3, Path = ./same/path.txt +- `twin.txt`, ID = 2$2!3, Path = ./same/path.txt +- `target.txt`, ID = 1$1!3, Path = ./same/path.txt, deleted +- `twin.txt`, ID = 2$2!3, Path = ./same/path.txt, deleted + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| ROOTSCOPE-01 | deletes only the one in the target root, then `name:"*target*"` | no match | no match | no match | ✅ | +| ROOTSCOPE-01 | deletes only the one in the target root, then `name:"*twin*"` | twin.txt | twin.txt | twin.txt | ✅ | +| ROOTSCOPE-02 | restores only the one in the target root, then `name:"*target*"` | target.txt | target.txt | target.txt | ✅ | +| ROOTSCOPE-02 | restores only the one in the target root, then `name:"*twin*"` | no match | no match | no match | ✅ | +| ROOTSCOPE-03 | moves only the one in the target root, then `path:"./moved.txt"` | moved.txt | moved.txt | moved.txt | ✅ | +| ROOTSCOPE-03 | moves only the one in the target root, then `path:"./same/path.txt"` | twin.txt | twin.txt | twin.txt | ✅ | + +### casepath + +Fixtures: + +- `Documents`, ID = 1$1!2, folder +- `Picture.jpg`, ID = 1$1!3, MimeType = image/jpeg, Path = ./Documents/Picture.jpg + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| CASEPATH-01 | takes the descendants along when deleting, then `path:"./Documents"` | no match | no match | Documents, Picture.jpg | ❌ known | +| CASEPATH-02 | takes the descendants along when moving, then `path:"./Other Documents"` | Other Documents, Picture.jpg | Other Documents | no match | ❌ known | +| CASEPATH-02 | takes the descendants along when moving, then `path:"./Documents"` | no match | no match | Documents, Picture.jpg | ❌ known | +| CASEPATH-03 | reaches the descendants when purging, then `path:"./Documents"` | no match | no match | Documents, Picture.jpg | ❌ known | +| CASEPATH-03 | reaches the descendants when purging, then `DocCount()` | 0 | 0 | 2 | ❌ known | + +### hidden + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf +- `parent`, ID = 1$1!2, folder, Path = ./.trash/parent, hidden +- `child.pdf`, ID = 1$1!3, Path = ./.trash/parent/child.pdf, hidden +- `parent`, ID = 1$1!2, folder, Path = ./.parent, hidden +- `child.pdf`, ID = 1$1!3, Path = ./.parent/child.pdf, hidden + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| HIDDEN-01 | follows a move into a dot folder, then `hidden:true` | child.pdf, parent | no match | child.pdf, parent | ❌ known | +| HIDDEN-02 | follows a move into a plain folder, then `hidden:true` | no match | no match | no match | ✅ | +| HIDDEN-03 | follows a move renamed with a leading dot, then `hidden:true` | .parent, child.pdf | no match | .parent, child.pdf | ❌ known | +| HIDDEN-04 | follows a move out of a dot folder, then `hidden:true` | no match | no match | no match | ✅ | +| HIDDEN-05 | follows a move renamed without the leading dot, then `hidden:true` | no match | no match | no match | ✅ | +| HIDDEN-06 | follows a move within the same dot folder, then `hidden:true` | child.pdf, moved | no match | child.pdf, moved | ❌ known | + +### upsert + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| UPSERT-01 | replaces the resource it already knows, then `name:"*parent*"` | parent | parent | parent | ✅ | +| UPSERT-01 | replaces the resource it already knows, then `name:"*child*"` | no match | no match | no match | ✅ | +| UPSERT-01 | replaces the resource it already knows, then `name:"*renamed*"` | renamed.pdf | renamed.pdf | renamed.pdf | ✅ | +| UPSERT-01 | replaces the resource it already knows, then `DocCount()` | 2 | 2 | 2 | ✅ | + +### idempotency + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| IDEMPOTENCY-01 | deleting the same resource twice is not an error, then `name:"*parent*"` | parent | parent | parent | ✅ | +| IDEMPOTENCY-01 | deleting the same resource twice is not an error, then `name:"*child*"` | no match | no match | no match | ✅ | +| IDEMPOTENCY-02 | deleting a resource the index does not have reports it, then `name:"*parent*"` | parent | parent | parent | ✅ | +| IDEMPOTENCY-02 | deleting a resource the index does not have reports it, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ | +| IDEMPOTENCY-03 | restoring a resource that was never deleted leaves it alone, then `name:"*parent*"` | parent | parent | parent | ✅ | +| IDEMPOTENCY-03 | restoring a resource that was never deleted leaves it alone, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ | +| IDEMPOTENCY-04 | purging the same resource twice reports the second one, then `name:"*parent*"` | parent | parent | parent | ✅ | +| IDEMPOTENCY-04 | purging the same resource twice reports the second one, then `name:"*child*"` | no match | no match | no match | ✅ | +| IDEMPOTENCY-04 | purging the same resource twice reports the second one, then `DocCount()` | 1 | 1 | 1 | ✅ | +| IDEMPOTENCY-05 | purging a resource the index does not have reports it, then `name:"*parent*"` | parent | parent | parent | ✅ | +| IDEMPOTENCY-05 | purging a resource the index does not have reports it, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ | +| IDEMPOTENCY-05 | purging a resource the index does not have reports it, then `DocCount()` | 2 | 2 | 2 | ✅ | +| IDEMPOTENCY-06 | moving a resource onto its own path leaves it where it is, then `path:"./parent/child.pdf"` | child.pdf | child.pdf | child.pdf | ✅ | +| IDEMPOTENCY-06 | moving a resource onto its own path leaves it where it is, then `name:"*parent*"` | parent | parent | parent | ✅ | +| IDEMPOTENCY-07 | purging a whole space that holds nothing is not an error, then `name:"*parent*"` | parent | parent | parent | ✅ | +| IDEMPOTENCY-07 | purging a whole space that holds nothing is not an error, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ | +| IDEMPOTENCY-07 | purging a whole space that holds nothing is not an error, then `DocCount()` | 2 | 2 | 2 | ✅ | + +### batch + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| BATCH-01 | holds what it was given until it is pushed, then `name:"*added*"` | no match | no match | no match | ✅ | +| BATCH-02 | writes what it was given once it is pushed, then `name:"*added*"` | added.pdf | added.pdf | added.pdf | ✅ | +| BATCH-02 | writes what it was given once it is pushed, then `DocCount()` | 3 | 3 | 3 | ✅ | +| BATCH-03 | takes a resource out the same way a delete does, then `name:"*parent*"` | parent | parent | parent | ✅ | +| BATCH-03 | takes a resource out the same way a delete does, then `name:"*child*"` | no match | no match | no match | ✅ | +| BATCH-04 | moves a resource the same way a move does, then `path:"./my/newname/child.pdf"` | child.pdf | child.pdf | child.pdf | ✅ | +| BATCH-04 | moves a resource the same way a move does, then `path:"./parent/child.pdf"` | no match | no match | no match | ✅ | +| BATCH-05 | keeps what another batch holds out of its push, then `name:"*added*"` | added.pdf | added.pdf | added.pdf | ✅ | +| BATCH-05 | keeps what another batch holds out of its push, then `name:"*other*"` | no match | no match | no match | ✅ | + +## Response + +### entity + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `bar.pdf`, ID = 1$1!3, MimeType = application/pdf, Path = ./parent/bar.pdf, Size = 1234 +- `notes.txt`, ID = 1$1!4, Content = "foo bar baz" + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| ENTITY-01 | `name:"bar.pdf"` reads `Ref.Path` | ./parent/bar.pdf | ./parent/bar.pdf | ./parent/bar.pdf | ✅ | +| ENTITY-02 | `name:"bar.pdf"` reads `Name` | bar.pdf | bar.pdf | bar.pdf | ✅ | +| ENTITY-03 | `name:"bar.pdf"` reads `Id` | 1$1!3 | 1$1!3 | 1$1!3 | ✅ | +| ENTITY-04 | `name:"bar.pdf"` reads `ParentId` | 1$1!2 | 1$1!2 | 1$1!2 | ✅ | +| ENTITY-05 | `name:"bar.pdf"` reads `Ref.ResourceId` | 1$1!1 | 1$1!1 | 1$1!1 | ✅ | +| ENTITY-06 | `name:"bar.pdf"` reads `Size` | 1234 | 1234 | 1234 | ✅ | +| ENTITY-07 | `name:"bar.pdf"` reads `Type` | 1 | 1 | 1 | ✅ | +| ENTITY-08 | `name:"bar.pdf"` reads `MimeType` | application/pdf | application/pdf | application/pdf | ✅ | +| ENTITY-09 | `name:"bar.pdf"` reads `Deleted` | false | false | false | ✅ | +| ENTITY-10 | `name:"bar.pdf"` reads `Score` | above zero | above zero | above zero | ✅ | +| ENTITY-11 | `path:"./parent"` reads `TotalMatches` | 2 | 1 | 2 | ❌ known | +| ENTITY-12 | `name:"*notes*"` reads `Highlights` | "" | "" | "" | ✅ | +| ENTITY-13 | `content:bar` reads `Highlights` | foo bar baz | foo bar baz | foo bar baz | ✅ | +| ENTITY-14 | moved to another parent, then `name:"newname"` reads `ParentId` | 1$1!9 | 1$1!9 | 1$1!9 | ✅ | +| ENTITY-15 | moved to another parent, then `name:"bar.pdf"` reads `ParentId` | 1$1!2 | 1$1!2 | 1$1!2 | ✅ | +| ENTITY-16 | moved to another parent, then `name:"bar.pdf"` reads `Ref.Path` | ./somewher.../bar.pdf | ./somewher.../bar.pdf | ./somewher.../bar.pdf | ✅ | + +### metadata + +Fixtures: + +- `some_song.mp3`, ID = 1$1!5, MimeType = audio/mpeg +- `team.jpg`, ID = 1$1!6, MimeType = image/jpeg + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| METADATA-01 | `*song*` reads `Audio` | all 16 fields unchanged | all 16 fields unchanged | all 16 fields unchanged | ✅ | +| METADATA-02 | `*team*` reads `Location` | all 3 fields unchanged | all 3 fields unchanged | all 3 fields unchanged | ✅ | +| METADATA-03 | `*team*` reads `Audio` | none | none | none | ✅ | diff --git a/services/search/pkg/parity/engines_test.go b/services/search/pkg/parity/engines_test.go new file mode 100644 index 0000000000..a77fb61008 --- /dev/null +++ b/services/search/pkg/parity/engines_test.go @@ -0,0 +1,194 @@ +package parity + +import ( + "context" + "fmt" + "strings" + + bleveSearch "github.com/blevesearch/bleve/v2" + sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/types" + "github.com/opencloud-eu/reva/v2/pkg/storagespace" + opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" + + "github.com/opencloud-eu/opencloud/pkg/log" + searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + bleveEngine "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +var engineNames = []string{"bleve", "opensearch"} + +type testEngine struct { + name string + backend search.Engine + settle func() + unavailable string +} + +// newEngines builds one backend per engine over the same fixtures. Call it +// from a BeforeAll: the cleanups it defers run when the container ends. +func newEngines(index string, fixtures []search.Resource) []testEngine { + GinkgoHelper() + + return []testEngine{ + newBleve(fixtures), + newOpenSearch(index, fixtures), + } +} + +// newEngine builds just the one engine a spec is about; its cleanup runs when +// that spec ends. +func newEngine(name, index string, fixtures []search.Resource) testEngine { + GinkgoHelper() + + switch name { + case "bleve": + return newBleve(fixtures) + case "opensearch": + return newOpenSearch(index, fixtures) + } + + Fail("no engine named " + name) + return testEngine{} +} + +func engineNamed(engines []testEngine, name string) testEngine { + GinkgoHelper() + + for _, e := range engines { + if e.name == name { + return e + } + } + + Fail("no engine named " + name) + return testEngine{} +} + +func newBleve(fixtures []search.Resource) testEngine { + GinkgoHelper() + + mapping, err := bleveEngine.NewMapping() + Expect(err).NotTo(HaveOccurred(), "failed to build the bleve mapping") + + idx, err := bleveSearch.NewMemOnly(mapping) + Expect(err).NotTo(HaveOccurred(), "failed to create the bleve index") + DeferCleanup(func() { _ = idx.Close() }) + + backend := bleveEngine.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}) + load(backend, fixtures) + + return testEngine{name: "bleve", backend: backend, settle: func() {}} +} + +func newOpenSearch(index string, fixtures []search.Resource) testEngine { + GinkgoHelper() + + tc := opensearchtest.NewDefaultTestClient(GinkgoTB(), openSearchClient) + + if err := tc.IndicesReset(context.Background(), []string{index}); err != nil { + return testEngine{name: "opensearch", unavailable: err.Error()} + } + + backend, err := opensearch.NewBackend(index, tc.Client()) + if err != nil { + return testEngine{name: "opensearch", unavailable: err.Error()} + } + + DeferCleanup(func() { + Expect(tc.IndicesDelete(context.Background(), []string{index})).To(Succeed()) + }) + + // every write waits for the next refresh (Refresh: wait_for), which is a + // second apart by default; the test index can refresh far more often + _, err = tc.Client().Indices.Settings.Put(context.Background(), opensearchgoAPI.SettingsPutReq{ + Indices: []string{index}, + Body: strings.NewReader(`{"index": {"refresh_interval": "50ms"}}`), + }) + Expect(err).NotTo(HaveOccurred(), "failed to speed up the refresh of %s", index) + + settle := func() { tc.Require.IndicesRefresh([]string{index}, nil) } + load(backend, fixtures) + settle() + + return testEngine{name: "opensearch", backend: backend, settle: settle} +} + +// load writes the fixtures the way the service does, in one batch. +func load(backend search.Engine, fixtures []search.Resource) { + GinkgoHelper() + + if len(fixtures) == 0 { + return + } + + batch, err := backend.NewBatch(len(fixtures) + 1) + Expect(err).NotTo(HaveOccurred(), "failed to open a batch for the fixtures") + for _, doc := range fixtures { + Expect(batch.Upsert(doc.ID, doc)).To(Succeed(), "upsert %s", doc.ID) + } + Expect(batch.Push()).To(Succeed(), "failed to write the fixtures") +} + +func badRequestAnswer(err error) []string { + if err == nil { + return []string{"no error"} + } + + return []string{"bad request"} +} + +func reads(read func(*searchMessage.Match) string) func(*searchService.SearchIndexResponse) []string { + return readsMany(func(m *searchMessage.Match) []string { return []string{read(m)} }) +} + +func readsMany(read func(*searchMessage.Match) []string) func(*searchService.SearchIndexResponse) []string { + return func(resp *searchService.SearchIndexResponse) []string { + if len(resp.Matches) != 1 { + return []string{fmt.Sprintf("%d matches", len(resp.Matches))} + } + + return read(resp.Matches[0]) + } +} + +func resourceID(id *searchMessage.ResourceID) string { + return storagespace.FormatResourceID(&sprovider.ResourceId{ + StorageId: id.GetStorageId(), + SpaceId: id.GetSpaceId(), + OpaqueId: id.GetOpaqueId(), + }) +} + +// ask runs a query and returns the matched names; an engine error is the +// answer "error", so a query the engine rejects still shows in the matrix. +func ask(e search.Engine, request *searchService.SearchIndexRequest) ([]string, error) { + resp, err := e.Search(context.Background(), request) + if err != nil { + return []string{"error"}, err + } + + names := make([]string, 0, len(resp.Matches)) + for _, m := range resp.Matches { + names = append(names, m.Entity.GetName()) + } + + return names, nil +} + +// matchNames is ConsistOf for a wanted list, spelled out for the empty case so +// a "no match" expectation reads as such in the failure. +func matchNames(want []string) types.GomegaMatcher { + if len(want) == 0 { + return BeEmpty() + } + + return ConsistOf(want) +} diff --git a/services/search/pkg/parity/fixtures_test.go b/services/search/pkg/parity/fixtures_test.go new file mode 100644 index 0000000000..4bfebd5e8c --- /dev/null +++ b/services/search/pkg/parity/fixtures_test.go @@ -0,0 +1,108 @@ +package parity + +import ( + "fmt" + "slices" + "strings" + "time" + + sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + libregraph "github.com/opencloud-eu/libre-graph-api-go" + + "github.com/opencloud-eu/opencloud/services/search/pkg/content" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +var fixtureLongName = strings.Repeat("a", 260) + "needle.txt" + +var fixtureNow = time.Now().UTC() + +type fixtureOption func(*search.Resource) + +func withPath(path string) fixtureOption { return func(r *search.Resource) { r.Path = path } } +func withMime(mime string) fixtureOption { return func(r *search.Resource) { r.MimeType = mime } } +func withTitle(t string) fixtureOption { return func(r *search.Resource) { r.Title = t } } +func withContent(c string) fixtureOption { return func(r *search.Resource) { r.Content = c } } +func withSize(s uint64) fixtureOption { return func(r *search.Resource) { r.Size = s } } +func withMtime(m string) fixtureOption { return func(r *search.Resource) { r.Mtime = m } } +func withID(id string) fixtureOption { return func(r *search.Resource) { r.ID = id } } +func withParent(id string) fixtureOption { return func(r *search.Resource) { r.ParentID = id } } +func withRoot(id string) fixtureOption { return func(r *search.Resource) { r.RootID = id } } +func isHidden() fixtureOption { return func(r *search.Resource) { r.Hidden = true } } +func isDeleted() fixtureOption { return func(r *search.Resource) { r.Deleted = true } } +func withTags(tags ...string) fixtureOption { + return func(r *search.Resource) { r.Tags = tags } +} + +func withFavorite(userID string) fixtureOption { + return func(r *search.Resource) { r.Favorites = []string{userID} } +} + +func withAudio(audio *libregraph.Audio) fixtureOption { + return func(r *search.Resource) { r.Audio = audio } +} + +func withLocation(location *libregraph.GeoCoordinates) fixtureOption { + return func(r *search.Resource) { r.Location = location } +} + +func fixtureDoc(name string, opts ...fixtureOption) search.Resource { + r := search.Resource{ + ID: "1$1!" + name, + RootID: "1$1!1", + ParentID: "1$1!1", + Path: "./" + name, + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{ + Name: name, + MimeType: "text/plain", + Mtime: fixtureNow.Format(time.RFC3339Nano), + Size: 1000, + }, + } + + for _, opt := range opts { + opt(&r) + } + + return r +} + +func fixtureFolder(name string, opts ...fixtureOption) search.Resource { + folder := []fixtureOption{withMime("httpd/unix-directory")} + r := fixtureDoc(name, append(folder, opts...)...) + r.Type = uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER) + + return r +} + +func fixtureBulk(count int) []search.Resource { + docs := make([]search.Resource, 0, count) + for i := range count { + docs = append(docs, fixtureDoc(fmt.Sprintf("bulk-%d.txt", i))) + } + + return docs +} + +func fixtureTree() (parent, child search.Resource) { + parent = fixtureFolder("parent", withID("1$1!2")) + child = fixtureDoc("child.pdf", withID("1$1!3"), withParent(parent.ID), withPath("./parent/child.pdf")) + + return parent, child +} + +func treeIsLeft(names ...string) []expectation { + left := func(name string) []string { + if slices.Contains(names, name) { + return []string{name} + } + + return nil + } + + return []expectation{ + {`name:"*parent*"`, left("parent")}, + {`name:"*child*"`, left("child.pdf")}, + } +} diff --git a/services/search/pkg/parity/lifecycle_batch_test.go b/services/search/pkg/parity/lifecycle_batch_test.go new file mode 100644 index 0000000000..eb718714db --- /dev/null +++ b/services/search/pkg/parity/lifecycle_batch_test.go @@ -0,0 +1,112 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func batchLifecycle() lifecycleGroup { + parent, child := fixtureTree() + + added := fixtureDoc("added.pdf", withID("1$1!7"), withParent(parent.ID), withPath("./parent/added.pdf")) + other := fixtureDoc("other.pdf", withID("1$1!8"), withParent(parent.ID), withPath("./parent/other.pdf")) + + return lifecycleGroup{ + name: "batch", + fixtures: []search.Resource{parent, child}, + cases: []lifecycleCase{ + { + id: 1, title: "holds what it was given until it is pushed", + do: func(e search.Engine) error { + batch, err := e.NewBatch(100) + if err != nil { + return err + } + + return batch.Upsert(added.ID, added) + }, + expect: []expectation{{`name:"*added*"`, nil}}, + }, + { + id: 2, title: "writes what it was given once it is pushed", + do: func(e search.Engine) error { + batch, err := e.NewBatch(100) + if err != nil { + return err + } + + if err := batch.Upsert(added.ID, added); err != nil { + return err + } + + return batch.Push() + }, + expect: []expectation{{`name:"*added*"`, []string{"added.pdf"}}}, + wantDocCount: conversions.ToPointer(uint64(3)), + }, + { + id: 3, title: "takes a resource out the same way a delete does", + do: func(e search.Engine) error { + batch, err := e.NewBatch(100) + if err != nil { + return err + } + + if err := batch.Delete(child.ID); err != nil { + return err + } + + return batch.Push() + }, + expect: treeIsLeft("parent"), + }, + { + id: 4, title: "moves a resource the same way a move does", + do: func(e search.Engine) error { + batch, err := e.NewBatch(100) + if err != nil { + return err + } + + if err := batch.Move(parent.ID, parent.ParentID, "./my/newname"); err != nil { + return err + } + + return batch.Push() + }, + expect: []expectation{ + {`path:"./my/newname/child.pdf"`, []string{"child.pdf"}}, + {`path:"./parent/child.pdf"`, nil}, + }, + }, + { + id: 5, title: "keeps what another batch holds out of its push", + do: func(e search.Engine) error { + first, err := e.NewBatch(100) + if err != nil { + return err + } + + second, err := e.NewBatch(100) + if err != nil { + return err + } + + if err := first.Upsert(added.ID, added); err != nil { + return err + } + + if err := second.Upsert(other.ID, other); err != nil { + return err + } + + return first.Push() + }, + expect: []expectation{ + {`name:"*added*"`, []string{"added.pdf"}}, + {`name:"*other*"`, nil}, + }, + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_casepath_test.go b/services/search/pkg/parity/lifecycle_casepath_test.go new file mode 100644 index 0000000000..8347fcf318 --- /dev/null +++ b/services/search/pkg/parity/lifecycle_casepath_test.go @@ -0,0 +1,61 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func casePathLifecycle() lifecycleGroup { + folder := fixtureFolder("Documents", withID("1$1!2"), withPath("./Documents")) + picture := fixtureDoc("Picture.jpg", + withID("1$1!3"), + withParent(folder.ID), + withMime("image/jpeg"), + withPath("./Documents/Picture.jpg"), + ) + + left := func(names ...string) []expectation { + return []expectation{ + {`path:"./Documents"`, names}, + } + } + + return lifecycleGroup{ + name: "casepath", + fixtures: []search.Resource{folder, picture}, + cases: []lifecycleCase{ + { + id: 1, title: "takes the descendants along when deleting", + do: func(e search.Engine) error { return e.Delete(folder.ID) }, + expect: left(), + engineOverrides: map[string]lifecycleOverride{ + "opensearch": {expect: map[string][]string{`path:"./Documents"`: {"Documents", "Picture.jpg"}}}, + }, + }, + { + id: 2, title: "takes the descendants along when moving", + do: func(e search.Engine) error { return e.Move(folder.ID, folder.ParentID, "./Other Documents") }, + expect: []expectation{ + {`path:"./Other Documents"`, []string{"Other Documents", "Picture.jpg"}}, + {`path:"./Documents"`, nil}, + }, + engineOverrides: map[string]lifecycleOverride{ + "bleve": {expect: map[string][]string{`path:"./Other Documents"`: {"Other Documents"}}}, + "opensearch": {expect: map[string][]string{`path:"./Other Documents"`: {}, `path:"./Documents"`: {"Documents", "Picture.jpg"}}}, + }, + }, + { + id: 3, title: "reaches the descendants when purging", + do: func(e search.Engine) error { return e.Purge(folder.ID, false) }, + expect: left(), + wantDocCount: conversions.ToPointer(uint64(0)), + engineOverrides: map[string]lifecycleOverride{ + "opensearch": { + expect: map[string][]string{`path:"./Documents"`: {"Documents", "Picture.jpg"}}, + wantDocCount: conversions.ToPointer(uint64(2)), + }, + }, + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_delete_test.go b/services/search/pkg/parity/lifecycle_delete_test.go new file mode 100644 index 0000000000..c483deb901 --- /dev/null +++ b/services/search/pkg/parity/lifecycle_delete_test.go @@ -0,0 +1,47 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func deleteLifecycle() lifecycleGroup { + parent, child := fixtureTree() + + return lifecycleGroup{ + name: "delete", + fixtures: []search.Resource{parent, child}, + cases: []lifecycleCase{ + { + id: 1, title: "takes the resource out of the results", + do: func(e search.Engine) error { return e.Delete(child.ID) }, + expect: treeIsLeft("parent"), + }, + { + id: 2, title: "takes the descendants along", + do: func(e search.Engine) error { return e.Delete(parent.ID) }, + expect: treeIsLeft(), + }, + { + id: 3, title: "leaves the resource in the index", + do: func(e search.Engine) error { return e.Delete(child.ID) }, + wantDocCount: conversions.ToPointer(uint64(2)), + engineOverrides: map[string]lifecycleOverride{ + "opensearch": {wantDocCount: conversions.ToPointer(uint64(1))}, + }, + }, + { + id: 4, title: "takes a resource out that was just written", + do: func(e search.Engine) error { + fresh := fixtureDoc("fresh.txt", withID("1$1!8")) + if err := e.Upsert(fresh.ID, fresh); err != nil { + return err + } + + return e.Delete(fresh.ID) + }, + expect: []expectation{{`name:"*fresh*"`, nil}}, + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_hidden_test.go b/services/search/pkg/parity/lifecycle_hidden_test.go new file mode 100644 index 0000000000..59ec242088 --- /dev/null +++ b/services/search/pkg/parity/lifecycle_hidden_test.go @@ -0,0 +1,54 @@ +package parity + +import ( + "path" + + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func hiddenLifecycle() lifecycleGroup { + group := lifecycleGroup{name: "hidden"} + + for _, move := range []struct { + id int + title string + from string + target string + hidden bool + }{ + {1, "into a dot folder", "./parent", "./.trash/parent", true}, + {2, "into a plain folder", "./parent", "./archive/parent", false}, + {3, "renamed with a leading dot", "./parent", "./.parent", true}, + {4, "out of a dot folder", "./.trash/parent", "./archive/parent", false}, + {5, "renamed without the leading dot", "./.parent", "./parent", false}, + {6, "within the same dot folder", "./.trash/parent", "./.trash/moved", true}, + } { + parent, child := fixtureTree() + parent.Path = move.from + parent.Hidden = search.IsHidden(move.from) + child.Path = move.from + "/child.pdf" + child.Hidden = parent.Hidden + + var ( + hidden []string + overrides map[string]lifecycleOverride + ) + if move.hidden { + hidden = []string{path.Base(move.target), "child.pdf"} + // bleve does not answer hidden:true at all today + overrides = map[string]lifecycleOverride{"bleve": {expect: map[string][]string{`hidden:true`: {}}}} + } + + group.cases = append(group.cases, lifecycleCase{ + id: move.id, title: "follows a move " + move.title, + fixtures: []search.Resource{parent, child}, + do: func(e search.Engine) error { + return e.Move(parent.ID, parent.ParentID, move.target) + }, + expect: []expectation{{`hidden:true`, hidden}}, + engineOverrides: overrides, + }) + } + + return group +} diff --git a/services/search/pkg/parity/lifecycle_idempotency_test.go b/services/search/pkg/parity/lifecycle_idempotency_test.go new file mode 100644 index 0000000000..73b32d86f3 --- /dev/null +++ b/services/search/pkg/parity/lifecycle_idempotency_test.go @@ -0,0 +1,76 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func idempotencyLifecycle() lifecycleGroup { + parent, child := fixtureTree() + missing := "1$1!gone" + + return lifecycleGroup{ + name: "idempotency", + fixtures: []search.Resource{parent, child}, + cases: []lifecycleCase{ + { + id: 1, title: "deleting the same resource twice is not an error", + do: func(e search.Engine) error { + if err := e.Delete(child.ID); err != nil { + return err + } + + return e.Delete(child.ID) + }, + expect: treeIsLeft("parent"), + }, + { + id: 2, title: "deleting a resource the index does not have reports it", + do: func(e search.Engine) error { return e.Delete(missing) }, + expect: treeIsLeft("parent", "child.pdf"), + wantErr: true, + }, + { + id: 3, title: "restoring a resource that was never deleted leaves it alone", + do: func(e search.Engine) error { return e.Restore(child.ID) }, + expect: treeIsLeft("parent", "child.pdf"), + }, + { + id: 4, title: "purging the same resource twice reports the second one", + do: func(e search.Engine) error { + if err := e.Purge(child.ID, false); err != nil { + return err + } + + return e.Purge(child.ID, false) + }, + expect: treeIsLeft("parent"), + wantDocCount: conversions.ToPointer(uint64(1)), + wantErr: true, + }, + { + id: 5, title: "purging a resource the index does not have reports it", + do: func(e search.Engine) error { return e.Purge(missing, false) }, + expect: treeIsLeft("parent", "child.pdf"), + wantDocCount: conversions.ToPointer(uint64(2)), + wantErr: true, + }, + { + id: 6, title: "moving a resource onto its own path leaves it where it is", + do: func(e search.Engine) error { + return e.Move(parent.ID, parent.ParentID, parent.Path) + }, + expect: []expectation{ + {`path:"./parent/child.pdf"`, []string{"child.pdf"}}, + {`name:"*parent*"`, []string{"parent"}}, + }, + }, + { + id: 7, title: "purging a whole space that holds nothing is not an error", + do: func(e search.Engine) error { return e.PurgeSpace("9$9!9") }, + expect: treeIsLeft("parent", "child.pdf"), + wantDocCount: conversions.ToPointer(uint64(2)), + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_move_test.go b/services/search/pkg/parity/lifecycle_move_test.go new file mode 100644 index 0000000000..eb54fd16b6 --- /dev/null +++ b/services/search/pkg/parity/lifecycle_move_test.go @@ -0,0 +1,37 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func moveLifecycle() lifecycleGroup { + parent, child := fixtureTree() + + return lifecycleGroup{ + name: "move", + fixtures: []search.Resource{parent, child}, + cases: []lifecycleCase{ + { + id: 1, title: "carries the descendants to the new path", + do: func(e search.Engine) error { + return e.Move(parent.ID, parent.ParentID, "./my/newname") + }, + expect: []expectation{ + {`path:"./my/newname/child.pdf"`, []string{"child.pdf"}}, + {`path:"./parent/child.pdf"`, nil}, + }, + }, + { + id: 2, title: "through the trash and back leaves the flag behind", + do: func(e search.Engine) error { + if err := e.Move(parent.ID, parent.ParentID, "./.trash/parent"); err != nil { + return err + } + + return e.Move(parent.ID, parent.ParentID, "./parent") + }, + expect: []expectation{{`hidden:true`, nil}}, + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_purge_test.go b/services/search/pkg/parity/lifecycle_purge_test.go new file mode 100644 index 0000000000..841d02c53d --- /dev/null +++ b/services/search/pkg/parity/lifecycle_purge_test.go @@ -0,0 +1,40 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func purgeLifecycle() lifecycleGroup { + parent, child := fixtureTree() + + return lifecycleGroup{ + name: "purge", + fixtures: []search.Resource{parent, child}, + cases: []lifecycleCase{ + { + id: 1, title: "removes one resource", + do: func(e search.Engine) error { return e.Purge(child.ID, false) }, + expect: treeIsLeft("parent"), + wantDocCount: conversions.ToPointer(uint64(1)), + }, + { + id: 2, title: "removes the tree", + do: func(e search.Engine) error { return e.Purge(parent.ID, false) }, + expect: treeIsLeft(), + wantDocCount: conversions.ToPointer(uint64(0)), + }, + { + id: 3, title: "takes only the deleted ones when it is told to", + do: func(e search.Engine) error { + if err := e.Delete(child.ID); err != nil { + return err + } + + return e.Purge(parent.ID, true) + }, + expect: treeIsLeft("parent"), + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_purgespace_test.go b/services/search/pkg/parity/lifecycle_purgespace_test.go new file mode 100644 index 0000000000..543de75b8d --- /dev/null +++ b/services/search/pkg/parity/lifecycle_purgespace_test.go @@ -0,0 +1,36 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func purgeSpaceLifecycle() lifecycleGroup { + inSpace := fixtureDoc("inSpace.txt", withID("1$1!4")) + elsewhere := fixtureDoc("elsewhere.txt", withID("2$2!4"), withRoot("2$2!1")) + + return lifecycleGroup{ + name: "purgespace", + fixtures: []search.Resource{inSpace, elsewhere}, + cases: []lifecycleCase{ + { + id: 1, title: "leaves the other space alone", + do: func(e search.Engine) error { return e.PurgeSpace(inSpace.RootID) }, + expect: []expectation{ + {`name:"*inSpace*"`, nil}, + {`name:"*elsewhere*"`, []string{"elsewhere.txt"}}, + }, + }, + { + id: 2, title: "takes a space out that holds more than one round", + fixtures: append(fixtureBulk(60), elsewhere), + do: func(e search.Engine) error { return e.PurgeSpace(inSpace.RootID) }, + expect: []expectation{ + {`name:"*bulk*"`, nil}, + {`name:"*elsewhere*"`, []string{"elsewhere.txt"}}, + }, + wantDocCount: conversions.ToPointer(uint64(1)), + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_restore_test.go b/services/search/pkg/parity/lifecycle_restore_test.go new file mode 100644 index 0000000000..79dd3e3f02 --- /dev/null +++ b/services/search/pkg/parity/lifecycle_restore_test.go @@ -0,0 +1,41 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func restoreLifecycle() lifecycleGroup { + parent, child := fixtureTree() + secret := fixtureDoc("file.txt", withID("1$1!5"), withPath("./.secret/file.txt"), isHidden()) + + return lifecycleGroup{ + name: "restore", + fixtures: []search.Resource{parent, child}, + cases: []lifecycleCase{ + { + id: 1, title: "brings the descendants back", + do: func(e search.Engine) error { + if err := e.Delete(parent.ID); err != nil { + return err + } + + return e.Restore(parent.ID) + }, + expect: treeIsLeft("parent", "child.pdf"), + }, + { + id: 2, title: "leaves the hidden flag alone", + fixtures: []search.Resource{secret}, + do: func(e search.Engine) error { + if err := e.Delete(secret.ID); err != nil { + return err + } + + return e.Restore(secret.ID) + }, + expect: []expectation{{`hidden:true`, []string{"file.txt"}}}, + engineOverrides: map[string]lifecycleOverride{"bleve": {expect: map[string][]string{`hidden:true`: {}}}}, + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_rootscope_test.go b/services/search/pkg/parity/lifecycle_rootscope_test.go new file mode 100644 index 0000000000..163df7d51f --- /dev/null +++ b/services/search/pkg/parity/lifecycle_rootscope_test.go @@ -0,0 +1,50 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func rootScopeLifecycle() lifecycleGroup { + const shared = "./same/path.txt" + + target := fixtureDoc("target.txt", withID("1$1!3"), withPath(shared)) + twin := fixtureDoc("twin.txt", withID("2$2!3"), withRoot("2$2!1"), withParent("2$2!2"), withPath(shared)) + + deleted := func(r search.Resource) search.Resource { + r.Deleted = true + + return r + } + + return lifecycleGroup{ + name: "rootscope", + fixtures: []search.Resource{target, twin}, + cases: []lifecycleCase{ + { + id: 1, title: "deletes only the one in the target root", + do: func(e search.Engine) error { return e.Delete(target.ID) }, + expect: []expectation{ + {`name:"*target*"`, nil}, + {`name:"*twin*"`, []string{"twin.txt"}}, + }, + }, + { + id: 2, title: "restores only the one in the target root", + fixtures: []search.Resource{deleted(target), deleted(twin)}, + do: func(e search.Engine) error { return e.Restore(target.ID) }, + expect: []expectation{ + {`name:"*target*"`, []string{"target.txt"}}, + {`name:"*twin*"`, nil}, + }, + }, + { + id: 3, title: "moves only the one in the target root", + do: func(e search.Engine) error { return e.Move(target.ID, target.ParentID, "./moved.txt") }, + expect: []expectation{ + {`path:"./moved.txt"`, []string{"moved.txt"}}, + {`path:"` + shared + `"`, []string{"twin.txt"}}, + }, + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_upsert_test.go b/services/search/pkg/parity/lifecycle_upsert_test.go new file mode 100644 index 0000000000..d867d1dc1d --- /dev/null +++ b/services/search/pkg/parity/lifecycle_upsert_test.go @@ -0,0 +1,27 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func upsertLifecycle() lifecycleGroup { + parent, child := fixtureTree() + + renamed := child + renamed.Name = "renamed.pdf" + renamed.Path = "./parent/renamed.pdf" + + return lifecycleGroup{ + name: "upsert", + fixtures: []search.Resource{parent, child}, + cases: []lifecycleCase{ + { + id: 1, title: "replaces the resource it already knows", + do: func(e search.Engine) error { return e.Upsert(renamed.ID, renamed) }, + expect: append(treeIsLeft("parent"), expectation{`name:"*renamed*"`, []string{"renamed.pdf"}}), + wantDocCount: conversions.ToPointer(uint64(2)), + }, + }, + } +} diff --git a/services/search/pkg/parity/matrix_test.go b/services/search/pkg/parity/matrix_test.go new file mode 100644 index 0000000000..4b4d65fca2 --- /dev/null +++ b/services/search/pkg/parity/matrix_test.go @@ -0,0 +1,457 @@ +package parity + +import ( + "encoding/json" + "fmt" + "os" + "sort" + "strings" + + sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + . "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/types" + "github.com/opencloud-eu/reva/v2/pkg/storagespace" + + searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +const ( + matrixFile = "README.md" + matrixEntry = "engine parity" +) + +// matrixRow is one line of the README. It travels in a report entry, which +// crosses processes as JSON under ginkgo -p, hence the exported fields and +// the scope already rendered to a string. +type matrixRow struct { + Section string + Group string + ID string + Query string + Scope string + Limit int32 + Reads string + Context string + Want []string + WantCount *int + WantBadRequest bool + // Overrides is what an engine is held to instead, rendered like expected + Overrides map[string]string + + GroupAt, CaseAt, QueryAt int +} + +func (r matrixRow) key() string { + return fmt.Sprintf("%s/%s/%s/%d", r.Group, r.ID, r.Query, r.QueryAt) +} + +// matrixAnswer is what a spec attaches to its report: one engine's answer to +// one row. A skipped engine leaves the README alone. +type matrixAnswer struct { + Row matrixRow + Engine string + Answer []string + Skipped bool +} + +func recordAnswer(row matrixRow, engine string, answer []string) { + AddReportEntry(matrixEntry, matrixAnswer{Row: row, Engine: engine, Answer: answer}, ReportEntryVisibilityNever) +} + +func recordSkip(row matrixRow, engine string) { + AddReportEntry(matrixEntry, matrixAnswer{Row: row, Engine: engine, Skipped: true}, ReportEntryVisibilityNever) +} + +// matrixPlanned is every row the spec tree announced while it was built, so +// the README is only written when each of them got an answer from every engine. +var matrixPlanned []matrixRow + +func planRow(rows ...matrixRow) { + matrixPlanned = append(matrixPlanned, rows...) +} + +type matrixResult struct { + matrixRow + answered map[string][]string + skipped map[string]bool +} + +// matrixAnswerOf reads an entry back: in-process it still carries the value, +// from another process only its JSON. +func matrixAnswerOf(entry types.ReportEntry) (matrixAnswer, bool) { + if answer, ok := entry.GetRawValue().(matrixAnswer); ok { + return answer, true + } + + var answer matrixAnswer + if err := json.Unmarshal([]byte(entry.Value.AsJSON), &answer); err != nil { + return matrixAnswer{}, false + } + + return answer, true +} + +func collectMatrix(report types.Report) []*matrixResult { + results := map[string]*matrixResult{} + for _, spec := range report.SpecReports { + for _, entry := range spec.ReportEntries { + if entry.Name != matrixEntry { + continue + } + + answer, ok := matrixAnswerOf(entry) + if !ok { + continue + } + + key := answer.Row.key() + result, known := results[key] + if !known { + result = &matrixResult{matrixRow: answer.Row, answered: map[string][]string{}, skipped: map[string]bool{}} + results[key] = result + } + + if answer.Skipped { + result.skipped[answer.Engine] = true + continue + } + + result.answered[answer.Engine] = answer.Answer + } + } + + rows := make([]*matrixResult, 0, len(results)) + for _, result := range results { + rows = append(rows, result) + } + + return rows +} + +func writeMatrix(report types.Report) { + rows := collectMatrix(report) + + answered := map[string]*matrixResult{} + for _, row := range rows { + answered[row.key()] = row + } + + var missing []string + for _, planned := range matrixPlanned { + result, ok := answered[planned.key()] + switch { + case !ok: + missing = append(missing, planned.ID+" "+planned.Query) + case len(result.skipped) > 0: + fmt.Fprintf(os.Stderr, "%s left alone, an engine was not reachable\n", matrixFile) + return + case len(result.answered) != len(engineNames): + missing = append(missing, planned.ID+" "+planned.Query) + } + } + + if len(missing) > 0 { + fmt.Fprintf(os.Stderr, "%s left alone, no answer from every engine for: %s\n", matrixFile, strings.Join(missing, "; ")) + return + } + + sort.Slice(rows, func(i, j int) bool { + a, b := rows[i], rows[j] + switch { + case a.GroupAt != b.GroupAt: + return a.GroupAt < b.GroupAt + case a.CaseAt != b.CaseAt: + return a.CaseAt < b.CaseAt + default: + return a.QueryAt < b.QueryAt + } + }) + + out := &strings.Builder{} + out.WriteString("# Engine parity\n\n") + out.WriteString("Written by the parity suite (`go test ./services/search/pkg/parity/`), do not edit.\n") + out.WriteString("Every case runs against bleve and OpenSearch. `same?` is ✅ when both answer as\n") + out.WriteString("expected, `❌ known` when an engine's divergence is documented in the case\n") + out.WriteString("(`engineOverrides`), `❌` when it is not.\n") + + group, section := "", "" + for _, row := range rows { + if row.Section != section { + section = row.Section + if out.Len() > 0 { + out.WriteString("\n") + } + + fmt.Fprintf(out, "## %s\n", section) + } + + if row.Group != group { + group = row.Group + fmt.Fprintf(out, "\n### %s\n\n%s\n\n", group, matrixFixtures(group)) + out.WriteString("| Case | Query | expected | bleve | OpenSearch | same? |\n") + out.WriteString("|---|---|---|---|---|---|\n") + } + + query := "`" + shortenQuery(row.Query) + "`" + if row.Scope != "" { + query += " " + row.Scope + } + + if row.Limit != 0 { + query = fmt.Sprintf("%s with a limit of %d", query, row.Limit) + } + + if row.Reads != "" { + query += " reads `" + row.Reads + "`" + } + + if row.Context != "" { + query = row.Context + ", then " + query + } + + expected := matrixNames(row.Want) + switch { + case row.WantBadRequest: + expected = "bad request" + case row.WantCount != nil: + expected = fmt.Sprintf("%d items", *row.WantCount) + } + + fmt.Fprintf(out, "| %s | %s | %s | %s | %s | %s |\n", + row.ID, query, expected, + matrixNames(row.answered["bleve"]), matrixNames(row.answered["opensearch"]), matrixVerdict(row)) + } + + if err := os.WriteFile(matrixFile, []byte(out.String()), 0o644); err != nil { + fmt.Fprintf(os.Stderr, "failed to write %s: %v\n", matrixFile, err) + } +} + +func matrixFixtures(group string) string { + var ( + fixtures []search.Resource + withIDs bool + ) + + for _, g := range queryGroups() { + if g.name != group { + continue + } + + fixtures = g.fixtures + for _, c := range g.cases { + withIDs = withIDs || strings.Contains(c.query, "id:") + } + } + + for _, g := range lifecycleGroups() { + if g.name != group { + continue + } + + withIDs = true + fixtures = g.fixtures + for _, c := range g.cases { + fixtures = append(fixtures, c.fixtures...) + } + } + + for _, g := range responseGroups() { + if g.name != group { + continue + } + + withIDs = true + fixtures = g.fixtures + } + + if len(fixtures) == 0 { + return "Fixtures: none" + } + + seen := map[string]bool{} + lines := []string{"Fixtures:", ""} + for _, f := range fixtures { + line := "- `" + shorten(f.Name) + "`" + if fields := fixtureFields(f, withIDs); fields != "" { + line += ", " + fields + } + + if seen[line] { + continue + } + + seen[line] = true + lines = append(lines, line) + } + + if listed := len(lines) - 2; listed > 12 { + lines = append(lines[:12], fmt.Sprintf("- ... and %d more of the same", listed-10)) + } + + return strings.Join(lines, "\n") +} + +func fixtureFields(f search.Resource, withID bool) string { + var fields []string + + add := func(format string, args ...any) { + fields = append(fields, fmt.Sprintf(format, args...)) + } + + if withID { + add("ID = %s", f.ID) + } + + if f.Type == uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER) { + add("folder") + } else if f.MimeType != "text/plain" { + add("MimeType = %s", f.MimeType) + } + + if f.Path != "./"+f.Name { + add("Path = %s", f.Path) + } + + if f.Title != "" { + add("Title = %q", f.Title) + } + + if f.Content != "" { + add("Content = %q", f.Content) + } + + if len(f.Tags) > 0 { + add("Tags = %s", strings.Join(f.Tags, ", ")) + } + + if len(f.Favorites) > 0 { + add("Favorites = %s", strings.Join(f.Favorites, ", ")) + } + + if f.Size != 1000 { + add("Size = %d", f.Size) + } + + if !strings.HasPrefix(f.Mtime, fixtureNow.Format("2006-01-02")) { + add("Mtime = %s", f.Mtime) + } + + if f.Hidden { + add("hidden") + } + + if f.Deleted { + add("deleted") + } + + return strings.Join(fields, ", ") +} + +func matrixScope(ref *searchMessage.Reference) string { + if ref == nil { + return "" + } + + space := storagespace.FormatResourceID(&sprovider.ResourceId{ + StorageId: ref.GetResourceId().GetStorageId(), + SpaceId: ref.GetResourceId().GetSpaceId(), + OpaqueId: ref.GetResourceId().GetOpaqueId(), + }) + + if path := ref.GetPath(); path != "" { + return "in " + space + " under " + path + } + + return "in " + space +} + +func matrixNames(names []string) string { + if len(names) == 0 { + return "no match" + } + + shortened := make([]string, 0, len(names)) + for _, name := range names { + shortened = append(shortened, shorten(name)) + } + sort.Strings(shortened) + + return strings.Join(shortened, ", ") +} + +func shorten(name string) string { + if len(name) <= 24 { + return name + } + + return name[:10] + "..." + name[len(name)-8:] +} + +func shortenQuery(q string) string { + if len(q) <= 64 { + return q + } + + return q[:32] + "..." + q[len(q)-24:] +} + +func matrixVerdict(row *matrixResult) string { + var off []string + known := true + for _, engine := range engineNames { + answer := matrixNames(row.answered[engine]) + + agrees := answer == matrixNames(row.Want) + switch { + case row.WantBadRequest: + agrees = answer == "bad request" + case row.WantCount != nil: + agrees = len(row.answered[engine]) == *row.WantCount + } + + if agrees { + continue + } + + off = append(off, engine) + if row.WantCount != nil { + answer = fmt.Sprintf("%d items", len(row.answered[engine])) + } + + if expected, ok := row.Overrides[engine]; !ok || expected != answer { + known = false + } + } + + switch { + case len(off) == 0: + return "✅" + case known: + return "❌ known" + default: + return "❌" + } +} + +func (c lifecycleCase) matrixRows(group string, groupAt, caseAt int) []matrixRow { + rows := make([]matrixRow, 0, len(c.expect)+1) + for i, expect := range c.expect { + rows = append(rows, matrixRow{ + Section: "Operations", Group: group, ID: c.label(group), Query: expect.query, Context: c.title, Want: expect.want, Overrides: renderOverrides(c.overridesFor(expect.query)), + GroupAt: groupAt, CaseAt: caseAt, QueryAt: i, + }) + } + + if c.wantDocCount != nil { + rows = append(rows, matrixRow{ + Section: "Operations", Group: group, ID: c.label(group), Query: "DocCount()", Context: c.title, + Want: []string{fmt.Sprint(*c.wantDocCount)}, + Overrides: renderOverrides(c.docCountOverrides()), + GroupAt: groupAt, CaseAt: caseAt, QueryAt: len(c.expect), + }) + } + + return rows +} diff --git a/services/search/pkg/parity/parity_suite_test.go b/services/search/pkg/parity/parity_suite_test.go new file mode 100644 index 0000000000..70baa137f3 --- /dev/null +++ b/services/search/pkg/parity/parity_suite_test.go @@ -0,0 +1,48 @@ +package parity + +import ( + "context" + "encoding/json" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/pkg/config" +) + +var ( + // openSearchClient points every process at the one OpenSearch started by + // process 1; the container handle stays there. + openSearchClient config.EngineOpenSearchClient + stopOpenSearch func() +) + +func TestEngineParity(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Search Engine Parity Suite") +} + +var _ = SynchronizedBeforeSuite(func() []byte { + cfg, done, err := opensearchtest.SetupTests(context.Background()) + Expect(err).NotTo(HaveOccurred(), "failed to set up the OpenSearch test container") + stopOpenSearch = done + + client, err := json.Marshal(cfg.Engine.OpenSearch.Client) + Expect(err).NotTo(HaveOccurred()) + + return client +}, func(client []byte) { + Expect(json.Unmarshal(client, &openSearchClient)).To(Succeed()) +}) + +var _ = SynchronizedAfterSuite(func() {}, func() { + if stopOpenSearch != nil { + stopOpenSearch() + } +}) + +var _ = ReportAfterSuite("engine parity matrix", func(report Report) { + writeMatrix(report) +}) diff --git a/services/search/pkg/parity/parity_test.go b/services/search/pkg/parity/parity_test.go new file mode 100644 index 0000000000..3bacdd762b --- /dev/null +++ b/services/search/pkg/parity/parity_test.go @@ -0,0 +1,466 @@ +package parity + +import ( + "context" + "fmt" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/types" + + "github.com/opencloud-eu/reva/v2/pkg/errtypes" + + searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +type queryCase struct { + id int + query string + want []string + limit int32 + wantCount *int + ref *searchMessage.Reference + wantBadRequest bool + + // engineOverrides holds, per engine, what that engine answers today where + // it still differs from the expectation above. The spec asserts the + // override and the README marks the row as a known divergence. Once the + // engine answers as expected the override fails and has to go. + engineOverrides map[string]override +} + +// override is one engine's current answer to a case, in the expectation's +// own terms. +type override struct { + want []string + wantCount *int + wantBadRequest bool +} + +func (o override) matcher() types.GomegaMatcher { + switch { + case o.wantBadRequest: + return ConsistOf("bad request") + case o.wantCount != nil: + return HaveLen(*o.wantCount) + default: + return matchNames(o.want) + } +} + +// rendered spells the override the way the README spells an expectation. +func (o override) rendered() string { + switch { + case o.wantBadRequest: + return "bad request" + case o.wantCount != nil: + return fmt.Sprintf("%d items", *o.wantCount) + default: + return matrixNames(o.want) + } +} + +func renderOverrides(overrides map[string]override) map[string]string { + if len(overrides) == 0 { + return nil + } + + out := make(map[string]string, len(overrides)) + for engine, o := range overrides { + out[engine] = o.rendered() + } + + return out +} + +func (c queryCase) label(group string) string { + return fmt.Sprintf("%s-%02d", strings.ToUpper(group), c.id) +} + +func (c queryCase) name(group string) string { + return c.label(group) + " " + c.query +} + +type queryGroup struct { + name string + fixtures []search.Resource + cases []queryCase +} + +func queryGroups() []queryGroup { + return []queryGroup{ + nameGroup(), + extensionGroup(), + tagsGroup(), + titleGroup(), + contentGroup(), + favoritesGroup(), + mediatypeGroup(), + pathGroup(), + fieldsGroup(), + deletedGroup(), + visibilityGroup(), + booleanGroup(), + samenameGroup(), + stressGroup(), + everythingGroup(), + rangeGroup(), + scopeGroup(), + invalidGroup(), + } +} + +type responseCase struct { + id int + do func(e search.Engine) error + after string + query string + reads string + read func(*searchService.SearchIndexResponse) []string + want []string + + engineOverrides map[string]override +} + +func (c responseCase) label(group string) string { + return fmt.Sprintf("%s-%02d", strings.ToUpper(group), c.id) +} + +func (c responseCase) name(group string) string { + return c.label(group) + " " + c.reads +} + +type responseGroup struct { + name string + fixtures []search.Resource + cases []responseCase +} + +func responseGroups() []responseGroup { + return []responseGroup{ + entityGroup(), + metadataGroup(), + } +} + +type lifecycleCase struct { + id int + title string + do func(e search.Engine) error + expect []expectation + wantErr bool + + fixtures []search.Resource + + wantDocCount *uint64 + + // engineOverrides is the lifecycle spelling of queryCase.engineOverrides: + // a case has several expectations, so an override answers them by query. + engineOverrides map[string]lifecycleOverride +} + +// lifecycleOverride is one engine's current answers to a lifecycle case. +type lifecycleOverride struct { + expect map[string][]string + wantDocCount *uint64 +} + +// overridesFor picks one expectation's overrides out of the case's. +func (c lifecycleCase) overridesFor(query string) map[string]override { + var overrides map[string]override + for engine, o := range c.engineOverrides { + if answer, ok := o.expect[query]; ok { + if overrides == nil { + overrides = map[string]override{} + } + + overrides[engine] = override{want: answer} + } + } + + return overrides +} + +// docCountOverrides renders the DocCount overrides as a query override. +func (c lifecycleCase) docCountOverrides() map[string]override { + var overrides map[string]override + for engine, o := range c.engineOverrides { + if o.wantDocCount != nil { + if overrides == nil { + overrides = map[string]override{} + } + + overrides[engine] = override{want: []string{fmt.Sprint(*o.wantDocCount)}} + } + } + + return overrides +} + +type expectation struct { + query string + want []string +} + +func (c lifecycleCase) label(group string) string { + return fmt.Sprintf("%s-%02d", strings.ToUpper(group), c.id) +} + +func (c lifecycleCase) name(group string) string { + return c.label(group) + " " + c.title +} + +type lifecycleGroup struct { + name string + fixtures []search.Resource + cases []lifecycleCase +} + +func lifecycleGroups() []lifecycleGroup { + return []lifecycleGroup{ + deleteLifecycle(), + restoreLifecycle(), + purgeLifecycle(), + purgeSpaceLifecycle(), + moveLifecycle(), + rootScopeLifecycle(), + casePathLifecycle(), + hiddenLifecycle(), + upsertLifecycle(), + idempotencyLifecycle(), + batchLifecycle(), + } +} + +// expectAnswer holds an engine to its override when it has one, to the +// expectation otherwise. An override that no longer holds fails on purpose: +// the engine answers as expected now, the override has to go. +func expectAnswer(engine string, answer []string, expected override, overrides map[string]override) { + GinkgoHelper() + + if o, ok := overrides[engine]; ok { + Expect(o.rendered()).NotTo(Equal(expected.rendered()), "an override that equals the expectation documents nothing, remove it") + Expect(answer).To(o.matcher(), "the override for %s no longer holds, remove it", engine) + + return + } + + Expect(answer).To(expected.matcher()) +} + +// Every case runs once per engine as its own spec, so one engine failing +// leaves the other's answer in the matrix. The groups are Ordered containers: +// their engines are built once in a BeforeAll and shared by the specs inside, +// and they carry on after a failure so every engine gets to answer. + +var _ = Describe("Queries", func() { + for groupAt, group := range queryGroups() { + Describe(group.name, Ordered, ContinueOnFailure, func() { + var engines []testEngine + + BeforeAll(func() { + engines = newEngines("opencloud-test-engine-parity-"+group.name, group.fixtures) + }) + + for caseAt, c := range group.cases { + row := matrixRow{ + Section: "Queries", Group: group.name, ID: c.label(group.name), + Query: c.query, Scope: matrixScope(c.ref), Limit: c.limit, + Want: c.want, WantCount: c.wantCount, WantBadRequest: c.wantBadRequest, Overrides: renderOverrides(c.engineOverrides), + GroupAt: groupAt, CaseAt: caseAt, + } + planRow(row) + + Describe(c.name(group.name), func() { + for _, name := range engineNames { + It("on "+name, func() { + e := engineNamed(engines, name) + if e.unavailable != "" { + recordSkip(row, name) + Skip(e.unavailable) + } + + request := &searchService.SearchIndexRequest{Query: c.query, PageSize: c.limit, Ref: c.ref} + + expected := override{want: c.want, wantCount: c.wantCount, wantBadRequest: c.wantBadRequest} + _, overridden := c.engineOverrides[name] + + if c.wantBadRequest { + _, err := e.backend.Search(context.Background(), request) + answer := badRequestAnswer(err) + recordAnswer(row, name, answer) + if !overridden { + Expect(err).To(BeAssignableToTypeOf(errtypes.BadRequest(""))) + } + + expectAnswer(name, answer, expected, c.engineOverrides) + + return + } + + answer, err := ask(e.backend, request) + recordAnswer(row, name, answer) + if !overridden { + Expect(err).NotTo(HaveOccurred(), "the query has to answer, an empty result is an answer") + } + + expectAnswer(name, answer, expected, c.engineOverrides) + }) + } + }) + } + }) + } +}) + +var _ = Describe("Operations", func() { + offset := len(queryGroups()) + + for groupAt, group := range lifecycleGroups() { + Describe(group.name, func() { + for caseAt, c := range group.cases { + fixtures := c.fixtures + if fixtures == nil { + fixtures = group.fixtures + } + + index := fmt.Sprintf("opencloud-test-engine-parity-%s-%d", group.name, c.id) + rows := c.matrixRows(group.name, offset+groupAt, caseAt) + planRow(rows...) + + // an operation changes its index, so every spec builds its own engine + Describe(c.name(group.name), func() { + for _, name := range engineNames { + It("on "+name, func() { + e := newEngine(name, index, fixtures) + if e.unavailable != "" { + for _, row := range rows { + recordSkip(row, name) + } + + Skip(e.unavailable) + } + + // every row gets its answer before anything is asserted, a + // failed assertion must not leave the later rows unanswered + err := c.do(e.backend) + if failed := err != nil; failed != c.wantErr { + for _, row := range rows { + recordAnswer(row, name, badRequestAnswer(err)) + } + } + + if c.wantErr { + Expect(err).To(HaveOccurred(), "the operation had to report that it did not find the resource") + } else { + Expect(err).NotTo(HaveOccurred(), "the operation under test failed") + } + + e.settle() + + answers := make([][]string, len(c.expect)) + errs := make([]error, len(c.expect)) + for i, expect := range c.expect { + answers[i], errs[i] = ask(e.backend, &searchService.SearchIndexRequest{Query: expect.query}) + recordAnswer(rows[i], name, answers[i]) + } + + var count uint64 + if c.wantDocCount != nil { + count, err = e.backend.DocCount() + Expect(err).NotTo(HaveOccurred()) + recordAnswer(rows[len(rows)-1], name, []string{fmt.Sprint(count)}) + } + + for i, expect := range c.expect { + overrides := c.overridesFor(expect.query) + if _, overridden := overrides[name]; !overridden { + Expect(errs[i]).NotTo(HaveOccurred(), "the query has to answer, an empty result is an answer") + } + + expectAnswer(name, answers[i], override{want: expect.want}, overrides) + } + + if c.wantDocCount != nil { + expected := override{want: []string{fmt.Sprint(*c.wantDocCount)}} + expectAnswer(name, []string{fmt.Sprint(count)}, expected, c.docCountOverrides()) + } + }) + } + }) + } + }) + } +}) + +var _ = Describe("Response", func() { + offset := len(queryGroups()) + len(lifecycleGroups()) + + for groupAt, group := range responseGroups() { + Describe(group.name, Ordered, ContinueOnFailure, func() { + var shared []testEngine + + BeforeAll(func() { + shared = newEngines("opencloud-test-engine-parity-"+group.name, group.fixtures) + }) + + for caseAt, c := range group.cases { + row := matrixRow{ + Section: "Response", Group: group.name, ID: c.label(group.name), + Query: c.query, Reads: c.reads, Context: c.after, Want: c.want, Overrides: renderOverrides(c.engineOverrides), + GroupAt: offset + groupAt, CaseAt: caseAt, + } + planRow(row) + + index := fmt.Sprintf("opencloud-test-engine-parity-%s-%d", group.name, c.id) + + Describe(c.name(group.name), func() { + for _, name := range engineNames { + It("on "+name, func() { + // a case with an operation changes its index, so it gets its own + e := engineNamed(shared, name) + if c.do != nil { + e = newEngine(name, index, group.fixtures) + } + + if e.unavailable != "" { + recordSkip(row, name) + Skip(e.unavailable) + } + + if c.do != nil { + err := c.do(e.backend) + if err != nil { + recordAnswer(row, name, []string{"error"}) + } + + Expect(err).NotTo(HaveOccurred(), "the operation under test failed") + e.settle() + } + + resp, err := e.backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: c.query}) + if err != nil { + recordAnswer(row, name, []string{"error"}) + } + Expect(err).NotTo(HaveOccurred(), "the query has to answer, an empty result is an answer") + + answer := c.read(resp) + recordAnswer(row, name, answer) + if o, overridden := c.engineOverrides[name]; overridden { + Expect(o.want).NotTo(Equal(c.want), "an override that equals the expectation documents nothing, remove it") + Expect(answer).To(Equal(o.want), "the override for %s no longer holds, remove it", name) + + return + } + + Expect(answer).To(Equal(c.want), c.reads) + }) + } + }) + } + }) + } +}) diff --git a/services/search/pkg/parity/query_boolean_test.go b/services/search/pkg/parity/query_boolean_test.go new file mode 100644 index 0000000000..4f4349989d --- /dev/null +++ b/services/search/pkg/parity/query_boolean_test.go @@ -0,0 +1,24 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func booleanGroup() queryGroup { + return queryGroup{ + name: "boolean", + fixtures: []search.Resource{ + fixtureDoc("alpha.txt", withTags("red")), + fixtureDoc("beta.txt", withTags("blue")), + fixtureDoc("gamma.md", withMime("text/markdown")), + }, + cases: []queryCase{ + {id: 1, query: `name:"*alpha*" AND name:"*txt*"`, want: []string{"alpha.txt"}}, + {id: 2, query: `name:"*alpha*" OR name:"*beta*"`, want: []string{"alpha.txt", "beta.txt"}}, + {id: 3, query: `name:"*a*" AND NOT name:"*alpha*"`, want: []string{"beta.txt", "gamma.md"}}, + {id: 4, query: `name:"*a*" AND tag:("red")`, want: []string{"alpha.txt"}}, + {id: 5, query: `(name:"*alpha*" OR name:"*beta*") AND mediatype:text/plain`, want: []string{"alpha.txt", "beta.txt"}}, + {id: 6, query: `name:"*a*" AND mediatype:text/markdown`, want: []string{"gamma.md"}}, + }, + } +} diff --git a/services/search/pkg/parity/query_content_test.go b/services/search/pkg/parity/query_content_test.go new file mode 100644 index 0000000000..d51d36d53b --- /dev/null +++ b/services/search/pkg/parity/query_content_test.go @@ -0,0 +1,27 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func contentGroup() queryGroup { + return queryGroup{ + name: "content", + fixtures: []search.Resource{ + fixtureDoc("monthly.txt", withContent("the monthly reports are due")), + fixtureDoc("links.txt", withContent("see https://opencloud.example.com/help or write to alan@example.org")), + }, + cases: []queryCase{ + {id: 1, query: `Content:report`, engineOverrides: map[string]override{"bleve": override{want: []string{"monthly.txt"}}}}, + {id: 2, query: `Content:REPORTS`, want: []string{"monthly.txt"}}, + {id: 3, query: `Content:"monthly reports"`, want: []string{"monthly.txt"}}, + {id: 4, query: `Content:"reports monthly"`, engineOverrides: map[string]override{"bleve": override{want: []string{"monthly.txt"}}}}, + {id: 5, query: `Content:report*`, want: []string{"monthly.txt"}}, + {id: 6, query: `Content:*eport*`, want: []string{"monthly.txt"}}, + {id: 7, query: `Content:month*`, want: []string{"monthly.txt"}}, + {id: 8, query: `Content:"https://opencloud.example.com/help"`, want: []string{"links.txt"}}, + {id: 9, query: `Content:"alan@example.org"`, want: []string{"links.txt"}}, + {id: 10, query: `Content:opencloud`, want: []string{"links.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + }, + } +} diff --git a/services/search/pkg/parity/query_deleted_test.go b/services/search/pkg/parity/query_deleted_test.go new file mode 100644 index 0000000000..905725bd8b --- /dev/null +++ b/services/search/pkg/parity/query_deleted_test.go @@ -0,0 +1,26 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func deletedGroup() queryGroup { + return queryGroup{ + name: "deleted", + fixtures: []search.Resource{ + fixtureDoc("trashed.txt", isDeleted()), + fixtureDoc("kept.txt"), + fixtureFolder("bin", isDeleted()), + fixtureDoc("receipt.txt", withParent("1$1!bin"), withPath("./bin/receipt.txt"), isDeleted()), + fixtureFolder("shelf"), + fixtureDoc("book.txt", withParent("1$1!shelf"), withPath("./shelf/book.txt")), + }, + cases: []queryCase{ + {id: 1, query: `name:"*trashed*"`}, + {id: 2, query: `name:"*.txt"`, want: []string{"kept.txt", "book.txt"}}, + {id: 3, query: `name:"*receipt*"`}, + {id: 4, query: `path:"./bin"`}, + {id: 5, query: `path:"./shelf"`, want: []string{"shelf", "book.txt"}, engineOverrides: map[string]override{"bleve": override{want: []string{"shelf"}}}}, + }, + } +} diff --git a/services/search/pkg/parity/query_everything_test.go b/services/search/pkg/parity/query_everything_test.go new file mode 100644 index 0000000000..449a93e8d9 --- /dev/null +++ b/services/search/pkg/parity/query_everything_test.go @@ -0,0 +1,23 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func everythingGroup() queryGroup { + return queryGroup{ + name: "everything", + fixtures: []search.Resource{ + fixtureDoc("alpha.txt"), + fixtureDoc("beta.txt"), + fixtureFolder("box"), + }, + cases: []queryCase{ + {id: 1, query: `*`, want: []string{"alpha.txt", "beta.txt", "box"}}, + {id: 2, query: `name:"*"`, want: []string{"alpha.txt", "beta.txt", "box"}}, + {id: 3, query: `*`, limit: 2, wantCount: conversions.ToPointer(2)}, + {id: 4, query: `*`, limit: -1, wantCount: conversions.ToPointer(3)}, + }, + } +} diff --git a/services/search/pkg/parity/query_extension_test.go b/services/search/pkg/parity/query_extension_test.go new file mode 100644 index 0000000000..e02b5e30fa --- /dev/null +++ b/services/search/pkg/parity/query_extension_test.go @@ -0,0 +1,22 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func extensionGroup() queryGroup { + return queryGroup{ + name: "extension", + fixtures: []search.Resource{ + fixtureDoc("report.txt"), + fixtureDoc("notes.md", withMime("text/markdown")), + fixtureFolder("archive"), + }, + cases: []queryCase{ + {id: 1, query: `txt`, want: []string{"report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 2, query: `md`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 3, query: `name:"*.txt"`, want: []string{"report.txt"}}, + {id: 4, query: `report`, want: []string{"report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + }, + } +} diff --git a/services/search/pkg/parity/query_favorites_test.go b/services/search/pkg/parity/query_favorites_test.go new file mode 100644 index 0000000000..ba1f4d7d1c --- /dev/null +++ b/services/search/pkg/parity/query_favorites_test.go @@ -0,0 +1,22 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func favoritesGroup() queryGroup { + return queryGroup{ + name: "favorites", + fixtures: []search.Resource{ + fixtureDoc("starred.txt", withFavorite("A1B2-Upper")), + fixtureDoc("plain.txt"), + fixtureFolder("keepsakes", withFavorite("A1B2-Upper")), + fixtureDoc("photo.jpg", withParent("1$1!keepsakes"), withPath("./keepsakes/photo.jpg"), withMime("image/jpeg")), + }, + cases: []queryCase{ + {id: 1, query: `Favorites:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 2, query: `favorite:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 3, query: `Favorites:"somebody-else"`}, + }, + } +} diff --git a/services/search/pkg/parity/query_fields_test.go b/services/search/pkg/parity/query_fields_test.go new file mode 100644 index 0000000000..a643f0cfae --- /dev/null +++ b/services/search/pkg/parity/query_fields_test.go @@ -0,0 +1,41 @@ +package parity + +import ( + libregraph "github.com/opencloud-eu/libre-graph-api-go" + + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func fieldsGroup() queryGroup { + return queryGroup{ + name: "fields", + fixtures: []search.Resource{ + fixtureDoc("small.txt", withSize(42)), + fixtureDoc("old.txt", withMtime("2020-01-01T00:00:00Z")), + fixtureDoc("known.txt", withID("1$1!23")), + fixtureDoc("cased.txt", withID("1$1!AB-23")), + fixtureDoc("hidden.txt", isHidden()), + fixtureDoc("plain.txt"), + fixtureFolder("box"), + fixtureDoc("boxed.txt", withParent("1$1!box"), withPath("./box/boxed.txt")), + fixtureDoc("song.mp3", withMime("audio/mpeg"), withAudio(&libregraph.Audio{Artist: libregraph.PtrString("Some Artist")})), + }, + cases: []queryCase{ + {id: 1, query: `size:42`, want: []string{"small.txt"}}, + {id: 2, query: `mtime<"2021-01-01T00:00:00Z"`, want: []string{"old.txt"}}, + {id: 3, query: `id:"1$1!23"`, want: []string{"known.txt"}}, + {id: 4, query: `hidden:true`, want: []string{"hidden.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 5, query: `type:file`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, + {id: 6, query: `type:folder`, want: []string{"box"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, + {id: 7, query: `unknown:field`}, + {id: 8, query: `type:File`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, + {id: 9, query: `type:FOLDER`, want: []string{"box"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, + {id: 10, query: `hidden:TRUE`, want: []string{"hidden.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, + {id: 11, query: `id:"1$1!AB-23"`, want: []string{"cased.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 12, query: `id:"1$1!ab-23"`}, + // a facet value keeps its case, the field is not marked lowercase + {id: 13, query: `audio.artist:"Some Artist"`, want: []string{"song.mp3"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 14, query: `audio.artist:"some artist"`}, + }, + } +} diff --git a/services/search/pkg/parity/query_invalid_test.go b/services/search/pkg/parity/query_invalid_test.go new file mode 100644 index 0000000000..87da4f0592 --- /dev/null +++ b/services/search/pkg/parity/query_invalid_test.go @@ -0,0 +1,18 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func invalidGroup() queryGroup { + return queryGroup{ + name: "invalid", + fixtures: []search.Resource{ + fixtureDoc("alpha.txt"), + }, + cases: []queryCase{ + {id: 1, query: `AND mediatype:document`, wantBadRequest: true}, + {id: 2, query: `mediatype:document AND`, want: []string{"alpha.txt"}}, + }, + } +} diff --git a/services/search/pkg/parity/query_mediatype_test.go b/services/search/pkg/parity/query_mediatype_test.go new file mode 100644 index 0000000000..11a5065d99 --- /dev/null +++ b/services/search/pkg/parity/query_mediatype_test.go @@ -0,0 +1,25 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func mediatypeGroup() queryGroup { + return queryGroup{ + name: "mediatype", + fixtures: []search.Resource{ + fixtureDoc("notes.md", withMime("text/markdown")), + fixtureDoc("photo.jpg", withMime("image/jpeg")), + fixtureFolder("albums"), + fixtureFolder("drafts"), + }, + cases: []queryCase{ + {id: 1, query: `mediatype:text/markdown`, want: []string{"notes.md"}}, + {id: 2, query: `mediatype:TEXT/MARKDOWN`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 3, query: `mediatype:image/jpeg`, want: []string{"photo.jpg"}}, + {id: 4, query: `mediatype:*jpeg`, want: []string{"photo.jpg"}}, + {id: 5, query: `mediatype:image`, want: []string{"photo.jpg"}}, + {id: 6, query: `mediatype:folder`, want: []string{"albums", "drafts"}}, + }, + } +} diff --git a/services/search/pkg/parity/query_name_test.go b/services/search/pkg/parity/query_name_test.go new file mode 100644 index 0000000000..807041166d --- /dev/null +++ b/services/search/pkg/parity/query_name_test.go @@ -0,0 +1,66 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func nameGroup() queryGroup { + return queryGroup{ + name: "name", + fixtures: []search.Resource{ + fixtureFolder("new-folder"), + fixtureDoc("quarterly notes.txt"), + fixtureDoc("Report.txt"), + fixtureDoc("Übung.txt"), + fixtureDoc("a+b.txt"), + fixtureDoc("c(d).txt"), + fixtureDoc("e&f.txt"), + fixtureDoc("v1.2.3.txt"), + fixtureDoc("foo bar.txt"), + fixtureDoc(fixtureLongName), + }, + cases: []queryCase{ + {id: 1, query: `new`, want: []string{"new-folder"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 2, query: `quarterly`, want: []string{"quarterly notes.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 3, query: `report`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 4, query: `name:"*new-folder*"`, want: []string{"new-folder"}}, + {id: 5, query: `name:"*w-fol*"`, want: []string{"new-folder"}}, + {id: 6, query: `name:"*oo ba*"`, want: []string{"foo bar.txt"}}, + {id: 7, query: `name:"*REPORT*"`, want: []string{"Report.txt"}}, + {id: 8, query: `name:"*übung*"`, want: []string{"Übung.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 9, query: `name:"*ÜBUNG*"`, want: []string{"Übung.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 10, query: `name:"*a+b*"`, want: []string{"a+b.txt"}}, + {id: 11, query: `name:"*c(d)*"`, want: []string{"c(d).txt"}}, + {id: 12, query: `name:"*e&f*"`, want: []string{"e&f.txt"}}, + {id: 13, query: `name:"*v1.2*"`, want: []string{"v1.2.3.txt"}}, + {id: 14, query: `new-folder`, want: []string{"new-folder"}}, + {id: 15, query: `*folder*`, want: []string{"new-folder"}}, + {id: 16, query: `name:"*foo bar*"`, want: []string{"foo bar.txt"}}, + {id: 17, query: `name:"foo bar.txt"`, want: []string{"foo bar.txt"}}, + {id: 18, query: `name:"*needle*"`, want: []string{fixtureLongName}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 19, query: `name:"report*"`, want: []string{"Report.txt"}}, + {id: 20, query: `name:"*report"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 21, query: `name:"Rep*rt.txt"`, want: []string{"Report.txt"}}, + {id: 22, query: `Name:"*report*"`, want: []string{"Report.txt"}}, + {id: 23, query: `NAME:"*report*"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 24, query: `name:Rep?rt.txt`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 25, query: `name:"*eport"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 26, query: `name:"repor*"`, want: []string{"Report.txt"}}, + {id: 27, query: `REPORT`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 28, query: `name:REPORT`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 29, query: `name:"REPORT.TXT"`, want: []string{"Report.txt"}}, + {id: 30, query: `name:"FOO BAR.TXT"`, want: []string{"foo bar.txt"}}, + {id: 31, query: `name:"ÜBUNG.TXT"`, want: []string{"Übung.txt"}}, + {id: 32, query: `name:"folder*"`}, + {id: 33, query: `name:"*new"`}, + {id: 34, query: `name:new`, want: []string{"new-folder"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 35, query: `name:"new"`, engineOverrides: map[string]override{"opensearch": override{want: []string{"new-folder"}}}}, + {id: 36, query: `name:"new-folder"`, want: []string{"new-folder"}}, + {id: 37, query: `name:"new-*"`, want: []string{"new-folder"}}, + {id: 38, query: `name:"new*"`, want: []string{"new-folder"}}, + {id: 39, query: `name:new-*`, want: []string{"new-folder"}}, + {id: 40, query: `name:"*-folder"`, want: []string{"new-folder"}}, + {id: 41, query: `name:"Rep?rt.txt"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + }, + } +} diff --git a/services/search/pkg/parity/query_path_test.go b/services/search/pkg/parity/query_path_test.go new file mode 100644 index 0000000000..828d298951 --- /dev/null +++ b/services/search/pkg/parity/query_path_test.go @@ -0,0 +1,28 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func pathGroup() queryGroup { + return queryGroup{ + name: "path", + fixtures: []search.Resource{ + fixtureFolder("parent"), + fixtureDoc("child.jpg", withMime("image/jpeg"), withPath("./parent/child.jpg")), + fixtureFolder("docs-lower", withPath("./documents")), + fixtureFolder("docs-upper", withPath("./DOCUMENTS")), + fixtureFolder("docs-mixed", withPath("./Documents")), + }, + cases: []queryCase{ + {id: 1, query: `path:"./parent"`, want: []string{"parent", "child.jpg"}, engineOverrides: map[string]override{"bleve": override{want: []string{"parent"}}}}, + {id: 2, query: `path:"./parent/child.jpg"`, want: []string{"child.jpg"}}, + {id: 3, query: `path:"./Parent"`, engineOverrides: map[string]override{"opensearch": override{want: []string{"child.jpg", "parent"}}}}, + {id: 4, query: `path:"*child*"`, want: []string{"child.jpg"}}, + {id: 5, query: `path:"./documents"`, want: []string{"docs-lower"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"docs-lower", "docs-mixed", "docs-upper"}}}}, + {id: 6, query: `path:"./DOCUMENTS"`, want: []string{"docs-upper"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"docs-lower", "docs-mixed", "docs-upper"}}}}, + {id: 7, query: `path:"./Documents"`, want: []string{"docs-mixed"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"docs-lower", "docs-mixed", "docs-upper"}}}}, + {id: 8, query: `path:"./parent/"`, want: []string{"parent", "child.jpg"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + }, + } +} diff --git a/services/search/pkg/parity/query_range_test.go b/services/search/pkg/parity/query_range_test.go new file mode 100644 index 0000000000..59a1618272 --- /dev/null +++ b/services/search/pkg/parity/query_range_test.go @@ -0,0 +1,26 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func rangeGroup() queryGroup { + return queryGroup{ + name: "range", + fixtures: []search.Resource{ + fixtureDoc("small.txt", withSize(50)), + fixtureDoc("big.txt", withSize(500)), + fixtureDoc("ancient.txt", withSize(10), withMtime("2020-01-01T00:00:00Z")), + }, + cases: []queryCase{ + {id: 1, query: `size>100`, want: []string{"big.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 2, query: `size<100`, want: []string{"small.txt", "ancient.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 3, query: `mtime>"2021-01-01T00:00:00Z"`, want: []string{"small.txt", "big.txt"}}, + {id: 4, query: `mtime<"2021-01-01T00:00:00Z"`, want: []string{"ancient.txt"}}, + {id: 5, query: `Mtime:"today"`, want: []string{"small.txt", "big.txt"}}, + {id: 6, query: `Mtime:"yesterday"`}, + {id: 7, query: `mtime>2021`}, + {id: 8, query: `name>100`}, + }, + } +} diff --git a/services/search/pkg/parity/query_samename_test.go b/services/search/pkg/parity/query_samename_test.go new file mode 100644 index 0000000000..46b07597b1 --- /dev/null +++ b/services/search/pkg/parity/query_samename_test.go @@ -0,0 +1,34 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func samenameGroup() queryGroup { + folder := fixtureFolder("doc", withID("1$1!2")) + under := func(path string) *searchMessage.Reference { + return &searchMessage.Reference{ + ResourceId: &searchMessage.ResourceID{StorageId: "1", SpaceId: "1", OpaqueId: "1"}, + Path: path, + } + } + + return queryGroup{ + name: "samename", + fixtures: []search.Resource{ + folder, + fixtureDoc("doc.pdf", withID("1$1!3")), + fixtureDoc("file.pdf", withID("1$1!4")), + fixtureDoc("doc.pdf", withID("1$1!5"), withParent(folder.ID), withPath("./doc/doc.pdf")), + fixtureDoc("file.pdf", withID("1$1!6"), withParent(folder.ID), withPath("./doc/file.pdf")), + }, + cases: []queryCase{ + {id: 1, query: `name:"*doc*"`, wantCount: conversions.ToPointer(3)}, + {id: 2, query: `name:"*doc*"`, ref: under("./doc"), wantCount: conversions.ToPointer(2)}, + {id: 3, query: `name:"*file*"`, wantCount: conversions.ToPointer(2)}, + {id: 4, query: `name:"*file*"`, ref: under("./doc"), wantCount: conversions.ToPointer(1)}, + }, + } +} diff --git a/services/search/pkg/parity/query_scope_test.go b/services/search/pkg/parity/query_scope_test.go new file mode 100644 index 0000000000..9bb01341f2 --- /dev/null +++ b/services/search/pkg/parity/query_scope_test.go @@ -0,0 +1,32 @@ +package parity + +import ( + searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func scopeGroup() queryGroup { + space := func(path string) *searchMessage.Reference { + return &searchMessage.Reference{ + ResourceId: &searchMessage.ResourceID{StorageId: "1", SpaceId: "1", OpaqueId: "1"}, + Path: path, + } + } + + return queryGroup{ + name: "scope", + fixtures: []search.Resource{ + fixtureFolder("parent"), + fixtureDoc("child.pdf", withPath("./parent/child.pdf")), + fixtureDoc("outside.txt"), + fixtureDoc("elsewhere.txt", withRoot("2$2!1")), + }, + cases: []queryCase{ + {id: 1, query: `*`, want: []string{"parent", "child.pdf", "outside.txt", "elsewhere.txt"}}, + {id: 2, query: `*`, ref: space(""), want: []string{"parent", "child.pdf", "outside.txt"}}, + {id: 3, query: `*`, ref: space("./parent"), want: []string{"parent", "child.pdf"}}, + {id: 4, query: `*`, ref: space("./parent/child.pdf"), want: []string{"child.pdf"}}, + {id: 5, query: `name:"*elsewhere*"`, ref: space(""), want: nil}, + }, + } +} diff --git a/services/search/pkg/parity/query_stress_test.go b/services/search/pkg/parity/query_stress_test.go new file mode 100644 index 0000000000..a739800ad2 --- /dev/null +++ b/services/search/pkg/parity/query_stress_test.go @@ -0,0 +1,38 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func stressGroup() queryGroup { + return queryGroup{ + name: "stress", + fixtures: []search.Resource{ + fixtureDoc("quarterly report.docx", + withMime("application/vnd.openxmlformats-officedocument.wordprocessingml.document"), + withTags("final"), withSize(2000)), + fixtureDoc("draft report.txt", withTags("draft"), withSize(50), withMtime("2020-01-01T00:00:00Z")), + fixtureDoc("photo.jpg", withMime("image/jpeg"), withTags("final")), + fixtureDoc("notes.md", withMime("text/markdown"), isHidden()), + fixtureFolder("archive"), + }, + cases: []queryCase{ + {id: 1, query: `name:"*report*" AND mediatype:document`, want: []string{"quarterly report.docx", "draft report.txt"}}, + {id: 2, query: `name:"*report*" AND NOT tag:("draft")`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{want: []string{"draft report.txt", "quarterly report.docx"}}}}, + {id: 3, query: `(tag:("final") OR tag:("draft")) AND mediatype:image`, want: []string{"photo.jpg"}}, + {id: 4, query: `mediatype:document AND mtime>"2021-01-01T00:00:00Z"`, want: []string{"quarterly report.docx", "notes.md"}}, + {id: 5, query: `name:"*report*" AND size>100`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 6, query: `tag:("final") AND NOT mediatype:folder`, want: []string{"quarterly report.docx", "photo.jpg"}}, + {id: 7, query: `hidden:true AND name:"*notes*"`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 8, query: `name:quarterly report`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 9, query: `name:"quarterly report"`}, + {id: 10, query: `name:"quarterly report.docx"`, want: []string{"quarterly report.docx"}}, + {id: 11, query: `NOT tag:("draft")`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md", "archive"}, engineOverrides: map[string]override{"bleve": override{want: []string{"archive", "draft report.txt", "notes.md", "photo.jpg", "quarterly report.docx"}}}}, + {id: 12, query: `tag:("final") OR hidden:true`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md"}, engineOverrides: map[string]override{"bleve": override{want: []string{"photo.jpg", "quarterly report.docx"}}}}, + {id: 13, query: `(name:"*report*" OR name:"*notes*") AND NOT (tag:("draft") OR hidden:true)`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{want: []string{"notes.md", "quarterly report.docx"}}}}, + {id: 14, query: `mediatype:image OR (mediatype:document AND tag:("draft"))`, want: []string{"photo.jpg", "draft report.txt"}}, + {id: 15, query: `NOT (mediatype:folder OR hidden:true)`, want: []string{"quarterly report.docx", "draft report.txt", "photo.jpg"}, engineOverrides: map[string]override{"bleve": override{want: []string{"draft report.txt", "notes.md", "photo.jpg", "quarterly report.docx"}}}}, + {id: 16, query: `name:"*report*" AND (size>100 OR tag:("draft"))`, want: []string{"quarterly report.docx", "draft report.txt"}, engineOverrides: map[string]override{"bleve": override{want: []string{"draft report.txt"}}, "opensearch": override{want: []string{"draft report.txt"}}}}, + }, + } +} diff --git a/services/search/pkg/parity/query_tags_test.go b/services/search/pkg/parity/query_tags_test.go new file mode 100644 index 0000000000..74cc3bc716 --- /dev/null +++ b/services/search/pkg/parity/query_tags_test.go @@ -0,0 +1,34 @@ +package parity + +import ( + "strings" + + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func tagsGroup() queryGroup { + longTag := strings.Repeat("z", 300) + "needle" + + return queryGroup{ + name: "tags", + fixtures: []search.Resource{ + fixtureDoc("invoice.txt", withTags("foo-bar")), + fixtureDoc("memo.txt", withTags("foo")), + fixtureDoc("spaced.txt", withTags("spaced tag")), + fixtureFolder("project", withTags("work")), + fixtureDoc("draft.txt", withParent("1$1!project"), withPath("./project/draft.txt")), + fixtureDoc("longtag.txt", withTags(longTag)), + }, + cases: []queryCase{ + {id: 1, query: `name:"*foo-bar*"`}, + {id: 2, query: `tag:("foo-bar")`, want: []string{"invoice.txt"}}, + {id: 3, query: `tag:("foo")`, want: []string{"memo.txt"}}, + {id: 4, query: `tag:("FOO-BAR")`, want: []string{"invoice.txt"}}, + {id: 5, query: `tag:("*foo*")`, want: []string{"invoice.txt", "memo.txt"}}, + {id: 6, query: `tag:("spaced tag")`, want: []string{"spaced.txt"}}, + {id: 7, query: `tag:("*paced ta*")`, want: []string{"spaced.txt"}}, + {id: 8, query: `tag:("work")`, want: []string{"project"}}, + {id: 9, query: `tag:("` + longTag + `")`, want: []string{"longtag.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + }, + } +} diff --git a/services/search/pkg/parity/query_title_test.go b/services/search/pkg/parity/query_title_test.go new file mode 100644 index 0000000000..8251c668db --- /dev/null +++ b/services/search/pkg/parity/query_title_test.go @@ -0,0 +1,23 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func titleGroup() queryGroup { + return queryGroup{ + name: "title", + fixtures: []search.Resource{ + fixtureDoc("q1.html", withMime("text/html"), withTitle("quarterly report")), + }, + cases: []queryCase{ + {id: 1, query: `Title:"quarterly report"`, want: []string{"q1.html"}}, + {id: 2, query: `Title:quarterly`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 3, query: `Title:QUARTERLY`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 4, query: `Title:quarterl*`, want: []string{"q1.html"}}, + {id: 5, query: `Title:"*ly rep*"`, want: []string{"q1.html"}}, + {id: 6, query: `title:quarterly`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 7, query: `Title:"QUARTERLY REPORT"`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}}, + }, + } +} diff --git a/services/search/pkg/parity/query_visibility_test.go b/services/search/pkg/parity/query_visibility_test.go new file mode 100644 index 0000000000..12939126c2 --- /dev/null +++ b/services/search/pkg/parity/query_visibility_test.go @@ -0,0 +1,26 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func visibilityGroup() queryGroup { + return queryGroup{ + name: "visibility", + fixtures: []search.Resource{ + fixtureDoc("visible.txt"), + fixtureDoc("dotfile.txt", isHidden()), + fixtureFolder(".private", isHidden()), + fixtureDoc("secret.txt", withParent("1$1!.private"), withPath("./.private/secret.txt"), isHidden()), + }, + cases: []queryCase{ + {id: 1, query: `hidden:true`, want: []string{"dotfile.txt", ".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 2, query: `hidden:TRUE`, want: []string{"dotfile.txt", ".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, + {id: 3, query: `hidden:false`, want: []string{"visible.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 4, query: `name:"*secret*"`, want: []string{"secret.txt"}}, + {id: 5, query: `path:"./.private"`, want: []string{".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{want: []string{".private"}}}}, + {id: 6, query: `hidden:banana`, engineOverrides: map[string]override{"opensearch": override{want: []string{"error"}}}}, + {id: 7, query: `hidden:"true"`, want: []string{"dotfile.txt", ".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, + }, + } +} diff --git a/services/search/pkg/parity/response_entity_test.go b/services/search/pkg/parity/response_entity_test.go new file mode 100644 index 0000000000..29ecffa8df --- /dev/null +++ b/services/search/pkg/parity/response_entity_test.go @@ -0,0 +1,102 @@ +package parity + +import ( + "fmt" + + searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func entityGroup() responseGroup { + parent := fixtureFolder("parent", withID("1$1!2")) + doc := fixtureDoc("bar.pdf", + withID("1$1!3"), + withParent(parent.ID), + withPath("./parent/bar.pdf"), + withMime("application/pdf"), + withSize(1234), + ) + notes := fixtureDoc("notes.txt", withID("1$1!4"), withContent("foo bar baz")) + + moved := func(e search.Engine) error { return e.Move(parent.ID, "1$1!9", "./somewhere/newname") } + + return responseGroup{ + name: "entity", + fixtures: []search.Resource{parent, doc, notes}, + cases: []responseCase{ + { + id: 1, query: `name:"bar.pdf"`, reads: "Ref.Path", want: []string{"./parent/bar.pdf"}, + read: reads(func(m *searchMessage.Match) string { return m.GetEntity().GetRef().GetPath() }), + }, + { + id: 2, query: `name:"bar.pdf"`, reads: "Name", want: []string{"bar.pdf"}, + read: reads(func(m *searchMessage.Match) string { return m.GetEntity().GetName() }), + }, + { + id: 3, query: `name:"bar.pdf"`, reads: "Id", want: []string{"1$1!3"}, + read: reads(func(m *searchMessage.Match) string { return resourceID(m.GetEntity().GetId()) }), + }, + { + id: 4, query: `name:"bar.pdf"`, reads: "ParentId", want: []string{"1$1!2"}, + read: reads(func(m *searchMessage.Match) string { return resourceID(m.GetEntity().GetParentId()) }), + }, + { + id: 5, query: `name:"bar.pdf"`, reads: "Ref.ResourceId", want: []string{"1$1!1"}, + read: reads(func(m *searchMessage.Match) string { return resourceID(m.GetEntity().GetRef().GetResourceId()) }), + }, + { + id: 6, query: `name:"bar.pdf"`, reads: "Size", want: []string{"1234"}, + read: reads(func(m *searchMessage.Match) string { return fmt.Sprint(m.GetEntity().GetSize()) }), + }, + { + id: 7, query: `name:"bar.pdf"`, reads: "Type", want: []string{"1"}, + read: reads(func(m *searchMessage.Match) string { return fmt.Sprint(m.GetEntity().GetType()) }), + }, + { + id: 8, query: `name:"bar.pdf"`, reads: "MimeType", want: []string{"application/pdf"}, + read: reads(func(m *searchMessage.Match) string { return m.GetEntity().GetMimeType() }), + }, + { + id: 9, query: `name:"bar.pdf"`, reads: "Deleted", want: []string{"false"}, + read: reads(func(m *searchMessage.Match) string { return fmt.Sprint(m.GetEntity().GetDeleted()) }), + }, + { + id: 10, query: `name:"bar.pdf"`, reads: "Score", want: []string{"above zero"}, + read: reads(func(m *searchMessage.Match) string { + if m.GetScore() > 0 { + return "above zero" + } + + return fmt.Sprint(m.GetScore()) + }), + }, + { + id: 11, query: `path:"./parent"`, reads: "TotalMatches", want: []string{"2"}, engineOverrides: map[string]override{"bleve": override{want: []string{"1"}}}, + read: func(resp *searchService.SearchIndexResponse) []string { + return []string{fmt.Sprint(resp.GetTotalMatches())} + }, + }, + { + id: 12, query: `name:"*notes*"`, reads: "Highlights", want: []string{`""`}, + read: reads(func(m *searchMessage.Match) string { return fmt.Sprintf("%q", m.GetEntity().GetHighlights()) }), + }, + { + id: 13, query: `content:bar`, reads: "Highlights", want: []string{"foo bar baz"}, + read: reads(func(m *searchMessage.Match) string { return m.GetEntity().GetHighlights() }), + }, + { + id: 14, do: moved, after: "moved to another parent", query: `name:"newname"`, reads: "ParentId", want: []string{"1$1!9"}, + read: reads(func(m *searchMessage.Match) string { return resourceID(m.GetEntity().GetParentId()) }), + }, + { + id: 15, do: moved, after: "moved to another parent", query: `name:"bar.pdf"`, reads: "ParentId", want: []string{"1$1!2"}, + read: reads(func(m *searchMessage.Match) string { return resourceID(m.GetEntity().GetParentId()) }), + }, + { + id: 16, do: moved, after: "moved to another parent", query: `name:"bar.pdf"`, reads: "Ref.Path", want: []string{"./somewhere/newname/bar.pdf"}, + read: reads(func(m *searchMessage.Match) string { return m.GetEntity().GetRef().GetPath() }), + }, + }, + } +} diff --git a/services/search/pkg/parity/response_metadata_test.go b/services/search/pkg/parity/response_metadata_test.go new file mode 100644 index 0000000000..e4bbca5e34 --- /dev/null +++ b/services/search/pkg/parity/response_metadata_test.go @@ -0,0 +1,143 @@ +package parity + +import ( + "fmt" + + libregraph "github.com/opencloud-eu/libre-graph-api-go" + + searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func metadataGroup() responseGroup { + song := fixtureDoc("some_song.mp3", + withID("1$1!5"), + withMime("audio/mpeg"), + withAudio(&libregraph.Audio{ + Album: libregraph.PtrString("Some Album"), + AlbumArtist: libregraph.PtrString("Some AlbumArtist"), + Artist: libregraph.PtrString("Some Artist"), + Bitrate: libregraph.PtrInt64(192), + Composers: libregraph.PtrString("Some Composers"), + Copyright: libregraph.PtrString(""), + Disc: libregraph.PtrInt32(2), + DiscCount: libregraph.PtrInt32(5), + Duration: libregraph.PtrInt64(225000), + Genre: libregraph.PtrString("Some Genre"), + HasDrm: libregraph.PtrBool(false), + IsVariableBitrate: libregraph.PtrBool(true), + Title: libregraph.PtrString("Some Title"), + Track: libregraph.PtrInt32(34), + TrackCount: libregraph.PtrInt32(99), + Year: libregraph.PtrInt32(2004), + }), + ) + + team := fixtureDoc("team.jpg", + withID("1$1!6"), + withMime("image/jpeg"), + withLocation(&libregraph.GeoCoordinates{ + Altitude: libregraph.PtrFloat64(1047.7), + Latitude: libregraph.PtrFloat64(49.48675890884328), + Longitude: libregraph.PtrFloat64(11.103870357204285), + }), + ) + + indexed := []string{ + "Album=Some Album", + "AlbumArtist=Some AlbumArtist", + "Artist=Some Artist", + "Bitrate=192", + "Composers=Some Composers", + "Copyright=", + "Disc=2", + "DiscCount=5", + "Duration=225000", + "Genre=Some Genre", + "HasDrm=false", + "IsVariableBitrate=true", + "Title=Some Title", + "Track=34", + "TrackCount=99", + "Year=2004", + } + + located := []string{ + "Altitude=1047.7", + "Latitude=49.48675890884328", + "Longitude=11.103870357204285", + } + + return responseGroup{ + name: "metadata", + fixtures: []search.Resource{song, team}, + cases: []responseCase{ + { + id: 1, query: `*song*`, reads: "Audio", want: unchanged(indexed), + read: readsMany(func(m *searchMessage.Match) []string { + audio := m.GetEntity().GetAudio() + + return changes(indexed, []string{ + fmt.Sprintf("Album=%s", audio.GetAlbum()), + fmt.Sprintf("AlbumArtist=%s", audio.GetAlbumArtist()), + fmt.Sprintf("Artist=%s", audio.GetArtist()), + fmt.Sprintf("Bitrate=%d", audio.GetBitrate()), + fmt.Sprintf("Composers=%s", audio.GetComposers()), + fmt.Sprintf("Copyright=%s", audio.GetCopyright()), + fmt.Sprintf("Disc=%d", audio.GetDisc()), + fmt.Sprintf("DiscCount=%d", audio.GetDiscCount()), + fmt.Sprintf("Duration=%d", audio.GetDuration()), + fmt.Sprintf("Genre=%s", audio.GetGenre()), + fmt.Sprintf("HasDrm=%t", audio.GetHasDrm()), + fmt.Sprintf("IsVariableBitrate=%t", audio.GetIsVariableBitrate()), + fmt.Sprintf("Title=%s", audio.GetTitle()), + fmt.Sprintf("Track=%d", audio.GetTrack()), + fmt.Sprintf("TrackCount=%d", audio.GetTrackCount()), + fmt.Sprintf("Year=%d", audio.GetYear()), + }) + }), + }, + { + id: 2, query: `*team*`, reads: "Location", want: unchanged(located), + read: readsMany(func(m *searchMessage.Match) []string { + location := m.GetEntity().GetLocation() + + return changes(located, []string{ + fmt.Sprintf("Altitude=%v", location.GetAltitude()), + fmt.Sprintf("Latitude=%v", location.GetLatitude()), + fmt.Sprintf("Longitude=%v", location.GetLongitude()), + }) + }), + }, + { + id: 3, query: `*team*`, reads: "Audio", want: []string{"none"}, + read: reads(func(m *searchMessage.Match) string { + if m.GetEntity().GetAudio() == nil { + return "none" + } + + return "set" + }), + }, + }, + } +} + +func unchanged(indexed []string) []string { + return []string{fmt.Sprintf("all %d fields unchanged", len(indexed))} +} + +func changes(indexed, got []string) []string { + var off []string + for i, field := range indexed { + if got[i] != field { + off = append(off, fmt.Sprintf("%s instead of %s", got[i], field)) + } + } + + if len(off) == 0 { + return unchanged(indexed) + } + + return off +} diff --git a/tests/acceptance/features/apiSearch1/search.feature b/tests/acceptance/features/apiSearch1/search.feature index 7f196c0e4f..122d8f2e98 100644 --- a/tests/acceptance/features/apiSearch1/search.feature +++ b/tests/acceptance/features/apiSearch1/search.feature @@ -501,3 +501,19 @@ Feature: Search | old | | new | | spaces | + + + Scenario: a deleted space leaves the search index + Given using spaces DAV path + And the administrator has assigned the role "Space Admin" to user "Alice" using the Graph API + And user "Alice" has created a space "index-space" with the default quota using the Graph API + And user "Alice" has uploaded a file inside space "index-space" with content "some data" to "inSpace.txt" + When user "Alice" searches for "*inSpace*" using the WebDAV API + Then the HTTP status code should be "207" + And the search result of user "Alice" should contain these entries: + | inSpace.txt | + When user "Alice" has disabled a space "index-space" + And user "Alice" has deleted a space "index-space" + And user "Alice" searches for "*inSpace*" using the WebDAV API + Then the HTTP status code should be "207" + And the search result should contain "0" entries