mirror/opencloud
1
0 Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-09-12 21:58:58 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3418 from opencloud-eu/test/search-parity-suite

Browse Source 
test(search): engine parity suite for bleve and opensearch
This commit is contained in:
Florian Schade authored and GitHub committed 2026-08-31 10:22:25 +02:00
2 parents 1bd59d9210 fef2df8671
commit ffbfe171c3
61 files changed
+3330 -1483
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.codacy.yml
+2
View File
@@ -18,6 +18,8 @@ exclude_paths:
- 'deployments/**'
- "release-config.ts"
- 'tests/acceptance/expected-failures-*.md'
# written by the search engine parity suite, table rows are as wide as they are
- 'services/search/pkg/parity/README.md'
- 'tests/acceptance/bootstrap/**'
- 'tests/acceptance/TestHelpers/**'
- 'tests/acceptance/scripts/run.sh'
services/search/pkg/opensearch/internal/test/helper.go → services/search/internal/opensearchtest/helper.go
View File
File renamed without changes.
services/search/pkg/opensearch/internal/test/os.go → services/search/internal/opensearchtest/os.go
View File
File renamed without changes.
services/search/pkg/opensearch/internal/test/suite.go → services/search/internal/opensearchtest/suite.go
+33 -3
View File
@@ -3,6 +3,7 @@ package opensearchtest
import (
"context"
"fmt"
"net/http"
"os"
"slices"
"strings"
@@ -18,7 +19,9 @@ import (
)
const (
openSearchImage = "opensearchproject/opensearch:2"
openSearchImage = "opensearchproject/opensearch:2"
openSearchPort = "9200"
openSearchStartupTimeout = 3 * time.Minute
)
func SetupTests(ctx context.Context) (*config.Config, func(), error) {
@@ -62,6 +65,14 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func
return func() {}, nil
}
keepContainer := os.Getenv("KEEP_TEST_CONTAINER") == "true"
if keepContainer {
// the reaper would take the kept container down with the session
if err := os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true"); err != nil {
return nil, fmt.Errorf("failed to disable the testcontainers reaper: %w", err)
}
}
containerName := fmt.Sprintf("opencloud/test/%s", openSearchImage)
containerName = strings.Replace(containerName, "/", "__", -1)
containerName = strings.Replace(containerName, ":", "_", -1)
@@ -72,9 +83,21 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func
opensearch.WithPassword(cfg.Engine.OpenSearch.Client.Password),
testcontainers.WithName(containerName),
testcontainers.WithReuseByName(containerName),
// test indexes are tiny; don't let a full host disk trip the flood-stage
// create-index / read-only blocks mid-run
testcontainers.WithEnv(map[string]string{
"cluster.routing.allocation.disk.threshold_enabled": "false",
}),
// a health probe answers at once on a reused container, a log wait
// would sit out the log timeout on it; a cold boot takes well over the
// previous 5s
testcontainers.WithWaitStrategy(
wait.ForLog("ML configuration initialized successfully").
WithStartupTimeout(5*time.Second),
wait.ForHTTP("/_cluster/health?wait_for_status=yellow&timeout=1s").
WithPort(openSearchPort).
WithTLS(false).
WithBasicAuth(cfg.Engine.OpenSearch.Client.Username, cfg.Engine.OpenSearch.Client.Password).
WithStatusCodeMatcher(func(status int) bool { return status == http.StatusOK }).
WithStartupTimeout(openSearchStartupTimeout),
),
)
if err != nil {
@@ -91,6 +114,13 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func
cfg.Engine.OpenSearch.Client.Addresses = []string{address}
return func() {
// KEEP_TEST_CONTAINER=true leaves the container up for the next run,
// which picks it up again by name instead of booting a fresh one
if keepContainer {
_, _ = fmt.Fprintf(os.Stderr, "keeping OpenSearch container %s\n", containerName)
return
}
err := container.Terminate(ctx)
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "failed to terminate OpenSearch container: %v\n", err)
services/search/pkg/opensearch/internal/test/test.go → services/search/internal/opensearchtest/test.go
View File
File renamed without changes.
services/search/pkg/opensearch/internal/test/testdata.go → services/search/internal/opensearchtest/testdata.go
View File
File renamed without changes.
services/search/pkg/opensearch/internal/test/testdata/resource_file.json → services/search/internal/opensearchtest/testdata/resource_file.json
Vendored
View File
File renamed without changes.
services/search/pkg/opensearch/internal/test/testdata/resource_folder.json → services/search/internal/opensearchtest/testdata/resource_folder.json
Vendored
View File
File renamed without changes.
services/search/pkg/opensearch/internal/test/testdata/resource_root.json → services/search/internal/opensearchtest/testdata/resource_root.json
Vendored
View File
File renamed without changes.
services/search/pkg/bleve/backend_test.go
-808
View File
@@ -1,808 +0,0 @@
package bleve_test
import (
"context"
"fmt"
bleveSearch "github.com/blevesearch/bleve/v2"
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/opencloud/pkg/log"
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func hiddenByID(idx bleveSearch.Index, id string) bool {
GinkgoHelper()
req := bleveSearch.NewSearchRequest(bleveSearch.NewDocIDQuery([]string{id}))
req.Fields = []string{"Hidden"}
res, err := idx.Search(req)
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits).To(HaveLen(1), "no record for %s", id)
hidden, _ := res.Hits[0].Fields["Hidden"].(bool)
return hidden
}
var _ = Describe("Bleve", func() {
var (
eng *bleve.Backend
idx bleveSearch.Index
doSearch = func(id string, query, path string) (*searchsvc.SearchIndexResponse, error) {
rID, err := storagespace.ParseID(id)
if err != nil {
return nil, err
}
return eng.Search(context.Background(), &searchsvc.SearchIndexRequest{
Query: query,
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: rID.StorageId,
SpaceId: rID.SpaceId,
OpaqueId: rID.OpaqueId,
},
Path: path,
},
})
}
assertDocCount = func(id string, query string, expectedCount int) []*searchmsg.Match {
res, err := doSearch(id, query, "")
ExpectWithOffset(1, err).ToNot(HaveOccurred())
ExpectWithOffset(1, len(res.Matches)).To(Equal(expectedCount), "query returned unexpected number of results: "+query)
return res.Matches
}
rootResource search.Resource
parentResource search.Resource
childResource search.Resource
childResource2 search.Resource
)
BeforeEach(func() {
mapping, err := bleve.NewMapping()
Expect(err).ToNot(HaveOccurred())
idx, err = bleveSearch.NewMemOnly(mapping)
Expect(err).ToNot(HaveOccurred())
eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{})
Expect(err).ToNot(HaveOccurred())
rootResource = search.Resource{
ID: "1$2!2",
RootID: "1$2!2",
Path: ".",
Document: content.Document{},
}
parentResource = search.Resource{
ID: "1$2!3",
ParentID: rootResource.ID,
RootID: rootResource.ID,
Path: "./parent d!r",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER),
Document: content.Document{Name: "parent d!r"},
}
childResource = search.Resource{
ID: "1$2!4",
ParentID: parentResource.ID,
RootID: rootResource.ID,
Path: "./parent d!r/child.pdf",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{Name: "child.pdf"},
}
childResource2 = search.Resource{
ID: "1$2!5",
ParentID: parentResource.ID,
RootID: rootResource.ID,
Path: "./parent d!r/child2.pdf",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{Name: "child2.pdf"},
}
})
Describe("PurgeSpace", func() {
It("takes every record of that space out of the index", func() {
otherSpace := search.Resource{
ID: "1$9!9",
RootID: "1$9!9",
Path: ".",
Document: content.Document{Name: "other"},
}
for _, resource := range []search.Resource{rootResource, parentResource, childResource, otherSpace} {
Expect(eng.Upsert(resource.ID, resource)).To(Succeed())
}
Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed())
count, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(1)), "only the records of that space are gone")
})
It("takes a space out that holds more records than one round", func() {
otherSpace := search.Resource{
ID: "1$9!9",
RootID: "1$9!9",
Path: ".",
Document: content.Document{Name: "other"},
}
Expect(eng.Upsert(otherSpace.ID, otherSpace)).To(Succeed())
for i := range 120 {
resource := search.Resource{
ID: fmt.Sprintf("%s!file-%d", rootResource.RootID, i),
RootID: rootResource.RootID,
Path: fmt.Sprintf("./file-%d", i),
Document: content.Document{Name: fmt.Sprintf("file-%d", i)},
}
Expect(eng.Upsert(resource.ID, resource)).To(Succeed())
}
Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed())
count, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(1)), "only the record of the other space is left")
})
})
Describe("New", func() {
It("returns a new index instance", func() {
b := bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{})
Expect(b).ToNot(BeNil())
})
})
Describe("Search", func() {
Context("by other fields than filename", func() {
It("finds files by tags", func() {
parentResource.Document.Tags = []string{"foo", "bar"}
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Tags:foo", 1)
assertDocCount(rootResource.ID, "Tags:bar", 1)
assertDocCount(rootResource.ID, "Tags:foo Tags:bar", 1)
assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1)
assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1)
assertDocCount(rootResource.ID, "Tags:baz", 0)
})
It("finds files by size", func() {
parentResource.Document.Size = 12345
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Size:12345", 1)
assertDocCount(rootResource.ID, "Size:>1000", 1)
assertDocCount(rootResource.ID, "Size:<100000", 1)
assertDocCount(rootResource.ID, "Size:12344", 0)
assertDocCount(rootResource.ID, "Size:<1000", 0)
assertDocCount(rootResource.ID, "Size:>100000", 0)
})
It("preserves value case for fields not explicitly marked lowercase", func() {
parentResource.Document.Audio = &libregraph.Audio{
Artist: libregraph.PtrString("Some Artist"),
}
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `audio.artist:"Some Artist"`, 1)
assertDocCount(rootResource.ID, `audio.artist:"some artist"`, 0)
})
})
Context("by filename", func() {
It("finds files with spaces in the filename", func() {
parentResource.Document.Name = "Foo oo.pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `name:"foo o*"`, 1)
})
It("finds files by digits in the filename", func() {
parentResource.Document.Name = "12345.pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:1234*", 1)
})
It("filters hidden files", func() {
childResource.Hidden = true
err := eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Hidden:T", 1)
assertDocCount(rootResource.ID, "Hidden:F", 0)
})
Context("with a file in the root of the space", func() {
It("scopes the search to the specified space", func() {
parentResource.Document.Name = "foo.pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:foo.pdf", 1)
assertDocCount("9$8!7", "Name:foo.pdf", 0)
})
})
It("limits the search to the specified fields", func() {
parentResource.Document.Name = "bar.pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:bar.pdf", 1)
assertDocCount(rootResource.ID, "Unknown:field", 0)
})
It("returns the total number of hits", func() {
parentResource.Document.Name = "bar.pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
res, err := doSearch(rootResource.ID, "Name:bar*", "")
Expect(err).ToNot(HaveOccurred())
Expect(res.TotalMatches).To(Equal(int32(1)))
})
It("returns all desired fields", func() {
parentResource.Document.Name = "bar.pdf"
parentResource.Type = 3
parentResource.MimeType = "application/pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
matches := assertDocCount(rootResource.ID, fmt.Sprintf("Name:%s", parentResource.Name), 1)
match := matches[0]
Expect(match.Entity.Ref.Path).To(Equal(parentResource.Path))
Expect(match.Entity.Name).To(Equal(parentResource.Name))
Expect(match.Entity.Size).To(Equal(parentResource.Size))
Expect(match.Entity.Type).To(Equal(parentResource.Type))
Expect(match.Entity.MimeType).To(Equal(parentResource.MimeType))
Expect(match.Entity.Deleted).To(BeFalse())
Expect(match.Score > 0).To(BeTrue())
})
It("finds files by name, prefix or substring match", func() {
parentResource.Document.Name = "foo.pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
queries := []string{"foo.pdf", "foo*", "*oo.p*"}
for _, query := range queries {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, query, 1)
}
})
It("does a case-insensitive search", func() {
parentResource.Document.Name = "foo.pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:foo*", 1)
assertDocCount(rootResource.ID, "Name:Foo*", 1)
})
Context("and an additional file in a subdirectory", func() {
BeforeEach(func() {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
})
It("finds files living deeper in the tree by filename, prefix or substring match", func() {
queries := []string{"child.pdf", "child*", "*ld.*"}
for _, query := range queries {
assertDocCount(rootResource.ID, query, 1)
}
})
})
})
Context("Highlights", func() {
It("highlights only for content searches", func() {
parentResource.Document.Name = "baz.pdf"
parentResource.Document.Content = "foo bar baz"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
res, err := doSearch(rootResource.ID, "Name:baz*", "")
Expect(err).ToNot(HaveOccurred())
Expect(res.TotalMatches).To(Equal(int32(1)))
Expect(res.Matches[0].Entity.Highlights).To(Equal(""))
})
It("highlights search terms", func() {
parentResource.Document.Name = "baz.pdf"
parentResource.Document.Content = "foo bar baz"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
res, err := doSearch(rootResource.ID, "Content:bar", "")
Expect(err).ToNot(HaveOccurred())
Expect(res.TotalMatches).To(Equal(int32(1)))
Expect(res.Matches[0].Entity.Highlights).To(Equal("foo <mark>bar</mark> baz"))
})
})
Context("with a file in the root of the space and folder with a file. all of them have the same name", func() {
BeforeEach(func() {
parentResource := search.Resource{
ID: "1$2!3",
ParentID: rootResource.ID,
RootID: rootResource.ID,
Path: "./doc",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER),
Document: content.Document{Name: "doc"},
}
childResource := search.Resource{
ID: "1$2!4",
ParentID: parentResource.ID,
RootID: rootResource.ID,
Path: "./doc/doc.pdf",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{Name: "doc.pdf"},
}
childResource2 := search.Resource{
ID: "1$2!7",
ParentID: parentResource.ID,
RootID: rootResource.ID,
Path: "./doc/file.pdf",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{Name: "file.pdf"},
}
rootChildResource := search.Resource{
ID: "1$2!5",
ParentID: rootResource.ID,
RootID: rootResource.ID,
Path: "./doc.pdf",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{Name: "doc.pdf"},
}
rootChildResource2 := search.Resource{
ID: "1$2!6",
ParentID: rootResource.ID,
RootID: rootResource.ID,
Path: "./file.pdf",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{Name: "file.pdf"},
}
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(rootChildResource.ID, rootChildResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(rootChildResource2.ID, rootChildResource2)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource2.ID, childResource2)
Expect(err).ToNot(HaveOccurred())
})
It("search *doc* in a root", func() {
res, err := doSearch(rootResource.ID, "Name:*doc*", "")
Expect(err).ToNot(HaveOccurred())
Expect(res.TotalMatches).To(Equal(int32(3)))
})
It("search *doc* in a subfolder", func() {
res, err := doSearch(rootResource.ID, "Name:*doc*", "./doc")
Expect(err).ToNot(HaveOccurred())
Expect(res.TotalMatches).To(Equal(int32(2)))
})
It("search *file* in a root", func() {
res, err := doSearch(rootResource.ID, "Name:*file*", "")
Expect(err).ToNot(HaveOccurred())
Expect(res.TotalMatches).To(Equal(int32(2)))
})
It("search *file* in a subfolder", func() {
res, err := doSearch(rootResource.ID, "Name:*file*", "./doc")
Expect(err).ToNot(HaveOccurred())
Expect(res.TotalMatches).To(Equal(int32(1)))
})
})
})
Describe("Upsert", func() {
It("adds a resourceInfo to the index", func() {
err := eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
count, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(1)))
query := bleveSearch.NewMatchQuery("child.pdf")
res, err := idx.Search(bleveSearch.NewSearchRequest(query))
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits.Len()).To(Equal(1))
})
It("updates an existing resource in the index", func() {
err := eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
countA, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(countA).To(Equal(uint64(1)))
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
countB, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(countB).To(Equal(uint64(1)))
})
})
Describe("Delete", func() {
It("marks a resource as deleted", func() {
err := eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:*child*", 1)
err = eng.Delete(childResource.ID)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:*child*", 0)
})
It("marks a child resources as deleted", func() {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1)
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
err = eng.Delete(parentResource.ID)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0)
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0)
})
})
Describe("Restore", func() {
It("also marks child resources as restored", func() {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Delete(parentResource.ID)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 0)
assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 0)
err = eng.Restore(parentResource.ID)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 1)
assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 1)
})
})
Describe("Purge", func() {
It("removes a resource from the index", func() {
err := eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:child.pdf", 1)
err = eng.Purge(childResource.ID, false)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:child.pdf", 0)
})
It("removes a resource and its children from the index", func() {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1)
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
err = eng.Purge(parentResource.ID, false)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0)
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0)
})
It("removes a resource and ignores its children from the index", func() {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1)
err = eng.Delete(parentResource.ID)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
err = eng.Purge(parentResource.ID, true)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0)
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
})
})
Describe("Move", func() {
It("renames the parent and its child resources", func() {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
parentResource.Path = "newname"
err = eng.Move(parentResource.ID, parentResource.ParentID, "./my/newname")
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, parentResource.Name, 0)
matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1)
Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3"))
Expect(matches[0].Entity.Ref.Path).To(Equal("./my/newname/child.pdf"))
})
DescribeTable("keeps the flag in step with the path",
func(from, target string, hidden bool) {
parentResource.Path = from
parentResource.Hidden = search.IsHidden(from)
childResource.Path = from + "/child.pdf"
childResource.Hidden = parentResource.Hidden
Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed())
Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed())
Expect(eng.Move(parentResource.ID, parentResource.ParentID, target)).To(Succeed())
for _, id := range []string{parentResource.ID, childResource.ID} {
Expect(hiddenByID(idx, id)).
To(Equal(hidden), "%s after moving from %s to %s", id, from, target)
}
},
Entry("into a dot folder", "./parent", "./.trash/parent", true),
Entry("into a plain folder", "./parent", "./archive/parent", false),
Entry("renamed with a leading dot", "./parent", "./.parent", true),
Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false),
Entry("renamed without the leading dot", "./.parent", "./parent", false),
Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true),
)
// the trash leaves the path alone, so the flag has to come through untouched
It("carries the flag through the trash and back", func() {
childResource.Path = "./.secret/file.txt"
childResource.Hidden = true
Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed())
Expect(eng.Delete(childResource.ID)).To(Succeed())
Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after trashing")
Expect(eng.Restore(childResource.ID)).To(Succeed())
Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after restoring")
})
It("moves the parent and its child resources", func() {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
parentResource.Path = " "
parentResource.ParentID = "1$2!somewhereopaqueid"
err = eng.Move(parentResource.ID, parentResource.ParentID, "./somewhere/else/newname")
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `parent d!r`, 0)
matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1)
Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3"))
Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname/child.pdf"))
matches = assertDocCount(rootResource.ID, `newname`, 1)
Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("somewhereopaqueid"))
Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname"))
})
})
Describe("StartBatch", func() {
It("starts a new batch", func() {
b, err := eng.NewBatch(100)
Expect(err).ToNot(HaveOccurred())
err = b.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
count, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(0)))
err = b.Push()
Expect(err).ToNot(HaveOccurred())
count, err = idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(1)))
query := bleveSearch.NewMatchQuery("child.pdf")
res, err := idx.Search(bleveSearch.NewSearchRequest(query))
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits.Len()).To(Equal(1))
})
It("doesn't intertwine different batches", func() {
b, err := eng.NewBatch(100)
Expect(err).ToNot(HaveOccurred())
err = b.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
count, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(0)))
b2, err := eng.NewBatch(100)
Expect(err).ToNot(HaveOccurred())
err = b2.Upsert(childResource2.ID, childResource2)
Expect(err).ToNot(HaveOccurred())
Expect(b.Push()).To(Succeed())
count, err = idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(1)))
Expect(b2.Push()).To(Succeed())
count, err = idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(2)))
})
})
Describe("File type specific metadata", func() {
Context("with audio metadata", func() {
BeforeEach(func() {
resource := search.Resource{
ID: "1$2!7",
ParentID: rootResource.ID,
RootID: rootResource.ID,
Path: "./some_song.mp3",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{
Name: "some_song.mp3",
MimeType: "audio/mpeg",
Audio: &libregraph.Audio{
Album: libregraph.PtrString("Some Album"),
AlbumArtist: libregraph.PtrString("Some AlbumArtist"),
Artist: libregraph.PtrString("Some Artist"),
Bitrate: libregraph.PtrInt64(192),
Composers: libregraph.PtrString("Some Composers"),
Copyright: libregraph.PtrString(""),
Disc: libregraph.PtrInt32(2),
DiscCount: libregraph.PtrInt32(5),
Duration: libregraph.PtrInt64(225000),
Genre: libregraph.PtrString("Some Genre"),
HasDrm: libregraph.PtrBool(false),
IsVariableBitrate: libregraph.PtrBool(true),
Title: libregraph.PtrString("Some Title"),
Track: libregraph.PtrInt32(34),
TrackCount: libregraph.PtrInt32(99),
Year: libregraph.PtrInt32(2004),
},
},
}
err := eng.Upsert(resource.ID, resource)
Expect(err).ToNot(HaveOccurred())
})
It("returns audio metadata for search", func() {
matches := assertDocCount(rootResource.ID, `*song*`, 1)
audio := matches[0].Entity.Audio
Expect(audio).ToNot(BeNil())
Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album")))
Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist")))
Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist")))
Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192)))
Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers")))
Expect(audio.Copyright).To(Equal(libregraph.PtrString("")))
Expect(audio.Disc).To(Equal(libregraph.PtrInt32(2)))
Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5)))
Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225000)))
Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre")))
Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false)))
Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true)))
Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title")))
Expect(audio.Track).To(Equal(libregraph.PtrInt32(34)))
Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(99)))
Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004)))
})
})
Context("with location metadata", func() {
BeforeEach(func() {
resource := search.Resource{
ID: "1$2!7",
ParentID: rootResource.ID,
RootID: rootResource.ID,
Path: "./team.jpg",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{
Name: "team.jpg",
MimeType: "image/jpeg",
Location: &libregraph.GeoCoordinates{
Altitude: libregraph.PtrFloat64(1047.7),
Latitude: libregraph.PtrFloat64(49.48675890884328),
Longitude: libregraph.PtrFloat64(11.103870357204285),
},
},
}
err := eng.Upsert(resource.ID, resource)
Expect(err).ToNot(HaveOccurred())
})
It("returns audio metadata for search", func() {
matches := assertDocCount(rootResource.ID, `*team*`, 1)
location := matches[0].Entity.Location
Expect(location).ToNot(BeNil())
Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(1047.7)))
Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328)))
Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285)))
})
})
})
})
services/search/pkg/opensearch/backend_test.go
+2 -660
View File
@@ -1,9 +1,6 @@
package opensearch_test
import (
"context"
"fmt"
"strings"
"testing"
. "github.com/onsi/ginkgo/v2"
@@ -11,12 +8,7 @@ import (
opensearchgo "github.com/opensearch-project/opensearch-go/v4"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func TestOpenSearchBackend(t *testing.T) {
@@ -24,56 +16,8 @@ func TestOpenSearchBackend(t *testing.T) {
RunSpecs(t, "OpenSearch Backend Suite")
}
func deleteIndexOnCleanup(tc *opensearchtest.TestClient, indexName string) {
DeferCleanup(func() {
Expect(tc.IndicesDelete(context.Background(), []string{indexName})).To(Succeed())
})
}
func resourceByID(tc *opensearchtest.TestClient, index, id string) search.Resource {
GinkgoHelper()
body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
"query": map[string]any{
"ids": map[string]any{
"values": []string{id},
},
},
})
resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(index, strings.NewReader(body)).Hits)
Expect(resources).To(HaveLen(1))
return resources[0]
}
// otherRoot returns a copy of the given resource that lives in a different root (space)
// while keeping the same path, so it can be used to assert that cross-root updates do
// not affect identically-named resources in other roots.
func otherRoot(r search.Resource) search.Resource {
r.ID = "2$2!3"
r.RootID = "2$2!1"
r.ParentID = "2$2!2"
return r
}
func newBackend(indexName string, resources ...search.Resource) (*opensearch.Backend, *opensearchtest.TestClient) {
GinkgoHelper()
tc := opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
backend, err := opensearch.NewBackend(indexName, tc.Client())
Expect(err).ToNot(HaveOccurred())
for _, r := range resources {
tc.Require.DocumentCreate(indexName, r.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), r)))
}
tc.Require.IndicesCount([]string{indexName}, nil, len(resources))
return backend, tc
}
// what the engine does with its index is covered for both engines by
// services/search/pkg/parity; this is the one thing only OpenSearch can do
var _ = Describe("Backend", func() {
Describe("NewBackend", func() {
It("fails to create if the cluster is not healthy", func() {
@@ -89,606 +33,4 @@ var _ = Describe("Backend", func() {
Expect(err).To(MatchError(opensearch.ErrUnhealthyCluster))
})
})
Describe("Search", func() {
const indexName = "opencloud-test-engine-search"
var (
tc *opensearchtest.TestClient
backend *opensearch.Backend
document search.Resource
)
BeforeEach(func() {
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
deleteIndexOnCleanup(tc, indexName)
var err error
backend, err = opensearch.NewBackend(indexName, tc.Client())
Expect(err).ToNot(HaveOccurred())
document = opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
})
It("performs the most simple search", func() {
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{
Query: fmt.Sprintf(`"%s"`, document.Name),
})
Expect(err).ToNot(HaveOccurred())
Expect(resp.Matches).To(HaveLen(1))
Expect(resp.TotalMatches).To(Equal(int32(1)))
Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID))
})
It("ignores files that are marked as deleted", func() {
deletedDocument := opensearchtest.Testdata.Resources.File
deletedDocument.ID = "1$2!4"
deletedDocument.Deleted = true
tc.Require.DocumentCreate(indexName, deletedDocument.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), deletedDocument)))
tc.Require.IndicesCount([]string{indexName}, nil, 2)
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{
Query: fmt.Sprintf(`"%s"`, document.Name),
})
Expect(err).ToNot(HaveOccurred())
Expect(resp.Matches).To(HaveLen(1))
Expect(resp.TotalMatches).To(Equal(int32(1)))
Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID))
})
})
Describe("Upsert", func() {
const indexName = "opencloud-test-engine-upsert"
var (
tc *opensearchtest.TestClient
backend *opensearch.Backend
)
BeforeEach(func() {
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
deleteIndexOnCleanup(tc, indexName)
var err error
backend, err = opensearch.NewBackend(indexName, tc.Client())
Expect(err).ToNot(HaveOccurred())
})
It("upserts a full document", func() {
document := opensearchtest.Testdata.Resources.File
Expect(backend.Upsert(document.ID, document)).To(Succeed())
tc.Require.IndicesCount([]string{indexName}, nil, 1)
})
})
Describe("Move", func() {
const indexName = "opencloud-test-engine-move"
var (
tc *opensearchtest.TestClient
backend *opensearch.Backend
)
BeforeEach(func() {
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
deleteIndexOnCleanup(tc, indexName)
var err error
backend, err = opensearch.NewBackend(indexName, tc.Client())
Expect(err).ToNot(HaveOccurred())
})
It("moves the document to a new path", func() {
document := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
"query": map[string]any{
"ids": map[string]any{
"values": []string{document.ID},
},
},
})
resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits)
Expect(resources).To(HaveLen(1))
Expect(resources[0].Path).To(Equal(document.Path))
document.Path = "./new/path/to/resource"
Expect(backend.Move(document.ID, document.ParentID, document.Path)).To(Succeed())
resources = opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits)
Expect(resources).To(HaveLen(1))
Expect(resources[0].Path).To(Equal(document.Path))
})
})
Describe("WriteVisibility", func() {
const indexName = "opencloud-test-engine-write-visibility"
It("deletes a record that was just written", func() {
document := opensearchtest.Testdata.Resources.File
document.ID = "1$1!95"
document.Name = "textfile.txt"
document.Path = "./textfile.txt"
backend, tc := newBackend(indexName)
deleteIndexOnCleanup(tc, indexName)
Expect(backend.Upsert(document.ID, document)).To(Succeed())
Expect(backend.Delete(document.ID)).To(Succeed())
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{
Query: fmt.Sprintf(`name:"%s"`, document.Name),
})
Expect(err).ToNot(HaveOccurred())
Expect(resp.Matches).To(BeEmpty())
})
})
Describe("Delete", func() {
const indexName = "opencloud-test-engine-delete"
var (
tc *opensearchtest.TestClient
backend *opensearch.Backend
)
BeforeEach(func() {
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
deleteIndexOnCleanup(tc, indexName)
var err error
backend, err = opensearch.NewBackend(indexName, tc.Client())
Expect(err).ToNot(HaveOccurred())
})
It("marks the document as deleted", func() {
document := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
"query": map[string]any{
"term": map[string]any{
"Deleted": map[string]any{
"value": true,
},
},
},
})
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0)
Expect(backend.Delete(document.ID)).To(Succeed())
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1)
})
})
Describe("Restore", func() {
const indexName = "opencloud-test-engine-restore"
var (
tc *opensearchtest.TestClient
backend *opensearch.Backend
)
BeforeEach(func() {
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
deleteIndexOnCleanup(tc, indexName)
var err error
backend, err = opensearch.NewBackend(indexName, tc.Client())
Expect(err).ToNot(HaveOccurred())
})
It("marks the document as not deleted", func() {
document := opensearchtest.Testdata.Resources.File
document.Deleted = true
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
"query": map[string]any{
"term": map[string]any{
"Deleted": map[string]any{
"value": true,
},
},
},
})
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1)
Expect(backend.Restore(document.ID)).To(Succeed())
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0)
})
})
Describe("Purge", func() {
const indexName = "opencloud-test-engine-purge"
var (
tc *opensearchtest.TestClient
backend *opensearch.Backend
)
BeforeEach(func() {
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
deleteIndexOnCleanup(tc, indexName)
var err error
backend, err = opensearch.NewBackend(indexName, tc.Client())
Expect(err).ToNot(HaveOccurred())
})
It("purges a full document", func() {
document := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
Expect(backend.Purge(document.ID, false)).To(Succeed())
tc.Require.IndicesCount([]string{indexName}, nil, 0)
})
It("purges resource trees", func() {
resourceFolder := opensearchtest.Testdata.Resources.Folder
tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder)))
resourceFile := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile)))
tc.Require.IndicesCount([]string{indexName}, nil, 2)
Expect(backend.Purge(resourceFolder.ID, false)).To(Succeed())
tc.Require.IndicesCount([]string{indexName}, nil, 0)
})
It("purges resource trees and ignores undeleted resources", func() {
resourceFolder := opensearchtest.Testdata.Resources.Folder
tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder)))
resourceFile := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile)))
tc.Require.IndicesCount([]string{indexName}, nil, 2)
Expect(backend.Delete(resourceFile.ID)).To(Succeed())
tc.Require.IndicesRefresh([]string{indexName}, nil)
Expect(backend.Purge(resourceFolder.ID, true)).To(Succeed())
tc.Require.IndicesCount([]string{indexName}, nil, 1)
})
})
Describe("PurgeSpace", func() {
const indexName = "opencloud-test-engine-purge-space"
var (
tc *opensearchtest.TestClient
backend *opensearch.Backend
)
BeforeEach(func() {
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
deleteIndexOnCleanup(tc, indexName)
var err error
backend, err = opensearch.NewBackend(indexName, tc.Client())
Expect(err).ToNot(HaveOccurred())
})
It("takes every record of that space out of the index", func() {
gone := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(indexName, gone.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), gone)))
stays := opensearchtest.Testdata.Resources.File
stays.ID = "1$2!3"
stays.RootID = "1$2!2"
tc.Require.DocumentCreate(indexName, stays.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), stays)))
tc.Require.IndicesCount([]string{indexName}, nil, 2)
Expect(backend.PurgeSpace(gone.RootID)).To(Succeed())
tc.Require.IndicesRefresh([]string{indexName}, nil)
left := opensearchtest.SearchHitsMustBeConverted[search.Resource](
GinkgoTB(),
tc.Require.Search(indexName, strings.NewReader(`{"query":{"match_all":{}}}`)).Hits,
)
Expect(left).To(HaveLen(1), "only the records of that space are gone")
Expect(left[0].ID).To(Equal(stays.ID))
})
})
Describe("Hidden", func() {
const indexName = "opencloud-test-engine-hidden"
DescribeTable("keeps the flag in step with the path",
func(from, target string, hidden bool) {
folder := opensearchtest.Testdata.Resources.Folder
folder.ID = "1$1!30"
folder.Name = "parent"
folder.Path = from
folder.Hidden = search.IsHidden(from)
child := opensearchtest.Testdata.Resources.File
child.ID = "1$1!31"
child.Name = "child.txt"
child.Path = from + "/child.txt"
child.ParentID = folder.ID
child.Hidden = folder.Hidden
backend, tc := newBackend(indexName, folder, child)
deleteIndexOnCleanup(tc, indexName)
tc.Require.IndicesRefresh([]string{indexName}, nil)
Expect(backend.Move(folder.ID, folder.ParentID, target)).To(Succeed())
tc.Require.IndicesRefresh([]string{indexName}, nil)
for _, id := range []string{folder.ID, child.ID} {
Expect(resourceByID(tc, indexName, id).Hidden).
To(Equal(hidden), "%s after moving from %s to %s", id, from, target)
}
},
Entry("into a dot folder", "./parent", "./.trash/parent", true),
Entry("into a plain folder", "./parent", "./archive/parent", false),
Entry("renamed with a leading dot", "./parent", "./.parent", true),
Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false),
Entry("renamed without the leading dot", "./.parent", "./parent", false),
Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true),
)
It("carries the flag through the trash and back", func() {
hidden := opensearchtest.Testdata.Resources.File
hidden.ID = "1$1!32"
hidden.Path = "./.secret/file.txt"
hidden.Hidden = true
backend, tc := newBackend(indexName, hidden)
deleteIndexOnCleanup(tc, indexName)
tc.Require.IndicesRefresh([]string{indexName}, nil)
Expect(backend.Delete(hidden.ID)).To(Succeed())
tc.Require.IndicesRefresh([]string{indexName}, nil)
Expect(resourceByID(tc, indexName, hidden.ID).Hidden).To(BeTrue(), "after trashing")
Expect(backend.Restore(hidden.ID)).To(Succeed())
tc.Require.IndicesRefresh([]string{indexName}, nil)
Expect(resourceByID(tc, indexName, hidden.ID).Hidden).To(BeTrue(), "after restoring")
})
})
Describe("DocCount", func() {
const indexName = "opencloud-test-engine-doc-count"
var (
tc *opensearchtest.TestClient
backend *opensearch.Backend
)
BeforeEach(func() {
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
deleteIndexOnCleanup(tc, indexName)
var err error
backend, err = opensearch.NewBackend(indexName, tc.Client())
Expect(err).ToNot(HaveOccurred())
})
It("ignores deleted documents", func() {
document := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
count, err := backend.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(1)))
tc.Require.Update(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
"doc": map[string]any{
"Deleted": true,
},
})))
tc.Require.IndicesCount([]string{indexName}, nil, 1)
count, err = backend.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(0)))
})
})
// The following specs ensure that updates which affect a resource and its descendants
// (Delete, Restore, Move) are scoped to the root (space) of the target resource. Two
// resources living in different roots may share the exact same path, so matching by
// path alone would incorrectly update the wrong resource.
Describe("updateSelfAndDescendants root scope", func() {
It("deletes only the resource in the target root", func() {
const indexName = "opencloud-test-engine-root-scope-delete"
target := opensearchtest.Testdata.Resources.File
other := otherRoot(target)
backend, tc := newBackend(indexName, target, other)
deleteIndexOnCleanup(tc, indexName)
Expect(backend.Delete(target.ID)).To(Succeed())
Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeTrue(), "target resource should be marked as deleted")
Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeFalse(), "resource in a different root must not be affected")
})
It("restores only the resource in the target root", func() {
const indexName = "opencloud-test-engine-root-scope-restore"
target := opensearchtest.Testdata.Resources.File
target.Deleted = true
other := otherRoot(target)
backend, tc := newBackend(indexName, target, other)
deleteIndexOnCleanup(tc, indexName)
Expect(backend.Restore(target.ID)).To(Succeed())
Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeFalse(), "target resource should be restored")
Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeTrue(), "resource in a different root must not be affected")
})
It("moves only the resource in the target root", func() {
const indexName = "opencloud-test-engine-root-scope-move"
target := opensearchtest.Testdata.Resources.File
other := otherRoot(target)
backend, tc := newBackend(indexName, target, other)
deleteIndexOnCleanup(tc, indexName)
Expect(backend.Move(target.ID, target.ParentID, "./new/path/to/resource")).To(Succeed())
Expect(resourceByID(tc, indexName, target.ID).Path).To(Equal("./new/path/to/resource"), "target resource should be moved")
Expect(resourceByID(tc, indexName, other.ID).Path).To(Equal(other.Path), "resource in a different root must not be moved")
})
})
Describe("SearchInAnalyzedFields", func() {
const indexName = "opencloud-test-engine-search-analyzed-fields"
var (
tc *opensearchtest.TestClient
backend *opensearch.Backend
)
BeforeEach(func() {
dashed := opensearchtest.Testdata.Resources.Folder
dashed.ID = "1$1!10"
dashed.Name = "new-folder"
dashed.Path = "./new-folder"
dashed.Title = "quarterly report"
plain := opensearchtest.Testdata.Resources.Folder
plain.ID = "1$1!11"
plain.Name = "documents"
plain.Path = "./documents"
plain.Title = "notes"
spaced := opensearchtest.Testdata.Resources.Folder
spaced.ID = "1$1!12"
spaced.Name = "foo bar"
spaced.Path = "./foo bar"
spaced.Title = "spaced out"
backend, tc = newBackend(indexName, dashed, plain, spaced)
deleteIndexOnCleanup(tc, indexName)
tc.Require.IndicesRefresh([]string{indexName}, nil)
})
DescribeTable("finds what the analyzer made of the value",
func(query string, want []string) {
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query})
Expect(err).ToNot(HaveOccurred())
names := make([]string, 0, len(resp.Matches))
for _, match := range resp.Matches {
names = append(names, match.Entity.Name)
}
Expect(names).To(ConsistOf(want))
},
Entry("the full name with the dash", "new-folder", []string{"new-folder"}),
Entry("one token of it", "new", []string{"new-folder"}),
Entry("a name without a dash", "documents", []string{"documents"}),
Entry("a wildcard", "*folder*", []string{"new-folder"}),
// the shape the web client sends for every name search
Entry("a wildcard around the whole dashed name", `name:"*new-folder*"`, []string{"new-folder"}),
Entry("a wildcard spanning the dash", `name:"*w-fol*"`, []string{"new-folder"}),
Entry("a wildcard in a different case", `name:"*NEW-FOLDER*"`, []string{"new-folder"}),
Entry("a wildcard spanning a space", `name:"*oo ba*"`, []string{"foo bar"}),
Entry("a wildcard around a name with a space", `name:"*foo bar*"`, []string{"foo bar"}),
Entry("a name with a space", `name:"foo bar"`, []string{"foo bar"}),
Entry("a title of two words", `Title:"quarterly report"`, []string{"new-folder"}),
Entry("one token of a title", "Title:quarterly", []string{"new-folder"}),
)
})
Describe("SearchByTag", func() {
const indexName = "opencloud-test-engine-search-by-tag"
var (
tc *opensearchtest.TestClient
backend *opensearch.Backend
)
BeforeEach(func() {
tagged := opensearchtest.Testdata.Resources.Folder
tagged.ID = "1$1!20"
tagged.Name = "tagged"
tagged.Path = "./tagged"
tagged.Tags = []string{"foo-bar"}
other := opensearchtest.Testdata.Resources.Folder
other.ID = "1$1!21"
other.Name = "other"
other.Path = "./other"
other.Tags = []string{"foo"}
backend, tc = newBackend(indexName, tagged, other)
deleteIndexOnCleanup(tc, indexName)
tc.Require.IndicesRefresh([]string{indexName}, nil)
})
// a tag is one label, not prose, so it matches as a whole or not at all
DescribeTable("matches a tag as a whole",
func(query string, want []string) {
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query})
Expect(err).ToNot(HaveOccurred())
names := make([]string, 0, len(resp.Matches))
for _, match := range resp.Matches {
names = append(names, match.Entity.Name)
}
Expect(names).To(ConsistOf(want))
},
Entry("the whole tag", `tag:("foo-bar")`, []string{"tagged"}),
Entry("a token of a tag does not match it", `tag:("foo")`, []string{"other"}),
Entry("a tag in a different case", `tag:("FOO-BAR")`, []string{"tagged"}),
Entry("a wildcard reaches both", `tag:("*foo*")`, []string{"tagged", "other"}),
)
})
Describe("SearchWithAnInvalidQuery", func() {
const indexName = "opencloud-test-engine-search-invalid-query"
It("answers with a bad request", func() {
backend, tc := newBackend(indexName)
deleteIndexOnCleanup(tc, indexName)
_, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: "AND mediatype:document"})
Expect(err).To(HaveOccurred())
Expect(err).To(BeAssignableToTypeOf(errtypes.BadRequest("")))
Expect(err.Error()).To(Equal(`error: bad request: the expression can't begin from a binary operator: 'AND'`))
})
})
})
services/search/pkg/opensearch/index_test.go
+1 -1
View File
@@ -7,8 +7,8 @@ import (
"github.com/stretchr/testify/require"
"github.com/tidwall/sjson"
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestIndexManager(t *testing.T) {
services/search/pkg/opensearch/internal/convert/kql_expand_test.go
+1 -1
View File
@@ -9,7 +9,7 @@ import (
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
)
func TestExpandKQLAST(t *testing.T) {
services/search/pkg/opensearch/internal/convert/kql_transpile_test.go
+1 -1
View File
@@ -7,9 +7,9 @@ import (
"github.com/stretchr/testify/assert"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestTranspileKQLToOpenSearch(t *testing.T) {
services/search/pkg/opensearch/internal/convert/opensearch_test.go
+1 -1
View File
@@ -10,8 +10,8 @@ import (
"github.com/opencloud-eu/opencloud/pkg/conversions"
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
services/search/pkg/opensearch/internal/osu/query_bool_test.go
+1 -1
View File
@@ -5,8 +5,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestBoolQuery(t *testing.T) {
services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase_test.go
+1 -1
View File
@@ -5,8 +5,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestNewMatchPhraseQuery(t *testing.T) {
services/search/pkg/opensearch/internal/osu/query_term_level_ids_test.go
+1 -1
View File
@@ -5,8 +5,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestIDsQuery(t *testing.T) {
services/search/pkg/opensearch/internal/osu/query_term_level_range_test.go
+1 -1
View File
@@ -8,8 +8,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestRangeQuery(t *testing.T) {
services/search/pkg/opensearch/internal/osu/query_term_level_term_test.go
+1 -1
View File
@@ -5,8 +5,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestTermQuery(t *testing.T) {
services/search/pkg/opensearch/internal/osu/query_term_level_wildcard_test.go
+1 -1
View File
@@ -5,8 +5,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestWildcardQuery(t *testing.T) {
services/search/pkg/opensearch/internal/osu/request_test.go
+1 -1
View File
@@ -8,8 +8,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestRequestBody(t *testing.T) {
services/search/pkg/opensearch/opensearch_test.go
+1 -1
View File
@@ -6,8 +6,8 @@ import (
"os"
"testing"
opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
"github.com/opencloud-eu/opencloud/services/search/pkg/config"
opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
var defaultConfig *config.Config
services/search/pkg/parity/README.md
+631
View File
@@ -0,0 +1,631 @@
# Engine parity
Written by the parity suite (`go test ./services/search/pkg/parity/`), do not edit.
Every case runs against bleve and OpenSearch. `same?` is ✅ when both answer as
expected, `❌ known` when an engine's divergence is documented in the case
(`engineOverrides`), `❌` when it is not.
## Queries
### name
Fixtures:
- `new-folder`, folder
- `quarterly notes.txt`
- `Report.txt`
- `Übung.txt`
- `a+b.txt`
- `c(d).txt`
- `e&f.txt`
- `v1.2.3.txt`
- `foo bar.txt`
- `aaaaaaaaaa...edle.txt`
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| NAME-01 | `new` | new-folder | no match | new-folder | ❌ known |
| NAME-02 | `quarterly` | quarterly notes.txt | no match | quarterly notes.txt | ❌ known |
| NAME-03 | `report` | Report.txt | no match | no match | ❌ known |
| NAME-04 | `name:"*new-folder*"` | new-folder | new-folder | new-folder | ✅ |
| NAME-05 | `name:"*w-fol*"` | new-folder | new-folder | new-folder | ✅ |
| NAME-06 | `name:"*oo ba*"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ |
| NAME-07 | `name:"*REPORT*"` | Report.txt | Report.txt | Report.txt | ✅ |
| NAME-08 | `name:"*übung*"` | Übung.txt | Übung.txt | no match | ❌ known |
| NAME-09 | `name:"*ÜBUNG*"` | Übung.txt | Übung.txt | no match | ❌ known |
| NAME-10 | `name:"*a+b*"` | a+b.txt | a+b.txt | a+b.txt | ✅ |
| NAME-11 | `name:"*c(d)*"` | c(d).txt | c(d).txt | c(d).txt | ✅ |
| NAME-12 | `name:"*e&f*"` | e&f.txt | e&f.txt | e&f.txt | ✅ |
| NAME-13 | `name:"*v1.2*"` | v1.2.3.txt | v1.2.3.txt | v1.2.3.txt | ✅ |
| NAME-14 | `new-folder` | new-folder | new-folder | new-folder | ✅ |
| NAME-15 | `*folder*` | new-folder | new-folder | new-folder | ✅ |
| NAME-16 | `name:"*foo bar*"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ |
| NAME-17 | `name:"foo bar.txt"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ |
| NAME-18 | `name:"*needle*"` | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | no match | ❌ known |
| NAME-19 | `name:"report*"` | Report.txt | Report.txt | Report.txt | ✅ |
| NAME-20 | `name:"*report"` | Report.txt | no match | no match | ❌ known |
| NAME-21 | `name:"Rep*rt.txt"` | Report.txt | Report.txt | Report.txt | ✅ |
| NAME-22 | `Name:"*report*"` | Report.txt | Report.txt | Report.txt | ✅ |
| NAME-23 | `NAME:"*report*"` | Report.txt | Report.txt | no match | ❌ known |
| NAME-24 | `name:Rep?rt.txt` | Report.txt | Report.txt | no match | ❌ known |
| NAME-25 | `name:"*eport"` | Report.txt | no match | no match | ❌ known |
| NAME-26 | `name:"repor*"` | Report.txt | Report.txt | Report.txt | ✅ |
| NAME-27 | `REPORT` | Report.txt | no match | no match | ❌ known |
| NAME-28 | `name:REPORT` | Report.txt | no match | no match | ❌ known |
| NAME-29 | `name:"REPORT.TXT"` | Report.txt | Report.txt | Report.txt | ✅ |
| NAME-30 | `name:"FOO BAR.TXT"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ |
| NAME-31 | `name:"ÜBUNG.TXT"` | Übung.txt | Übung.txt | Übung.txt | ✅ |
| NAME-32 | `name:"folder*"` | no match | no match | no match | ✅ |
| NAME-33 | `name:"*new"` | no match | no match | no match | ✅ |
| NAME-34 | `name:new` | new-folder | no match | new-folder | ❌ known |
| NAME-35 | `name:"new"` | no match | no match | new-folder | ❌ known |
| NAME-36 | `name:"new-folder"` | new-folder | new-folder | new-folder | ✅ |
| NAME-37 | `name:"new-*"` | new-folder | new-folder | new-folder | ✅ |
| NAME-38 | `name:"new*"` | new-folder | new-folder | new-folder | ✅ |
| NAME-39 | `name:new-*` | new-folder | new-folder | new-folder | ✅ |
| NAME-40 | `name:"*-folder"` | new-folder | new-folder | new-folder | ✅ |
| NAME-41 | `name:"Rep?rt.txt"` | Report.txt | Report.txt | no match | ❌ known |
### extension
Fixtures:
- `report.txt`
- `notes.md`, MimeType = text/markdown
- `archive`, folder
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| EXTENSION-01 | `txt` | report.txt | no match | no match | ❌ known |
| EXTENSION-02 | `md` | notes.md | no match | no match | ❌ known |
| EXTENSION-03 | `name:"*.txt"` | report.txt | report.txt | report.txt | ✅ |
| EXTENSION-04 | `report` | report.txt | no match | no match | ❌ known |
### tags
Fixtures:
- `invoice.txt`, Tags = foo-bar
- `memo.txt`, Tags = foo
- `spaced.txt`, Tags = spaced tag
- `project`, folder, Tags = work
- `draft.txt`, Path = ./project/draft.txt
- `longtag.txt`, Tags = zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzneedle
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| TAGS-01 | `name:"*foo-bar*"` | no match | no match | no match | ✅ |
| TAGS-02 | `tag:("foo-bar")` | invoice.txt | invoice.txt | invoice.txt | ✅ |
| TAGS-03 | `tag:("foo")` | memo.txt | memo.txt | memo.txt | ✅ |
| TAGS-04 | `tag:("FOO-BAR")` | invoice.txt | invoice.txt | invoice.txt | ✅ |
| TAGS-05 | `tag:("*foo*")` | invoice.txt, memo.txt | invoice.txt, memo.txt | invoice.txt, memo.txt | ✅ |
| TAGS-06 | `tag:("spaced tag")` | spaced.txt | spaced.txt | spaced.txt | ✅ |
| TAGS-07 | `tag:("*paced ta*")` | spaced.txt | spaced.txt | spaced.txt | ✅ |
| TAGS-08 | `tag:("work")` | project | project | project | ✅ |
| TAGS-09 | `tag:("zzzzzzzzzzzzzzzzzzzzzzzzzz...zzzzzzzzzzzzzzzzneedle")` | longtag.txt | longtag.txt | no match | ❌ known |
### title
Fixtures:
- `q1.html`, MimeType = text/html, Title = "quarterly report"
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| TITLE-01 | `Title:"quarterly report"` | q1.html | q1.html | q1.html | ✅ |
| TITLE-02 | `Title:quarterly` | q1.html | no match | q1.html | ❌ known |
| TITLE-03 | `Title:QUARTERLY` | q1.html | no match | q1.html | ❌ known |
| TITLE-04 | `Title:quarterl*` | q1.html | q1.html | q1.html | ✅ |
| TITLE-05 | `Title:"*ly rep*"` | q1.html | q1.html | q1.html | ✅ |
| TITLE-06 | `title:quarterly` | q1.html | no match | no match | ❌ known |
| TITLE-07 | `Title:"QUARTERLY REPORT"` | q1.html | no match | q1.html | ❌ known |
### content
Fixtures:
- `monthly.txt`, Content = "the monthly reports are due"
- `links.txt`, Content = "see https://opencloud.example.com/help or write to alan@example.org"
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| CONTENT-01 | `Content:report` | no match | monthly.txt | no match | ❌ known |
| CONTENT-02 | `Content:REPORTS` | monthly.txt | monthly.txt | monthly.txt | ✅ |
| CONTENT-03 | `Content:"monthly reports"` | monthly.txt | monthly.txt | monthly.txt | ✅ |
| CONTENT-04 | `Content:"reports monthly"` | no match | monthly.txt | no match | ❌ known |
| CONTENT-05 | `Content:report*` | monthly.txt | monthly.txt | monthly.txt | ✅ |
| CONTENT-06 | `Content:*eport*` | monthly.txt | monthly.txt | monthly.txt | ✅ |
| CONTENT-07 | `Content:month*` | monthly.txt | monthly.txt | monthly.txt | ✅ |
| CONTENT-08 | `Content:"https://opencloud.example.com/help"` | links.txt | links.txt | links.txt | ✅ |
| CONTENT-09 | `Content:"alan@example.org"` | links.txt | links.txt | links.txt | ✅ |
| CONTENT-10 | `Content:opencloud` | links.txt | no match | no match | ❌ known |
### favorites
Fixtures:
- `starred.txt`, Favorites = A1B2-Upper
- `plain.txt`
- `keepsakes`, folder, Favorites = A1B2-Upper
- `photo.jpg`, MimeType = image/jpeg, Path = ./keepsakes/photo.jpg
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| FAVORITES-01 | `Favorites:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | no match | ❌ known |
| FAVORITES-02 | `favorite:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | no match | ❌ known |
| FAVORITES-03 | `Favorites:"somebody-else"` | no match | no match | no match | ✅ |
### mediatype
Fixtures:
- `notes.md`, MimeType = text/markdown
- `photo.jpg`, MimeType = image/jpeg
- `albums`, folder
- `drafts`, folder
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| MEDIATYPE-01 | `mediatype:text/markdown` | notes.md | notes.md | notes.md | ✅ |
| MEDIATYPE-02 | `mediatype:TEXT/MARKDOWN` | notes.md | no match | notes.md | ❌ known |
| MEDIATYPE-03 | `mediatype:image/jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ |
| MEDIATYPE-04 | `mediatype:*jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ |
| MEDIATYPE-05 | `mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ |
| MEDIATYPE-06 | `mediatype:folder` | albums, drafts | albums, drafts | albums, drafts | ✅ |
### path
Fixtures:
- `parent`, folder
- `child.jpg`, MimeType = image/jpeg, Path = ./parent/child.jpg
- `docs-lower`, folder, Path = ./documents
- `docs-upper`, folder, Path = ./DOCUMENTS
- `docs-mixed`, folder, Path = ./Documents
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| PATH-01 | `path:"./parent"` | child.jpg, parent | parent | child.jpg, parent | ❌ known |
| PATH-02 | `path:"./parent/child.jpg"` | child.jpg | child.jpg | child.jpg | ✅ |
| PATH-03 | `path:"./Parent"` | no match | no match | child.jpg, parent | ❌ known |
| PATH-04 | `path:"*child*"` | child.jpg | child.jpg | child.jpg | ✅ |
| PATH-05 | `path:"./documents"` | docs-lower | docs-lower | docs-lower, docs-mixed, docs-upper | ❌ known |
| PATH-06 | `path:"./DOCUMENTS"` | docs-upper | docs-upper | docs-lower, docs-mixed, docs-upper | ❌ known |
| PATH-07 | `path:"./Documents"` | docs-mixed | docs-mixed | docs-lower, docs-mixed, docs-upper | ❌ known |
| PATH-08 | `path:"./parent/"` | child.jpg, parent | no match | no match | ❌ known |
### fields
Fixtures:
- `small.txt`, ID = 1$1!small.txt, Size = 42
- `old.txt`, ID = 1$1!old.txt, Mtime = 2020-01-01T00:00:00Z
- `known.txt`, ID = 1$1!23
- `cased.txt`, ID = 1$1!AB-23
- `hidden.txt`, ID = 1$1!hidden.txt, hidden
- `plain.txt`, ID = 1$1!plain.txt
- `box`, ID = 1$1!box, folder
- `boxed.txt`, ID = 1$1!boxed.txt, Path = ./box/boxed.txt
- `song.mp3`, ID = 1$1!song.mp3, MimeType = audio/mpeg
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| FIELDS-01 | `size:42` | small.txt | small.txt | small.txt | ✅ |
| FIELDS-02 | `mtime<"2021-01-01T00:00:00Z"` | old.txt | old.txt | old.txt | ✅ |
| FIELDS-03 | `id:"1$1!23"` | known.txt | known.txt | known.txt | ✅ |
| FIELDS-04 | `hidden:true` | hidden.txt | no match | hidden.txt | ❌ known |
| FIELDS-05 | `type:file` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | no match | error | ❌ known |
| FIELDS-06 | `type:folder` | box | no match | error | ❌ known |
| FIELDS-07 | `unknown:field` | no match | no match | no match | ✅ |
| FIELDS-08 | `type:File` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | no match | error | ❌ known |
| FIELDS-09 | `type:FOLDER` | box | no match | error | ❌ known |
| FIELDS-10 | `hidden:TRUE` | hidden.txt | no match | error | ❌ known |
| FIELDS-11 | `id:"1$1!AB-23"` | cased.txt | cased.txt | no match | ❌ known |
| FIELDS-12 | `id:"1$1!ab-23"` | no match | no match | no match | ✅ |
| FIELDS-13 | `audio.artist:"Some Artist"` | song.mp3 | song.mp3 | no match | ❌ known |
| FIELDS-14 | `audio.artist:"some artist"` | no match | no match | no match | ✅ |
### deleted
Fixtures:
- `trashed.txt`, deleted
- `kept.txt`
- `bin`, folder, deleted
- `receipt.txt`, Path = ./bin/receipt.txt, deleted
- `shelf`, folder
- `book.txt`, Path = ./shelf/book.txt
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| DELETED-01 | `name:"*trashed*"` | no match | no match | no match | ✅ |
| DELETED-02 | `name:"*.txt"` | book.txt, kept.txt | book.txt, kept.txt | book.txt, kept.txt | ✅ |
| DELETED-03 | `name:"*receipt*"` | no match | no match | no match | ✅ |
| DELETED-04 | `path:"./bin"` | no match | no match | no match | ✅ |
| DELETED-05 | `path:"./shelf"` | book.txt, shelf | shelf | book.txt, shelf | ❌ known |
### visibility
Fixtures:
- `visible.txt`
- `dotfile.txt`, hidden
- `.private`, folder, hidden
- `secret.txt`, Path = ./.private/secret.txt, hidden
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| VISIBILITY-01 | `hidden:true` | .private, dotfile.txt, secret.txt | no match | .private, dotfile.txt, secret.txt | ❌ known |
| VISIBILITY-02 | `hidden:TRUE` | .private, dotfile.txt, secret.txt | no match | error | ❌ known |
| VISIBILITY-03 | `hidden:false` | visible.txt | no match | visible.txt | ❌ known |
| VISIBILITY-04 | `name:"*secret*"` | secret.txt | secret.txt | secret.txt | ✅ |
| VISIBILITY-05 | `path:"./.private"` | .private, secret.txt | .private | .private, secret.txt | ❌ known |
| VISIBILITY-06 | `hidden:banana` | no match | no match | error | ❌ known |
| VISIBILITY-07 | `hidden:"true"` | .private, dotfile.txt, secret.txt | no match | .private, dotfile.txt, secret.txt | ❌ known |
### boolean
Fixtures:
- `alpha.txt`, Tags = red
- `beta.txt`, Tags = blue
- `gamma.md`, MimeType = text/markdown
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| BOOLEAN-01 | `name:"*alpha*" AND name:"*txt*"` | alpha.txt | alpha.txt | alpha.txt | ✅ |
| BOOLEAN-02 | `name:"*alpha*" OR name:"*beta*"` | alpha.txt, beta.txt | alpha.txt, beta.txt | alpha.txt, beta.txt | ✅ |
| BOOLEAN-03 | `name:"*a*" AND NOT name:"*alpha*"` | beta.txt, gamma.md | beta.txt, gamma.md | beta.txt, gamma.md | ✅ |
| BOOLEAN-04 | `name:"*a*" AND tag:("red")` | alpha.txt | alpha.txt | alpha.txt | ✅ |
| BOOLEAN-05 | `(name:"*alpha*" OR name:"*beta*") AND mediatype:text/plain` | alpha.txt, beta.txt | alpha.txt, beta.txt | alpha.txt, beta.txt | ✅ |
| BOOLEAN-06 | `name:"*a*" AND mediatype:text/markdown` | gamma.md | gamma.md | gamma.md | ✅ |
### samename
Fixtures:
- `doc`, folder
- `doc.pdf`
- `file.pdf`
- `doc.pdf`, Path = ./doc/doc.pdf
- `file.pdf`, Path = ./doc/file.pdf
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| SAMENAME-01 | `name:"*doc*"` | 3 items | doc, doc.pdf, doc.pdf | doc, doc.pdf, doc.pdf | ✅ |
| SAMENAME-02 | `name:"*doc*"` in 1$1!1 under ./doc | 2 items | doc, doc.pdf | doc, doc.pdf | ✅ |
| SAMENAME-03 | `name:"*file*"` | 2 items | file.pdf, file.pdf | file.pdf, file.pdf | ✅ |
| SAMENAME-04 | `name:"*file*"` in 1$1!1 under ./doc | 1 items | file.pdf | file.pdf | ✅ |
### stress
Fixtures:
- `quarterly report.docx`, MimeType = application/vnd.openxmlformats-officedocument.wordprocessingml.document, Tags = final, Size = 2000
- `draft report.txt`, Tags = draft, Size = 50, Mtime = 2020-01-01T00:00:00Z
- `photo.jpg`, MimeType = image/jpeg, Tags = final
- `notes.md`, MimeType = text/markdown, hidden
- `archive`, folder
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| STRESS-01 | `name:"*report*" AND mediatype:document` | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | ✅ |
| STRESS-02 | `name:"*report*" AND NOT tag:("draft")` | quarterly report.docx | draft report.txt, quarterly report.docx | quarterly report.docx | ❌ known |
| STRESS-03 | `(tag:("final") OR tag:("draft")) AND mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ |
| STRESS-04 | `mediatype:document AND mtime>"2021-01-01T00:00:00Z"` | notes.md, quarterly report.docx | notes.md, quarterly report.docx | notes.md, quarterly report.docx | ✅ |
| STRESS-05 | `name:"*report*" AND size>100` | quarterly report.docx | no match | no match | ❌ known |
| STRESS-06 | `tag:("final") AND NOT mediatype:folder` | photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | ✅ |
| STRESS-07 | `hidden:true AND name:"*notes*"` | notes.md | no match | notes.md | ❌ known |
| STRESS-08 | `name:quarterly report` | quarterly report.docx | no match | no match | ❌ known |
| STRESS-09 | `name:"quarterly report"` | no match | no match | no match | ✅ |
| STRESS-10 | `name:"quarterly report.docx"` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ |
| STRESS-11 | `NOT tag:("draft")` | archive, notes.md, photo.jpg, quarterly report.docx | archive, draft report.txt, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | ❌ known |
| STRESS-12 | `tag:("final") OR hidden:true` | notes.md, photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | ❌ known |
| STRESS-13 | `(name:"*report*" OR name:"*notes..."draft") OR hidden:true)` | quarterly report.docx | notes.md, quarterly report.docx | quarterly report.docx | ❌ known |
| STRESS-14 | `mediatype:image OR (mediatype:document AND tag:("draft"))` | draft report.txt, photo.jpg | draft report.txt, photo.jpg | draft report.txt, photo.jpg | ✅ |
| STRESS-15 | `NOT (mediatype:folder OR hidden:true)` | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, notes.md, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | ❌ known |
| STRESS-16 | `name:"*report*" AND (size>100 OR tag:("draft"))` | draft report.txt, quarterly report.docx | draft report.txt | draft report.txt | ❌ known |
### everything
Fixtures:
- `alpha.txt`
- `beta.txt`
- `box`, folder
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| EVERYTHING-01 | `*` | alpha.txt, beta.txt, box | alpha.txt, beta.txt, box | alpha.txt, beta.txt, box | ✅ |
| EVERYTHING-02 | `name:"*"` | alpha.txt, beta.txt, box | alpha.txt, beta.txt, box | alpha.txt, beta.txt, box | ✅ |
| EVERYTHING-03 | `*` with a limit of 2 | 2 items | alpha.txt, beta.txt | alpha.txt, beta.txt | ✅ |
| EVERYTHING-04 | `*` with a limit of -1 | 3 items | alpha.txt, beta.txt, box | alpha.txt, beta.txt, box | ✅ |
### range
Fixtures:
- `small.txt`, Size = 50
- `big.txt`, Size = 500
- `ancient.txt`, Size = 10, Mtime = 2020-01-01T00:00:00Z
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| RANGE-01 | `size>100` | big.txt | no match | no match | ❌ known |
| RANGE-02 | `size<100` | ancient.txt, small.txt | no match | no match | ❌ known |
| RANGE-03 | `mtime>"2021-01-01T00:00:00Z"` | big.txt, small.txt | big.txt, small.txt | big.txt, small.txt | ✅ |
| RANGE-04 | `mtime<"2021-01-01T00:00:00Z"` | ancient.txt | ancient.txt | ancient.txt | ✅ |
| RANGE-05 | `Mtime:"today"` | big.txt, small.txt | big.txt, small.txt | big.txt, small.txt | ✅ |
| RANGE-06 | `Mtime:"yesterday"` | no match | no match | no match | ✅ |
| RANGE-07 | `mtime>2021` | no match | no match | no match | ✅ |
| RANGE-08 | `name>100` | no match | no match | no match | ✅ |
### scope
Fixtures:
- `parent`, folder
- `child.pdf`, Path = ./parent/child.pdf
- `outside.txt`
- `elsewhere.txt`
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| SCOPE-01 | `*` | child.pdf, elsewhere.txt, outside.txt, parent | child.pdf, elsewhere.txt, outside.txt, parent | child.pdf, elsewhere.txt, outside.txt, parent | ✅ |
| SCOPE-02 | `*` in 1$1!1 | child.pdf, outside.txt, parent | child.pdf, outside.txt, parent | child.pdf, outside.txt, parent | ✅ |
| SCOPE-03 | `*` in 1$1!1 under ./parent | child.pdf, parent | child.pdf, parent | child.pdf, parent | ✅ |
| SCOPE-04 | `*` in 1$1!1 under ./parent/child.pdf | child.pdf | child.pdf | child.pdf | ✅ |
| SCOPE-05 | `name:"*elsewhere*"` in 1$1!1 | no match | no match | no match | ✅ |
### invalid
Fixtures:
- `alpha.txt`
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| INVALID-01 | `AND mediatype:document` | bad request | bad request | bad request | ✅ |
| INVALID-02 | `mediatype:document AND` | alpha.txt | alpha.txt | alpha.txt | ✅ |
## Operations
### delete
Fixtures:
- `parent`, ID = 1$1!2, folder
- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| DELETE-01 | takes the resource out of the results, then `name:"*parent*"` | parent | parent | parent | ✅ |
| DELETE-01 | takes the resource out of the results, then `name:"*child*"` | no match | no match | no match | ✅ |
| DELETE-02 | takes the descendants along, then `name:"*parent*"` | no match | no match | no match | ✅ |
| DELETE-02 | takes the descendants along, then `name:"*child*"` | no match | no match | no match | ✅ |
| DELETE-03 | leaves the resource in the index, then `DocCount()` | 2 | 2 | 1 | ❌ known |
| DELETE-04 | takes a resource out that was just written, then `name:"*fresh*"` | no match | no match | no match | ✅ |
### restore
Fixtures:
- `parent`, ID = 1$1!2, folder
- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf
- `file.txt`, ID = 1$1!5, Path = ./.secret/file.txt, hidden
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| RESTORE-01 | brings the descendants back, then `name:"*parent*"` | parent | parent | parent | ✅ |
| RESTORE-01 | brings the descendants back, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ |
| RESTORE-02 | leaves the hidden flag alone, then `hidden:true` | file.txt | no match | file.txt | ❌ known |
### purge
Fixtures:
- `parent`, ID = 1$1!2, folder
- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| PURGE-01 | removes one resource, then `name:"*parent*"` | parent | parent | parent | ✅ |
| PURGE-01 | removes one resource, then `name:"*child*"` | no match | no match | no match | ✅ |
| PURGE-01 | removes one resource, then `DocCount()` | 1 | 1 | 1 | ✅ |
| PURGE-02 | removes the tree, then `name:"*parent*"` | no match | no match | no match | ✅ |
| PURGE-02 | removes the tree, then `name:"*child*"` | no match | no match | no match | ✅ |
| PURGE-02 | removes the tree, then `DocCount()` | 0 | 0 | 0 | ✅ |
| PURGE-03 | takes only the deleted ones when it is told to, then `name:"*parent*"` | parent | parent | parent | ✅ |
| PURGE-03 | takes only the deleted ones when it is told to, then `name:"*child*"` | no match | no match | no match | ✅ |
### purgespace
Fixtures:
- `inSpace.txt`, ID = 1$1!4
- `elsewhere.txt`, ID = 2$2!4
- `bulk-0.txt`, ID = 1$1!bulk-0.txt
- `bulk-1.txt`, ID = 1$1!bulk-1.txt
- `bulk-2.txt`, ID = 1$1!bulk-2.txt
- `bulk-3.txt`, ID = 1$1!bulk-3.txt
- `bulk-4.txt`, ID = 1$1!bulk-4.txt
- `bulk-5.txt`, ID = 1$1!bulk-5.txt
- `bulk-6.txt`, ID = 1$1!bulk-6.txt
- `bulk-7.txt`, ID = 1$1!bulk-7.txt
- ... and 52 more of the same
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| PURGESPACE-01 | leaves the other space alone, then `name:"*inSpace*"` | no match | no match | no match | ✅ |
| PURGESPACE-01 | leaves the other space alone, then `name:"*elsewhere*"` | elsewhere.txt | elsewhere.txt | elsewhere.txt | ✅ |
| PURGESPACE-02 | takes a space out that holds more than one round, then `name:"*bulk*"` | no match | no match | no match | ✅ |
| PURGESPACE-02 | takes a space out that holds more than one round, then `name:"*elsewhere*"` | elsewhere.txt | elsewhere.txt | elsewhere.txt | ✅ |
| PURGESPACE-02 | takes a space out that holds more than one round, then `DocCount()` | 1 | 1 | 1 | ✅ |
### move
Fixtures:
- `parent`, ID = 1$1!2, folder
- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| MOVE-01 | carries the descendants to the new path, then `path:"./my/newname/child.pdf"` | child.pdf | child.pdf | child.pdf | ✅ |
| MOVE-01 | carries the descendants to the new path, then `path:"./parent/child.pdf"` | no match | no match | no match | ✅ |
| MOVE-02 | through the trash and back leaves the flag behind, then `hidden:true` | no match | no match | no match | ✅ |
### rootscope
Fixtures:
- `target.txt`, ID = 1$1!3, Path = ./same/path.txt
- `twin.txt`, ID = 2$2!3, Path = ./same/path.txt
- `target.txt`, ID = 1$1!3, Path = ./same/path.txt, deleted
- `twin.txt`, ID = 2$2!3, Path = ./same/path.txt, deleted
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| ROOTSCOPE-01 | deletes only the one in the target root, then `name:"*target*"` | no match | no match | no match | ✅ |
| ROOTSCOPE-01 | deletes only the one in the target root, then `name:"*twin*"` | twin.txt | twin.txt | twin.txt | ✅ |
| ROOTSCOPE-02 | restores only the one in the target root, then `name:"*target*"` | target.txt | target.txt | target.txt | ✅ |
| ROOTSCOPE-02 | restores only the one in the target root, then `name:"*twin*"` | no match | no match | no match | ✅ |
| ROOTSCOPE-03 | moves only the one in the target root, then `path:"./moved.txt"` | moved.txt | moved.txt | moved.txt | ✅ |
| ROOTSCOPE-03 | moves only the one in the target root, then `path:"./same/path.txt"` | twin.txt | twin.txt | twin.txt | ✅ |
### casepath
Fixtures:
- `Documents`, ID = 1$1!2, folder
- `Picture.jpg`, ID = 1$1!3, MimeType = image/jpeg, Path = ./Documents/Picture.jpg
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| CASEPATH-01 | takes the descendants along when deleting, then `path:"./Documents"` | no match | no match | Documents, Picture.jpg | ❌ known |
| CASEPATH-02 | takes the descendants along when moving, then `path:"./Other Documents"` | Other Documents, Picture.jpg | Other Documents | no match | ❌ known |
| CASEPATH-02 | takes the descendants along when moving, then `path:"./Documents"` | no match | no match | Documents, Picture.jpg | ❌ known |
| CASEPATH-03 | reaches the descendants when purging, then `path:"./Documents"` | no match | no match | Documents, Picture.jpg | ❌ known |
| CASEPATH-03 | reaches the descendants when purging, then `DocCount()` | 0 | 0 | 2 | ❌ known |
### hidden
Fixtures:
- `parent`, ID = 1$1!2, folder
- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf
- `parent`, ID = 1$1!2, folder, Path = ./.trash/parent, hidden
- `child.pdf`, ID = 1$1!3, Path = ./.trash/parent/child.pdf, hidden
- `parent`, ID = 1$1!2, folder, Path = ./.parent, hidden
- `child.pdf`, ID = 1$1!3, Path = ./.parent/child.pdf, hidden
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| HIDDEN-01 | follows a move into a dot folder, then `hidden:true` | child.pdf, parent | no match | child.pdf, parent | ❌ known |
| HIDDEN-02 | follows a move into a plain folder, then `hidden:true` | no match | no match | no match | ✅ |
| HIDDEN-03 | follows a move renamed with a leading dot, then `hidden:true` | .parent, child.pdf | no match | .parent, child.pdf | ❌ known |
| HIDDEN-04 | follows a move out of a dot folder, then `hidden:true` | no match | no match | no match | ✅ |
| HIDDEN-05 | follows a move renamed without the leading dot, then `hidden:true` | no match | no match | no match | ✅ |
| HIDDEN-06 | follows a move within the same dot folder, then `hidden:true` | child.pdf, moved | no match | child.pdf, moved | ❌ known |
### upsert
Fixtures:
- `parent`, ID = 1$1!2, folder
- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| UPSERT-01 | replaces the resource it already knows, then `name:"*parent*"` | parent | parent | parent | ✅ |
| UPSERT-01 | replaces the resource it already knows, then `name:"*child*"` | no match | no match | no match | ✅ |
| UPSERT-01 | replaces the resource it already knows, then `name:"*renamed*"` | renamed.pdf | renamed.pdf | renamed.pdf | ✅ |
| UPSERT-01 | replaces the resource it already knows, then `DocCount()` | 2 | 2 | 2 | ✅ |
### idempotency
Fixtures:
- `parent`, ID = 1$1!2, folder
- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| IDEMPOTENCY-01 | deleting the same resource twice is not an error, then `name:"*parent*"` | parent | parent | parent | ✅ |
| IDEMPOTENCY-01 | deleting the same resource twice is not an error, then `name:"*child*"` | no match | no match | no match | ✅ |
| IDEMPOTENCY-02 | deleting a resource the index does not have reports it, then `name:"*parent*"` | parent | parent | parent | ✅ |
| IDEMPOTENCY-02 | deleting a resource the index does not have reports it, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ |
| IDEMPOTENCY-03 | restoring a resource that was never deleted leaves it alone, then `name:"*parent*"` | parent | parent | parent | ✅ |
| IDEMPOTENCY-03 | restoring a resource that was never deleted leaves it alone, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ |
| IDEMPOTENCY-04 | purging the same resource twice reports the second one, then `name:"*parent*"` | parent | parent | parent | ✅ |
| IDEMPOTENCY-04 | purging the same resource twice reports the second one, then `name:"*child*"` | no match | no match | no match | ✅ |
| IDEMPOTENCY-04 | purging the same resource twice reports the second one, then `DocCount()` | 1 | 1 | 1 | ✅ |
| IDEMPOTENCY-05 | purging a resource the index does not have reports it, then `name:"*parent*"` | parent | parent | parent | ✅ |
| IDEMPOTENCY-05 | purging a resource the index does not have reports it, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ |
| IDEMPOTENCY-05 | purging a resource the index does not have reports it, then `DocCount()` | 2 | 2 | 2 | ✅ |
| IDEMPOTENCY-06 | moving a resource onto its own path leaves it where it is, then `path:"./parent/child.pdf"` | child.pdf | child.pdf | child.pdf | ✅ |
| IDEMPOTENCY-06 | moving a resource onto its own path leaves it where it is, then `name:"*parent*"` | parent | parent | parent | ✅ |
| IDEMPOTENCY-07 | purging a whole space that holds nothing is not an error, then `name:"*parent*"` | parent | parent | parent | ✅ |
| IDEMPOTENCY-07 | purging a whole space that holds nothing is not an error, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ |
| IDEMPOTENCY-07 | purging a whole space that holds nothing is not an error, then `DocCount()` | 2 | 2 | 2 | ✅ |
### batch
Fixtures:
- `parent`, ID = 1$1!2, folder
- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| BATCH-01 | holds what it was given until it is pushed, then `name:"*added*"` | no match | no match | no match | ✅ |
| BATCH-02 | writes what it was given once it is pushed, then `name:"*added*"` | added.pdf | added.pdf | added.pdf | ✅ |
| BATCH-02 | writes what it was given once it is pushed, then `DocCount()` | 3 | 3 | 3 | ✅ |
| BATCH-03 | takes a resource out the same way a delete does, then `name:"*parent*"` | parent | parent | parent | ✅ |
| BATCH-03 | takes a resource out the same way a delete does, then `name:"*child*"` | no match | no match | no match | ✅ |
| BATCH-04 | moves a resource the same way a move does, then `path:"./my/newname/child.pdf"` | child.pdf | child.pdf | child.pdf | ✅ |
| BATCH-04 | moves a resource the same way a move does, then `path:"./parent/child.pdf"` | no match | no match | no match | ✅ |
| BATCH-05 | keeps what another batch holds out of its push, then `name:"*added*"` | added.pdf | added.pdf | added.pdf | ✅ |
| BATCH-05 | keeps what another batch holds out of its push, then `name:"*other*"` | no match | no match | no match | ✅ |
## Response
### entity
Fixtures:
- `parent`, ID = 1$1!2, folder
- `bar.pdf`, ID = 1$1!3, MimeType = application/pdf, Path = ./parent/bar.pdf, Size = 1234
- `notes.txt`, ID = 1$1!4, Content = "foo bar baz"
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| ENTITY-01 | `name:"bar.pdf"` reads `Ref.Path` | ./parent/bar.pdf | ./parent/bar.pdf | ./parent/bar.pdf | ✅ |
| ENTITY-02 | `name:"bar.pdf"` reads `Name` | bar.pdf | bar.pdf | bar.pdf | ✅ |
| ENTITY-03 | `name:"bar.pdf"` reads `Id` | 1$1!3 | 1$1!3 | 1$1!3 | ✅ |
| ENTITY-04 | `name:"bar.pdf"` reads `ParentId` | 1$1!2 | 1$1!2 | 1$1!2 | ✅ |
| ENTITY-05 | `name:"bar.pdf"` reads `Ref.ResourceId` | 1$1!1 | 1$1!1 | 1$1!1 | ✅ |
| ENTITY-06 | `name:"bar.pdf"` reads `Size` | 1234 | 1234 | 1234 | ✅ |
| ENTITY-07 | `name:"bar.pdf"` reads `Type` | 1 | 1 | 1 | ✅ |
| ENTITY-08 | `name:"bar.pdf"` reads `MimeType` | application/pdf | application/pdf | application/pdf | ✅ |
| ENTITY-09 | `name:"bar.pdf"` reads `Deleted` | false | false | false | ✅ |
| ENTITY-10 | `name:"bar.pdf"` reads `Score` | above zero | above zero | above zero | ✅ |
| ENTITY-11 | `path:"./parent"` reads `TotalMatches` | 2 | 1 | 2 | ❌ known |
| ENTITY-12 | `name:"*notes*"` reads `Highlights` | "" | "" | "" | ✅ |
| ENTITY-13 | `content:bar` reads `Highlights` | foo <mark>bar</mark> baz | foo <mark>bar</mark> baz | foo <mark>bar</mark> baz | ✅ |
| ENTITY-14 | moved to another parent, then `name:"newname"` reads `ParentId` | 1$1!9 | 1$1!9 | 1$1!9 | ✅ |
| ENTITY-15 | moved to another parent, then `name:"bar.pdf"` reads `ParentId` | 1$1!2 | 1$1!2 | 1$1!2 | ✅ |
| ENTITY-16 | moved to another parent, then `name:"bar.pdf"` reads `Ref.Path` | ./somewher.../bar.pdf | ./somewher.../bar.pdf | ./somewher.../bar.pdf | ✅ |
### metadata
Fixtures:
- `some_song.mp3`, ID = 1$1!5, MimeType = audio/mpeg
- `team.jpg`, ID = 1$1!6, MimeType = image/jpeg
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| METADATA-01 | `*song*` reads `Audio` | all 16 fields unchanged | all 16 fields unchanged | all 16 fields unchanged | ✅ |
| METADATA-02 | `*team*` reads `Location` | all 3 fields unchanged | all 3 fields unchanged | all 3 fields unchanged | ✅ |
| METADATA-03 | `*team*` reads `Audio` | none | none | none | ✅ |
services/search/pkg/parity/engines_test.go
+194
View File
@@ -0,0 +1,194 @@
package parity
import (
"context"
"fmt"
"strings"
bleveSearch "github.com/blevesearch/bleve/v2"
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/types"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/opencloud-eu/opencloud/pkg/log"
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
bleveEngine "github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
var engineNames = []string{"bleve", "opensearch"}
type testEngine struct {
name string
backend search.Engine
settle func()
unavailable string
}
// newEngines builds one backend per engine over the same fixtures. Call it
// from a BeforeAll: the cleanups it defers run when the container ends.
func newEngines(index string, fixtures []search.Resource) []testEngine {
GinkgoHelper()
return []testEngine{
newBleve(fixtures),
newOpenSearch(index, fixtures),
}
}
// newEngine builds just the one engine a spec is about; its cleanup runs when
// that spec ends.
func newEngine(name, index string, fixtures []search.Resource) testEngine {
GinkgoHelper()
switch name {
case "bleve":
return newBleve(fixtures)
case "opensearch":
return newOpenSearch(index, fixtures)
}
Fail("no engine named " + name)
return testEngine{}
}
func engineNamed(engines []testEngine, name string) testEngine {
GinkgoHelper()
for _, e := range engines {
if e.name == name {
return e
}
}
Fail("no engine named " + name)
return testEngine{}
}
func newBleve(fixtures []search.Resource) testEngine {
GinkgoHelper()
mapping, err := bleveEngine.NewMapping()
Expect(err).NotTo(HaveOccurred(), "failed to build the bleve mapping")
idx, err := bleveSearch.NewMemOnly(mapping)
Expect(err).NotTo(HaveOccurred(), "failed to create the bleve index")
DeferCleanup(func() { _ = idx.Close() })
backend := bleveEngine.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{})
load(backend, fixtures)
return testEngine{name: "bleve", backend: backend, settle: func() {}}
}
func newOpenSearch(index string, fixtures []search.Resource) testEngine {
GinkgoHelper()
tc := opensearchtest.NewDefaultTestClient(GinkgoTB(), openSearchClient)
if err := tc.IndicesReset(context.Background(), []string{index}); err != nil {
return testEngine{name: "opensearch", unavailable: err.Error()}
}
backend, err := opensearch.NewBackend(index, tc.Client())
if err != nil {
return testEngine{name: "opensearch", unavailable: err.Error()}
}
DeferCleanup(func() {
Expect(tc.IndicesDelete(context.Background(), []string{index})).To(Succeed())
})
// every write waits for the next refresh (Refresh: wait_for), which is a
// second apart by default; the test index can refresh far more often
_, err = tc.Client().Indices.Settings.Put(context.Background(), opensearchgoAPI.SettingsPutReq{
Indices: []string{index},
Body: strings.NewReader(`{"index": {"refresh_interval": "50ms"}}`),
})
Expect(err).NotTo(HaveOccurred(), "failed to speed up the refresh of %s", index)
settle := func() { tc.Require.IndicesRefresh([]string{index}, nil) }
load(backend, fixtures)
settle()
return testEngine{name: "opensearch", backend: backend, settle: settle}
}
// load writes the fixtures the way the service does, in one batch.
func load(backend search.Engine, fixtures []search.Resource) {
GinkgoHelper()
if len(fixtures) == 0 {
return
}
batch, err := backend.NewBatch(len(fixtures) + 1)
Expect(err).NotTo(HaveOccurred(), "failed to open a batch for the fixtures")
for _, doc := range fixtures {
Expect(batch.Upsert(doc.ID, doc)).To(Succeed(), "upsert %s", doc.ID)
}
Expect(batch.Push()).To(Succeed(), "failed to write the fixtures")
}
func badRequestAnswer(err error) []string {
if err == nil {
return []string{"no error"}
}
return []string{"bad request"}
}
func reads(read func(*searchMessage.Match) string) func(*searchService.SearchIndexResponse) []string {
return readsMany(func(m *searchMessage.Match) []string { return []string{read(m)} })
}
func readsMany(read func(*searchMessage.Match) []string) func(*searchService.SearchIndexResponse) []string {
return func(resp *searchService.SearchIndexResponse) []string {
if len(resp.Matches) != 1 {
return []string{fmt.Sprintf("%d matches", len(resp.Matches))}
}
return read(resp.Matches[0])
}
}
func resourceID(id *searchMessage.ResourceID) string {
return storagespace.FormatResourceID(&sprovider.ResourceId{
StorageId: id.GetStorageId(),
SpaceId: id.GetSpaceId(),
OpaqueId: id.GetOpaqueId(),
})
}
// ask runs a query and returns the matched names; an engine error is the
// answer "error", so a query the engine rejects still shows in the matrix.
func ask(e search.Engine, request *searchService.SearchIndexRequest) ([]string, error) {
resp, err := e.Search(context.Background(), request)
if err != nil {
return []string{"error"}, err
}
names := make([]string, 0, len(resp.Matches))
for _, m := range resp.Matches {
names = append(names, m.Entity.GetName())
}
return names, nil
}
// matchNames is ConsistOf for a wanted list, spelled out for the empty case so
// a "no match" expectation reads as such in the failure.
func matchNames(want []string) types.GomegaMatcher {
if len(want) == 0 {
return BeEmpty()
}
return ConsistOf(want)
}
services/search/pkg/parity/fixtures_test.go
+108
View File
@@ -0,0 +1,108 @@
package parity
import (
"fmt"
"slices"
"strings"
"time"
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
var fixtureLongName = strings.Repeat("a", 260) + "needle.txt"
var fixtureNow = time.Now().UTC()
type fixtureOption func(*search.Resource)
func withPath(path string) fixtureOption { return func(r *search.Resource) { r.Path = path } }
func withMime(mime string) fixtureOption { return func(r *search.Resource) { r.MimeType = mime } }
func withTitle(t string) fixtureOption { return func(r *search.Resource) { r.Title = t } }
func withContent(c string) fixtureOption { return func(r *search.Resource) { r.Content = c } }
func withSize(s uint64) fixtureOption { return func(r *search.Resource) { r.Size = s } }
func withMtime(m string) fixtureOption { return func(r *search.Resource) { r.Mtime = m } }
func withID(id string) fixtureOption { return func(r *search.Resource) { r.ID = id } }
func withParent(id string) fixtureOption { return func(r *search.Resource) { r.ParentID = id } }
func withRoot(id string) fixtureOption { return func(r *search.Resource) { r.RootID = id } }
func isHidden() fixtureOption { return func(r *search.Resource) { r.Hidden = true } }
func isDeleted() fixtureOption { return func(r *search.Resource) { r.Deleted = true } }
func withTags(tags ...string) fixtureOption {
return func(r *search.Resource) { r.Tags = tags }
}
func withFavorite(userID string) fixtureOption {
return func(r *search.Resource) { r.Favorites = []string{userID} }
}
func withAudio(audio *libregraph.Audio) fixtureOption {
return func(r *search.Resource) { r.Audio = audio }
}
func withLocation(location *libregraph.GeoCoordinates) fixtureOption {
return func(r *search.Resource) { r.Location = location }
}
func fixtureDoc(name string, opts ...fixtureOption) search.Resource {
r := search.Resource{
ID: "1$1!" + name,
RootID: "1$1!1",
ParentID: "1$1!1",
Path: "./" + name,
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{
Name: name,
MimeType: "text/plain",
Mtime: fixtureNow.Format(time.RFC3339Nano),
Size: 1000,
},
}
for _, opt := range opts {
opt(&r)
}
return r
}
func fixtureFolder(name string, opts ...fixtureOption) search.Resource {
folder := []fixtureOption{withMime("httpd/unix-directory")}
r := fixtureDoc(name, append(folder, opts...)...)
r.Type = uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER)
return r
}
func fixtureBulk(count int) []search.Resource {
docs := make([]search.Resource, 0, count)
for i := range count {
docs = append(docs, fixtureDoc(fmt.Sprintf("bulk-%d.txt", i)))
}
return docs
}
func fixtureTree() (parent, child search.Resource) {
parent = fixtureFolder("parent", withID("1$1!2"))
child = fixtureDoc("child.pdf", withID("1$1!3"), withParent(parent.ID), withPath("./parent/child.pdf"))
return parent, child
}
func treeIsLeft(names ...string) []expectation {
left := func(name string) []string {
if slices.Contains(names, name) {
return []string{name}
}
return nil
}
return []expectation{
{`name:"*parent*"`, left("parent")},
{`name:"*child*"`, left("child.pdf")},
}
}
services/search/pkg/parity/lifecycle_batch_test.go
+112
View File
@@ -0,0 +1,112 @@
package parity
import (
"github.com/opencloud-eu/opencloud/pkg/conversions"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func batchLifecycle() lifecycleGroup {
parent, child := fixtureTree()
added := fixtureDoc("added.pdf", withID("1$1!7"), withParent(parent.ID), withPath("./parent/added.pdf"))
other := fixtureDoc("other.pdf", withID("1$1!8"), withParent(parent.ID), withPath("./parent/other.pdf"))
return lifecycleGroup{
name: "batch",
fixtures: []search.Resource{parent, child},
cases: []lifecycleCase{
{
id: 1, title: "holds what it was given until it is pushed",
do: func(e search.Engine) error {
batch, err := e.NewBatch(100)
if err != nil {
return err
}
return batch.Upsert(added.ID, added)
},
expect: []expectation{{`name:"*added*"`, nil}},
},
{
id: 2, title: "writes what it was given once it is pushed",
do: func(e search.Engine) error {
batch, err := e.NewBatch(100)
if err != nil {
return err
}
if err := batch.Upsert(added.ID, added); err != nil {
return err
}
return batch.Push()
},
expect: []expectation{{`name:"*added*"`, []string{"added.pdf"}}},
wantDocCount: conversions.ToPointer(uint64(3)),
},
{
id: 3, title: "takes a resource out the same way a delete does",
do: func(e search.Engine) error {
batch, err := e.NewBatch(100)
if err != nil {
return err
}
if err := batch.Delete(child.ID); err != nil {
return err
}
return batch.Push()
},
expect: treeIsLeft("parent"),
},
{
id: 4, title: "moves a resource the same way a move does",
do: func(e search.Engine) error {
batch, err := e.NewBatch(100)
if err != nil {
return err
}
if err := batch.Move(parent.ID, parent.ParentID, "./my/newname"); err != nil {
return err
}
return batch.Push()
},
expect: []expectation{
{`path:"./my/newname/child.pdf"`, []string{"child.pdf"}},
{`path:"./parent/child.pdf"`, nil},
},
},
{
id: 5, title: "keeps what another batch holds out of its push",
do: func(e search.Engine) error {
first, err := e.NewBatch(100)
if err != nil {
return err
}
second, err := e.NewBatch(100)
if err != nil {
return err
}
if err := first.Upsert(added.ID, added); err != nil {
return err
}
if err := second.Upsert(other.ID, other); err != nil {
return err
}
return first.Push()
},
expect: []expectation{
{`name:"*added*"`, []string{"added.pdf"}},
{`name:"*other*"`, nil},
},
},
},
}
}
services/search/pkg/parity/lifecycle_casepath_test.go
+61
View File
@@ -0,0 +1,61 @@
package parity
import (
"github.com/opencloud-eu/opencloud/pkg/conversions"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func casePathLifecycle() lifecycleGroup {
folder := fixtureFolder("Documents", withID("1$1!2"), withPath("./Documents"))
picture := fixtureDoc("Picture.jpg",
withID("1$1!3"),
withParent(folder.ID),
withMime("image/jpeg"),
withPath("./Documents/Picture.jpg"),
)
left := func(names ...string) []expectation {
return []expectation{
{`path:"./Documents"`, names},
}
}
return lifecycleGroup{
name: "casepath",
fixtures: []search.Resource{folder, picture},
cases: []lifecycleCase{
{
id: 1, title: "takes the descendants along when deleting",
do: func(e search.Engine) error { return e.Delete(folder.ID) },
expect: left(),
engineOverrides: map[string]lifecycleOverride{
"opensearch": {expect: map[string][]string{`path:"./Documents"`: {"Documents", "Picture.jpg"}}},
},
},
{
id: 2, title: "takes the descendants along when moving",
do: func(e search.Engine) error { return e.Move(folder.ID, folder.ParentID, "./Other Documents") },
expect: []expectation{
{`path:"./Other Documents"`, []string{"Other Documents", "Picture.jpg"}},
{`path:"./Documents"`, nil},
},
engineOverrides: map[string]lifecycleOverride{
"bleve": {expect: map[string][]string{`path:"./Other Documents"`: {"Other Documents"}}},
"opensearch": {expect: map[string][]string{`path:"./Other Documents"`: {}, `path:"./Documents"`: {"Documents", "Picture.jpg"}}},
},
},
{
id: 3, title: "reaches the descendants when purging",
do: func(e search.Engine) error { return e.Purge(folder.ID, false) },
expect: left(),
wantDocCount: conversions.ToPointer(uint64(0)),
engineOverrides: map[string]lifecycleOverride{
"opensearch": {
expect: map[string][]string{`path:"./Documents"`: {"Documents", "Picture.jpg"}},
wantDocCount: conversions.ToPointer(uint64(2)),
},
},
},
},
}
}
services/search/pkg/parity/lifecycle_delete_test.go
+47
View File
@@ -0,0 +1,47 @@
package parity
import (
"github.com/opencloud-eu/opencloud/pkg/conversions"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func deleteLifecycle() lifecycleGroup {
parent, child := fixtureTree()
return lifecycleGroup{
name: "delete",
fixtures: []search.Resource{parent, child},
cases: []lifecycleCase{
{
id: 1, title: "takes the resource out of the results",
do: func(e search.Engine) error { return e.Delete(child.ID) },
expect: treeIsLeft("parent"),
},
{
id: 2, title: "takes the descendants along",
do: func(e search.Engine) error { return e.Delete(parent.ID) },
expect: treeIsLeft(),
},
{
id: 3, title: "leaves the resource in the index",
do: func(e search.Engine) error { return e.Delete(child.ID) },
wantDocCount: conversions.ToPointer(uint64(2)),
engineOverrides: map[string]lifecycleOverride{
"opensearch": {wantDocCount: conversions.ToPointer(uint64(1))},
},
},
{
id: 4, title: "takes a resource out that was just written",
do: func(e search.Engine) error {
fresh := fixtureDoc("fresh.txt", withID("1$1!8"))
if err := e.Upsert(fresh.ID, fresh); err != nil {
return err
}
return e.Delete(fresh.ID)
},
expect: []expectation{{`name:"*fresh*"`, nil}},
},
},
}
}
services/search/pkg/parity/lifecycle_hidden_test.go
+54
View File
@@ -0,0 +1,54 @@
package parity
import (
"path"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func hiddenLifecycle() lifecycleGroup {
group := lifecycleGroup{name: "hidden"}
for _, move := range []struct {
id int
title string
from string
target string
hidden bool
}{
{1, "into a dot folder", "./parent", "./.trash/parent", true},
{2, "into a plain folder", "./parent", "./archive/parent", false},
{3, "renamed with a leading dot", "./parent", "./.parent", true},
{4, "out of a dot folder", "./.trash/parent", "./archive/parent", false},
{5, "renamed without the leading dot", "./.parent", "./parent", false},
{6, "within the same dot folder", "./.trash/parent", "./.trash/moved", true},
} {
parent, child := fixtureTree()
parent.Path = move.from
parent.Hidden = search.IsHidden(move.from)
child.Path = move.from + "/child.pdf"
child.Hidden = parent.Hidden
var (
hidden []string
overrides map[string]lifecycleOverride
)
if move.hidden {
hidden = []string{path.Base(move.target), "child.pdf"}
// bleve does not answer hidden:true at all today
overrides = map[string]lifecycleOverride{"bleve": {expect: map[string][]string{`hidden:true`: {}}}}
}
group.cases = append(group.cases, lifecycleCase{
id: move.id, title: "follows a move " + move.title,
fixtures: []search.Resource{parent, child},
do: func(e search.Engine) error {
return e.Move(parent.ID, parent.ParentID, move.target)
},
expect: []expectation{{`hidden:true`, hidden}},
engineOverrides: overrides,
})
}
return group
}
services/search/pkg/parity/lifecycle_idempotency_test.go
+76
View File
@@ -0,0 +1,76 @@
package parity
import (
"github.com/opencloud-eu/opencloud/pkg/conversions"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func idempotencyLifecycle() lifecycleGroup {
parent, child := fixtureTree()
missing := "1$1!gone"
return lifecycleGroup{
name: "idempotency",
fixtures: []search.Resource{parent, child},
cases: []lifecycleCase{
{
id: 1, title: "deleting the same resource twice is not an error",
do: func(e search.Engine) error {
if err := e.Delete(child.ID); err != nil {
return err
}
return e.Delete(child.ID)
},
expect: treeIsLeft("parent"),
},
{
id: 2, title: "deleting a resource the index does not have reports it",
do: func(e search.Engine) error { return e.Delete(missing) },
expect: treeIsLeft("parent", "child.pdf"),
wantErr: true,
},
{
id: 3, title: "restoring a resource that was never deleted leaves it alone",
do: func(e search.Engine) error { return e.Restore(child.ID) },
expect: treeIsLeft("parent", "child.pdf"),
},
{
id: 4, title: "purging the same resource twice reports the second one",
do: func(e search.Engine) error {
if err := e.Purge(child.ID, false); err != nil {
return err
}
return e.Purge(child.ID, false)
},
expect: treeIsLeft("parent"),
wantDocCount: conversions.ToPointer(uint64(1)),
wantErr: true,
},
{
id: 5, title: "purging a resource the index does not have reports it",
do: func(e search.Engine) error { return e.Purge(missing, false) },
expect: treeIsLeft("parent", "child.pdf"),
wantDocCount: conversions.ToPointer(uint64(2)),
wantErr: true,
},
{
id: 6, title: "moving a resource onto its own path leaves it where it is",
do: func(e search.Engine) error {
return e.Move(parent.ID, parent.ParentID, parent.Path)
},
expect: []expectation{
{`path:"./parent/child.pdf"`, []string{"child.pdf"}},
{`name:"*parent*"`, []string{"parent"}},
},
},
{
id: 7, title: "purging a whole space that holds nothing is not an error",
do: func(e search.Engine) error { return e.PurgeSpace("9$9!9") },
expect: treeIsLeft("parent", "child.pdf"),
wantDocCount: conversions.ToPointer(uint64(2)),
},
},
}
}
services/search/pkg/parity/lifecycle_move_test.go
+37
View File
@@ -0,0 +1,37 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func moveLifecycle() lifecycleGroup {
parent, child := fixtureTree()
return lifecycleGroup{
name: "move",
fixtures: []search.Resource{parent, child},
cases: []lifecycleCase{
{
id: 1, title: "carries the descendants to the new path",
do: func(e search.Engine) error {
return e.Move(parent.ID, parent.ParentID, "./my/newname")
},
expect: []expectation{
{`path:"./my/newname/child.pdf"`, []string{"child.pdf"}},
{`path:"./parent/child.pdf"`, nil},
},
},
{
id: 2, title: "through the trash and back leaves the flag behind",
do: func(e search.Engine) error {
if err := e.Move(parent.ID, parent.ParentID, "./.trash/parent"); err != nil {
return err
}
return e.Move(parent.ID, parent.ParentID, "./parent")
},
expect: []expectation{{`hidden:true`, nil}},
},
},
}
}
services/search/pkg/parity/lifecycle_purge_test.go
+40
View File
@@ -0,0 +1,40 @@
package parity
import (
"github.com/opencloud-eu/opencloud/pkg/conversions"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func purgeLifecycle() lifecycleGroup {
parent, child := fixtureTree()
return lifecycleGroup{
name: "purge",
fixtures: []search.Resource{parent, child},
cases: []lifecycleCase{
{
id: 1, title: "removes one resource",
do: func(e search.Engine) error { return e.Purge(child.ID, false) },
expect: treeIsLeft("parent"),
wantDocCount: conversions.ToPointer(uint64(1)),
},
{
id: 2, title: "removes the tree",
do: func(e search.Engine) error { return e.Purge(parent.ID, false) },
expect: treeIsLeft(),
wantDocCount: conversions.ToPointer(uint64(0)),
},
{
id: 3, title: "takes only the deleted ones when it is told to",
do: func(e search.Engine) error {
if err := e.Delete(child.ID); err != nil {
return err
}
return e.Purge(parent.ID, true)
},
expect: treeIsLeft("parent"),
},
},
}
}
services/search/pkg/parity/lifecycle_purgespace_test.go
+36
View File
@@ -0,0 +1,36 @@
package parity
import (
"github.com/opencloud-eu/opencloud/pkg/conversions"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func purgeSpaceLifecycle() lifecycleGroup {
inSpace := fixtureDoc("inSpace.txt", withID("1$1!4"))
elsewhere := fixtureDoc("elsewhere.txt", withID("2$2!4"), withRoot("2$2!1"))
return lifecycleGroup{
name: "purgespace",
fixtures: []search.Resource{inSpace, elsewhere},
cases: []lifecycleCase{
{
id: 1, title: "leaves the other space alone",
do: func(e search.Engine) error { return e.PurgeSpace(inSpace.RootID) },
expect: []expectation{
{`name:"*inSpace*"`, nil},
{`name:"*elsewhere*"`, []string{"elsewhere.txt"}},
},
},
{
id: 2, title: "takes a space out that holds more than one round",
fixtures: append(fixtureBulk(60), elsewhere),
do: func(e search.Engine) error { return e.PurgeSpace(inSpace.RootID) },
expect: []expectation{
{`name:"*bulk*"`, nil},
{`name:"*elsewhere*"`, []string{"elsewhere.txt"}},
},
wantDocCount: conversions.ToPointer(uint64(1)),
},
},
}
}
services/search/pkg/parity/lifecycle_restore_test.go
+41
View File
@@ -0,0 +1,41 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func restoreLifecycle() lifecycleGroup {
parent, child := fixtureTree()
secret := fixtureDoc("file.txt", withID("1$1!5"), withPath("./.secret/file.txt"), isHidden())
return lifecycleGroup{
name: "restore",
fixtures: []search.Resource{parent, child},
cases: []lifecycleCase{
{
id: 1, title: "brings the descendants back",
do: func(e search.Engine) error {
if err := e.Delete(parent.ID); err != nil {
return err
}
return e.Restore(parent.ID)
},
expect: treeIsLeft("parent", "child.pdf"),
},
{
id: 2, title: "leaves the hidden flag alone",
fixtures: []search.Resource{secret},
do: func(e search.Engine) error {
if err := e.Delete(secret.ID); err != nil {
return err
}
return e.Restore(secret.ID)
},
expect: []expectation{{`hidden:true`, []string{"file.txt"}}},
engineOverrides: map[string]lifecycleOverride{"bleve": {expect: map[string][]string{`hidden:true`: {}}}},
},
},
}
}
services/search/pkg/parity/lifecycle_rootscope_test.go
+50
View File
@@ -0,0 +1,50 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func rootScopeLifecycle() lifecycleGroup {
const shared = "./same/path.txt"
target := fixtureDoc("target.txt", withID("1$1!3"), withPath(shared))
twin := fixtureDoc("twin.txt", withID("2$2!3"), withRoot("2$2!1"), withParent("2$2!2"), withPath(shared))
deleted := func(r search.Resource) search.Resource {
r.Deleted = true
return r
}
return lifecycleGroup{
name: "rootscope",
fixtures: []search.Resource{target, twin},
cases: []lifecycleCase{
{
id: 1, title: "deletes only the one in the target root",
do: func(e search.Engine) error { return e.Delete(target.ID) },
expect: []expectation{
{`name:"*target*"`, nil},
{`name:"*twin*"`, []string{"twin.txt"}},
},
},
{
id: 2, title: "restores only the one in the target root",
fixtures: []search.Resource{deleted(target), deleted(twin)},
do: func(e search.Engine) error { return e.Restore(target.ID) },
expect: []expectation{
{`name:"*target*"`, []string{"target.txt"}},
{`name:"*twin*"`, nil},
},
},
{
id: 3, title: "moves only the one in the target root",
do: func(e search.Engine) error { return e.Move(target.ID, target.ParentID, "./moved.txt") },
expect: []expectation{
{`path:"./moved.txt"`, []string{"moved.txt"}},
{`path:"` + shared + `"`, []string{"twin.txt"}},
},
},
},
}
}
services/search/pkg/parity/lifecycle_upsert_test.go
+27
View File
@@ -0,0 +1,27 @@
package parity
import (
"github.com/opencloud-eu/opencloud/pkg/conversions"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func upsertLifecycle() lifecycleGroup {
parent, child := fixtureTree()
renamed := child
renamed.Name = "renamed.pdf"
renamed.Path = "./parent/renamed.pdf"
return lifecycleGroup{
name: "upsert",
fixtures: []search.Resource{parent, child},
cases: []lifecycleCase{
{
id: 1, title: "replaces the resource it already knows",
do: func(e search.Engine) error { return e.Upsert(renamed.ID, renamed) },
expect: append(treeIsLeft("parent"), expectation{`name:"*renamed*"`, []string{"renamed.pdf"}}),
wantDocCount: conversions.ToPointer(uint64(2)),
},
},
}
}
services/search/pkg/parity/matrix_test.go
+457
View File
@@ -0,0 +1,457 @@
package parity
import (
"encoding/json"
"fmt"
"os"
"sort"
"strings"
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
. "github.com/onsi/ginkgo/v2"
"github.com/onsi/ginkgo/v2/types"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
const (
matrixFile = "README.md"
matrixEntry = "engine parity"
)
// matrixRow is one line of the README. It travels in a report entry, which
// crosses processes as JSON under ginkgo -p, hence the exported fields and
// the scope already rendered to a string.
type matrixRow struct {
Section string
Group string
ID string
Query string
Scope string
Limit int32
Reads string
Context string
Want []string
WantCount *int
WantBadRequest bool
// Overrides is what an engine is held to instead, rendered like expected
Overrides map[string]string
GroupAt, CaseAt, QueryAt int
}
func (r matrixRow) key() string {
return fmt.Sprintf("%s/%s/%s/%d", r.Group, r.ID, r.Query, r.QueryAt)
}
// matrixAnswer is what a spec attaches to its report: one engine's answer to
// one row. A skipped engine leaves the README alone.
type matrixAnswer struct {
Row matrixRow
Engine string
Answer []string
Skipped bool
}
func recordAnswer(row matrixRow, engine string, answer []string) {
AddReportEntry(matrixEntry, matrixAnswer{Row: row, Engine: engine, Answer: answer}, ReportEntryVisibilityNever)
}
func recordSkip(row matrixRow, engine string) {
AddReportEntry(matrixEntry, matrixAnswer{Row: row, Engine: engine, Skipped: true}, ReportEntryVisibilityNever)
}
// matrixPlanned is every row the spec tree announced while it was built, so
// the README is only written when each of them got an answer from every engine.
var matrixPlanned []matrixRow
func planRow(rows ...matrixRow) {
matrixPlanned = append(matrixPlanned, rows...)
}
type matrixResult struct {
matrixRow
answered map[string][]string
skipped map[string]bool
}
// matrixAnswerOf reads an entry back: in-process it still carries the value,
// from another process only its JSON.
func matrixAnswerOf(entry types.ReportEntry) (matrixAnswer, bool) {
if answer, ok := entry.GetRawValue().(matrixAnswer); ok {
return answer, true
}
var answer matrixAnswer
if err := json.Unmarshal([]byte(entry.Value.AsJSON), &answer); err != nil {
return matrixAnswer{}, false
}
return answer, true
}
func collectMatrix(report types.Report) []*matrixResult {
results := map[string]*matrixResult{}
for _, spec := range report.SpecReports {
for _, entry := range spec.ReportEntries {
if entry.Name != matrixEntry {
continue
}
answer, ok := matrixAnswerOf(entry)
if !ok {
continue
}
key := answer.Row.key()
result, known := results[key]
if !known {
result = &matrixResult{matrixRow: answer.Row, answered: map[string][]string{}, skipped: map[string]bool{}}
results[key] = result
}
if answer.Skipped {
result.skipped[answer.Engine] = true
continue
}
result.answered[answer.Engine] = answer.Answer
}
}
rows := make([]*matrixResult, 0, len(results))
for _, result := range results {
rows = append(rows, result)
}
return rows
}
func writeMatrix(report types.Report) {
rows := collectMatrix(report)
answered := map[string]*matrixResult{}
for _, row := range rows {
answered[row.key()] = row
}
var missing []string
for _, planned := range matrixPlanned {
result, ok := answered[planned.key()]
switch {
case !ok:
missing = append(missing, planned.ID+" "+planned.Query)
case len(result.skipped) > 0:
fmt.Fprintf(os.Stderr, "%s left alone, an engine was not reachable\n", matrixFile)
return
case len(result.answered) != len(engineNames):
missing = append(missing, planned.ID+" "+planned.Query)
}
}
if len(missing) > 0 {
fmt.Fprintf(os.Stderr, "%s left alone, no answer from every engine for: %s\n", matrixFile, strings.Join(missing, "; "))
return
}
sort.Slice(rows, func(i, j int) bool {
a, b := rows[i], rows[j]
switch {
case a.GroupAt != b.GroupAt:
return a.GroupAt < b.GroupAt
case a.CaseAt != b.CaseAt:
return a.CaseAt < b.CaseAt
default:
return a.QueryAt < b.QueryAt
}
})
out := &strings.Builder{}
out.WriteString("# Engine parity\n\n")
out.WriteString("Written by the parity suite (`go test ./services/search/pkg/parity/`), do not edit.\n")
out.WriteString("Every case runs against bleve and OpenSearch. `same?` is ✅ when both answer as\n")
out.WriteString("expected, `❌ known` when an engine's divergence is documented in the case\n")
out.WriteString("(`engineOverrides`), `❌` when it is not.\n")
group, section := "", ""
for _, row := range rows {
if row.Section != section {
section = row.Section
if out.Len() > 0 {
out.WriteString("\n")
}
fmt.Fprintf(out, "## %s\n", section)
}
if row.Group != group {
group = row.Group
fmt.Fprintf(out, "\n### %s\n\n%s\n\n", group, matrixFixtures(group))
out.WriteString("| Case | Query | expected | bleve | OpenSearch | same? |\n")
out.WriteString("|---|---|---|---|---|---|\n")
}
query := "`" + shortenQuery(row.Query) + "`"
if row.Scope != "" {
query += " " + row.Scope
}
if row.Limit != 0 {
query = fmt.Sprintf("%s with a limit of %d", query, row.Limit)
}
if row.Reads != "" {
query += " reads `" + row.Reads + "`"
}
if row.Context != "" {
query = row.Context + ", then " + query
}
expected := matrixNames(row.Want)
switch {
case row.WantBadRequest:
expected = "bad request"
case row.WantCount != nil:
expected = fmt.Sprintf("%d items", *row.WantCount)
}
fmt.Fprintf(out, "| %s | %s | %s | %s | %s | %s |\n",
row.ID, query, expected,
matrixNames(row.answered["bleve"]), matrixNames(row.answered["opensearch"]), matrixVerdict(row))
}
if err := os.WriteFile(matrixFile, []byte(out.String()), 0o644); err != nil {
fmt.Fprintf(os.Stderr, "failed to write %s: %v\n", matrixFile, err)
}
}
func matrixFixtures(group string) string {
var (
fixtures []search.Resource
withIDs bool
)
for _, g := range queryGroups() {
if g.name != group {
continue
}
fixtures = g.fixtures
for _, c := range g.cases {
withIDs = withIDs || strings.Contains(c.query, "id:")
}
}
for _, g := range lifecycleGroups() {
if g.name != group {
continue
}
withIDs = true
fixtures = g.fixtures
for _, c := range g.cases {
fixtures = append(fixtures, c.fixtures...)
}
}
for _, g := range responseGroups() {
if g.name != group {
continue
}
withIDs = true
fixtures = g.fixtures
}
if len(fixtures) == 0 {
return "Fixtures: none"
}
seen := map[string]bool{}
lines := []string{"Fixtures:", ""}
for _, f := range fixtures {
line := "- `" + shorten(f.Name) + "`"
if fields := fixtureFields(f, withIDs); fields != "" {
line += ", " + fields
}
if seen[line] {
continue
}
seen[line] = true
lines = append(lines, line)
}
if listed := len(lines) - 2; listed > 12 {
lines = append(lines[:12], fmt.Sprintf("- ... and %d more of the same", listed-10))
}
return strings.Join(lines, "\n")
}
func fixtureFields(f search.Resource, withID bool) string {
var fields []string
add := func(format string, args ...any) {
fields = append(fields, fmt.Sprintf(format, args...))
}
if withID {
add("ID = %s", f.ID)
}
if f.Type == uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER) {
add("folder")
} else if f.MimeType != "text/plain" {
add("MimeType = %s", f.MimeType)
}
if f.Path != "./"+f.Name {
add("Path = %s", f.Path)
}
if f.Title != "" {
add("Title = %q", f.Title)
}
if f.Content != "" {
add("Content = %q", f.Content)
}
if len(f.Tags) > 0 {
add("Tags = %s", strings.Join(f.Tags, ", "))
}
if len(f.Favorites) > 0 {
add("Favorites = %s", strings.Join(f.Favorites, ", "))
}
if f.Size != 1000 {
add("Size = %d", f.Size)
}
if !strings.HasPrefix(f.Mtime, fixtureNow.Format("2006-01-02")) {
add("Mtime = %s", f.Mtime)
}
if f.Hidden {
add("hidden")
}
if f.Deleted {
add("deleted")
}
return strings.Join(fields, ", ")
}
func matrixScope(ref *searchMessage.Reference) string {
if ref == nil {
return ""
}
space := storagespace.FormatResourceID(&sprovider.ResourceId{
StorageId: ref.GetResourceId().GetStorageId(),
SpaceId: ref.GetResourceId().GetSpaceId(),
OpaqueId: ref.GetResourceId().GetOpaqueId(),
})
if path := ref.GetPath(); path != "" {
return "in " + space + " under " + path
}
return "in " + space
}
func matrixNames(names []string) string {
if len(names) == 0 {
return "no match"
}
shortened := make([]string, 0, len(names))
for _, name := range names {
shortened = append(shortened, shorten(name))
}
sort.Strings(shortened)
return strings.Join(shortened, ", ")
}
func shorten(name string) string {
if len(name) <= 24 {
return name
}
return name[:10] + "..." + name[len(name)-8:]
}
func shortenQuery(q string) string {
if len(q) <= 64 {
return q
}
return q[:32] + "..." + q[len(q)-24:]
}
func matrixVerdict(row *matrixResult) string {
var off []string
known := true
for _, engine := range engineNames {
answer := matrixNames(row.answered[engine])
agrees := answer == matrixNames(row.Want)
switch {
case row.WantBadRequest:
agrees = answer == "bad request"
case row.WantCount != nil:
agrees = len(row.answered[engine]) == *row.WantCount
}
if agrees {
continue
}
off = append(off, engine)
if row.WantCount != nil {
answer = fmt.Sprintf("%d items", len(row.answered[engine]))
}
if expected, ok := row.Overrides[engine]; !ok || expected != answer {
known = false
}
}
switch {
case len(off) == 0:
return "✅"
case known:
return "❌ known"
default:
return "❌"
}
}
func (c lifecycleCase) matrixRows(group string, groupAt, caseAt int) []matrixRow {
rows := make([]matrixRow, 0, len(c.expect)+1)
for i, expect := range c.expect {
rows = append(rows, matrixRow{
Section: "Operations", Group: group, ID: c.label(group), Query: expect.query, Context: c.title, Want: expect.want, Overrides: renderOverrides(c.overridesFor(expect.query)),
GroupAt: groupAt, CaseAt: caseAt, QueryAt: i,
})
}
if c.wantDocCount != nil {
rows = append(rows, matrixRow{
Section: "Operations", Group: group, ID: c.label(group), Query: "DocCount()", Context: c.title,
Want: []string{fmt.Sprint(*c.wantDocCount)},
Overrides: renderOverrides(c.docCountOverrides()),
GroupAt: groupAt, CaseAt: caseAt, QueryAt: len(c.expect),
})
}
return rows
}
services/search/pkg/parity/parity_suite_test.go
+48
View File
@@ -0,0 +1,48 @@
package parity
import (
"context"
"encoding/json"
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
"github.com/opencloud-eu/opencloud/services/search/pkg/config"
)
var (
// openSearchClient points every process at the one OpenSearch started by
// process 1; the container handle stays there.
openSearchClient config.EngineOpenSearchClient
stopOpenSearch func()
)
func TestEngineParity(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Search Engine Parity Suite")
}
var _ = SynchronizedBeforeSuite(func() []byte {
cfg, done, err := opensearchtest.SetupTests(context.Background())
Expect(err).NotTo(HaveOccurred(), "failed to set up the OpenSearch test container")
stopOpenSearch = done
client, err := json.Marshal(cfg.Engine.OpenSearch.Client)
Expect(err).NotTo(HaveOccurred())
return client
}, func(client []byte) {
Expect(json.Unmarshal(client, &openSearchClient)).To(Succeed())
})
var _ = SynchronizedAfterSuite(func() {}, func() {
if stopOpenSearch != nil {
stopOpenSearch()
}
})
var _ = ReportAfterSuite("engine parity matrix", func(report Report) {
writeMatrix(report)
})
services/search/pkg/parity/parity_test.go
+466
View File
@@ -0,0 +1,466 @@
package parity
import (
"context"
"fmt"
"strings"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/types"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
type queryCase struct {
id int
query string
want []string
limit int32
wantCount *int
ref *searchMessage.Reference
wantBadRequest bool
// engineOverrides holds, per engine, what that engine answers today where
// it still differs from the expectation above. The spec asserts the
// override and the README marks the row as a known divergence. Once the
// engine answers as expected the override fails and has to go.
engineOverrides map[string]override
}
// override is one engine's current answer to a case, in the expectation's
// own terms.
type override struct {
want []string
wantCount *int
wantBadRequest bool
}
func (o override) matcher() types.GomegaMatcher {
switch {
case o.wantBadRequest:
return ConsistOf("bad request")
case o.wantCount != nil:
return HaveLen(*o.wantCount)
default:
return matchNames(o.want)
}
}
// rendered spells the override the way the README spells an expectation.
func (o override) rendered() string {
switch {
case o.wantBadRequest:
return "bad request"
case o.wantCount != nil:
return fmt.Sprintf("%d items", *o.wantCount)
default:
return matrixNames(o.want)
}
}
func renderOverrides(overrides map[string]override) map[string]string {
if len(overrides) == 0 {
return nil
}
out := make(map[string]string, len(overrides))
for engine, o := range overrides {
out[engine] = o.rendered()
}
return out
}
func (c queryCase) label(group string) string {
return fmt.Sprintf("%s-%02d", strings.ToUpper(group), c.id)
}
func (c queryCase) name(group string) string {
return c.label(group) + " " + c.query
}
type queryGroup struct {
name string
fixtures []search.Resource
cases []queryCase
}
func queryGroups() []queryGroup {
return []queryGroup{
nameGroup(),
extensionGroup(),
tagsGroup(),
titleGroup(),
contentGroup(),
favoritesGroup(),
mediatypeGroup(),
pathGroup(),
fieldsGroup(),
deletedGroup(),
visibilityGroup(),
booleanGroup(),
samenameGroup(),
stressGroup(),
everythingGroup(),
rangeGroup(),
scopeGroup(),
invalidGroup(),
}
}
type responseCase struct {
id int
do func(e search.Engine) error
after string
query string
reads string
read func(*searchService.SearchIndexResponse) []string
want []string
engineOverrides map[string]override
}
func (c responseCase) label(group string) string {
return fmt.Sprintf("%s-%02d", strings.ToUpper(group), c.id)
}
func (c responseCase) name(group string) string {
return c.label(group) + " " + c.reads
}
type responseGroup struct {
name string
fixtures []search.Resource
cases []responseCase
}
func responseGroups() []responseGroup {
return []responseGroup{
entityGroup(),
metadataGroup(),
}
}
type lifecycleCase struct {
id int
title string
do func(e search.Engine) error
expect []expectation
wantErr bool
fixtures []search.Resource
wantDocCount *uint64
// engineOverrides is the lifecycle spelling of queryCase.engineOverrides:
// a case has several expectations, so an override answers them by query.
engineOverrides map[string]lifecycleOverride
}
// lifecycleOverride is one engine's current answers to a lifecycle case.
type lifecycleOverride struct {
expect map[string][]string
wantDocCount *uint64
}
// overridesFor picks one expectation's overrides out of the case's.
func (c lifecycleCase) overridesFor(query string) map[string]override {
var overrides map[string]override
for engine, o := range c.engineOverrides {
if answer, ok := o.expect[query]; ok {
if overrides == nil {
overrides = map[string]override{}
}
overrides[engine] = override{want: answer}
}
}
return overrides
}
// docCountOverrides renders the DocCount overrides as a query override.
func (c lifecycleCase) docCountOverrides() map[string]override {
var overrides map[string]override
for engine, o := range c.engineOverrides {
if o.wantDocCount != nil {
if overrides == nil {
overrides = map[string]override{}
}
overrides[engine] = override{want: []string{fmt.Sprint(*o.wantDocCount)}}
}
}
return overrides
}
type expectation struct {
query string
want []string
}
func (c lifecycleCase) label(group string) string {
return fmt.Sprintf("%s-%02d", strings.ToUpper(group), c.id)
}
func (c lifecycleCase) name(group string) string {
return c.label(group) + " " + c.title
}
type lifecycleGroup struct {
name string
fixtures []search.Resource
cases []lifecycleCase
}
func lifecycleGroups() []lifecycleGroup {
return []lifecycleGroup{
deleteLifecycle(),
restoreLifecycle(),
purgeLifecycle(),
purgeSpaceLifecycle(),
moveLifecycle(),
rootScopeLifecycle(),
casePathLifecycle(),
hiddenLifecycle(),
upsertLifecycle(),
idempotencyLifecycle(),
batchLifecycle(),
}
}
// expectAnswer holds an engine to its override when it has one, to the
// expectation otherwise. An override that no longer holds fails on purpose:
// the engine answers as expected now, the override has to go.
func expectAnswer(engine string, answer []string, expected override, overrides map[string]override) {
GinkgoHelper()
if o, ok := overrides[engine]; ok {
Expect(o.rendered()).NotTo(Equal(expected.rendered()), "an override that equals the expectation documents nothing, remove it")
Expect(answer).To(o.matcher(), "the override for %s no longer holds, remove it", engine)
return
}
Expect(answer).To(expected.matcher())
}
// Every case runs once per engine as its own spec, so one engine failing
// leaves the other's answer in the matrix. The groups are Ordered containers:
// their engines are built once in a BeforeAll and shared by the specs inside,
// and they carry on after a failure so every engine gets to answer.
var _ = Describe("Queries", func() {
for groupAt, group := range queryGroups() {
Describe(group.name, Ordered, ContinueOnFailure, func() {
var engines []testEngine
BeforeAll(func() {
engines = newEngines("opencloud-test-engine-parity-"+group.name, group.fixtures)
})
for caseAt, c := range group.cases {
row := matrixRow{
Section: "Queries", Group: group.name, ID: c.label(group.name),
Query: c.query, Scope: matrixScope(c.ref), Limit: c.limit,
Want: c.want, WantCount: c.wantCount, WantBadRequest: c.wantBadRequest, Overrides: renderOverrides(c.engineOverrides),
GroupAt: groupAt, CaseAt: caseAt,
}
planRow(row)
Describe(c.name(group.name), func() {
for _, name := range engineNames {
It("on "+name, func() {
e := engineNamed(engines, name)
if e.unavailable != "" {
recordSkip(row, name)
Skip(e.unavailable)
}
request := &searchService.SearchIndexRequest{Query: c.query, PageSize: c.limit, Ref: c.ref}
expected := override{want: c.want, wantCount: c.wantCount, wantBadRequest: c.wantBadRequest}
_, overridden := c.engineOverrides[name]
if c.wantBadRequest {
_, err := e.backend.Search(context.Background(), request)
answer := badRequestAnswer(err)
recordAnswer(row, name, answer)
if !overridden {
Expect(err).To(BeAssignableToTypeOf(errtypes.BadRequest("")))
}
expectAnswer(name, answer, expected, c.engineOverrides)
return
}
answer, err := ask(e.backend, request)
recordAnswer(row, name, answer)
if !overridden {
Expect(err).NotTo(HaveOccurred(), "the query has to answer, an empty result is an answer")
}
expectAnswer(name, answer, expected, c.engineOverrides)
})
}
})
}
})
}
})
var _ = Describe("Operations", func() {
offset := len(queryGroups())
for groupAt, group := range lifecycleGroups() {
Describe(group.name, func() {
for caseAt, c := range group.cases {
fixtures := c.fixtures
if fixtures == nil {
fixtures = group.fixtures
}
index := fmt.Sprintf("opencloud-test-engine-parity-%s-%d", group.name, c.id)
rows := c.matrixRows(group.name, offset+groupAt, caseAt)
planRow(rows...)
// an operation changes its index, so every spec builds its own engine
Describe(c.name(group.name), func() {
for _, name := range engineNames {
It("on "+name, func() {
e := newEngine(name, index, fixtures)
if e.unavailable != "" {
for _, row := range rows {
recordSkip(row, name)
}
Skip(e.unavailable)
}
// every row gets its answer before anything is asserted, a
// failed assertion must not leave the later rows unanswered
err := c.do(e.backend)
if failed := err != nil; failed != c.wantErr {
for _, row := range rows {
recordAnswer(row, name, badRequestAnswer(err))
}
}
if c.wantErr {
Expect(err).To(HaveOccurred(), "the operation had to report that it did not find the resource")
} else {
Expect(err).NotTo(HaveOccurred(), "the operation under test failed")
}
e.settle()
answers := make([][]string, len(c.expect))
errs := make([]error, len(c.expect))
for i, expect := range c.expect {
answers[i], errs[i] = ask(e.backend, &searchService.SearchIndexRequest{Query: expect.query})
recordAnswer(rows[i], name, answers[i])
}
var count uint64
if c.wantDocCount != nil {
count, err = e.backend.DocCount()
Expect(err).NotTo(HaveOccurred())
recordAnswer(rows[len(rows)-1], name, []string{fmt.Sprint(count)})
}
for i, expect := range c.expect {
overrides := c.overridesFor(expect.query)
if _, overridden := overrides[name]; !overridden {
Expect(errs[i]).NotTo(HaveOccurred(), "the query has to answer, an empty result is an answer")
}
expectAnswer(name, answers[i], override{want: expect.want}, overrides)
}
if c.wantDocCount != nil {
expected := override{want: []string{fmt.Sprint(*c.wantDocCount)}}
expectAnswer(name, []string{fmt.Sprint(count)}, expected, c.docCountOverrides())
}
})
}
})
}
})
}
})
var _ = Describe("Response", func() {
offset := len(queryGroups()) + len(lifecycleGroups())
for groupAt, group := range responseGroups() {
Describe(group.name, Ordered, ContinueOnFailure, func() {
var shared []testEngine
BeforeAll(func() {
shared = newEngines("opencloud-test-engine-parity-"+group.name, group.fixtures)
})
for caseAt, c := range group.cases {
row := matrixRow{
Section: "Response", Group: group.name, ID: c.label(group.name),
Query: c.query, Reads: c.reads, Context: c.after, Want: c.want, Overrides: renderOverrides(c.engineOverrides),
GroupAt: offset + groupAt, CaseAt: caseAt,
}
planRow(row)
index := fmt.Sprintf("opencloud-test-engine-parity-%s-%d", group.name, c.id)
Describe(c.name(group.name), func() {
for _, name := range engineNames {
It("on "+name, func() {
// a case with an operation changes its index, so it gets its own
e := engineNamed(shared, name)
if c.do != nil {
e = newEngine(name, index, group.fixtures)
}
if e.unavailable != "" {
recordSkip(row, name)
Skip(e.unavailable)
}
if c.do != nil {
err := c.do(e.backend)
if err != nil {
recordAnswer(row, name, []string{"error"})
}
Expect(err).NotTo(HaveOccurred(), "the operation under test failed")
e.settle()
}
resp, err := e.backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: c.query})
if err != nil {
recordAnswer(row, name, []string{"error"})
}
Expect(err).NotTo(HaveOccurred(), "the query has to answer, an empty result is an answer")
answer := c.read(resp)
recordAnswer(row, name, answer)
if o, overridden := c.engineOverrides[name]; overridden {
Expect(o.want).NotTo(Equal(c.want), "an override that equals the expectation documents nothing, remove it")
Expect(answer).To(Equal(o.want), "the override for %s no longer holds, remove it", name)
return
}
Expect(answer).To(Equal(c.want), c.reads)
})
}
})
}
})
}
})
services/search/pkg/parity/query_boolean_test.go
+24
View File
@@ -0,0 +1,24 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func booleanGroup() queryGroup {
return queryGroup{
name: "boolean",
fixtures: []search.Resource{
fixtureDoc("alpha.txt", withTags("red")),
fixtureDoc("beta.txt", withTags("blue")),
fixtureDoc("gamma.md", withMime("text/markdown")),
},
cases: []queryCase{
{id: 1, query: `name:"*alpha*" AND name:"*txt*"`, want: []string{"alpha.txt"}},
{id: 2, query: `name:"*alpha*" OR name:"*beta*"`, want: []string{"alpha.txt", "beta.txt"}},
{id: 3, query: `name:"*a*" AND NOT name:"*alpha*"`, want: []string{"beta.txt", "gamma.md"}},
{id: 4, query: `name:"*a*" AND tag:("red")`, want: []string{"alpha.txt"}},
{id: 5, query: `(name:"*alpha*" OR name:"*beta*") AND mediatype:text/plain`, want: []string{"alpha.txt", "beta.txt"}},
{id: 6, query: `name:"*a*" AND mediatype:text/markdown`, want: []string{"gamma.md"}},
},
}
}
services/search/pkg/parity/query_content_test.go
+27
View File
@@ -0,0 +1,27 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func contentGroup() queryGroup {
return queryGroup{
name: "content",
fixtures: []search.Resource{
fixtureDoc("monthly.txt", withContent("the monthly reports are due")),
fixtureDoc("links.txt", withContent("see https://opencloud.example.com/help or write to alan@example.org")),
},
cases: []queryCase{
{id: 1, query: `Content:report`, engineOverrides: map[string]override{"bleve": override{want: []string{"monthly.txt"}}}},
{id: 2, query: `Content:REPORTS`, want: []string{"monthly.txt"}},
{id: 3, query: `Content:"monthly reports"`, want: []string{"monthly.txt"}},
{id: 4, query: `Content:"reports monthly"`, engineOverrides: map[string]override{"bleve": override{want: []string{"monthly.txt"}}}},
{id: 5, query: `Content:report*`, want: []string{"monthly.txt"}},
{id: 6, query: `Content:*eport*`, want: []string{"monthly.txt"}},
{id: 7, query: `Content:month*`, want: []string{"monthly.txt"}},
{id: 8, query: `Content:"https://opencloud.example.com/help"`, want: []string{"links.txt"}},
{id: 9, query: `Content:"alan@example.org"`, want: []string{"links.txt"}},
{id: 10, query: `Content:opencloud`, want: []string{"links.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}},
},
}
}
services/search/pkg/parity/query_deleted_test.go
+26
View File
@@ -0,0 +1,26 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func deletedGroup() queryGroup {
return queryGroup{
name: "deleted",
fixtures: []search.Resource{
fixtureDoc("trashed.txt", isDeleted()),
fixtureDoc("kept.txt"),
fixtureFolder("bin", isDeleted()),
fixtureDoc("receipt.txt", withParent("1$1!bin"), withPath("./bin/receipt.txt"), isDeleted()),
fixtureFolder("shelf"),
fixtureDoc("book.txt", withParent("1$1!shelf"), withPath("./shelf/book.txt")),
},
cases: []queryCase{
{id: 1, query: `name:"*trashed*"`},
{id: 2, query: `name:"*.txt"`, want: []string{"kept.txt", "book.txt"}},
{id: 3, query: `name:"*receipt*"`},
{id: 4, query: `path:"./bin"`},
{id: 5, query: `path:"./shelf"`, want: []string{"shelf", "book.txt"}, engineOverrides: map[string]override{"bleve": override{want: []string{"shelf"}}}},
},
}
}
services/search/pkg/parity/query_everything_test.go
+23
View File
@@ -0,0 +1,23 @@
package parity
import (
"github.com/opencloud-eu/opencloud/pkg/conversions"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func everythingGroup() queryGroup {
return queryGroup{
name: "everything",
fixtures: []search.Resource{
fixtureDoc("alpha.txt"),
fixtureDoc("beta.txt"),
fixtureFolder("box"),
},
cases: []queryCase{
{id: 1, query: `*`, want: []string{"alpha.txt", "beta.txt", "box"}},
{id: 2, query: `name:"*"`, want: []string{"alpha.txt", "beta.txt", "box"}},
{id: 3, query: `*`, limit: 2, wantCount: conversions.ToPointer(2)},
{id: 4, query: `*`, limit: -1, wantCount: conversions.ToPointer(3)},
},
}
}
services/search/pkg/parity/query_extension_test.go
+22
View File
@@ -0,0 +1,22 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func extensionGroup() queryGroup {
return queryGroup{
name: "extension",
fixtures: []search.Resource{
fixtureDoc("report.txt"),
fixtureDoc("notes.md", withMime("text/markdown")),
fixtureFolder("archive"),
},
cases: []queryCase{
{id: 1, query: `txt`, want: []string{"report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}},
{id: 2, query: `md`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}},
{id: 3, query: `name:"*.txt"`, want: []string{"report.txt"}},
{id: 4, query: `report`, want: []string{"report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}},
},
}
}
services/search/pkg/parity/query_favorites_test.go
+22
View File
@@ -0,0 +1,22 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func favoritesGroup() queryGroup {
return queryGroup{
name: "favorites",
fixtures: []search.Resource{
fixtureDoc("starred.txt", withFavorite("A1B2-Upper")),
fixtureDoc("plain.txt"),
fixtureFolder("keepsakes", withFavorite("A1B2-Upper")),
fixtureDoc("photo.jpg", withParent("1$1!keepsakes"), withPath("./keepsakes/photo.jpg"), withMime("image/jpeg")),
},
cases: []queryCase{
{id: 1, query: `Favorites:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}, engineOverrides: map[string]override{"opensearch": override{}}},
{id: 2, query: `favorite:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}, engineOverrides: map[string]override{"opensearch": override{}}},
{id: 3, query: `Favorites:"somebody-else"`},
},
}
}
services/search/pkg/parity/query_fields_test.go
+41
View File
@@ -0,0 +1,41 @@
package parity
import (
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func fieldsGroup() queryGroup {
return queryGroup{
name: "fields",
fixtures: []search.Resource{
fixtureDoc("small.txt", withSize(42)),
fixtureDoc("old.txt", withMtime("2020-01-01T00:00:00Z")),
fixtureDoc("known.txt", withID("1$1!23")),
fixtureDoc("cased.txt", withID("1$1!AB-23")),
fixtureDoc("hidden.txt", isHidden()),
fixtureDoc("plain.txt"),
fixtureFolder("box"),
fixtureDoc("boxed.txt", withParent("1$1!box"), withPath("./box/boxed.txt")),
fixtureDoc("song.mp3", withMime("audio/mpeg"), withAudio(&libregraph.Audio{Artist: libregraph.PtrString("Some Artist")})),
},
cases: []queryCase{
{id: 1, query: `size:42`, want: []string{"small.txt"}},
{id: 2, query: `mtime<"2021-01-01T00:00:00Z"`, want: []string{"old.txt"}},
{id: 3, query: `id:"1$1!23"`, want: []string{"known.txt"}},
{id: 4, query: `hidden:true`, want: []string{"hidden.txt"}, engineOverrides: map[string]override{"bleve": override{}}},
{id: 5, query: `type:file`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}},
{id: 6, query: `type:folder`, want: []string{"box"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}},
{id: 7, query: `unknown:field`},
{id: 8, query: `type:File`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}},
{id: 9, query: `type:FOLDER`, want: []string{"box"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}},
{id: 10, query: `hidden:TRUE`, want: []string{"hidden.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}},
{id: 11, query: `id:"1$1!AB-23"`, want: []string{"cased.txt"}, engineOverrides: map[string]override{"opensearch": override{}}},
{id: 12, query: `id:"1$1!ab-23"`},
// a facet value keeps its case, the field is not marked lowercase
{id: 13, query: `audio.artist:"Some Artist"`, want: []string{"song.mp3"}, engineOverrides: map[string]override{"opensearch": override{}}},
{id: 14, query: `audio.artist:"some artist"`},
},
}
}
services/search/pkg/parity/query_invalid_test.go
+18
View File
@@ -0,0 +1,18 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func invalidGroup() queryGroup {
return queryGroup{
name: "invalid",
fixtures: []search.Resource{
fixtureDoc("alpha.txt"),
},
cases: []queryCase{
{id: 1, query: `AND mediatype:document`, wantBadRequest: true},
{id: 2, query: `mediatype:document AND`, want: []string{"alpha.txt"}},
},
}
}
services/search/pkg/parity/query_mediatype_test.go
+25
View File
@@ -0,0 +1,25 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func mediatypeGroup() queryGroup {
return queryGroup{
name: "mediatype",
fixtures: []search.Resource{
fixtureDoc("notes.md", withMime("text/markdown")),
fixtureDoc("photo.jpg", withMime("image/jpeg")),
fixtureFolder("albums"),
fixtureFolder("drafts"),
},
cases: []queryCase{
{id: 1, query: `mediatype:text/markdown`, want: []string{"notes.md"}},
{id: 2, query: `mediatype:TEXT/MARKDOWN`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}}},
{id: 3, query: `mediatype:image/jpeg`, want: []string{"photo.jpg"}},
{id: 4, query: `mediatype:*jpeg`, want: []string{"photo.jpg"}},
{id: 5, query: `mediatype:image`, want: []string{"photo.jpg"}},
{id: 6, query: `mediatype:folder`, want: []string{"albums", "drafts"}},
},
}
}
services/search/pkg/parity/query_name_test.go
+66
View File
@@ -0,0 +1,66 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func nameGroup() queryGroup {
return queryGroup{
name: "name",
fixtures: []search.Resource{
fixtureFolder("new-folder"),
fixtureDoc("quarterly notes.txt"),
fixtureDoc("Report.txt"),
fixtureDoc("Übung.txt"),
fixtureDoc("a+b.txt"),
fixtureDoc("c(d).txt"),
fixtureDoc("e&f.txt"),
fixtureDoc("v1.2.3.txt"),
fixtureDoc("foo bar.txt"),
fixtureDoc(fixtureLongName),
},
cases: []queryCase{
{id: 1, query: `new`, want: []string{"new-folder"}, engineOverrides: map[string]override{"bleve": override{}}},
{id: 2, query: `quarterly`, want: []string{"quarterly notes.txt"}, engineOverrides: map[string]override{"bleve": override{}}},
{id: 3, query: `report`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}},
{id: 4, query: `name:"*new-folder*"`, want: []string{"new-folder"}},
{id: 5, query: `name:"*w-fol*"`, want: []string{"new-folder"}},
{id: 6, query: `name:"*oo ba*"`, want: []string{"foo bar.txt"}},
{id: 7, query: `name:"*REPORT*"`, want: []string{"Report.txt"}},
{id: 8, query: `name:"*übung*"`, want: []string{"Übung.txt"}, engineOverrides: map[string]override{"opensearch": override{}}},
{id: 9, query: `name:"*ÜBUNG*"`, want: []string{"Übung.txt"}, engineOverrides: map[string]override{"opensearch": override{}}},
{id: 10, query: `name:"*a+b*"`, want: []string{"a+b.txt"}},
{id: 11, query: `name:"*c(d)*"`, want: []string{"c(d).txt"}},
{id: 12, query: `name:"*e&f*"`, want: []string{"e&f.txt"}},
{id: 13, query: `name:"*v1.2*"`, want: []string{"v1.2.3.txt"}},
{id: 14, query: `new-folder`, want: []string{"new-folder"}},
{id: 15, query: `*folder*`, want: []string{"new-folder"}},
{id: 16, query: `name:"*foo bar*"`, want: []string{"foo bar.txt"}},
{id: 17, query: `name:"foo bar.txt"`, want: []string{"foo bar.txt"}},
{id: 18, query: `name:"*needle*"`, want: []string{fixtureLongName}, engineOverrides: map[string]override{"opensearch": override{}}},
{id: 19, query: `name:"report*"`, want: []string{"Report.txt"}},
{id: 20, query: `name:"*report"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}},
{id: 21, query: `name:"Rep*rt.txt"`, want: []string{"Report.txt"}},
{id: 22, query: `Name:"*report*"`, want: []string{"Report.txt"}},
{id: 23, query: `NAME:"*report*"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}},
{id: 24, query: `name:Rep?rt.txt`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}},
{id: 25, query: `name:"*eport"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}},
{id: 26, query: `name:"repor*"`, want: []string{"Report.txt"}},
{id: 27, query: `REPORT`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}},
{id: 28, query: `name:REPORT`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}},
{id: 29, query: `name:"REPORT.TXT"`, want: []string{"Report.txt"}},
{id: 30, query: `name:"FOO BAR.TXT"`, want: []string{"foo bar.txt"}},
{id: 31, query: `name:"ÜBUNG.TXT"`, want: []string{"Übung.txt"}},
{id: 32, query: `name:"folder*"`},
{id: 33, query: `name:"*new"`},
{id: 34, query: `name:new`, want: []string{"new-folder"}, engineOverrides: map[string]override{"bleve": override{}}},
{id: 35, query: `name:"new"`, engineOverrides: map[string]override{"opensearch": override{want: []string{"new-folder"}}}},
{id: 36, query: `name:"new-folder"`, want: []string{"new-folder"}},
{id: 37, query: `name:"new-*"`, want: []string{"new-folder"}},
{id: 38, query: `name:"new*"`, want: []string{"new-folder"}},
{id: 39, query: `name:new-*`, want: []string{"new-folder"}},
{id: 40, query: `name:"*-folder"`, want: []string{"new-folder"}},
{id: 41, query: `name:"Rep?rt.txt"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}},
},
}
}
services/search/pkg/parity/query_path_test.go
+28
View File
@@ -0,0 +1,28 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func pathGroup() queryGroup {
return queryGroup{
name: "path",
fixtures: []search.Resource{
fixtureFolder("parent"),
fixtureDoc("child.jpg", withMime("image/jpeg"), withPath("./parent/child.jpg")),
fixtureFolder("docs-lower", withPath("./documents")),
fixtureFolder("docs-upper", withPath("./DOCUMENTS")),
fixtureFolder("docs-mixed", withPath("./Documents")),
},
cases: []queryCase{
{id: 1, query: `path:"./parent"`, want: []string{"parent", "child.jpg"}, engineOverrides: map[string]override{"bleve": override{want: []string{"parent"}}}},
{id: 2, query: `path:"./parent/child.jpg"`, want: []string{"child.jpg"}},
{id: 3, query: `path:"./Parent"`, engineOverrides: map[string]override{"opensearch": override{want: []string{"child.jpg", "parent"}}}},
{id: 4, query: `path:"*child*"`, want: []string{"child.jpg"}},
{id: 5, query: `path:"./documents"`, want: []string{"docs-lower"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"docs-lower", "docs-mixed", "docs-upper"}}}},
{id: 6, query: `path:"./DOCUMENTS"`, want: []string{"docs-upper"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"docs-lower", "docs-mixed", "docs-upper"}}}},
{id: 7, query: `path:"./Documents"`, want: []string{"docs-mixed"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"docs-lower", "docs-mixed", "docs-upper"}}}},
{id: 8, query: `path:"./parent/"`, want: []string{"parent", "child.jpg"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}},
},
}
}
services/search/pkg/parity/query_range_test.go
+26
View File
@@ -0,0 +1,26 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func rangeGroup() queryGroup {
return queryGroup{
name: "range",
fixtures: []search.Resource{
fixtureDoc("small.txt", withSize(50)),
fixtureDoc("big.txt", withSize(500)),
fixtureDoc("ancient.txt", withSize(10), withMtime("2020-01-01T00:00:00Z")),
},
cases: []queryCase{
{id: 1, query: `size>100`, want: []string{"big.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}},
{id: 2, query: `size<100`, want: []string{"small.txt", "ancient.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}},
{id: 3, query: `mtime>"2021-01-01T00:00:00Z"`, want: []string{"small.txt", "big.txt"}},
{id: 4, query: `mtime<"2021-01-01T00:00:00Z"`, want: []string{"ancient.txt"}},
{id: 5, query: `Mtime:"today"`, want: []string{"small.txt", "big.txt"}},
{id: 6, query: `Mtime:"yesterday"`},
{id: 7, query: `mtime>2021`},
{id: 8, query: `name>100`},
},
}
}
services/search/pkg/parity/query_samename_test.go
+34
View File
@@ -0,0 +1,34 @@
package parity
import (
"github.com/opencloud-eu/opencloud/pkg/conversions"
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func samenameGroup() queryGroup {
folder := fixtureFolder("doc", withID("1$1!2"))
under := func(path string) *searchMessage.Reference {
return &searchMessage.Reference{
ResourceId: &searchMessage.ResourceID{StorageId: "1", SpaceId: "1", OpaqueId: "1"},
Path: path,
}
}
return queryGroup{
name: "samename",
fixtures: []search.Resource{
folder,
fixtureDoc("doc.pdf", withID("1$1!3")),
fixtureDoc("file.pdf", withID("1$1!4")),
fixtureDoc("doc.pdf", withID("1$1!5"), withParent(folder.ID), withPath("./doc/doc.pdf")),
fixtureDoc("file.pdf", withID("1$1!6"), withParent(folder.ID), withPath("./doc/file.pdf")),
},
cases: []queryCase{
{id: 1, query: `name:"*doc*"`, wantCount: conversions.ToPointer(3)},
{id: 2, query: `name:"*doc*"`, ref: under("./doc"), wantCount: conversions.ToPointer(2)},
{id: 3, query: `name:"*file*"`, wantCount: conversions.ToPointer(2)},
{id: 4, query: `name:"*file*"`, ref: under("./doc"), wantCount: conversions.ToPointer(1)},
},
}
}
services/search/pkg/parity/query_scope_test.go
+32
View File
@@ -0,0 +1,32 @@
package parity
import (
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func scopeGroup() queryGroup {
space := func(path string) *searchMessage.Reference {
return &searchMessage.Reference{
ResourceId: &searchMessage.ResourceID{StorageId: "1", SpaceId: "1", OpaqueId: "1"},
Path: path,
}
}
return queryGroup{
name: "scope",
fixtures: []search.Resource{
fixtureFolder("parent"),
fixtureDoc("child.pdf", withPath("./parent/child.pdf")),
fixtureDoc("outside.txt"),
fixtureDoc("elsewhere.txt", withRoot("2$2!1")),
},
cases: []queryCase{
{id: 1, query: `*`, want: []string{"parent", "child.pdf", "outside.txt", "elsewhere.txt"}},
{id: 2, query: `*`, ref: space(""), want: []string{"parent", "child.pdf", "outside.txt"}},
{id: 3, query: `*`, ref: space("./parent"), want: []string{"parent", "child.pdf"}},
{id: 4, query: `*`, ref: space("./parent/child.pdf"), want: []string{"child.pdf"}},
{id: 5, query: `name:"*elsewhere*"`, ref: space(""), want: nil},
},
}
}
services/search/pkg/parity/query_stress_test.go
+38
View File
@@ -0,0 +1,38 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func stressGroup() queryGroup {
return queryGroup{
name: "stress",
fixtures: []search.Resource{
fixtureDoc("quarterly report.docx",
withMime("application/vnd.openxmlformats-officedocument.wordprocessingml.document"),
withTags("final"), withSize(2000)),
fixtureDoc("draft report.txt", withTags("draft"), withSize(50), withMtime("2020-01-01T00:00:00Z")),
fixtureDoc("photo.jpg", withMime("image/jpeg"), withTags("final")),
fixtureDoc("notes.md", withMime("text/markdown"), isHidden()),
fixtureFolder("archive"),
},
cases: []queryCase{
{id: 1, query: `name:"*report*" AND mediatype:document`, want: []string{"quarterly report.docx", "draft report.txt"}},
{id: 2, query: `name:"*report*" AND NOT tag:("draft")`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{want: []string{"draft report.txt", "quarterly report.docx"}}}},
{id: 3, query: `(tag:("final") OR tag:("draft")) AND mediatype:image`, want: []string{"photo.jpg"}},
{id: 4, query: `mediatype:document AND mtime>"2021-01-01T00:00:00Z"`, want: []string{"quarterly report.docx", "notes.md"}},
{id: 5, query: `name:"*report*" AND size>100`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}},
{id: 6, query: `tag:("final") AND NOT mediatype:folder`, want: []string{"quarterly report.docx", "photo.jpg"}},
{id: 7, query: `hidden:true AND name:"*notes*"`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}}},
{id: 8, query: `name:quarterly report`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}},
{id: 9, query: `name:"quarterly report"`},
{id: 10, query: `name:"quarterly report.docx"`, want: []string{"quarterly report.docx"}},
{id: 11, query: `NOT tag:("draft")`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md", "archive"}, engineOverrides: map[string]override{"bleve": override{want: []string{"archive", "draft report.txt", "notes.md", "photo.jpg", "quarterly report.docx"}}}},
{id: 12, query: `tag:("final") OR hidden:true`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md"}, engineOverrides: map[string]override{"bleve": override{want: []string{"photo.jpg", "quarterly report.docx"}}}},
{id: 13, query: `(name:"*report*" OR name:"*notes*") AND NOT (tag:("draft") OR hidden:true)`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{want: []string{"notes.md", "quarterly report.docx"}}}},
{id: 14, query: `mediatype:image OR (mediatype:document AND tag:("draft"))`, want: []string{"photo.jpg", "draft report.txt"}},
{id: 15, query: `NOT (mediatype:folder OR hidden:true)`, want: []string{"quarterly report.docx", "draft report.txt", "photo.jpg"}, engineOverrides: map[string]override{"bleve": override{want: []string{"draft report.txt", "notes.md", "photo.jpg", "quarterly report.docx"}}}},
{id: 16, query: `name:"*report*" AND (size>100 OR tag:("draft"))`, want: []string{"quarterly report.docx", "draft report.txt"}, engineOverrides: map[string]override{"bleve": override{want: []string{"draft report.txt"}}, "opensearch": override{want: []string{"draft report.txt"}}}},
},
}
}
services/search/pkg/parity/query_tags_test.go
+34
View File
@@ -0,0 +1,34 @@
package parity
import (
"strings"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func tagsGroup() queryGroup {
longTag := strings.Repeat("z", 300) + "needle"
return queryGroup{
name: "tags",
fixtures: []search.Resource{
fixtureDoc("invoice.txt", withTags("foo-bar")),
fixtureDoc("memo.txt", withTags("foo")),
fixtureDoc("spaced.txt", withTags("spaced tag")),
fixtureFolder("project", withTags("work")),
fixtureDoc("draft.txt", withParent("1$1!project"), withPath("./project/draft.txt")),
fixtureDoc("longtag.txt", withTags(longTag)),
},
cases: []queryCase{
{id: 1, query: `name:"*foo-bar*"`},
{id: 2, query: `tag:("foo-bar")`, want: []string{"invoice.txt"}},
{id: 3, query: `tag:("foo")`, want: []string{"memo.txt"}},
{id: 4, query: `tag:("FOO-BAR")`, want: []string{"invoice.txt"}},
{id: 5, query: `tag:("*foo*")`, want: []string{"invoice.txt", "memo.txt"}},
{id: 6, query: `tag:("spaced tag")`, want: []string{"spaced.txt"}},
{id: 7, query: `tag:("*paced ta*")`, want: []string{"spaced.txt"}},
{id: 8, query: `tag:("work")`, want: []string{"project"}},
{id: 9, query: `tag:("` + longTag + `")`, want: []string{"longtag.txt"}, engineOverrides: map[string]override{"opensearch": override{}}},
},
}
}
services/search/pkg/parity/query_title_test.go
+23
View File
@@ -0,0 +1,23 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func titleGroup() queryGroup {
return queryGroup{
name: "title",
fixtures: []search.Resource{
fixtureDoc("q1.html", withMime("text/html"), withTitle("quarterly report")),
},
cases: []queryCase{
{id: 1, query: `Title:"quarterly report"`, want: []string{"q1.html"}},
{id: 2, query: `Title:quarterly`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}},
{id: 3, query: `Title:QUARTERLY`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}},
{id: 4, query: `Title:quarterl*`, want: []string{"q1.html"}},
{id: 5, query: `Title:"*ly rep*"`, want: []string{"q1.html"}},
{id: 6, query: `title:quarterly`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}},
{id: 7, query: `Title:"QUARTERLY REPORT"`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}},
},
}
}
services/search/pkg/parity/query_visibility_test.go
+26
View File
@@ -0,0 +1,26 @@
package parity
import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func visibilityGroup() queryGroup {
return queryGroup{
name: "visibility",
fixtures: []search.Resource{
fixtureDoc("visible.txt"),
fixtureDoc("dotfile.txt", isHidden()),
fixtureFolder(".private", isHidden()),
fixtureDoc("secret.txt", withParent("1$1!.private"), withPath("./.private/secret.txt"), isHidden()),
},
cases: []queryCase{
{id: 1, query: `hidden:true`, want: []string{"dotfile.txt", ".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{}}},
{id: 2, query: `hidden:TRUE`, want: []string{"dotfile.txt", ".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}},
{id: 3, query: `hidden:false`, want: []string{"visible.txt"}, engineOverrides: map[string]override{"bleve": override{}}},
{id: 4, query: `name:"*secret*"`, want: []string{"secret.txt"}},
{id: 5, query: `path:"./.private"`, want: []string{".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{want: []string{".private"}}}},
{id: 6, query: `hidden:banana`, engineOverrides: map[string]override{"opensearch": override{want: []string{"error"}}}},
{id: 7, query: `hidden:"true"`, want: []string{"dotfile.txt", ".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{}}},
},
}
}
services/search/pkg/parity/response_entity_test.go
+102
View File
@@ -0,0 +1,102 @@
package parity
import (
"fmt"
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func entityGroup() responseGroup {
parent := fixtureFolder("parent", withID("1$1!2"))
doc := fixtureDoc("bar.pdf",
withID("1$1!3"),
withParent(parent.ID),
withPath("./parent/bar.pdf"),
withMime("application/pdf"),
withSize(1234),
)
notes := fixtureDoc("notes.txt", withID("1$1!4"), withContent("foo bar baz"))
moved := func(e search.Engine) error { return e.Move(parent.ID, "1$1!9", "./somewhere/newname") }
return responseGroup{
name: "entity",
fixtures: []search.Resource{parent, doc, notes},
cases: []responseCase{
{
id: 1, query: `name:"bar.pdf"`, reads: "Ref.Path", want: []string{"./parent/bar.pdf"},
read: reads(func(m *searchMessage.Match) string { return m.GetEntity().GetRef().GetPath() }),
},
{
id: 2, query: `name:"bar.pdf"`, reads: "Name", want: []string{"bar.pdf"},
read: reads(func(m *searchMessage.Match) string { return m.GetEntity().GetName() }),
},
{
id: 3, query: `name:"bar.pdf"`, reads: "Id", want: []string{"1$1!3"},
read: reads(func(m *searchMessage.Match) string { return resourceID(m.GetEntity().GetId()) }),
},
{
id: 4, query: `name:"bar.pdf"`, reads: "ParentId", want: []string{"1$1!2"},
read: reads(func(m *searchMessage.Match) string { return resourceID(m.GetEntity().GetParentId()) }),
},
{
id: 5, query: `name:"bar.pdf"`, reads: "Ref.ResourceId", want: []string{"1$1!1"},
read: reads(func(m *searchMessage.Match) string { return resourceID(m.GetEntity().GetRef().GetResourceId()) }),
},
{
id: 6, query: `name:"bar.pdf"`, reads: "Size", want: []string{"1234"},
read: reads(func(m *searchMessage.Match) string { return fmt.Sprint(m.GetEntity().GetSize()) }),
},
{
id: 7, query: `name:"bar.pdf"`, reads: "Type", want: []string{"1"},
read: reads(func(m *searchMessage.Match) string { return fmt.Sprint(m.GetEntity().GetType()) }),
},
{
id: 8, query: `name:"bar.pdf"`, reads: "MimeType", want: []string{"application/pdf"},
read: reads(func(m *searchMessage.Match) string { return m.GetEntity().GetMimeType() }),
},
{
id: 9, query: `name:"bar.pdf"`, reads: "Deleted", want: []string{"false"},
read: reads(func(m *searchMessage.Match) string { return fmt.Sprint(m.GetEntity().GetDeleted()) }),
},
{
id: 10, query: `name:"bar.pdf"`, reads: "Score", want: []string{"above zero"},
read: reads(func(m *searchMessage.Match) string {
if m.GetScore() > 0 {
return "above zero"
}
return fmt.Sprint(m.GetScore())
}),
},
{
id: 11, query: `path:"./parent"`, reads: "TotalMatches", want: []string{"2"}, engineOverrides: map[string]override{"bleve": override{want: []string{"1"}}},
read: func(resp *searchService.SearchIndexResponse) []string {
return []string{fmt.Sprint(resp.GetTotalMatches())}
},
},
{
id: 12, query: `name:"*notes*"`, reads: "Highlights", want: []string{`""`},
read: reads(func(m *searchMessage.Match) string { return fmt.Sprintf("%q", m.GetEntity().GetHighlights()) }),
},
{
id: 13, query: `content:bar`, reads: "Highlights", want: []string{"foo <mark>bar</mark> baz"},
read: reads(func(m *searchMessage.Match) string { return m.GetEntity().GetHighlights() }),
},
{
id: 14, do: moved, after: "moved to another parent", query: `name:"newname"`, reads: "ParentId", want: []string{"1$1!9"},
read: reads(func(m *searchMessage.Match) string { return resourceID(m.GetEntity().GetParentId()) }),
},
{
id: 15, do: moved, after: "moved to another parent", query: `name:"bar.pdf"`, reads: "ParentId", want: []string{"1$1!2"},
read: reads(func(m *searchMessage.Match) string { return resourceID(m.GetEntity().GetParentId()) }),
},
{
id: 16, do: moved, after: "moved to another parent", query: `name:"bar.pdf"`, reads: "Ref.Path", want: []string{"./somewhere/newname/bar.pdf"},
read: reads(func(m *searchMessage.Match) string { return m.GetEntity().GetRef().GetPath() }),
},
},
}
}
services/search/pkg/parity/response_metadata_test.go
+143
View File
@@ -0,0 +1,143 @@
package parity
import (
"fmt"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func metadataGroup() responseGroup {
song := fixtureDoc("some_song.mp3",
withID("1$1!5"),
withMime("audio/mpeg"),
withAudio(&libregraph.Audio{
Album: libregraph.PtrString("Some Album"),
AlbumArtist: libregraph.PtrString("Some AlbumArtist"),
Artist: libregraph.PtrString("Some Artist"),
Bitrate: libregraph.PtrInt64(192),
Composers: libregraph.PtrString("Some Composers"),
Copyright: libregraph.PtrString(""),
Disc: libregraph.PtrInt32(2),
DiscCount: libregraph.PtrInt32(5),
Duration: libregraph.PtrInt64(225000),
Genre: libregraph.PtrString("Some Genre"),
HasDrm: libregraph.PtrBool(false),
IsVariableBitrate: libregraph.PtrBool(true),
Title: libregraph.PtrString("Some Title"),
Track: libregraph.PtrInt32(34),
TrackCount: libregraph.PtrInt32(99),
Year: libregraph.PtrInt32(2004),
}),
)
team := fixtureDoc("team.jpg",
withID("1$1!6"),
withMime("image/jpeg"),
withLocation(&libregraph.GeoCoordinates{
Altitude: libregraph.PtrFloat64(1047.7),
Latitude: libregraph.PtrFloat64(49.48675890884328),
Longitude: libregraph.PtrFloat64(11.103870357204285),
}),
)
indexed := []string{
"Album=Some Album",
"AlbumArtist=Some AlbumArtist",
"Artist=Some Artist",
"Bitrate=192",
"Composers=Some Composers",
"Copyright=",
"Disc=2",
"DiscCount=5",
"Duration=225000",
"Genre=Some Genre",
"HasDrm=false",
"IsVariableBitrate=true",
"Title=Some Title",
"Track=34",
"TrackCount=99",
"Year=2004",
}
located := []string{
"Altitude=1047.7",
"Latitude=49.48675890884328",
"Longitude=11.103870357204285",
}
return responseGroup{
name: "metadata",
fixtures: []search.Resource{song, team},
cases: []responseCase{
{
id: 1, query: `*song*`, reads: "Audio", want: unchanged(indexed),
read: readsMany(func(m *searchMessage.Match) []string {
audio := m.GetEntity().GetAudio()
return changes(indexed, []string{
fmt.Sprintf("Album=%s", audio.GetAlbum()),
fmt.Sprintf("AlbumArtist=%s", audio.GetAlbumArtist()),
fmt.Sprintf("Artist=%s", audio.GetArtist()),
fmt.Sprintf("Bitrate=%d", audio.GetBitrate()),
fmt.Sprintf("Composers=%s", audio.GetComposers()),
fmt.Sprintf("Copyright=%s", audio.GetCopyright()),
fmt.Sprintf("Disc=%d", audio.GetDisc()),
fmt.Sprintf("DiscCount=%d", audio.GetDiscCount()),
fmt.Sprintf("Duration=%d", audio.GetDuration()),
fmt.Sprintf("Genre=%s", audio.GetGenre()),
fmt.Sprintf("HasDrm=%t", audio.GetHasDrm()),
fmt.Sprintf("IsVariableBitrate=%t", audio.GetIsVariableBitrate()),
fmt.Sprintf("Title=%s", audio.GetTitle()),
fmt.Sprintf("Track=%d", audio.GetTrack()),
fmt.Sprintf("TrackCount=%d", audio.GetTrackCount()),
fmt.Sprintf("Year=%d", audio.GetYear()),
})
}),
},
{
id: 2, query: `*team*`, reads: "Location", want: unchanged(located),
read: readsMany(func(m *searchMessage.Match) []string {
location := m.GetEntity().GetLocation()
return changes(located, []string{
fmt.Sprintf("Altitude=%v", location.GetAltitude()),
fmt.Sprintf("Latitude=%v", location.GetLatitude()),
fmt.Sprintf("Longitude=%v", location.GetLongitude()),
})
}),
},
{
id: 3, query: `*team*`, reads: "Audio", want: []string{"none"},
read: reads(func(m *searchMessage.Match) string {
if m.GetEntity().GetAudio() == nil {
return "none"
}
return "set"
}),
},
},
}
}
func unchanged(indexed []string) []string {
return []string{fmt.Sprintf("all %d fields unchanged", len(indexed))}
}
func changes(indexed, got []string) []string {
var off []string
for i, field := range indexed {
if got[i] != field {
off = append(off, fmt.Sprintf("%s instead of %s", got[i], field))
}
}
if len(off) == 0 {
return unchanged(indexed)
}
return off
}
tests/acceptance/features/apiSearch1/search.feature
+16
View File
@@ -501,3 +501,19 @@ Feature: Search
| old |
| new |
| spaces |
Scenario: a deleted space leaves the search index
Given using spaces DAV path
And the administrator has assigned the role "Space Admin" to user "Alice" using the Graph API
And user "Alice" has created a space "index-space" with the default quota using the Graph API
And user "Alice" has uploaded a file inside space "index-space" with content "some data" to "inSpace.txt"
When user "Alice" searches for "*inSpace*" using the WebDAV API
Then the HTTP status code should be "207"
And the search result of user "Alice" should contain these entries:
| inSpace.txt |
When user "Alice" has disabled a space "index-space"
And user "Alice" has deleted a space "index-space"
And user "Alice" searches for "*inSpace*" using the WebDAV API
Then the HTTP status code should be "207"
And the search result should contain "0" entries