mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-09-12 21:58:58 -04:00
test(search): leave behavior to the parity suite
The engine suites keep what is engine-specific (index setup, health, purge-space batching); every behavior answer lives in the parity suite once. The shared test client learns IndicesCount, and FIELDS-15 pins the Size/Type gate for number queries.
This commit is contained in:
Dominik Schmidt committed
1 parent
b92d671c02
commit
2383bcddcc
5 files changed
+37
-1678
No files matched your search
@@ -137,6 +137,19 @@ func (tc *TestClient) IndicesCreate(ctx context.Context, index string, body io.R
|
||||
}
|
||||
}
|
||||
|
||||
// IndicesCount returns the number of documents in the given indices.
|
||||
func (tc *TestClient) IndicesCount(ctx context.Context, indices []string, body io.Reader) (int, error) {
|
||||
resp, err := tc.c.Indices.Count(ctx, &opensearchgoAPI.IndicesCountReq{
|
||||
Indices: indices,
|
||||
Body: body,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to count documents in %v: %w", indices, err)
|
||||
}
|
||||
|
||||
return resp.Count, nil
|
||||
}
|
||||
|
||||
type testRequireClient struct {
|
||||
tc *TestClient
|
||||
t testing.TB
|
||||
@@ -157,3 +170,9 @@ func (trc *testRequireClient) IndicesCreate(index string, body io.Reader) {
|
||||
func (trc *testRequireClient) IndicesDelete(indices []string) {
|
||||
require.NoError(trc.t, trc.tc.IndicesDelete(trc.t.Context(), indices))
|
||||
}
|
||||
|
||||
func (trc *testRequireClient) IndicesCount(indices []string, body io.Reader, want int) {
|
||||
got, err := trc.tc.IndicesCount(trc.t.Context(), indices, body)
|
||||
require.NoError(trc.t, err)
|
||||
require.Equal(trc.t, want, got)
|
||||
}
|
||||
@@ -1,75 +1,28 @@
|
||||
package bleve_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
bleveSearch "github.com/blevesearch/bleve/v2"
|
||||
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
func hiddenByID(idx bleveSearch.Index, id string) bool {
|
||||
GinkgoHelper()
|
||||
|
||||
req := bleveSearch.NewSearchRequest(bleveSearch.NewDocIDQuery([]string{id}))
|
||||
req.Fields = []string{"Hidden"}
|
||||
|
||||
res, err := idx.Search(req)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits).To(HaveLen(1), "no record for %s", id)
|
||||
|
||||
hidden, _ := res.Hits[0].Fields["Hidden"].(bool)
|
||||
return hidden
|
||||
}
|
||||
|
||||
var _ = Describe("Bleve", func() {
|
||||
var (
|
||||
eng *bleve.Backend
|
||||
idx bleveSearch.Index
|
||||
|
||||
doSearch = func(id string, query, path string) (*searchsvc.SearchIndexResponse, error) {
|
||||
rID, err := storagespace.ParseID(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return eng.Search(context.Background(), &searchsvc.SearchIndexRequest{
|
||||
Query: query,
|
||||
Ref: &searchmsg.Reference{
|
||||
ResourceId: &searchmsg.ResourceID{
|
||||
StorageId: rID.StorageId,
|
||||
SpaceId: rID.SpaceId,
|
||||
OpaqueId: rID.OpaqueId,
|
||||
},
|
||||
Path: path,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
assertDocCount = func(id string, query string, expectedCount int) []*searchmsg.Match {
|
||||
res, err := doSearch(id, query, "")
|
||||
|
||||
ExpectWithOffset(1, err).ToNot(HaveOccurred())
|
||||
ExpectWithOffset(1, len(res.Matches)).To(Equal(expectedCount), "query returned unexpected number of results: "+query)
|
||||
return res.Matches
|
||||
}
|
||||
|
||||
rootResource search.Resource
|
||||
parentResource search.Resource
|
||||
childResource search.Resource
|
||||
childResource2 search.Resource
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
@@ -107,14 +60,6 @@ var _ = Describe("Bleve", func() {
|
||||
Document: content.Document{Name: "child.pdf"},
|
||||
}
|
||||
|
||||
childResource2 = search.Resource{
|
||||
ID: "1$2!5",
|
||||
ParentID: parentResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./parent d!r/child2.pdf",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{Name: "child2.pdf"},
|
||||
}
|
||||
})
|
||||
|
||||
Describe("PurgeSpace", func() {
|
||||
@@ -170,809 +115,4 @@ var _ = Describe("Bleve", func() {
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Search", func() {
|
||||
Context("by other fields than filename", func() {
|
||||
It("finds files by tags", func() {
|
||||
parentResource.Document.Tags = []string{"foo", "bar"}
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Tags:foo", 1)
|
||||
assertDocCount(rootResource.ID, "Tags:bar", 1)
|
||||
assertDocCount(rootResource.ID, "Tags:foo Tags:bar", 1)
|
||||
assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1)
|
||||
assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1)
|
||||
assertDocCount(rootResource.ID, "Tags:baz", 0)
|
||||
})
|
||||
|
||||
It("finds files by tags case-insensitively", func() {
|
||||
// exercises the []string/[]any sibling-lowercasing branch end-to-end.
|
||||
parentResource.Document.Tags = []string{"Work", "Urgent"}
|
||||
Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed())
|
||||
|
||||
assertDocCount(rootResource.ID, "tag:work", 1) // stored "Work", queried lower
|
||||
assertDocCount(rootResource.ID, "tag:WORK", 1) // queried upper
|
||||
assertDocCount(rootResource.ID, "Tags:Urgent", 1)
|
||||
assertDocCount(rootResource.ID, "tag:missing", 0)
|
||||
})
|
||||
|
||||
It("binds a leading NOT to the term right after it, combined with AND", func() {
|
||||
// regression: a leading NOT next to AND dropped the AND'd term, so
|
||||
// `NOT tag:x AND name:y` matched nothing (a self-contradicting clause).
|
||||
parentResource.Document.Tags = []string{"physik"}
|
||||
childResource.Document.Tags = []string{"mathe"}
|
||||
Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed())
|
||||
Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed())
|
||||
|
||||
assertDocCount(rootResource.ID, "NOT tag:physik AND name:child.pdf", 1) // the mathe child
|
||||
assertDocCount(rootResource.ID, "NOT tag:mathe AND name:parent*", 1) // the physik parent
|
||||
})
|
||||
|
||||
It("finds files by size", func() {
|
||||
parentResource.Document.Size = 12345
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Size:12345", 1)
|
||||
assertDocCount(rootResource.ID, "Size:>1000", 1)
|
||||
assertDocCount(rootResource.ID, "Size:<100000", 1)
|
||||
assertDocCount(rootResource.ID, "Size:12344", 0)
|
||||
assertDocCount(rootResource.ID, "Size:<1000", 0)
|
||||
assertDocCount(rootResource.ID, "Size:>100000", 0)
|
||||
})
|
||||
|
||||
It("matches facet values case-insensitively", func() {
|
||||
parentResource.Document.Audio = &libregraph.Audio{
|
||||
Artist: libregraph.PtrString("Some Artist"),
|
||||
}
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `audio.artist:"Some Artist"`, 1)
|
||||
assertDocCount(rootResource.ID, `audio.artist:"some artist"`, 1)
|
||||
})
|
||||
})
|
||||
|
||||
Context("by filename", func() {
|
||||
It("finds files with spaces in the filename", func() {
|
||||
parentResource.Document.Name = "Foo oo.pdf"
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `name:"foo o*"`, 1)
|
||||
})
|
||||
|
||||
It("finds files by digits in the filename", func() {
|
||||
parentResource.Document.Name = "12345.pdf"
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Name:1234*", 1)
|
||||
})
|
||||
|
||||
It("filters hidden files", func() {
|
||||
childResource.Hidden = true
|
||||
err := eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Hidden:T", 1)
|
||||
assertDocCount(rootResource.ID, "Hidden:F", 0)
|
||||
})
|
||||
|
||||
Context("with a file in the root of the space", func() {
|
||||
It("scopes the search to the specified space", func() {
|
||||
parentResource.Document.Name = "foo.pdf"
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Name:foo.pdf", 1)
|
||||
assertDocCount("9$8!7", "Name:foo.pdf", 0)
|
||||
})
|
||||
})
|
||||
|
||||
It("limits the search to the specified fields", func() {
|
||||
parentResource.Document.Name = "bar.pdf"
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Name:bar.pdf", 1)
|
||||
assertDocCount(rootResource.ID, "Unknown:field", 0)
|
||||
})
|
||||
|
||||
It("returns the total number of hits", func() {
|
||||
parentResource.Document.Name = "bar.pdf"
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
res, err := doSearch(rootResource.ID, "Name:bar*", "")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(1)))
|
||||
})
|
||||
|
||||
It("returns all desired fields", func() {
|
||||
parentResource.Document.Name = "bar.pdf"
|
||||
parentResource.Type = 3
|
||||
parentResource.MimeType = "application/pdf"
|
||||
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
matches := assertDocCount(rootResource.ID, fmt.Sprintf("Name:%s", parentResource.Name), 1)
|
||||
match := matches[0]
|
||||
Expect(match.Entity.Ref.Path).To(Equal(parentResource.Path))
|
||||
Expect(match.Entity.Name).To(Equal(parentResource.Name))
|
||||
Expect(match.Entity.Size).To(Equal(parentResource.Size))
|
||||
Expect(match.Entity.Type).To(Equal(parentResource.Type))
|
||||
Expect(match.Entity.MimeType).To(Equal(parentResource.MimeType))
|
||||
Expect(match.Entity.Deleted).To(BeFalse())
|
||||
Expect(match.Score > 0).To(BeTrue())
|
||||
})
|
||||
|
||||
It("finds files by name, prefix or substring match", func() {
|
||||
parentResource.Document.Name = "foo.pdf"
|
||||
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
queries := []string{"foo.pdf", "foo*", "*oo.p*"}
|
||||
for _, query := range queries {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, query, 1)
|
||||
}
|
||||
})
|
||||
|
||||
It("does a case-insensitive search", func() {
|
||||
parentResource.Document.Name = "foo.pdf"
|
||||
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Name:foo*", 1)
|
||||
assertDocCount(rootResource.ID, "Name:Foo*", 1)
|
||||
})
|
||||
|
||||
Context("and an additional file in a subdirectory", func() {
|
||||
BeforeEach(func() {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("finds files living deeper in the tree by filename, prefix or substring match", func() {
|
||||
queries := []string{"child.pdf", "child*", "*ld.*"}
|
||||
for _, query := range queries {
|
||||
assertDocCount(rootResource.ID, query, 1)
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Context("by path", func() {
|
||||
BeforeEach(func() {
|
||||
for _, r := range []search.Resource{parentResource, childResource, childResource2} {
|
||||
Expect(eng.Upsert(r.ID, r)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
||||
It("matches a folder and its descendants", func() {
|
||||
assertDocCount(rootResource.ID, `path:"./parent d!r"`, 3)
|
||||
})
|
||||
|
||||
It("matches a descendant path only itself", func() {
|
||||
assertDocCount(rootResource.ID, `path:"./parent d!r/child.pdf"`, 1)
|
||||
})
|
||||
|
||||
It("matches case-sensitively", func() {
|
||||
// paths act as references: /Foo and /foo are distinct siblings,
|
||||
// so a wrong-cased path must not match
|
||||
assertDocCount(rootResource.ID, `path:"./PARENT D!R"`, 0)
|
||||
})
|
||||
|
||||
It("applies an AND filter to the folder itself, not only descendants", func() {
|
||||
// regression: the folder-itself clause used to match unconditionally
|
||||
// under an AND, so the parent leaked in despite the name filter.
|
||||
matches := assertDocCount(rootResource.ID, `path:"./parent d!r" AND name:child.pdf`, 1)
|
||||
Expect(matches[0].Entity.Name).To(Equal("child.pdf"))
|
||||
})
|
||||
})
|
||||
|
||||
Context("by content", func() {
|
||||
It("matches full-text case-insensitively, without stemming", func() {
|
||||
parentResource.Document.Content = "Running Foxes"
|
||||
Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed())
|
||||
|
||||
assertDocCount(rootResource.ID, "content:running", 1)
|
||||
assertDocCount(rootResource.ID, "content:RUNNING", 1) // case-insensitive
|
||||
assertDocCount(rootResource.ID, "content:run", 0) // no stemming
|
||||
assertDocCount(rootResource.ID, "content:run*", 1) // wildcard over the word
|
||||
assertDocCount(rootResource.ID, "content:cat", 0)
|
||||
})
|
||||
})
|
||||
|
||||
Context("by mediatype", func() {
|
||||
It("matches categories and literal MIME types (incl. + and /)", func() {
|
||||
childResource.Document.MimeType = "image/svg+xml"
|
||||
childResource2.Document.MimeType = "image/png"
|
||||
for _, r := range []search.Resource{childResource, childResource2} {
|
||||
Expect(eng.Upsert(r.ID, r)).To(Succeed())
|
||||
}
|
||||
|
||||
assertDocCount(rootResource.ID, "mediatype:image", 2) // image/* wildcard -> both
|
||||
assertDocCount(rootResource.ID, "mediatype:IMAGE", 2) // categories are case-insensitive
|
||||
assertDocCount(rootResource.ID, "mediatype:pdf", 0)
|
||||
// literal MIME with + and /, must hit only the svg doc, not the png
|
||||
assertDocCount(rootResource.ID, "mediatype:image/svg+xml", 1)
|
||||
assertDocCount(rootResource.ID, "mediatype:image/png", 1)
|
||||
// the same literal via the raw field name (no mediatype alias)
|
||||
assertDocCount(rootResource.ID, "MimeType:image/svg+xml", 1)
|
||||
assertDocCount(rootResource.ID, "MimeType:image/png", 1)
|
||||
})
|
||||
|
||||
It("combines mediatype:file with another term", func() {
|
||||
// regression: mediatype:file (a NOT) next to an operator dropped the
|
||||
// other operand, so mediatype:file AND name:x matched nothing.
|
||||
parentResource.Document.MimeType = "httpd/unix-directory" // a folder
|
||||
childResource.Document.MimeType = "image/png" // a file
|
||||
for _, r := range []search.Resource{parentResource, childResource} {
|
||||
Expect(eng.Upsert(r.ID, r)).To(Succeed())
|
||||
}
|
||||
assertDocCount(rootResource.ID, "mediatype:file", 1) // only the file
|
||||
assertDocCount(rootResource.ID, "mediatype:file AND name:child.pdf", 1) // file AND its name
|
||||
assertDocCount(rootResource.ID, "mediatype:file AND name:nope", 0)
|
||||
})
|
||||
})
|
||||
|
||||
Context("Highlights", func() {
|
||||
|
||||
It("highlights only for content searches", func() {
|
||||
parentResource.Document.Name = "baz.pdf"
|
||||
parentResource.Document.Content = "foo bar baz"
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
res, err := doSearch(rootResource.ID, "Name:baz*", "")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(1)))
|
||||
Expect(res.Matches[0].Entity.Highlights).To(Equal(""))
|
||||
})
|
||||
|
||||
It("highlights search terms", func() {
|
||||
parentResource.Document.Name = "baz.pdf"
|
||||
parentResource.Document.Content = "foo bar baz"
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
res, err := doSearch(rootResource.ID, "Content:bar", "")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(1)))
|
||||
Expect(res.Matches[0].Entity.Highlights).To(Equal("foo <mark>bar</mark> baz"))
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
Context("with a file in the root of the space and folder with a file. all of them have the same name", func() {
|
||||
BeforeEach(func() {
|
||||
parentResource := search.Resource{
|
||||
ID: "1$2!3",
|
||||
ParentID: rootResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./doc",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER),
|
||||
Document: content.Document{Name: "doc"},
|
||||
}
|
||||
|
||||
childResource := search.Resource{
|
||||
ID: "1$2!4",
|
||||
ParentID: parentResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./doc/doc.pdf",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{Name: "doc.pdf"},
|
||||
}
|
||||
|
||||
childResource2 := search.Resource{
|
||||
ID: "1$2!7",
|
||||
ParentID: parentResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./doc/file.pdf",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{Name: "file.pdf"},
|
||||
}
|
||||
|
||||
rootChildResource := search.Resource{
|
||||
ID: "1$2!5",
|
||||
ParentID: rootResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./doc.pdf",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{Name: "doc.pdf"},
|
||||
}
|
||||
|
||||
rootChildResource2 := search.Resource{
|
||||
ID: "1$2!6",
|
||||
ParentID: rootResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./file.pdf",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{Name: "file.pdf"},
|
||||
}
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(rootChildResource.ID, rootChildResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = eng.Upsert(rootChildResource2.ID, rootChildResource2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = eng.Upsert(childResource2.ID, childResource2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
It("search *doc* in a root", func() {
|
||||
res, err := doSearch(rootResource.ID, "Name:*doc*", "")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(3)))
|
||||
})
|
||||
It("search *doc* in a subfolder", func() {
|
||||
res, err := doSearch(rootResource.ID, "Name:*doc*", "./doc")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(2)))
|
||||
})
|
||||
It("search *file* in a root", func() {
|
||||
res, err := doSearch(rootResource.ID, "Name:*file*", "")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(2)))
|
||||
})
|
||||
It("search *file* in a subfolder", func() {
|
||||
res, err := doSearch(rootResource.ID, "Name:*file*", "./doc")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(1)))
|
||||
})
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
Describe("path scoped searches", func() {
|
||||
BeforeEach(func() {
|
||||
Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed())
|
||||
Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed())
|
||||
Expect(eng.Upsert(childResource2.ID, childResource2)).To(Succeed())
|
||||
outside := search.Resource{
|
||||
ID: "1$2!6",
|
||||
ParentID: rootResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./other/child3.pdf",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{Name: "child3.pdf"},
|
||||
}
|
||||
Expect(eng.Upsert(outside.ID, outside)).To(Succeed())
|
||||
})
|
||||
|
||||
It("restricts hits and totals to the scope at query level", func() {
|
||||
// without the scope all three children match
|
||||
res, err := doSearch(rootResource.ID, "name:child*", "")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(3)))
|
||||
|
||||
res, err = doSearch(rootResource.ID, "name:child*", "./parent d!r")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(2)))
|
||||
Expect(len(res.Matches)).To(Equal(2))
|
||||
})
|
||||
|
||||
It("keeps totals right on a small page", func() {
|
||||
// the scope is part of the query, so totals cover the full scope
|
||||
// even when the page holds a single hit
|
||||
rID, err := storagespace.ParseID(rootResource.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
res, err := eng.Search(context.Background(), &searchsvc.SearchIndexRequest{
|
||||
Query: "name:child*",
|
||||
PageSize: 1,
|
||||
Ref: &searchmsg.Reference{
|
||||
ResourceId: &searchmsg.ResourceID{
|
||||
StorageId: rID.StorageId, SpaceId: rID.SpaceId, OpaqueId: rID.OpaqueId,
|
||||
},
|
||||
Path: "./parent d!r",
|
||||
},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(2)))
|
||||
Expect(len(res.Matches)).To(Equal(1))
|
||||
})
|
||||
|
||||
It("matches the scope case-sensitively", func() {
|
||||
res, err := doSearch(rootResource.ID, "name:child*", "./PARENT D!R")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(0)))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Upsert", func() {
|
||||
It("adds a resourceInfo to the index", func() {
|
||||
err := eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
count, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(1)))
|
||||
|
||||
query := bleveSearch.NewMatchQuery("child.pdf")
|
||||
query.SetField("Name")
|
||||
res, err := idx.Search(bleveSearch.NewSearchRequest(query))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits.Len()).To(Equal(1))
|
||||
})
|
||||
|
||||
It("updates an existing resource in the index", func() {
|
||||
|
||||
err := eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
countA, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(countA).To(Equal(uint64(1)))
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
countB, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(countB).To(Equal(uint64(1)))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Delete", func() {
|
||||
It("marks a resource as deleted", func() {
|
||||
err := eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Name:*child*", 1)
|
||||
|
||||
err = eng.Delete(childResource.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Name:*child*", 0)
|
||||
})
|
||||
|
||||
It("marks a child resources as deleted", func() {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1)
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
|
||||
|
||||
err = eng.Delete(parentResource.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0)
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Restore", func() {
|
||||
It("also marks child resources as restored", func() {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Delete(parentResource.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 0)
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 0)
|
||||
|
||||
err = eng.Restore(parentResource.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 1)
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 1)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Purge", func() {
|
||||
It("removes a resource from the index", func() {
|
||||
err := eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
assertDocCount(rootResource.ID, "Name:child.pdf", 1)
|
||||
|
||||
err = eng.Purge(childResource.ID, false)
|
||||
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
assertDocCount(rootResource.ID, "Name:child.pdf", 0)
|
||||
})
|
||||
It("removes a resource and its children from the index", func() {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1)
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
|
||||
|
||||
err = eng.Purge(parentResource.ID, false)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0)
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0)
|
||||
})
|
||||
It("removes a resource and ignores its children from the index", func() {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1)
|
||||
|
||||
err = eng.Delete(parentResource.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
|
||||
|
||||
err = eng.Purge(parentResource.ID, true)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0)
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Move", func() {
|
||||
It("renames the parent and its child resources", func() {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
parentResource.Path = "newname"
|
||||
err = eng.Move(parentResource.ID, parentResource.ParentID, "./my/newname")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, parentResource.Name, 0)
|
||||
|
||||
matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1)
|
||||
Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3"))
|
||||
Expect(matches[0].Entity.Ref.Path).To(Equal("./my/newname/child.pdf"))
|
||||
})
|
||||
|
||||
DescribeTable("keeps the flag in step with the path",
|
||||
func(from, target string, hidden bool) {
|
||||
parentResource.Path = from
|
||||
parentResource.Hidden = search.IsHidden(from)
|
||||
childResource.Path = from + "/child.pdf"
|
||||
childResource.Hidden = parentResource.Hidden
|
||||
|
||||
Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed())
|
||||
Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed())
|
||||
|
||||
Expect(eng.Move(parentResource.ID, parentResource.ParentID, target)).To(Succeed())
|
||||
|
||||
for _, id := range []string{parentResource.ID, childResource.ID} {
|
||||
Expect(hiddenByID(idx, id)).
|
||||
To(Equal(hidden), "%s after moving from %s to %s", id, from, target)
|
||||
}
|
||||
},
|
||||
Entry("into a dot folder", "./parent", "./.trash/parent", true),
|
||||
Entry("into a plain folder", "./parent", "./archive/parent", false),
|
||||
Entry("renamed with a leading dot", "./parent", "./.parent", true),
|
||||
Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false),
|
||||
Entry("renamed without the leading dot", "./.parent", "./parent", false),
|
||||
Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true),
|
||||
)
|
||||
|
||||
// the trash leaves the path alone, so the flag has to come through untouched
|
||||
It("carries the flag through the trash and back", func() {
|
||||
childResource.Path = "./.secret/file.txt"
|
||||
childResource.Hidden = true
|
||||
Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed())
|
||||
|
||||
Expect(eng.Delete(childResource.ID)).To(Succeed())
|
||||
Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after trashing")
|
||||
|
||||
Expect(eng.Restore(childResource.ID)).To(Succeed())
|
||||
Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after restoring")
|
||||
})
|
||||
|
||||
It("moves the parent and its child resources", func() {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
parentResource.Path = " "
|
||||
parentResource.ParentID = "1$2!somewhereopaqueid"
|
||||
|
||||
err = eng.Move(parentResource.ID, parentResource.ParentID, "./somewhere/else/newname")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
assertDocCount(rootResource.ID, `parent d!r`, 0)
|
||||
|
||||
matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1)
|
||||
Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3"))
|
||||
Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname/child.pdf"))
|
||||
|
||||
matches = assertDocCount(rootResource.ID, `newname`, 1)
|
||||
Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("somewhereopaqueid"))
|
||||
Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname"))
|
||||
|
||||
})
|
||||
|
||||
It("keeps case-insensitive search working after a move", func() {
|
||||
Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed())
|
||||
Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed())
|
||||
|
||||
Expect(eng.Move(parentResource.ID, parentResource.ParentID, "./my/NewName")).To(Succeed())
|
||||
|
||||
// the lowercased name sibling is rebuilt, so a case-insensitive
|
||||
// name query still works; the path is case-sensitive by design, so
|
||||
// only the exact new path matches (and the old one no longer does).
|
||||
assertDocCount(rootResource.ID, "name:NEWNAME", 1)
|
||||
assertDocCount(rootResource.ID, `path:"./my/NewName"`, 2)
|
||||
assertDocCount(rootResource.ID, `path:"./MY/NEWNAME"`, 0)
|
||||
assertDocCount(rootResource.ID, `path:"./parent d!r"`, 0)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("StartBatch", func() {
|
||||
It("starts a new batch", func() {
|
||||
b, err := eng.NewBatch(100)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = b.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
count, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(0)))
|
||||
|
||||
err = b.Push()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
count, err = idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(1)))
|
||||
|
||||
query := bleveSearch.NewMatchQuery("child.pdf")
|
||||
query.SetField("Name")
|
||||
res, err := idx.Search(bleveSearch.NewSearchRequest(query))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits.Len()).To(Equal(1))
|
||||
})
|
||||
|
||||
It("doesn't intertwine different batches", func() {
|
||||
b, err := eng.NewBatch(100)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = b.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
count, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(0)))
|
||||
|
||||
b2, err := eng.NewBatch(100)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = b2.Upsert(childResource2.ID, childResource2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
Expect(b.Push()).To(Succeed())
|
||||
count, err = idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(1)))
|
||||
|
||||
Expect(b2.Push()).To(Succeed())
|
||||
count, err = idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(2)))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("File type specific metadata", func() {
|
||||
|
||||
Context("with audio metadata", func() {
|
||||
BeforeEach(func() {
|
||||
resource := search.Resource{
|
||||
ID: "1$2!7",
|
||||
ParentID: rootResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./some_song.mp3",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{
|
||||
Name: "some_song.mp3",
|
||||
MimeType: "audio/mpeg",
|
||||
Audio: &libregraph.Audio{
|
||||
Album: libregraph.PtrString("Some Album"),
|
||||
AlbumArtist: libregraph.PtrString("Some AlbumArtist"),
|
||||
Artist: libregraph.PtrString("Some Artist"),
|
||||
Bitrate: libregraph.PtrInt64(192),
|
||||
Composers: libregraph.PtrString("Some Composers"),
|
||||
Copyright: libregraph.PtrString(""),
|
||||
Disc: libregraph.PtrInt32(2),
|
||||
DiscCount: libregraph.PtrInt32(5),
|
||||
Duration: libregraph.PtrInt64(225000),
|
||||
Genre: libregraph.PtrString("Some Genre"),
|
||||
HasDrm: libregraph.PtrBool(false),
|
||||
IsVariableBitrate: libregraph.PtrBool(true),
|
||||
Title: libregraph.PtrString("Some Title"),
|
||||
Track: libregraph.PtrInt32(34),
|
||||
TrackCount: libregraph.PtrInt32(99),
|
||||
Year: libregraph.PtrInt32(2004),
|
||||
},
|
||||
},
|
||||
}
|
||||
err := eng.Upsert(resource.ID, resource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("returns audio metadata for search", func() {
|
||||
matches := assertDocCount(rootResource.ID, `*song*`, 1)
|
||||
audio := matches[0].Entity.Audio
|
||||
|
||||
Expect(audio).ToNot(BeNil())
|
||||
|
||||
Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album")))
|
||||
Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist")))
|
||||
Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist")))
|
||||
Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192)))
|
||||
Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers")))
|
||||
Expect(audio.Copyright).To(Equal(libregraph.PtrString("")))
|
||||
Expect(audio.Disc).To(Equal(libregraph.PtrInt32(2)))
|
||||
Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5)))
|
||||
Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225000)))
|
||||
Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre")))
|
||||
Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false)))
|
||||
Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true)))
|
||||
Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title")))
|
||||
Expect(audio.Track).To(Equal(libregraph.PtrInt32(34)))
|
||||
Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(99)))
|
||||
Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004)))
|
||||
})
|
||||
})
|
||||
|
||||
Context("with location metadata", func() {
|
||||
BeforeEach(func() {
|
||||
resource := search.Resource{
|
||||
ID: "1$2!7",
|
||||
ParentID: rootResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./team.jpg",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{
|
||||
Name: "team.jpg",
|
||||
MimeType: "image/jpeg",
|
||||
Location: &libregraph.GeoCoordinates{
|
||||
Altitude: libregraph.PtrFloat64(1047.7),
|
||||
Latitude: libregraph.PtrFloat64(49.48675890884328),
|
||||
Longitude: libregraph.PtrFloat64(11.103870357204285),
|
||||
},
|
||||
},
|
||||
}
|
||||
err := eng.Upsert(resource.ID, resource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("returns audio metadata for search", func() {
|
||||
matches := assertDocCount(rootResource.ID, `*team*`, 1)
|
||||
location := matches[0].Entity.Location
|
||||
|
||||
Expect(location).ToNot(BeNil())
|
||||
|
||||
Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(1047.7)))
|
||||
Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328)))
|
||||
Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285)))
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,6 +1,7 @@
|
||||
package opensearch_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
@@ -8,10 +9,7 @@ import (
|
||||
opensearchgo "github.com/opensearch-project/opensearch-go/v4"
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
|
||||
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
|
||||
)
|
||||
|
||||
@@ -20,6 +18,12 @@ func TestOpenSearchBackend(t *testing.T) {
|
||||
RunSpecs(t, "OpenSearch Backend Suite")
|
||||
}
|
||||
|
||||
func deleteIndexOnCleanup(tc *opensearchtest.TestClient, indexName string) {
|
||||
DeferCleanup(func() {
|
||||
Expect(tc.IndicesDelete(context.Background(), []string{indexName})).To(Succeed())
|
||||
})
|
||||
}
|
||||
|
||||
var _ = Describe("Backend", func() {
|
||||
Describe("NewBackend", func() {
|
||||
It("fails to create if the cluster is not healthy", func() {
|
||||
@@ -36,256 +40,6 @@ var _ = Describe("Backend", func() {
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Search", func() {
|
||||
const indexName = "opencloud-test-engine-search"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
document search.Resource
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
document = opensearchtest.Testdata.Resources.File
|
||||
Expect(backend.Upsert(document.ID, document)).To(Succeed())
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
})
|
||||
|
||||
It("performs the most simple search", func() {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{
|
||||
Query: fmt.Sprintf(`"%s"`, document.Name),
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp.Matches).To(HaveLen(1))
|
||||
Expect(resp.TotalMatches).To(Equal(int32(1)))
|
||||
Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID))
|
||||
})
|
||||
|
||||
It("ignores files that are marked as deleted", func() {
|
||||
deletedDocument := opensearchtest.Testdata.Resources.File
|
||||
deletedDocument.ID = "1$2!4"
|
||||
deletedDocument.Deleted = true
|
||||
|
||||
Expect(backend.Upsert(deletedDocument.ID, deletedDocument)).To(Succeed())
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{
|
||||
Query: fmt.Sprintf(`"%s"`, document.Name),
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp.Matches).To(HaveLen(1))
|
||||
Expect(resp.TotalMatches).To(Equal(int32(1)))
|
||||
Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID))
|
||||
})
|
||||
|
||||
It("restricts hits and totals to the path scope", func() {
|
||||
outside := opensearchtest.Testdata.Resources.File
|
||||
outside.ID = "1$1!5"
|
||||
outside.Path = "./other folder/else.jpg"
|
||||
Expect(backend.Upsert(outside.ID, outside)).To(Succeed())
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
|
||||
scoped := &searchMessage.Reference{
|
||||
ResourceId: &searchMessage.ResourceID{StorageId: "1", SpaceId: "1", OpaqueId: "1"},
|
||||
Path: "./parent d!r",
|
||||
}
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{
|
||||
Query: fmt.Sprintf(`"%s"`, document.Name),
|
||||
Ref: scoped,
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp.Matches).To(HaveLen(1))
|
||||
Expect(resp.TotalMatches).To(Equal(int32(1)))
|
||||
Expect(resp.Matches[0].Entity.Ref.Path).To(Equal("./parent d!r/child.jpg"))
|
||||
|
||||
// the scope is a reference and matches case-sensitively
|
||||
wrongCase := &searchMessage.Reference{
|
||||
ResourceId: &searchMessage.ResourceID{StorageId: "1", SpaceId: "1", OpaqueId: "1"},
|
||||
Path: "./PARENT D!R",
|
||||
}
|
||||
respWrongCase, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{
|
||||
Query: fmt.Sprintf(`"%s"`, document.Name),
|
||||
Ref: wrongCase,
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(respWrongCase.Matches).To(HaveLen(0))
|
||||
Expect(respWrongCase.TotalMatches).To(Equal(int32(0)))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("FullTextSearch", func() {
|
||||
const indexName = "opencloud-test-engine-fulltext"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
document.Content = "Running Foxes"
|
||||
Expect(backend.Upsert(document.ID, document)).To(Succeed())
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
})
|
||||
|
||||
It("searches content case-insensitively and stemmed, like bleve", func() {
|
||||
// case-folded and porter-stemmed by the fulltext analyzer; the match
|
||||
// query analyzes the query value the same way. "content:run*" is an
|
||||
// unanalyzed wildcard over the stemmed term "run", so it must still
|
||||
// route to a wildcard query (not degrade to a phrase match).
|
||||
for _, q := range []string{"content:running", "content:RUNNING", "content:run", "content:run*"} {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: q})
|
||||
Expect(err).ToNot(HaveOccurred(), q)
|
||||
Expect(resp.Matches).To(HaveLen(1), q)
|
||||
}
|
||||
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: "content:cat"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp.Matches).To(HaveLen(0))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("CaseInsensitiveSearch", func() {
|
||||
const indexName = "opencloud-test-engine-ci"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
folder := opensearchtest.Testdata.Resources.Folder
|
||||
folder.ID = "1$2!cifolder"
|
||||
folder.Path = "./My Dir"
|
||||
folder.Tags = []string{"Work", "Urgent"}
|
||||
Expect(backend.Upsert(folder.ID, folder)).To(Succeed())
|
||||
|
||||
child := opensearchtest.Testdata.Resources.File
|
||||
child.ID = "1$2!cichild"
|
||||
child.ParentID = folder.ID
|
||||
child.Path = "./My Dir/report.pdf"
|
||||
child.Tags = nil
|
||||
Expect(backend.Upsert(child.ID, child)).To(Succeed())
|
||||
|
||||
// a doc outside the folder, so the path assertions below discriminate:
|
||||
// a phrase-matched path query would analyze into the "." prefix and
|
||||
// match this one too
|
||||
outside := opensearchtest.Testdata.Resources.File
|
||||
outside.ID = "1$2!cioutside"
|
||||
outside.Path = "./other.pdf"
|
||||
outside.Tags = nil
|
||||
Expect(backend.Upsert(outside.ID, outside)).To(Succeed())
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
})
|
||||
|
||||
It("matches tags case-insensitively (array sibling)", func() {
|
||||
for _, q := range []string{"tag:work", "tag:WORK", "Tags:Urgent"} {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: q})
|
||||
Expect(err).ToNot(HaveOccurred(), q)
|
||||
Expect(resp.Matches).To(HaveLen(1), q)
|
||||
}
|
||||
})
|
||||
|
||||
It("matches a spaced path on the folder and its descendants case-sensitively", func() {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./My Dir"`})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp.Matches).To(HaveLen(2)) // folder itself + the descendant, not the outside doc
|
||||
|
||||
// paths act as references, a wrong-cased path must not match
|
||||
respWrongCase, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./MY DIR"`})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(respWrongCase.Matches).To(HaveLen(0))
|
||||
})
|
||||
|
||||
It("matches a spaced descendant path only itself", func() {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./My Dir/report.pdf"`})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp.Matches).To(HaveLen(1))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("MediaTypeSearch", func() {
|
||||
const indexName = "opencloud-test-engine-mediatype"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
svg := opensearchtest.Testdata.Resources.File
|
||||
svg.ID = "1$2!svg"
|
||||
svg.MimeType = "image/svg+xml"
|
||||
Expect(backend.Upsert(svg.ID, svg)).To(Succeed())
|
||||
|
||||
png := opensearchtest.Testdata.Resources.File
|
||||
png.ID = "1$2!png"
|
||||
png.MimeType = "image/png"
|
||||
Expect(backend.Upsert(png.ID, png)).To(Succeed())
|
||||
|
||||
folder := opensearchtest.Testdata.Resources.Folder
|
||||
folder.ID = "1$2!dir"
|
||||
folder.MimeType = "httpd/unix-directory"
|
||||
Expect(backend.Upsert(folder.ID, folder)).To(Succeed())
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
})
|
||||
|
||||
DescribeTable("resolves the media type query",
|
||||
func(query string, want int) {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query})
|
||||
Expect(err).ToNot(HaveOccurred(), query)
|
||||
Expect(resp.Matches).To(HaveLen(want), query)
|
||||
},
|
||||
Entry("image/* wildcard matches both files", "mediatype:image", 2),
|
||||
Entry("categories are case-insensitive", "mediatype:IMAGE", 2),
|
||||
Entry("literal MIME (+ and /) via mediatype", "mediatype:image/svg+xml", 1),
|
||||
Entry("same literal via the raw field name", "MimeType:image/svg+xml", 1),
|
||||
Entry("literal png MIME", "mediatype:image/png", 1),
|
||||
Entry("no pdf documents", "mediatype:pdf", 0),
|
||||
Entry("folder category matches the directory only", "mediatype:folder", 1),
|
||||
Entry("file category matches both files, not the directory", "mediatype:file", 2),
|
||||
Entry("file category combined with a term", "mediatype:file AND MimeType:image/png", 1),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("Upsert", func() {
|
||||
const indexName = "opencloud-test-engine-upsert"
|
||||
|
||||
@@ -295,10 +49,12 @@ var _ = Describe("Backend", func() {
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
// the backend versions the physical index by schema generation
|
||||
physical := opensearch.VersionedIndexName(indexName)
|
||||
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
tc.Require.IndicesReset([]string{physical})
|
||||
deleteIndexOnCleanup(tc, physical)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
@@ -309,7 +65,7 @@ var _ = Describe("Backend", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
Expect(backend.Upsert(document.ID, document)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
tc.Require.IndicesCount([]string{opensearch.VersionedIndexName(indexName)}, nil, 1)
|
||||
})
|
||||
|
||||
It("upserts a document without an mtime", func() {
|
||||
@@ -319,566 +75,8 @@ var _ = Describe("Backend", func() {
|
||||
document.Mtime = nil
|
||||
Expect(backend.Upsert(document.ID, document)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
tc.Require.IndicesCount([]string{opensearch.VersionedIndexName(indexName)}, nil, 1)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Move", func() {
|
||||
const indexName = "opencloud-test-engine-move"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("moves the document to a new path", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
|
||||
"query": map[string]any{
|
||||
"ids": map[string]any{
|
||||
"values": []string{document.ID},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits)
|
||||
Expect(resources).To(HaveLen(1))
|
||||
Expect(resources[0].Path).To(Equal(document.Path))
|
||||
|
||||
document.Path = "./new/path/to/resource"
|
||||
Expect(backend.Move(document.ID, document.ParentID, document.Path)).To(Succeed())
|
||||
|
||||
resources = opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits)
|
||||
Expect(resources).To(HaveLen(1))
|
||||
Expect(resources[0].Path).To(Equal(document.Path))
|
||||
})
|
||||
|
||||
It("keeps case-sensitive path search working after a move", func() {
|
||||
// Spaced paths so the queries only stay exact as term queries; a phrase
|
||||
// match would analyze into the "." prefix and match regardless.
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
document.ID = "1$2!cimove"
|
||||
document.Path = "./Foo Dir/Bar"
|
||||
Expect(backend.Upsert(document.ID, document)).To(Succeed())
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
|
||||
document.Path = "./Moved Dir/Bar"
|
||||
Expect(backend.Move(document.ID, document.ParentID, document.Path)).To(Succeed())
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
|
||||
// Path is case-sensitive by design: the exact new path matches, a
|
||||
// wrong-cased query does not, and the old path no longer matches.
|
||||
respNew, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./Moved Dir/Bar"`})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(respNew.Matches).To(HaveLen(1))
|
||||
|
||||
respWrongCase, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./MOVED DIR/BAR"`})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(respWrongCase.Matches).To(HaveLen(0))
|
||||
|
||||
respOld, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./Foo Dir/Bar"`})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(respOld.Matches).To(HaveLen(0))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("WriteVisibility", func() {
|
||||
const indexName = "opencloud-test-engine-write-visibility"
|
||||
|
||||
It("deletes a record that was just written", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
document.ID = "1$1!95"
|
||||
document.Name = "textfile.txt"
|
||||
document.Path = "./textfile.txt"
|
||||
|
||||
backend, tc := newBackend(indexName)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
Expect(backend.Upsert(document.ID, document)).To(Succeed())
|
||||
Expect(backend.Delete(document.ID)).To(Succeed())
|
||||
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{
|
||||
Query: fmt.Sprintf(`name:"%s"`, document.Name),
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(resp.Matches).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Delete", func() {
|
||||
const indexName = "opencloud-test-engine-delete"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("marks the document as deleted", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
|
||||
"query": map[string]any{
|
||||
"term": map[string]any{
|
||||
"Deleted": map[string]any{
|
||||
"value": true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0)
|
||||
|
||||
Expect(backend.Delete(document.ID)).To(Succeed())
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Restore", func() {
|
||||
const indexName = "opencloud-test-engine-restore"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("marks the document as not deleted", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
document.Deleted = true
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
|
||||
"query": map[string]any{
|
||||
"term": map[string]any{
|
||||
"Deleted": map[string]any{
|
||||
"value": true,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1)
|
||||
|
||||
Expect(backend.Restore(document.ID)).To(Succeed())
|
||||
tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Purge", func() {
|
||||
const indexName = "opencloud-test-engine-purge"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("purges a full document", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
Expect(backend.Purge(document.ID, false)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
})
|
||||
|
||||
It("purges resource trees", func() {
|
||||
resourceFolder := opensearchtest.Testdata.Resources.Folder
|
||||
tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder)))
|
||||
|
||||
resourceFile := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile)))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
|
||||
Expect(backend.Purge(resourceFolder.ID, false)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
})
|
||||
|
||||
It("purges resource trees and ignores undeleted resources", func() {
|
||||
resourceFolder := opensearchtest.Testdata.Resources.Folder
|
||||
tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder)))
|
||||
|
||||
resourceFile := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile)))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
|
||||
Expect(backend.Delete(resourceFile.ID)).To(Succeed())
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
Expect(backend.Purge(resourceFolder.ID, true)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("PurgeSpace", func() {
|
||||
const indexName = "opencloud-test-engine-purge-space"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("takes every record of that space out of the index", func() {
|
||||
gone := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, gone.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), gone)))
|
||||
|
||||
stays := opensearchtest.Testdata.Resources.File
|
||||
stays.ID = "1$2!3"
|
||||
stays.RootID = "1$2!2"
|
||||
tc.Require.DocumentCreate(indexName, stays.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), stays)))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
|
||||
Expect(backend.PurgeSpace(gone.RootID)).To(Succeed())
|
||||
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
left := opensearchtest.SearchHitsMustBeConverted[search.Resource](
|
||||
GinkgoTB(),
|
||||
tc.Require.Search(indexName, strings.NewReader(`{"query":{"match_all":{}}}`)).Hits,
|
||||
)
|
||||
Expect(left).To(HaveLen(1), "only the records of that space are gone")
|
||||
Expect(left[0].ID).To(Equal(stays.ID))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Hidden", func() {
|
||||
const indexName = "opencloud-test-engine-hidden"
|
||||
|
||||
DescribeTable("keeps the flag in step with the path",
|
||||
func(from, target string, hidden bool) {
|
||||
folder := opensearchtest.Testdata.Resources.Folder
|
||||
folder.ID = "1$1!30"
|
||||
folder.Name = "parent"
|
||||
folder.Path = from
|
||||
folder.Hidden = search.IsHidden(from)
|
||||
|
||||
child := opensearchtest.Testdata.Resources.File
|
||||
child.ID = "1$1!31"
|
||||
child.Name = "child.txt"
|
||||
child.Path = from + "/child.txt"
|
||||
child.ParentID = folder.ID
|
||||
child.Hidden = folder.Hidden
|
||||
|
||||
backend, tc := newBackend(indexName, folder, child)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
|
||||
Expect(backend.Move(folder.ID, folder.ParentID, target)).To(Succeed())
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
|
||||
for _, id := range []string{folder.ID, child.ID} {
|
||||
Expect(resourceByID(tc, indexName, id).Hidden).
|
||||
To(Equal(hidden), "%s after moving from %s to %s", id, from, target)
|
||||
}
|
||||
},
|
||||
Entry("into a dot folder", "./parent", "./.trash/parent", true),
|
||||
Entry("into a plain folder", "./parent", "./archive/parent", false),
|
||||
Entry("renamed with a leading dot", "./parent", "./.parent", true),
|
||||
Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false),
|
||||
Entry("renamed without the leading dot", "./.parent", "./parent", false),
|
||||
Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true),
|
||||
)
|
||||
|
||||
It("carries the flag through the trash and back", func() {
|
||||
hidden := opensearchtest.Testdata.Resources.File
|
||||
hidden.ID = "1$1!32"
|
||||
hidden.Path = "./.secret/file.txt"
|
||||
hidden.Hidden = true
|
||||
|
||||
backend, tc := newBackend(indexName, hidden)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
|
||||
Expect(backend.Delete(hidden.ID)).To(Succeed())
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
Expect(resourceByID(tc, indexName, hidden.ID).Hidden).To(BeTrue(), "after trashing")
|
||||
|
||||
Expect(backend.Restore(hidden.ID)).To(Succeed())
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
Expect(resourceByID(tc, indexName, hidden.ID).Hidden).To(BeTrue(), "after restoring")
|
||||
})
|
||||
})
|
||||
|
||||
Describe("DocCount", func() {
|
||||
const indexName = "opencloud-test-engine-doc-count"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 0)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
var err error
|
||||
backend, err = opensearch.NewBackend(indexName, tc.Client())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("ignores deleted documents", func() {
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document)))
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
count, err := backend.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(1)))
|
||||
|
||||
tc.Require.Update(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{
|
||||
"doc": map[string]any{
|
||||
"Deleted": true,
|
||||
},
|
||||
})))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
|
||||
count, err = backend.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(0)))
|
||||
})
|
||||
})
|
||||
|
||||
// The following specs ensure that updates which affect a resource and its descendants
|
||||
// (Delete, Restore, Move) are scoped to the root (space) of the target resource. Two
|
||||
// resources living in different roots may share the exact same path, so matching by
|
||||
// path alone would incorrectly update the wrong resource.
|
||||
Describe("updateSelfAndDescendants root scope", func() {
|
||||
It("deletes only the resource in the target root", func() {
|
||||
const indexName = "opencloud-test-engine-root-scope-delete"
|
||||
|
||||
target := opensearchtest.Testdata.Resources.File
|
||||
other := otherRoot(target)
|
||||
|
||||
backend, tc := newBackend(indexName, target, other)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
Expect(backend.Delete(target.ID)).To(Succeed())
|
||||
|
||||
Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeTrue(), "target resource should be marked as deleted")
|
||||
Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeFalse(), "resource in a different root must not be affected")
|
||||
})
|
||||
|
||||
It("restores only the resource in the target root", func() {
|
||||
const indexName = "opencloud-test-engine-root-scope-restore"
|
||||
|
||||
target := opensearchtest.Testdata.Resources.File
|
||||
target.Deleted = true
|
||||
other := otherRoot(target)
|
||||
|
||||
backend, tc := newBackend(indexName, target, other)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
Expect(backend.Restore(target.ID)).To(Succeed())
|
||||
|
||||
Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeFalse(), "target resource should be restored")
|
||||
Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeTrue(), "resource in a different root must not be affected")
|
||||
})
|
||||
|
||||
It("moves only the resource in the target root", func() {
|
||||
const indexName = "opencloud-test-engine-root-scope-move"
|
||||
|
||||
target := opensearchtest.Testdata.Resources.File
|
||||
other := otherRoot(target)
|
||||
|
||||
backend, tc := newBackend(indexName, target, other)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
Expect(backend.Move(target.ID, target.ParentID, "./new/path/to/resource")).To(Succeed())
|
||||
|
||||
Expect(resourceByID(tc, indexName, target.ID).Path).To(Equal("./new/path/to/resource"), "target resource should be moved")
|
||||
Expect(resourceByID(tc, indexName, other.ID).Path).To(Equal(other.Path), "resource in a different root must not be moved")
|
||||
})
|
||||
})
|
||||
|
||||
Describe("SearchInAnalyzedFields", func() {
|
||||
const indexName = "opencloud-test-engine-search-analyzed-fields"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
dashed := opensearchtest.Testdata.Resources.Folder
|
||||
dashed.ID = "1$1!10"
|
||||
dashed.Name = "new-folder"
|
||||
dashed.Path = "./new-folder"
|
||||
dashed.Title = "quarterly report"
|
||||
|
||||
plain := opensearchtest.Testdata.Resources.Folder
|
||||
plain.ID = "1$1!11"
|
||||
plain.Name = "documents"
|
||||
plain.Path = "./documents"
|
||||
plain.Title = "notes"
|
||||
|
||||
spaced := opensearchtest.Testdata.Resources.Folder
|
||||
spaced.ID = "1$1!12"
|
||||
spaced.Name = "foo bar"
|
||||
spaced.Path = "./foo bar"
|
||||
spaced.Title = "spaced out"
|
||||
|
||||
backend, tc = newBackend(indexName)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
for _, r := range []search.Resource{dashed, plain, spaced} {
|
||||
Expect(backend.Upsert(r.ID, r)).To(Succeed())
|
||||
}
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 3)
|
||||
})
|
||||
|
||||
DescribeTable("finds what the analyzer made of the value",
|
||||
func(query string, want []string) {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
names := make([]string, 0, len(resp.Matches))
|
||||
for _, match := range resp.Matches {
|
||||
names = append(names, match.Entity.Name)
|
||||
}
|
||||
Expect(names).To(ConsistOf(want))
|
||||
},
|
||||
Entry("the full name with the dash", "new-folder", []string{"new-folder"}),
|
||||
Entry("one token of it", "new", []string{"new-folder"}),
|
||||
Entry("a name without a dash", "documents", []string{"documents"}),
|
||||
Entry("a wildcard", "*folder*", []string{"new-folder"}),
|
||||
// the shape the web client sends for every name search
|
||||
Entry("a wildcard around the whole dashed name", `name:"*new-folder*"`, []string{"new-folder"}),
|
||||
Entry("a wildcard spanning the dash", `name:"*w-fol*"`, []string{"new-folder"}),
|
||||
Entry("a wildcard in a different case", `name:"*NEW-FOLDER*"`, []string{"new-folder"}),
|
||||
Entry("a wildcard spanning a space", `name:"*oo ba*"`, []string{"foo bar"}),
|
||||
Entry("a wildcard around a name with a space", `name:"*foo bar*"`, []string{"foo bar"}),
|
||||
Entry("a name with a space", `name:"foo bar"`, []string{"foo bar"}),
|
||||
Entry("a title of two words", `Title:"quarterly report"`, []string{"new-folder"}),
|
||||
Entry("one token of a title", "Title:quarterly", []string{"new-folder"}),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("SearchByTag", func() {
|
||||
const indexName = "opencloud-test-engine-search-by-tag"
|
||||
|
||||
var (
|
||||
tc *opensearchtest.TestClient
|
||||
backend *opensearch.Backend
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
tagged := opensearchtest.Testdata.Resources.Folder
|
||||
tagged.ID = "1$1!20"
|
||||
tagged.Name = "tagged"
|
||||
tagged.Path = "./tagged"
|
||||
tagged.Tags = []string{"foo-bar"}
|
||||
|
||||
other := opensearchtest.Testdata.Resources.Folder
|
||||
other.ID = "1$1!21"
|
||||
other.Name = "other"
|
||||
other.Path = "./other"
|
||||
other.Tags = []string{"foo"}
|
||||
|
||||
backend, tc = newBackend(indexName)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
for _, r := range []search.Resource{tagged, other} {
|
||||
Expect(backend.Upsert(r.ID, r)).To(Succeed())
|
||||
}
|
||||
tc.Require.IndicesRefresh([]string{indexName}, nil)
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
})
|
||||
|
||||
// a tag is one label, not prose, so it matches as a whole or not at all
|
||||
DescribeTable("matches a tag as a whole",
|
||||
func(query string, want []string) {
|
||||
resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
names := make([]string, 0, len(resp.Matches))
|
||||
for _, match := range resp.Matches {
|
||||
names = append(names, match.Entity.Name)
|
||||
}
|
||||
Expect(names).To(ConsistOf(want))
|
||||
},
|
||||
Entry("the whole tag", `tag:("foo-bar")`, []string{"tagged"}),
|
||||
Entry("a token of a tag does not match it", `tag:("foo")`, []string{"other"}),
|
||||
Entry("a tag in a different case", `tag:("FOO-BAR")`, []string{"tagged"}),
|
||||
Entry("a wildcard reaches both", `tag:("*foo*")`, []string{"tagged", "other"}),
|
||||
)
|
||||
})
|
||||
|
||||
Describe("SearchWithAnInvalidQuery", func() {
|
||||
const indexName = "opencloud-test-engine-search-invalid-query"
|
||||
|
||||
It("answers with a bad request", func() {
|
||||
backend, tc := newBackend(indexName)
|
||||
deleteIndexOnCleanup(tc, indexName)
|
||||
|
||||
_, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: "AND mediatype:document"})
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err).To(BeAssignableToTypeOf(errtypes.BadRequest("")))
|
||||
Expect(err.Error()).To(Equal(`error: bad request: the expression can't begin from a binary operator: 'AND'`))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -233,6 +233,7 @@ Fixtures:
|
||||
| FIELDS-12 | `id:"1$1!ab-23"` | no match | no match | no match | ✅ |
|
||||
| FIELDS-13 | `audio.artist:"Some Artist"` | song.mp3 | song.mp3 | song.mp3 | ✅ |
|
||||
| FIELDS-14 | `audio.artist:"some artist"` | song.mp3 | song.mp3 | song.mp3 | ✅ |
|
||||
| FIELDS-15 | `audio.duration>100` | no match | no match | no match | ✅ |
|
||||
|
||||
### deleted
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ func fieldsGroup() queryGroup {
|
||||
fixtureDoc("plain.txt"),
|
||||
fixtureFolder("box"),
|
||||
fixtureDoc("boxed.txt", withParent("1$1!box"), withPath("./box/boxed.txt")),
|
||||
fixtureDoc("song.mp3", withMime("audio/mpeg"), withAudio(&libregraph.Audio{Artist: libregraph.PtrString("Some Artist")})),
|
||||
fixtureDoc("song.mp3", withMime("audio/mpeg"), withAudio(&libregraph.Audio{Artist: libregraph.PtrString("Some Artist"), Duration: libregraph.PtrInt64(200)})),
|
||||
},
|
||||
cases: []queryCase{
|
||||
{id: 1, query: `size:42`, want: []string{"small.txt"}},
|
||||
@@ -36,6 +36,7 @@ func fieldsGroup() queryGroup {
|
||||
// a facet value keeps its case, the field is not marked lowercase
|
||||
{id: 13, query: `audio.artist:"Some Artist"`, want: []string{"song.mp3"}},
|
||||
{id: 14, query: `audio.artist:"some artist"`, want: []string{"song.mp3"}}, // facets search case-insensitively
|
||||
{id: 15, query: `audio.duration>100`}, // number queries are gated to Size and Type on both engines
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user