mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-09-12 21:58:58 -04:00
feat(search): per-field case-insensitive search via _lowercase siblings
Keyword and path fields always index their case-preserved base and, when CaseInsensitive is set, an additional <field>_lowercase sibling used only for matching. The KQL lowering marks a restriction case-insensitive; each backend searches the sibling and lowercases the query value the same way the sibling is precomputed at index time (Go strings.ToLower on both sides, so non-ASCII stays consistent). Search always returns the case-preserved base, so the sibling never has to be read back. In bleve it is indexed but not stored, kept out of _all, and without doc values. In OpenSearch it deliberately stays in _source: excluding it would make every update-by-query script rebuild all siblings from the document via painless toLowerCase, which lowercases differently than Go and would drift from the query side. Keeping it in _source avoids that, and a lowercased copy of a name or path is negligible disk in a cluster. The OpenSearch move script keeps the base and its sibling in sync by swapping the moved prefix in Path_lowercase and setting Name_lowercase from Go-lowercased params, so case-insensitive search still finds a file after it moves (previously the sibling went stale). bleve re-indexes the whole document on move/delete/restore, so its siblings stay fresh for free. This also repairs OpenSearch path search (the query value was no longer folded to lowercase, so path:<Foo> returned nothing) and makes bleve path queries match a folder and its descendants like OpenSearch's path_hierarchy. The Path base stays case-preserved so the move/delete descendant update (an exact TermQuery on Path) matches mixed-case folders.
This commit is contained in:
1 parent
fb22dd81a4
commit
ec58861e4e
24 files changed
+1309
-440
No files matched your search
@@ -44,6 +44,9 @@ type StringNode struct {
|
||||
Key string
|
||||
Value string
|
||||
Exact bool
|
||||
// CaseInsensitive marks a case-insensitive restriction; set by the search
|
||||
// lowering pass, a backend routes it to the field's lowercased form.
|
||||
CaseInsensitive bool
|
||||
}
|
||||
|
||||
// BooleanNode represents a bool value
|
||||
|
||||
@@ -0,0 +1,842 @@
|
||||
package bleve_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
bleveSearch "github.com/blevesearch/bleve/v2"
|
||||
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
func hiddenByID(idx bleveSearch.Index, id string) bool {
|
||||
GinkgoHelper()
|
||||
|
||||
req := bleveSearch.NewSearchRequest(bleveSearch.NewDocIDQuery([]string{id}))
|
||||
req.Fields = []string{"Hidden"}
|
||||
|
||||
res, err := idx.Search(req)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits).To(HaveLen(1), "no record for %s", id)
|
||||
|
||||
hidden, _ := res.Hits[0].Fields["Hidden"].(bool)
|
||||
return hidden
|
||||
}
|
||||
|
||||
var _ = Describe("Bleve", func() {
|
||||
var (
|
||||
eng *bleve.Backend
|
||||
idx bleveSearch.Index
|
||||
|
||||
doSearch = func(id string, query, path string) (*searchsvc.SearchIndexResponse, error) {
|
||||
rID, err := storagespace.ParseID(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return eng.Search(context.Background(), &searchsvc.SearchIndexRequest{
|
||||
Query: query,
|
||||
Ref: &searchmsg.Reference{
|
||||
ResourceId: &searchmsg.ResourceID{
|
||||
StorageId: rID.StorageId,
|
||||
SpaceId: rID.SpaceId,
|
||||
OpaqueId: rID.OpaqueId,
|
||||
},
|
||||
Path: path,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
assertDocCount = func(id string, query string, expectedCount int) []*searchmsg.Match {
|
||||
res, err := doSearch(id, query, "")
|
||||
|
||||
ExpectWithOffset(1, err).ToNot(HaveOccurred())
|
||||
ExpectWithOffset(1, len(res.Matches)).To(Equal(expectedCount), "query returned unexpected number of results: "+query)
|
||||
return res.Matches
|
||||
}
|
||||
|
||||
rootResource search.Resource
|
||||
parentResource search.Resource
|
||||
childResource search.Resource
|
||||
childResource2 search.Resource
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
mapping, err := bleve.NewMapping()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
idx, err = bleveSearch.NewMemOnly(mapping)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
rootResource = search.Resource{
|
||||
ID: "1$2!2",
|
||||
RootID: "1$2!2",
|
||||
Path: ".",
|
||||
Document: content.Document{},
|
||||
}
|
||||
|
||||
parentResource = search.Resource{
|
||||
ID: "1$2!3",
|
||||
ParentID: rootResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./parent d!r",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER),
|
||||
Document: content.Document{Name: "parent d!r"},
|
||||
}
|
||||
|
||||
childResource = search.Resource{
|
||||
ID: "1$2!4",
|
||||
ParentID: parentResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./parent d!r/child.pdf",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{Name: "child.pdf"},
|
||||
}
|
||||
|
||||
childResource2 = search.Resource{
|
||||
ID: "1$2!5",
|
||||
ParentID: parentResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./parent d!r/child2.pdf",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{Name: "child2.pdf"},
|
||||
}
|
||||
})
|
||||
|
||||
Describe("PurgeSpace", func() {
|
||||
It("takes every record of that space out of the index", func() {
|
||||
otherSpace := search.Resource{
|
||||
ID: "1$9!9",
|
||||
RootID: "1$9!9",
|
||||
Path: ".",
|
||||
Document: content.Document{Name: "other"},
|
||||
}
|
||||
for _, resource := range []search.Resource{rootResource, parentResource, childResource, otherSpace} {
|
||||
Expect(eng.Upsert(resource.ID, resource)).To(Succeed())
|
||||
}
|
||||
|
||||
Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed())
|
||||
|
||||
count, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(1)), "only the records of that space are gone")
|
||||
})
|
||||
|
||||
It("takes a space out that holds more records than one round", func() {
|
||||
otherSpace := search.Resource{
|
||||
ID: "1$9!9",
|
||||
RootID: "1$9!9",
|
||||
Path: ".",
|
||||
Document: content.Document{Name: "other"},
|
||||
}
|
||||
Expect(eng.Upsert(otherSpace.ID, otherSpace)).To(Succeed())
|
||||
|
||||
for i := range 120 {
|
||||
resource := search.Resource{
|
||||
ID: fmt.Sprintf("%s!file-%d", rootResource.RootID, i),
|
||||
RootID: rootResource.RootID,
|
||||
Path: fmt.Sprintf("./file-%d", i),
|
||||
Document: content.Document{Name: fmt.Sprintf("file-%d", i)},
|
||||
}
|
||||
Expect(eng.Upsert(resource.ID, resource)).To(Succeed())
|
||||
}
|
||||
|
||||
Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed())
|
||||
|
||||
count, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(1)), "only the record of the other space is left")
|
||||
})
|
||||
})
|
||||
|
||||
Describe("New", func() {
|
||||
It("returns a new index instance", func() {
|
||||
b := bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{})
|
||||
Expect(b).ToNot(BeNil())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Search", func() {
|
||||
Context("by other fields than filename", func() {
|
||||
It("finds files by tags", func() {
|
||||
parentResource.Document.Tags = []string{"foo", "bar"}
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Tags:foo", 1)
|
||||
assertDocCount(rootResource.ID, "Tags:bar", 1)
|
||||
assertDocCount(rootResource.ID, "Tags:foo Tags:bar", 1)
|
||||
assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1)
|
||||
assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1)
|
||||
assertDocCount(rootResource.ID, "Tags:baz", 0)
|
||||
})
|
||||
|
||||
It("finds files by size", func() {
|
||||
parentResource.Document.Size = 12345
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Size:12345", 1)
|
||||
assertDocCount(rootResource.ID, "Size:>1000", 1)
|
||||
assertDocCount(rootResource.ID, "Size:<100000", 1)
|
||||
assertDocCount(rootResource.ID, "Size:12344", 0)
|
||||
assertDocCount(rootResource.ID, "Size:<1000", 0)
|
||||
assertDocCount(rootResource.ID, "Size:>100000", 0)
|
||||
})
|
||||
|
||||
It("preserves value case for fields not explicitly marked lowercase", func() {
|
||||
parentResource.Document.Audio = &libregraph.Audio{
|
||||
Artist: libregraph.PtrString("Some Artist"),
|
||||
}
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `audio.artist:"Some Artist"`, 1)
|
||||
assertDocCount(rootResource.ID, `audio.artist:"some artist"`, 0)
|
||||
})
|
||||
})
|
||||
|
||||
Context("by filename", func() {
|
||||
It("finds files with spaces in the filename", func() {
|
||||
parentResource.Document.Name = "Foo oo.pdf"
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `name:"foo o*"`, 1)
|
||||
})
|
||||
|
||||
It("finds files by digits in the filename", func() {
|
||||
parentResource.Document.Name = "12345.pdf"
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Name:1234*", 1)
|
||||
})
|
||||
|
||||
It("filters hidden files", func() {
|
||||
childResource.Hidden = true
|
||||
err := eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Hidden:T", 1)
|
||||
assertDocCount(rootResource.ID, "Hidden:F", 0)
|
||||
})
|
||||
|
||||
Context("with a file in the root of the space", func() {
|
||||
It("scopes the search to the specified space", func() {
|
||||
parentResource.Document.Name = "foo.pdf"
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Name:foo.pdf", 1)
|
||||
assertDocCount("9$8!7", "Name:foo.pdf", 0)
|
||||
})
|
||||
})
|
||||
|
||||
It("limits the search to the specified fields", func() {
|
||||
parentResource.Document.Name = "bar.pdf"
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Name:bar.pdf", 1)
|
||||
assertDocCount(rootResource.ID, "Unknown:field", 0)
|
||||
})
|
||||
|
||||
It("returns the total number of hits", func() {
|
||||
parentResource.Document.Name = "bar.pdf"
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
res, err := doSearch(rootResource.ID, "Name:bar*", "")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(1)))
|
||||
})
|
||||
|
||||
It("returns all desired fields", func() {
|
||||
parentResource.Document.Name = "bar.pdf"
|
||||
parentResource.Type = 3
|
||||
parentResource.MimeType = "application/pdf"
|
||||
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
matches := assertDocCount(rootResource.ID, fmt.Sprintf("Name:%s", parentResource.Name), 1)
|
||||
match := matches[0]
|
||||
Expect(match.Entity.Ref.Path).To(Equal(parentResource.Path))
|
||||
Expect(match.Entity.Name).To(Equal(parentResource.Name))
|
||||
Expect(match.Entity.Size).To(Equal(parentResource.Size))
|
||||
Expect(match.Entity.Type).To(Equal(parentResource.Type))
|
||||
Expect(match.Entity.MimeType).To(Equal(parentResource.MimeType))
|
||||
Expect(match.Entity.Deleted).To(BeFalse())
|
||||
Expect(match.Score > 0).To(BeTrue())
|
||||
})
|
||||
|
||||
It("finds files by name, prefix or substring match", func() {
|
||||
parentResource.Document.Name = "foo.pdf"
|
||||
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
queries := []string{"foo.pdf", "foo*", "*oo.p*"}
|
||||
for _, query := range queries {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, query, 1)
|
||||
}
|
||||
})
|
||||
|
||||
It("does a case-insensitive search", func() {
|
||||
parentResource.Document.Name = "foo.pdf"
|
||||
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Name:foo*", 1)
|
||||
assertDocCount(rootResource.ID, "Name:Foo*", 1)
|
||||
})
|
||||
|
||||
Context("and an additional file in a subdirectory", func() {
|
||||
BeforeEach(func() {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("finds files living deeper in the tree by filename, prefix or substring match", func() {
|
||||
queries := []string{"child.pdf", "child*", "*ld.*"}
|
||||
for _, query := range queries {
|
||||
assertDocCount(rootResource.ID, query, 1)
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
Context("by path", func() {
|
||||
BeforeEach(func() {
|
||||
for _, r := range []search.Resource{parentResource, childResource, childResource2} {
|
||||
Expect(eng.Upsert(r.ID, r)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
||||
It("matches a folder and its descendants", func() {
|
||||
assertDocCount(rootResource.ID, `path:"./parent d!r"`, 3)
|
||||
})
|
||||
|
||||
It("matches a descendant path only itself", func() {
|
||||
assertDocCount(rootResource.ID, `path:"./parent d!r/child.pdf"`, 1)
|
||||
})
|
||||
|
||||
It("matches case-insensitively", func() {
|
||||
assertDocCount(rootResource.ID, `path:"./PARENT D!R"`, 3)
|
||||
})
|
||||
})
|
||||
|
||||
Context("Highlights", func() {
|
||||
|
||||
It("highlights only for content searches", func() {
|
||||
parentResource.Document.Name = "baz.pdf"
|
||||
parentResource.Document.Content = "foo bar baz"
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
res, err := doSearch(rootResource.ID, "Name:baz*", "")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(1)))
|
||||
Expect(res.Matches[0].Entity.Highlights).To(Equal(""))
|
||||
})
|
||||
|
||||
It("highlights search terms", func() {
|
||||
parentResource.Document.Name = "baz.pdf"
|
||||
parentResource.Document.Content = "foo bar baz"
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
res, err := doSearch(rootResource.ID, "Content:bar", "")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(1)))
|
||||
Expect(res.Matches[0].Entity.Highlights).To(Equal("foo <mark>bar</mark> baz"))
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
Context("with a file in the root of the space and folder with a file. all of them have the same name", func() {
|
||||
BeforeEach(func() {
|
||||
parentResource := search.Resource{
|
||||
ID: "1$2!3",
|
||||
ParentID: rootResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./doc",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER),
|
||||
Document: content.Document{Name: "doc"},
|
||||
}
|
||||
|
||||
childResource := search.Resource{
|
||||
ID: "1$2!4",
|
||||
ParentID: parentResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./doc/doc.pdf",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{Name: "doc.pdf"},
|
||||
}
|
||||
|
||||
childResource2 := search.Resource{
|
||||
ID: "1$2!7",
|
||||
ParentID: parentResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./doc/file.pdf",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{Name: "file.pdf"},
|
||||
}
|
||||
|
||||
rootChildResource := search.Resource{
|
||||
ID: "1$2!5",
|
||||
ParentID: rootResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./doc.pdf",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{Name: "doc.pdf"},
|
||||
}
|
||||
|
||||
rootChildResource2 := search.Resource{
|
||||
ID: "1$2!6",
|
||||
ParentID: rootResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./file.pdf",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{Name: "file.pdf"},
|
||||
}
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(rootChildResource.ID, rootChildResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = eng.Upsert(rootChildResource2.ID, rootChildResource2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = eng.Upsert(childResource2.ID, childResource2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
It("search *doc* in a root", func() {
|
||||
res, err := doSearch(rootResource.ID, "Name:*doc*", "")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(3)))
|
||||
})
|
||||
It("search *doc* in a subfolder", func() {
|
||||
res, err := doSearch(rootResource.ID, "Name:*doc*", "./doc")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(2)))
|
||||
})
|
||||
It("search *file* in a root", func() {
|
||||
res, err := doSearch(rootResource.ID, "Name:*file*", "")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(2)))
|
||||
})
|
||||
It("search *file* in a subfolder", func() {
|
||||
res, err := doSearch(rootResource.ID, "Name:*file*", "./doc")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.TotalMatches).To(Equal(int32(1)))
|
||||
})
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
Describe("Upsert", func() {
|
||||
It("adds a resourceInfo to the index", func() {
|
||||
err := eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
count, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(1)))
|
||||
|
||||
query := bleveSearch.NewMatchQuery("child.pdf")
|
||||
res, err := idx.Search(bleveSearch.NewSearchRequest(query))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits.Len()).To(Equal(1))
|
||||
})
|
||||
|
||||
It("updates an existing resource in the index", func() {
|
||||
|
||||
err := eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
countA, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(countA).To(Equal(uint64(1)))
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
countB, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(countB).To(Equal(uint64(1)))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Delete", func() {
|
||||
It("marks a resource as deleted", func() {
|
||||
err := eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Name:*child*", 1)
|
||||
|
||||
err = eng.Delete(childResource.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, "Name:*child*", 0)
|
||||
})
|
||||
|
||||
It("marks a child resources as deleted", func() {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1)
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
|
||||
|
||||
err = eng.Delete(parentResource.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0)
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Restore", func() {
|
||||
It("also marks child resources as restored", func() {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Delete(parentResource.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 0)
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 0)
|
||||
|
||||
err = eng.Restore(parentResource.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 1)
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 1)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Purge", func() {
|
||||
It("removes a resource from the index", func() {
|
||||
err := eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
assertDocCount(rootResource.ID, "Name:child.pdf", 1)
|
||||
|
||||
err = eng.Purge(childResource.ID, false)
|
||||
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
assertDocCount(rootResource.ID, "Name:child.pdf", 0)
|
||||
})
|
||||
It("removes a resource and its children from the index", func() {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1)
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
|
||||
|
||||
err = eng.Purge(parentResource.ID, false)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0)
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0)
|
||||
})
|
||||
It("removes a resource and ignores its children from the index", func() {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1)
|
||||
|
||||
err = eng.Delete(parentResource.ID)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
|
||||
|
||||
err = eng.Purge(parentResource.ID, true)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0)
|
||||
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Move", func() {
|
||||
It("renames the parent and its child resources", func() {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
parentResource.Path = "newname"
|
||||
err = eng.Move(parentResource.ID, parentResource.ParentID, "./my/newname")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
assertDocCount(rootResource.ID, parentResource.Name, 0)
|
||||
|
||||
matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1)
|
||||
Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3"))
|
||||
Expect(matches[0].Entity.Ref.Path).To(Equal("./my/newname/child.pdf"))
|
||||
})
|
||||
|
||||
DescribeTable("keeps the flag in step with the path",
|
||||
func(from, target string, hidden bool) {
|
||||
parentResource.Path = from
|
||||
parentResource.Hidden = search.IsHidden(from)
|
||||
childResource.Path = from + "/child.pdf"
|
||||
childResource.Hidden = parentResource.Hidden
|
||||
|
||||
Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed())
|
||||
Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed())
|
||||
|
||||
Expect(eng.Move(parentResource.ID, parentResource.ParentID, target)).To(Succeed())
|
||||
|
||||
for _, id := range []string{parentResource.ID, childResource.ID} {
|
||||
Expect(hiddenByID(idx, id)).
|
||||
To(Equal(hidden), "%s after moving from %s to %s", id, from, target)
|
||||
}
|
||||
},
|
||||
Entry("into a dot folder", "./parent", "./.trash/parent", true),
|
||||
Entry("into a plain folder", "./parent", "./archive/parent", false),
|
||||
Entry("renamed with a leading dot", "./parent", "./.parent", true),
|
||||
Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false),
|
||||
Entry("renamed without the leading dot", "./.parent", "./parent", false),
|
||||
Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true),
|
||||
)
|
||||
|
||||
// the trash leaves the path alone, so the flag has to come through untouched
|
||||
It("carries the flag through the trash and back", func() {
|
||||
childResource.Path = "./.secret/file.txt"
|
||||
childResource.Hidden = true
|
||||
Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed())
|
||||
|
||||
Expect(eng.Delete(childResource.ID)).To(Succeed())
|
||||
Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after trashing")
|
||||
|
||||
Expect(eng.Restore(childResource.ID)).To(Succeed())
|
||||
Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after restoring")
|
||||
})
|
||||
|
||||
It("moves the parent and its child resources", func() {
|
||||
err := eng.Upsert(parentResource.ID, parentResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = eng.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
parentResource.Path = " "
|
||||
parentResource.ParentID = "1$2!somewhereopaqueid"
|
||||
|
||||
err = eng.Move(parentResource.ID, parentResource.ParentID, "./somewhere/else/newname")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
assertDocCount(rootResource.ID, `parent d!r`, 0)
|
||||
|
||||
matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1)
|
||||
Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3"))
|
||||
Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname/child.pdf"))
|
||||
|
||||
matches = assertDocCount(rootResource.ID, `newname`, 1)
|
||||
Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("somewhereopaqueid"))
|
||||
Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname"))
|
||||
|
||||
})
|
||||
|
||||
It("keeps case-insensitive search working after a move", func() {
|
||||
Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed())
|
||||
Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed())
|
||||
|
||||
Expect(eng.Move(parentResource.ID, parentResource.ParentID, "./my/NewName")).To(Succeed())
|
||||
|
||||
// the lowercased siblings are rebuilt at the new path, so a
|
||||
// case-insensitive query finds the folder under its new name and path,
|
||||
// including the descendant, and no longer under the old path.
|
||||
assertDocCount(rootResource.ID, "name:NEWNAME", 1)
|
||||
assertDocCount(rootResource.ID, `path:"./MY/NEWNAME"`, 2)
|
||||
assertDocCount(rootResource.ID, `path:"./parent d!r"`, 0)
|
||||
})
|
||||
})
|
||||
|
||||
Describe("StartBatch", func() {
|
||||
It("starts a new batch", func() {
|
||||
b, err := eng.NewBatch(100)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = b.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
count, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(0)))
|
||||
|
||||
err = b.Push()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
count, err = idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(1)))
|
||||
|
||||
query := bleveSearch.NewMatchQuery("child.pdf")
|
||||
res, err := idx.Search(bleveSearch.NewSearchRequest(query))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits.Len()).To(Equal(1))
|
||||
})
|
||||
|
||||
It("doesn't intertwine different batches", func() {
|
||||
b, err := eng.NewBatch(100)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = b.Upsert(childResource.ID, childResource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
count, err := idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(0)))
|
||||
|
||||
b2, err := eng.NewBatch(100)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
err = b2.Upsert(childResource2.ID, childResource2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
Expect(b.Push()).To(Succeed())
|
||||
count, err = idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(1)))
|
||||
|
||||
Expect(b2.Push()).To(Succeed())
|
||||
count, err = idx.DocCount()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(uint64(2)))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("File type specific metadata", func() {
|
||||
|
||||
Context("with audio metadata", func() {
|
||||
BeforeEach(func() {
|
||||
resource := search.Resource{
|
||||
ID: "1$2!7",
|
||||
ParentID: rootResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./some_song.mp3",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{
|
||||
Name: "some_song.mp3",
|
||||
MimeType: "audio/mpeg",
|
||||
Audio: &libregraph.Audio{
|
||||
Album: libregraph.PtrString("Some Album"),
|
||||
AlbumArtist: libregraph.PtrString("Some AlbumArtist"),
|
||||
Artist: libregraph.PtrString("Some Artist"),
|
||||
Bitrate: libregraph.PtrInt64(192),
|
||||
Composers: libregraph.PtrString("Some Composers"),
|
||||
Copyright: libregraph.PtrString(""),
|
||||
Disc: libregraph.PtrInt32(2),
|
||||
DiscCount: libregraph.PtrInt32(5),
|
||||
Duration: libregraph.PtrInt64(225000),
|
||||
Genre: libregraph.PtrString("Some Genre"),
|
||||
HasDrm: libregraph.PtrBool(false),
|
||||
IsVariableBitrate: libregraph.PtrBool(true),
|
||||
Title: libregraph.PtrString("Some Title"),
|
||||
Track: libregraph.PtrInt32(34),
|
||||
TrackCount: libregraph.PtrInt32(99),
|
||||
Year: libregraph.PtrInt32(2004),
|
||||
},
|
||||
},
|
||||
}
|
||||
err := eng.Upsert(resource.ID, resource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("returns audio metadata for search", func() {
|
||||
matches := assertDocCount(rootResource.ID, `*song*`, 1)
|
||||
audio := matches[0].Entity.Audio
|
||||
|
||||
Expect(audio).ToNot(BeNil())
|
||||
|
||||
Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album")))
|
||||
Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist")))
|
||||
Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist")))
|
||||
Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192)))
|
||||
Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers")))
|
||||
Expect(audio.Copyright).To(Equal(libregraph.PtrString("")))
|
||||
Expect(audio.Disc).To(Equal(libregraph.PtrInt32(2)))
|
||||
Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5)))
|
||||
Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225000)))
|
||||
Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre")))
|
||||
Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false)))
|
||||
Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true)))
|
||||
Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title")))
|
||||
Expect(audio.Track).To(Equal(libregraph.PtrInt32(34)))
|
||||
Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(99)))
|
||||
Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004)))
|
||||
})
|
||||
})
|
||||
|
||||
Context("with location metadata", func() {
|
||||
BeforeEach(func() {
|
||||
resource := search.Resource{
|
||||
ID: "1$2!7",
|
||||
ParentID: rootResource.ID,
|
||||
RootID: rootResource.ID,
|
||||
Path: "./team.jpg",
|
||||
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
|
||||
Document: content.Document{
|
||||
Name: "team.jpg",
|
||||
MimeType: "image/jpeg",
|
||||
Location: &libregraph.GeoCoordinates{
|
||||
Altitude: libregraph.PtrFloat64(1047.7),
|
||||
Latitude: libregraph.PtrFloat64(49.48675890884328),
|
||||
Longitude: libregraph.PtrFloat64(11.103870357204285),
|
||||
},
|
||||
},
|
||||
}
|
||||
err := eng.Upsert(resource.ID, resource)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("returns audio metadata for search", func() {
|
||||
matches := assertDocCount(rootResource.ID, `*team*`, 1)
|
||||
location := matches[0].Entity.Location
|
||||
|
||||
Expect(location).ToNot(BeNil())
|
||||
|
||||
Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(1047.7)))
|
||||
Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328)))
|
||||
Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285)))
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -10,10 +10,8 @@ import (
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
|
||||
"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
|
||||
regexpCharFilter "github.com/blevesearch/bleve/v2/analysis/char/regexp"
|
||||
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
|
||||
"github.com/blevesearch/bleve/v2/analysis/token/porter"
|
||||
"github.com/blevesearch/bleve/v2/analysis/tokenizer/single"
|
||||
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
|
||||
"github.com/blevesearch/bleve/v2/mapping"
|
||||
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
@@ -60,44 +58,6 @@ func NewMapping() (mapping.IndexMapping, error) {
|
||||
indexMapping := bleve.NewIndexMapping()
|
||||
indexMapping.DefaultAnalyzer = keyword.Name
|
||||
indexMapping.DefaultMapping = docMapping
|
||||
err = indexMapping.AddCustomCharFilter("dotToSpace",
|
||||
map[string]any{
|
||||
"type": regexpCharFilter.Name,
|
||||
"regexp": `\.`,
|
||||
"replace": " ",
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = indexMapping.AddCustomAnalyzer("lowercaseWords",
|
||||
map[string]any{
|
||||
"type": custom.Name,
|
||||
"char_filters": []string{"dotToSpace"},
|
||||
"tokenizer": unicode.Name,
|
||||
"token_filters": []string{
|
||||
lowercase.Name,
|
||||
},
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = indexMapping.AddCustomAnalyzer("lowercaseKeyword",
|
||||
map[string]any{
|
||||
"type": custom.Name,
|
||||
"tokenizer": single.Name,
|
||||
"token_filters": []string{
|
||||
lowercase.Name,
|
||||
},
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = indexMapping.AddCustomAnalyzer("fulltext",
|
||||
map[string]any{
|
||||
"type": custom.Name,
|
||||
|
||||
@@ -12,10 +12,8 @@ import (
|
||||
// struct via reflection. Field names come from json tags; overrides are
|
||||
// keyed by those names (or dotted paths for nested fields).
|
||||
//
|
||||
// The returned mapping references analyzer names (Analyzer field on the
|
||||
// FieldOpts, plus "fulltext" / "path_hierarchy" for the corresponding Types);
|
||||
// the caller is responsible for registering those analyzers on the enclosing
|
||||
// IndexMapping.
|
||||
// The returned mapping references the "fulltext" analyzer for Fulltext fields;
|
||||
// the caller registers it on the enclosing IndexMapping.
|
||||
func BleveBuildMapping(t reflect.Type, overrides map[string]FieldOpts) (*bleveMapping.DocumentMapping, error) {
|
||||
return buildBleveDocMapping(t, overrides, "")
|
||||
}
|
||||
@@ -61,6 +59,16 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix
|
||||
return nil
|
||||
}
|
||||
|
||||
if fieldType == TypeKeyword || fieldType == TypePath {
|
||||
// bleve has no path tokenizer, so a path is a plain keyword here.
|
||||
base := bleveKeywordMapping(fieldType, opts)
|
||||
doc.AddFieldMappingsAt(fi.Name, base)
|
||||
if opts.caseInsensitive() {
|
||||
doc.AddFieldMappingsAt(fi.Name+LowercaseSuffix, lowercaseSibling(base))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
fm, err := bleveFieldMapping(fieldType, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mapping: field %q: %w", key, err)
|
||||
@@ -71,26 +79,46 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix
|
||||
return doc, err
|
||||
}
|
||||
|
||||
// bleveKeywordMapping is a case-preserving keyword field; path fields stay out
|
||||
// of _all by default.
|
||||
func bleveKeywordMapping(fieldType string, opts FieldOpts) *bleveMapping.FieldMapping {
|
||||
fm := bleve.NewKeywordFieldMapping()
|
||||
switch {
|
||||
case opts.IncludeInAll != nil:
|
||||
fm.IncludeInAll = *opts.IncludeInAll
|
||||
case fieldType == TypePath:
|
||||
fm.IncludeInAll = false
|
||||
}
|
||||
return fm
|
||||
}
|
||||
|
||||
// lowercaseSibling derives the lowercased shadow of a keyword/path field from its
|
||||
// base mapping: used only for case-insensitive matching, so indexed but never
|
||||
// stored, kept out of _all, and without doc values, since the case-preserved base
|
||||
// field is what we return and aggregate on.
|
||||
func lowercaseSibling(base *bleveMapping.FieldMapping) *bleveMapping.FieldMapping {
|
||||
fm := *base
|
||||
fm.Store = false
|
||||
fm.IncludeInAll = false
|
||||
fm.DocValues = false
|
||||
return &fm
|
||||
}
|
||||
|
||||
func bleveFieldMapping(fieldType string, opts FieldOpts) (*bleveMapping.FieldMapping, error) {
|
||||
switch fieldType {
|
||||
case TypeWildcard:
|
||||
// bleve has no wildcard type; fall back to keyword-ish text.
|
||||
fieldType = TypeKeyword
|
||||
fallthrough
|
||||
case TypeKeyword, TypeFulltext, TypePath:
|
||||
case TypeKeyword, TypeFulltext:
|
||||
fm := bleve.NewTextFieldMapping()
|
||||
switch {
|
||||
case opts.Analyzer != "":
|
||||
fm.Analyzer = opts.Analyzer
|
||||
case fieldType == TypeFulltext:
|
||||
if fieldType == TypeFulltext {
|
||||
fm.Analyzer = "fulltext"
|
||||
case fieldType == TypePath:
|
||||
fm.Analyzer = "path_hierarchy"
|
||||
}
|
||||
switch {
|
||||
case opts.IncludeInAll != nil:
|
||||
fm.IncludeInAll = *opts.IncludeInAll
|
||||
case fieldType == TypeFulltext, fieldType == TypePath:
|
||||
case fieldType == TypeFulltext:
|
||||
fm.IncludeInAll = false
|
||||
}
|
||||
return fm, nil
|
||||
|
||||
@@ -65,21 +65,31 @@ var _ = Describe("BleveBuildMapping", func() {
|
||||
})
|
||||
|
||||
It("applies field overrides", func() {
|
||||
includeInAllFalse := false
|
||||
True, False := true, false
|
||||
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{
|
||||
"Name": {Analyzer: "lowercaseKeyword"},
|
||||
"Name": {CaseInsensitive: &True},
|
||||
"Content": {Type: TypeFulltext},
|
||||
"Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &includeInAllFalse},
|
||||
"Tags": {CaseInsensitive: &True, IncludeInAll: &False},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
nameField := dm.Properties["Name"].Fields[0]
|
||||
Expect(nameField.Analyzer).To(Equal("lowercaseKeyword"), "Name analyzer")
|
||||
Expect(nameField.IncludeInAll).To(BeTrue(), "Name IncludeInAll should stay default-true when not overridden")
|
||||
// Name: case-preserved base keyword + lowercased sibling.
|
||||
Expect(dm.Properties["Name"]).ToNot(BeNil(), "Name base field")
|
||||
Expect(dm.Properties["Name"].Fields[0].Analyzer).To(Equal("keyword"), "Name base is a keyword")
|
||||
Expect(dm.Properties["Name"].Fields[0].Store).To(BeTrue(), "Name base is stored (returned)")
|
||||
Expect(dm.Properties["Name_lowercase"]).ToNot(BeNil(), "Name_lowercase sibling")
|
||||
// The sibling is a search-only shadow: indexed but never stored, kept out
|
||||
// of _all, no doc values (the base is what we return).
|
||||
sibling := dm.Properties["Name_lowercase"].Fields[0]
|
||||
Expect(sibling.Index).To(BeTrue(), "Name_lowercase is indexed")
|
||||
Expect(sibling.Store).To(BeFalse(), "Name_lowercase is not stored")
|
||||
Expect(sibling.IncludeInAll).To(BeFalse(), "Name_lowercase is out of _all")
|
||||
Expect(sibling.DocValues).To(BeFalse(), "Name_lowercase has no doc values")
|
||||
contentField := dm.Properties["Content"].Fields[0]
|
||||
Expect(contentField.Analyzer).To(Equal("fulltext"), "Content analyzer")
|
||||
Expect(contentField.IncludeInAll).To(BeFalse(), "Content IncludeInAll should default to false for fulltext type")
|
||||
tagsField := dm.Properties["Tags"].Fields[0]
|
||||
Expect(tagsField.IncludeInAll).To(BeFalse(), "Tags IncludeInAll should honor the explicit false override")
|
||||
// Tags: base + lowercased sibling, both honoring the IncludeInAll override.
|
||||
Expect(dm.Properties["Tags"].Fields[0].IncludeInAll).To(BeFalse(), "Tags base IncludeInAll honored")
|
||||
Expect(dm.Properties["Tags_lowercase"].Fields[0].IncludeInAll).To(BeFalse(), "Tags sibling IncludeInAll honored")
|
||||
})
|
||||
|
||||
It("builds an object sub-document plus a geopoint sibling", func() {
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package mapping
|
||||
|
||||
import "strings"
|
||||
|
||||
func addLowercaseSiblings(m map[string]any, overrides map[string]FieldOpts) {
|
||||
for key, opts := range overrides {
|
||||
if !opts.caseInsensitive() || !isCasedType(opts) {
|
||||
continue
|
||||
}
|
||||
parent, leaf, ok := resolveLeaf(m, key)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
addLowercaseSibling(parent, leaf)
|
||||
}
|
||||
}
|
||||
|
||||
func isCasedType(opts FieldOpts) bool {
|
||||
return opts.Type == "" || opts.Type == TypeKeyword || opts.Type == TypePath
|
||||
}
|
||||
|
||||
func resolveLeaf(m map[string]any, dottedPath string) (map[string]any, string, bool) {
|
||||
parts := strings.Split(dottedPath, ".")
|
||||
parent := m
|
||||
for _, p := range parts[:len(parts)-1] {
|
||||
next, ok := parent[p].(map[string]any)
|
||||
if !ok {
|
||||
return nil, "", false
|
||||
}
|
||||
parent = next
|
||||
}
|
||||
return parent, parts[len(parts)-1], true
|
||||
}
|
||||
|
||||
// addLowercaseSibling writes a <leaf>_lowercase sibling; no-op for non-strings.
|
||||
func addLowercaseSibling(parent map[string]any, leaf string) {
|
||||
switch v := parent[leaf].(type) {
|
||||
case string:
|
||||
parent[leaf+LowercaseSuffix] = strings.ToLower(v)
|
||||
case []any:
|
||||
out := make([]any, 0, len(v))
|
||||
for _, item := range v {
|
||||
if s, ok := item.(string); ok {
|
||||
out = append(out, strings.ToLower(s))
|
||||
}
|
||||
}
|
||||
if len(out) > 0 {
|
||||
parent[leaf+LowercaseSuffix] = out
|
||||
}
|
||||
case []string:
|
||||
out := make([]string, len(v))
|
||||
for i, s := range v {
|
||||
out[i] = strings.ToLower(s)
|
||||
}
|
||||
parent[leaf+LowercaseSuffix] = out
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("PrepareForIndex casing", func() {
|
||||
It("adds lowercased siblings for CaseInsensitive keyword and path fields", func() {
|
||||
True := true
|
||||
type doc struct {
|
||||
Name string `json:"Name"`
|
||||
Path string `json:"Path"`
|
||||
Tags []string `json:"Tags"`
|
||||
}
|
||||
d := doc{Name: "Report FINAL", Path: "/Foo/Bar", Tags: []string{"Work", "Urgent"}}
|
||||
m, err := PrepareForIndex(d, map[string]FieldOpts{
|
||||
"Name": {CaseInsensitive: &True},
|
||||
"Path": {Type: TypePath, CaseInsensitive: &True},
|
||||
"Tags": {CaseInsensitive: &True},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Originals stay for the case-preserved base fields and the cascade.
|
||||
Expect(m["Name"]).To(Equal("Report FINAL"))
|
||||
Expect(m["Path"]).To(Equal("/Foo/Bar"))
|
||||
|
||||
Expect(m["Name_lowercase"]).To(Equal("report final"))
|
||||
Expect(m["Path_lowercase"]).To(Equal("/foo/bar"))
|
||||
Expect(m["Tags_lowercase"]).To(Equal([]any{"work", "urgent"}))
|
||||
})
|
||||
|
||||
It("writes no sibling without CaseInsensitive", func() {
|
||||
type doc struct {
|
||||
ID string `json:"ID"`
|
||||
}
|
||||
m, err := PrepareForIndex(doc{ID: "ABC"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(m).ToNot(HaveKey("ID" + LowercaseSuffix))
|
||||
})
|
||||
})
|
||||
@@ -56,7 +56,20 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p
|
||||
return nil
|
||||
}
|
||||
|
||||
fm, err := openSearchFieldMapping(fieldType, opts, fi.GoField.Type)
|
||||
if fieldType == TypeKeyword || fieldType == TypePath {
|
||||
// path_hierarchy is case-preserving here; casing lives in the value.
|
||||
m := map[string]any{"type": "keyword"}
|
||||
if fieldType == TypePath {
|
||||
m = map[string]any{"type": "text", "analyzer": "path_hierarchy"}
|
||||
}
|
||||
props[fi.Name] = m
|
||||
if opts.caseInsensitive() {
|
||||
props[fi.Name+LowercaseSuffix] = m
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
fm, err := openSearchFieldMapping(fieldType, fi.GoField.Type)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mapping: field %q: %w", key, err)
|
||||
}
|
||||
@@ -66,32 +79,15 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p
|
||||
return props, err
|
||||
}
|
||||
|
||||
func openSearchFieldMapping(fieldType string, opts FieldOpts, goType reflect.Type) (map[string]any, error) {
|
||||
// openSearchFieldMapping handles the non-keyword/path types; keyword and path
|
||||
// are emitted (with their cased forms) by buildOpenSearchProperties directly.
|
||||
func openSearchFieldMapping(fieldType string, goType reflect.Type) (map[string]any, error) {
|
||||
switch fieldType {
|
||||
case TypeKeyword:
|
||||
m := map[string]any{"type": "keyword"}
|
||||
if opts.Analyzer != "" {
|
||||
m["type"] = "text"
|
||||
m["analyzer"] = opts.Analyzer
|
||||
}
|
||||
return m, nil
|
||||
case TypeFulltext:
|
||||
m := map[string]any{
|
||||
return map[string]any{
|
||||
"type": "text",
|
||||
"term_vector": "with_positions_offsets",
|
||||
}
|
||||
if opts.Analyzer != "" {
|
||||
m["analyzer"] = opts.Analyzer
|
||||
}
|
||||
return m, nil
|
||||
case TypePath:
|
||||
m := map[string]any{"type": "text"}
|
||||
if opts.Analyzer != "" {
|
||||
m["analyzer"] = opts.Analyzer
|
||||
} else {
|
||||
m["analyzer"] = "path_hierarchy"
|
||||
}
|
||||
return m, nil
|
||||
}, nil
|
||||
case TypeWildcard:
|
||||
// OpenSearch stores wildcard fields with doc_values=false by
|
||||
// default, so emit it explicitly to keep local and remote
|
||||
|
||||
@@ -78,6 +78,7 @@ var _ = Describe("OpenSearchBuildMapping", func() {
|
||||
})
|
||||
|
||||
It("applies field overrides", func() {
|
||||
True := true
|
||||
type doc struct {
|
||||
Name string `json:"Name"`
|
||||
Content string `json:"Content"`
|
||||
@@ -85,23 +86,23 @@ var _ = Describe("OpenSearchBuildMapping", func() {
|
||||
MimeType string `json:"MimeType"`
|
||||
}
|
||||
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{
|
||||
"Name": {Analyzer: "lowercaseKeyword"},
|
||||
"Name": {CaseInsensitive: &True},
|
||||
"Content": {Type: TypeFulltext},
|
||||
"Path": {Type: TypePath},
|
||||
"Path": {Type: TypePath, CaseInsensitive: &True},
|
||||
"MimeType": {Type: TypeWildcard},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
name := props["Name"].(map[string]any)
|
||||
Expect(name["type"]).To(Equal("text"), "Name: %#v", name)
|
||||
Expect(name["analyzer"]).To(Equal("lowercaseKeyword"), "Name: %#v", name)
|
||||
// Name: case-preserved keyword base + lowercased keyword sibling.
|
||||
Expect(props["Name"]).To(Equal(map[string]any{"type": "keyword"}))
|
||||
Expect(props["Name_lowercase"]).To(Equal(map[string]any{"type": "keyword"}))
|
||||
content := props["Content"].(map[string]any)
|
||||
Expect(content["type"]).To(Equal("text"), "Content: %#v", content)
|
||||
Expect(content["term_vector"]).To(Equal("with_positions_offsets"), "Content: %#v", content)
|
||||
_, ok := content["analyzer"]
|
||||
Expect(ok).To(BeFalse(), "Content should leave analyzer unset (use OpenSearch default)")
|
||||
path := props["Path"].(map[string]any)
|
||||
Expect(path["type"]).To(Equal("text"), "Path: %#v", path)
|
||||
Expect(path["analyzer"]).To(Equal("path_hierarchy"), "Path: %#v", path)
|
||||
// Path: path_hierarchy base + lowercased sibling, both case-preserving.
|
||||
Expect(props["Path"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"}))
|
||||
Expect(props["Path_lowercase"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"}))
|
||||
mime := props["MimeType"].(map[string]any)
|
||||
Expect(mime["type"]).To(Equal("wildcard"), "MimeType: %#v", mime)
|
||||
})
|
||||
|
||||
@@ -17,6 +17,9 @@ const (
|
||||
TypeGeopoint = "geopoint"
|
||||
)
|
||||
|
||||
// LowercaseSuffix names the lowercased sibling of a keyword/path field.
|
||||
const LowercaseSuffix = "_lowercase"
|
||||
|
||||
// FieldOpts overrides the default type inference for a struct field. Keys in
|
||||
// the override map are json-tag names (e.g. "Name", "location", "audio.artist"),
|
||||
// not Go field names.
|
||||
@@ -24,12 +27,14 @@ type FieldOpts struct {
|
||||
// Type is one of the Type* constants. Empty means "infer from Go type".
|
||||
Type string
|
||||
|
||||
// Analyzer is the name of a custom analyzer registered on the bleve
|
||||
// IndexMapping (e.g. "lowercaseKeyword", "fulltext"). For OpenSearch it
|
||||
// becomes the analyzer attribute on the field.
|
||||
Analyzer string
|
||||
// CaseInsensitive additionally indexes a lowercased <name>_lowercase sibling
|
||||
// for case-insensitive search; the case-preserved base is always indexed.
|
||||
// Nil/false means off. Keyword/path only.
|
||||
CaseInsensitive *bool
|
||||
|
||||
// IncludeInAll controls bleve's _all field inclusion. Nil means "use the
|
||||
// bleve default for this field type". Has no effect on OpenSearch.
|
||||
IncludeInAll *bool
|
||||
}
|
||||
|
||||
func (o FieldOpts) caseInsensitive() bool { return o.CaseInsensitive != nil && *o.CaseInsensitive }
|
||||
@@ -19,5 +19,6 @@ func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, err
|
||||
return out, nil
|
||||
}
|
||||
addGeopointSiblings(out, overrides)
|
||||
addLowercaseSiblings(out, overrides)
|
||||
return out, nil
|
||||
}
|
||||
@@ -23,9 +23,9 @@ type sample struct {
|
||||
var _ = Describe("Validate", func() {
|
||||
It("accepts known override keys", func() {
|
||||
err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{
|
||||
"Name": {Analyzer: "lowercaseKeyword"},
|
||||
"Name": {},
|
||||
"audio": {Type: TypeObject},
|
||||
"audio.artist": {Analyzer: "lowercaseKeyword"},
|
||||
"audio.artist": {},
|
||||
"location": {Type: TypeGeopoint},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
@@ -64,27 +64,45 @@ func (b *Batch) Upsert(id string, r search.Resource) error {
|
||||
})
|
||||
}
|
||||
|
||||
func (b *Batch) Move(id string, parentID string, targetPath string) error {
|
||||
func (b *Batch) Move(id string, parentID string, location string) error {
|
||||
return b.withSizeLimit(func() error {
|
||||
op := func() error {
|
||||
return updateSelfAndDescendants(context.Background(), b.client, b.index, id, func(rootResource search.Resource) *osu.BodyParamScript {
|
||||
newPath := utils.MakeRelativePath(location)
|
||||
newName := path.Base(newPath)
|
||||
return &osu.BodyParamScript{
|
||||
Source: `
|
||||
if (ctx._source.ID == params.id ) { ctx._source.Name = params.newName; ctx._source.ParentID = params.parentID; }
|
||||
ctx._source.Path = ctx._source.Path.replace(params.oldPath, params.newPath);
|
||||
boolean hidden = false;
|
||||
for (String name : ctx._source.Path.splitOnToken('/')) {
|
||||
if (!name.equals('.') && !name.equals('..') && name.startsWith('.')) { hidden = true; break; }
|
||||
}
|
||||
ctx._source.Hidden = hidden;
|
||||
`,
|
||||
// Keep the case-preserved base fields and their lowercased
|
||||
// search siblings in sync: swap the moved prefix in both. The
|
||||
// lowercased new values come from Go's strings.ToLower via
|
||||
// params, so the sibling stays byte-identical to what
|
||||
// PrepareForIndex writes on upsert (painless toLowerCase would
|
||||
// lowercase differently than Go).
|
||||
Source: fmt.Sprintf(`
|
||||
if (ctx._source.ID == params.id) {
|
||||
ctx._source.Name = params.newName;
|
||||
ctx._source.ParentID = params.parentID;
|
||||
if (ctx._source.Name%[1]s != null) { ctx._source.Name%[1]s = params.newNameLower; }
|
||||
}
|
||||
ctx._source.Path = ctx._source.Path.replace(params.oldPath, params.newPath);
|
||||
if (ctx._source.Path%[1]s != null) {
|
||||
ctx._source.Path%[1]s = ctx._source.Path%[1]s.replace(params.oldPathLower, params.newPathLower);
|
||||
}
|
||||
boolean hidden = false;
|
||||
for (String name : ctx._source.Path.splitOnToken('/')) {
|
||||
if (!name.equals('.') && !name.equals('..') && name.startsWith('.')) { hidden = true; break; }
|
||||
}
|
||||
ctx._source.Hidden = hidden;
|
||||
`, mapping.LowercaseSuffix),
|
||||
Lang: "painless",
|
||||
Params: map[string]any{
|
||||
"id": id,
|
||||
"parentID": parentID,
|
||||
"oldPath": rootResource.Path,
|
||||
"newPath": utils.MakeRelativePath(targetPath),
|
||||
"newName": path.Base(utils.MakeRelativePath(targetPath)),
|
||||
"id": id,
|
||||
"parentID": parentID,
|
||||
"oldPath": rootResource.Path,
|
||||
"newPath": newPath,
|
||||
"newName": newName,
|
||||
"oldPathLower": strings.ToLower(rootResource.Path),
|
||||
"newPathLower": strings.ToLower(newPath),
|
||||
"newNameLower": strings.ToLower(newName),
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
@@ -61,7 +61,6 @@ func buildResourceMapping() ([]byte, error) {
|
||||
resourceType := reflect.TypeFor[search.Resource]()
|
||||
overrides := maps.Clone(search.Resource{}.SearchFieldOverrides())
|
||||
overrides["MimeType"] = searchmapping.FieldOpts{Type: searchmapping.TypeWildcard}
|
||||
overrides["Path"] = searchmapping.FieldOpts{Type: searchmapping.TypePath}
|
||||
if err := searchmapping.Validate(resourceType, overrides); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -75,16 +74,11 @@ func buildResourceMapping() ([]byte, error) {
|
||||
"number_of_shards": "1",
|
||||
"number_of_replicas": "1",
|
||||
"analysis": map[string]any{
|
||||
// path_hierarchy is case-preserving; casing lives in the value.
|
||||
"analyzer": map[string]any{
|
||||
"path_hierarchy": map[string]any{
|
||||
"type": "custom",
|
||||
"tokenizer": "path_hierarchy",
|
||||
"filter": []string{"lowercase"},
|
||||
},
|
||||
"lowercaseKeyword": map[string]any{
|
||||
"type": "custom",
|
||||
"tokenizer": "keyword",
|
||||
"filter": []string{"lowercase"},
|
||||
},
|
||||
},
|
||||
"tokenizer": map[string]any{
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
package convert
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
// LowerValues folds restriction values for fields whose index analyzer
|
||||
// lowercases (search.LowercaseValueFields, shared with bleve); case-preserved
|
||||
// fields keep their casing. Runs after query.Normalize, so keys are resolved.
|
||||
func LowerValues(nodes []ast.Node) []ast.Node {
|
||||
for _, n := range nodes {
|
||||
switch node := n.(type) {
|
||||
case *ast.StringNode:
|
||||
if _, ok := search.LowercaseValueFields()[node.Key]; ok {
|
||||
node.Value = strings.ToLower(node.Value)
|
||||
}
|
||||
case *ast.GroupNode:
|
||||
LowerValues(node.Nodes)
|
||||
}
|
||||
}
|
||||
return nodes
|
||||
}
|
||||
@@ -18,11 +18,10 @@ func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// shared lowering (field resolution + media-type), then value lowercasing.
|
||||
// shared lowering: field resolution, media-type expansion, value lowercasing.
|
||||
kqlAst = query.Normalize(kqlAst, query.ResolveField)
|
||||
kqlNodes := LowerValues(kqlAst.Nodes)
|
||||
|
||||
builder, err := TranspileKQLToOpenSearch(kqlNodes)
|
||||
builder, err := TranspileKQLToOpenSearch(kqlAst.Nodes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to compile query: %w", err)
|
||||
}
|
||||
|
||||
@@ -4,12 +4,12 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/pkg/kql"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
|
||||
)
|
||||
|
||||
@@ -99,7 +99,28 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
|
||||
case *ast.BooleanNode:
|
||||
return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil
|
||||
case *ast.StringNode:
|
||||
return stringNodeQuery(node), nil
|
||||
field, value := node.Key, node.Value
|
||||
if node.CaseInsensitive {
|
||||
field += mapping.LowercaseSuffix
|
||||
value = strings.ToLower(value)
|
||||
}
|
||||
|
||||
isWildcard := strings.Contains(value, "*")
|
||||
if isWildcard {
|
||||
return osu.NewWildcardQuery(field).Value(value), nil
|
||||
}
|
||||
|
||||
totalTerms := strings.Split(value, " ")
|
||||
isSingleTerm := len(totalTerms) == 1
|
||||
isMultiTerm := len(totalTerms) >= 1
|
||||
switch {
|
||||
case isSingleTerm:
|
||||
return osu.NewTermQuery[string](field).Value(value), nil
|
||||
case isMultiTerm:
|
||||
return osu.NewMatchPhraseQuery(field).Query(value), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported string node value: %s", value)
|
||||
case *ast.DateTimeNode:
|
||||
return dateTimeNodeQuery(node)
|
||||
case *ast.NumberNode:
|
||||
@@ -116,67 +137,26 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
|
||||
return nil, fmt.Errorf("%w: %T", ErrUnsupportedNodeType, node)
|
||||
}
|
||||
|
||||
// stringNodeQuery picks the query a string node turns into.
|
||||
func stringNodeQuery(node *ast.StringNode) osu.Builder {
|
||||
isWildcard := strings.ContainsAny(node.Value, "*?")
|
||||
|
||||
switch {
|
||||
// Name: "*oo-bar", "*oo ba*", "*OO*"
|
||||
// Title: "*rterly rep*"
|
||||
// Tags: "*spaced tag*"
|
||||
case isWildcard && slices.Contains([]string{"Name", "Title"}, node.Key):
|
||||
patterns := []osu.Builder{wildcardOn(node.Key+".wildcard", node.Value)}
|
||||
if !strings.HasSuffix(node.Value, "*") {
|
||||
patterns = append(patterns, wildcardOn(node.Key+".wildcard", node.Value+".*"))
|
||||
}
|
||||
|
||||
return osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Should(patterns...)
|
||||
// Tags: "*foo*", "*paced ta*"
|
||||
case isWildcard && node.Key == "Tags":
|
||||
return wildcardOn(node.Key+".wildcard", node.Value)
|
||||
// Path: "./foo*", MimeType: "*plain"
|
||||
case isWildcard:
|
||||
return osu.NewWildcardQuery(node.Key).Value(node.Value)
|
||||
// Name: =new, Title: ="quarterly report"
|
||||
case node.Exact && slices.Contains([]string{"Name", "Title"}, node.Key):
|
||||
return osu.NewTermQuery[string](node.Key + ".wildcard").
|
||||
Value(node.Value).
|
||||
Params(&osu.TermQueryParams{CaseInsensitive: true})
|
||||
// Tags: "foo-bar", "spaced tag", "FOO-BAR"
|
||||
case node.Key == "Tags":
|
||||
return osu.NewTermQuery[string](node.Key + ".wildcard").
|
||||
Value(node.Value).
|
||||
Params(&osu.TermQueryParams{CaseInsensitive: true})
|
||||
// Name: "foo-bar", "foo bar"
|
||||
// Title: "quarterly report"
|
||||
// Content: "foo bar"
|
||||
case slices.Contains([]string{"Name", "Title", "Content"}, node.Key):
|
||||
return osu.NewMatchPhraseQuery(node.Key).Query(node.Value)
|
||||
// Size: "42", Type: "1"
|
||||
case slices.Contains([]string{"Size", "Type"}, node.Key):
|
||||
number, err := strconv.ParseInt(node.Value, 10, 64)
|
||||
if err != nil {
|
||||
return osu.NewMatchNoneQuery()
|
||||
}
|
||||
|
||||
return osu.NewTermQuery[int64](node.Key).Value(number)
|
||||
// Path: "./foo bar/", the hierarchy tokens carry no trailing slash
|
||||
case node.Key == "Path":
|
||||
return osu.NewTermQuery[string](node.Key).Value(strings.TrimSuffix(node.Value, "/"))
|
||||
// Hidden: "TRUE" arrives lowered, anything that is no bool matches nothing
|
||||
case node.Key == "Hidden":
|
||||
value, err := strconv.ParseBool(node.Value)
|
||||
if err != nil {
|
||||
return osu.NewMatchNoneQuery()
|
||||
}
|
||||
|
||||
return osu.NewTermQuery[bool](node.Key).Value(value)
|
||||
// MimeType: "text/plain"
|
||||
default:
|
||||
return osu.NewTermQuery[string](node.Key).Value(node.Value)
|
||||
// dateTimeNodeQuery turns a date time node into a range query.
|
||||
func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) {
|
||||
if node.Operator == nil {
|
||||
return nil, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
|
||||
}
|
||||
|
||||
query := osu.NewRangeQuery[time.Time](node.Key)
|
||||
|
||||
switch node.Operator.Value {
|
||||
case ">":
|
||||
return query.Gt(node.Value), nil
|
||||
case ">=":
|
||||
return query.Gte(node.Value), nil
|
||||
case "<":
|
||||
return query.Lt(node.Value), nil
|
||||
case "<=":
|
||||
return query.Lte(node.Value), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
|
||||
}
|
||||
|
||||
func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) {
|
||||
@@ -203,31 +183,3 @@ func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) {
|
||||
|
||||
return nil, fmt.Errorf("unsupported operator %s for number node: %w", node.Operator.Value, ErrUnsupportedNodeType)
|
||||
}
|
||||
|
||||
func wildcardOn(field, value string) osu.Builder {
|
||||
return osu.NewWildcardQuery(field).
|
||||
Value(value).
|
||||
Params(&osu.WildcardQueryParams{CaseInsensitive: true})
|
||||
}
|
||||
|
||||
// dateTimeNodeQuery turns a date time node into a range query.
|
||||
func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) {
|
||||
if node.Operator == nil {
|
||||
return nil, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
|
||||
}
|
||||
|
||||
query := osu.NewRangeQuery[time.Time](node.Key)
|
||||
|
||||
switch node.Operator.Value {
|
||||
case ">":
|
||||
return query.Gt(node.Value), nil
|
||||
case ">=":
|
||||
return query.Gte(node.Value), nil
|
||||
case "<":
|
||||
return query.Lt(node.Value), nil
|
||||
case "<=":
|
||||
return query.Lte(node.Value), nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
|
||||
}
|
||||
@@ -16,13 +16,31 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{
|
||||
// kql to os dsl - type tests
|
||||
{
|
||||
Name: "match phrase query - string node on an analyzed field",
|
||||
Name: "term query - string node",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "openCloud"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
Want: osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
},
|
||||
{
|
||||
Name: "case-insensitive term routes to the lowercased sibling",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "openCloud", CaseInsensitive: true},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[string]("Name_lowercase").Value("opencloud"),
|
||||
},
|
||||
{
|
||||
Name: "case-insensitive wildcard routes to the lowercased sibling",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "Open*", CaseInsensitive: true},
|
||||
},
|
||||
},
|
||||
Want: osu.NewWildcardQuery("Name_lowercase").Value("open*"),
|
||||
},
|
||||
{
|
||||
Name: "term query - boolean node - true",
|
||||
@@ -58,16 +76,10 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
&ast.StringNode{Key: "Name", Value: "open*"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
osu.NewWildcardQuery("Name.wildcard").
|
||||
Value("open*").
|
||||
Params(&osu.WildcardQueryParams{CaseInsensitive: true}),
|
||||
),
|
||||
Want: osu.NewWildcardQuery("Name").Value("open*"),
|
||||
},
|
||||
{
|
||||
Name: "wildcard query - string node without an unanalyzed sub field",
|
||||
Name: "wildcard query - fulltext field",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Content", Value: "open*"},
|
||||
@@ -142,8 +154,8 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().Must(
|
||||
osu.NewMatchPhraseQuery("Name").Query("a"),
|
||||
osu.NewMatchPhraseQuery("Name").Query("b"),
|
||||
osu.NewTermQuery[string]("Name").Value("a"),
|
||||
osu.NewTermQuery[string]("Name").Value("b"),
|
||||
),
|
||||
},
|
||||
{
|
||||
@@ -155,7 +167,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
}},
|
||||
},
|
||||
},
|
||||
Want: osu.NewMatchPhraseQuery("Name").Query("any"),
|
||||
Want: osu.NewTermQuery[string]("Name").Value("any"),
|
||||
},
|
||||
{
|
||||
Name: "range query >",
|
||||
@@ -217,7 +229,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
&ast.StringNode{Key: "Name", Value: "openCloud"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
Want: osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
},
|
||||
{
|
||||
Name: "[* *]",
|
||||
@@ -229,7 +241,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
@@ -244,7 +256,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
@@ -260,7 +272,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
Want: osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
@@ -288,7 +300,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
).
|
||||
MustNot(
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
@@ -308,7 +320,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
Want: osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
osu.NewTermQuery[string]("age").Value("44"),
|
||||
),
|
||||
|
||||
@@ -12,31 +12,10 @@ import (
|
||||
bleveQuery "github.com/blevesearch/bleve/v2/search/query"
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/pkg/kql"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
searchQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query"
|
||||
)
|
||||
|
||||
// lowercaseFields holds the fields whose query-side value is pre-lowercased so
|
||||
// it matches the index-time lowercasing analyzer. Shared with the OpenSearch
|
||||
// backend via search.LowercaseValueFields; every other field keeps its casing.
|
||||
var lowercaseFields = search.LowercaseValueFields()
|
||||
|
||||
var _fields = map[string]string{
|
||||
"rootid": "RootID",
|
||||
"path": "Path",
|
||||
"id": "ID",
|
||||
"name": "Name",
|
||||
"size": "Size",
|
||||
"mtime": "Mtime",
|
||||
"mediatype": "MimeType",
|
||||
"type": "Type",
|
||||
"tag": "Tags",
|
||||
"tags": "Tags",
|
||||
"content": "Content",
|
||||
"title": "Title",
|
||||
"hidden": "Hidden",
|
||||
"favorite": "Favorites",
|
||||
}
|
||||
|
||||
// The following quoted string enumerates the characters which may be escaped: "+-=&|><!(){}[]^\"~*?:\\/ "
|
||||
// based on bleve docs https://blevesearch.com/docs/Query-String-Query/
|
||||
// Wildcards * and ? are excluded
|
||||
@@ -102,12 +81,21 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
|
||||
if k != "ID" && k != "Size" && k != "MimeType" {
|
||||
v = bleveEscaper.Replace(n.Value)
|
||||
}
|
||||
|
||||
if _, ok := lowercaseFields[k]; ok {
|
||||
if n.CaseInsensitive {
|
||||
k += mapping.LowercaseSuffix
|
||||
v = strings.ToLower(v)
|
||||
}
|
||||
|
||||
q := bleveQuery.NewQueryStringQuery(k + ":" + v)
|
||||
var q bleveQuery.Query = bleveQuery.NewQueryStringQuery(k + ":" + v)
|
||||
if searchQuery.FieldIsPath(n.Key) {
|
||||
// bleve has no path hierarchy analyzer, unlike OpenSearch: match
|
||||
// the folder itself and its descendants (`\/*`, a trailing
|
||||
// wildcard on the value).
|
||||
q = bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{
|
||||
q,
|
||||
bleveQuery.NewQueryStringQuery(k + ":" + v + `\/*`),
|
||||
})
|
||||
}
|
||||
|
||||
if prev == nil {
|
||||
prev = q
|
||||
@@ -151,7 +139,7 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
|
||||
}
|
||||
case *ast.NumberNode:
|
||||
var q bleveQuery.Query
|
||||
if field := getField(n.Key); slices.Contains([]string{"Size", "Type"}, field) {
|
||||
if field := n.Key; slices.Contains([]string{"Size", "Type"}, field) {
|
||||
q = numberRange(field, n.Operator, n.Value)
|
||||
} else {
|
||||
// same answer as the OpenSearch backend: unknown numeric keys
|
||||
@@ -169,7 +157,7 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
|
||||
}
|
||||
case *ast.BooleanNode:
|
||||
q := bleveQuery.NewBoolFieldQuery(n.Value)
|
||||
q.SetField(getField(n.Key))
|
||||
q.SetField(n.Key)
|
||||
if prev == nil {
|
||||
prev = q
|
||||
} else {
|
||||
@@ -340,16 +328,6 @@ func phrase(field, value string) bleveQuery.Query {
|
||||
return q
|
||||
}
|
||||
|
||||
func getField(name string) string {
|
||||
if name == "" {
|
||||
return "Name"
|
||||
}
|
||||
if _, ok := _fields[strings.ToLower(name)]; ok {
|
||||
return _fields[strings.ToLower(name)]
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode {
|
||||
for _, n := range group.Nodes {
|
||||
if onode, ok := n.(*ast.StringNode); ok {
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package bleve
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -20,29 +19,11 @@ var timeMustParse = func(t *testing.T, ts string) time.Time {
|
||||
return tp
|
||||
}
|
||||
|
||||
func wildcardQuery(field, value string) query.Query {
|
||||
patterns := []query.Query{query.NewQueryStringQuery(field + ".wildcard:" + value)}
|
||||
if !strings.HasSuffix(value, "*") {
|
||||
patterns = append(patterns, query.NewQueryStringQuery(field+".wildcard:"+value+".*"))
|
||||
}
|
||||
|
||||
return query.NewConjunctionQuery([]query.Query{query.NewDisjunctionQuery(patterns)})
|
||||
}
|
||||
|
||||
func phraseQuery(field, value string) query.Query {
|
||||
q := query.NewMatchPhraseQuery(value)
|
||||
q.SetField(field)
|
||||
|
||||
return q
|
||||
}
|
||||
|
||||
func boolFieldQuery(field string, value bool) query.Query {
|
||||
q := query.NewBoolFieldQuery(value)
|
||||
q.SetField(field)
|
||||
|
||||
return q
|
||||
}
|
||||
|
||||
// TODO(followup): make this a pure compiler test. Field resolution and
|
||||
// media-type expansion live in query.Normalize, so this test could feed
|
||||
// canonical ASTs (real field names, media-type already expanded) and call
|
||||
// compile() directly, dropping the query.Normalize wrapper and the mediatype
|
||||
// cases.
|
||||
func Test_compile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -58,7 +39,22 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
phraseQuery("Name", "federated"),
|
||||
query.NewQueryStringQuery(`Name_lowercase:federated`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
// path fields expand to match the folder itself and its descendants,
|
||||
// since bleve has no path hierarchy analyzer.
|
||||
name: `path:/Foo`,
|
||||
args: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "path", Value: "/Foo"},
|
||||
},
|
||||
},
|
||||
want: query.NewDisjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Path_lowercase:\/foo`),
|
||||
query.NewQueryStringQuery(`Path_lowercase:\/foo\/*`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -70,7 +66,7 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
phraseQuery("Name", "John Smith"),
|
||||
query.NewQueryStringQuery(`Name_lowercase:john\ smith`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -84,8 +80,8 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
phraseQuery("Name", "John Smith"),
|
||||
phraseQuery("Name", "Jane"),
|
||||
query.NewQueryStringQuery(`Name_lowercase:john\ smith`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:jane`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -99,8 +95,8 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Tags:bestseller`),
|
||||
query.NewQueryStringQuery(`Tags:book`),
|
||||
query.NewQueryStringQuery(`Tags_lowercase:bestseller`),
|
||||
query.NewQueryStringQuery(`Tags_lowercase:book`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -116,10 +112,10 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewDisjunctionQuery([]query.Query{
|
||||
wildcardQuery("Name", `moby\ di*`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:moby\ di*`),
|
||||
query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Tags:bestseller`),
|
||||
query.NewQueryStringQuery(`Tags:book`),
|
||||
query.NewQueryStringQuery(`Tags_lowercase:bestseller`),
|
||||
query.NewQueryStringQuery(`Tags_lowercase:book`),
|
||||
}),
|
||||
}),
|
||||
wantErr: false,
|
||||
@@ -137,10 +133,10 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
want: query.NewDisjunctionQuery([]query.Query{
|
||||
query.NewConjunctionQuery([]query.Query{
|
||||
phraseQuery("Name", "a"),
|
||||
phraseQuery("Name", "b"),
|
||||
query.NewQueryStringQuery(`Name_lowercase:a`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:b`),
|
||||
}),
|
||||
phraseQuery("Name", "c"),
|
||||
query.NewQueryStringQuery(`Name_lowercase:c`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -156,10 +152,10 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewDisjunctionQuery([]query.Query{
|
||||
phraseQuery("Name", "a"),
|
||||
query.NewQueryStringQuery(`Name_lowercase:a`),
|
||||
query.NewConjunctionQuery([]query.Query{
|
||||
phraseQuery("Name", "b"),
|
||||
phraseQuery("Name", "c"),
|
||||
query.NewQueryStringQuery(`Name_lowercase:b`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:c`),
|
||||
}),
|
||||
}),
|
||||
wantErr: false,
|
||||
@@ -181,11 +177,11 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewDisjunctionQuery([]query.Query{
|
||||
phraseQuery("Name", "a"),
|
||||
phraseQuery("Name", "b"),
|
||||
phraseQuery("Name", "c"),
|
||||
query.NewQueryStringQuery(`Name_lowercase:a`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:b`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:c`),
|
||||
}),
|
||||
phraseQuery("Name", "d"),
|
||||
query.NewQueryStringQuery(`Name_lowercase:d`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -204,10 +200,10 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewDisjunctionQuery([]query.Query{
|
||||
wildcardQuery("Name", `moby\ di*`),
|
||||
query.NewQueryStringQuery(`Tags:bestseller`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:moby\ di*`),
|
||||
query.NewQueryStringQuery(`Tags_lowercase:bestseller`),
|
||||
}),
|
||||
query.NewQueryStringQuery(`Tags:book`),
|
||||
query.NewQueryStringQuery(`Tags_lowercase:book`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -229,11 +225,11 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewDisjunctionQuery([]query.Query{
|
||||
wildcardQuery("Name", `moby\ di*`),
|
||||
query.NewQueryStringQuery(`Tags:bestseller`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:moby\ di*`),
|
||||
query.NewQueryStringQuery(`Tags_lowercase:bestseller`),
|
||||
}),
|
||||
query.NewQueryStringQuery(`Tags:book`),
|
||||
query.NewBooleanQuery(nil, nil, []query.Query{query.NewQueryStringQuery(`Tags:read`)}),
|
||||
query.NewQueryStringQuery(`Tags_lowercase:book`),
|
||||
query.NewBooleanQuery(nil, nil, []query.Query{query.NewQueryStringQuery(`Tags_lowercase:read`)}),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -252,7 +248,7 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
phraseQuery("author", "John Smith"),
|
||||
query.NewQueryStringQuery(`author:John\ Smith`),
|
||||
query.NewQueryStringQuery(`author:Jane`),
|
||||
}),
|
||||
wantErr: false,
|
||||
@@ -274,9 +270,9 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
phraseQuery("author", "John Smith"),
|
||||
query.NewQueryStringQuery(`author:John\ Smith`),
|
||||
query.NewQueryStringQuery(`author:Jane`),
|
||||
query.NewQueryStringQuery(`Tags:bestseller`),
|
||||
query.NewQueryStringQuery(`Tags_lowercase:bestseller`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -318,44 +314,9 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
phraseQuery("Name", "John Smith"),
|
||||
boolFieldQuery("Hidden", true),
|
||||
boolFieldQuery("Hidden", true),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: `hidden:banana`,
|
||||
args: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "hidden", Value: "banana"},
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{query.NewMatchNoneQuery()}),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: `name="Report.txt"`,
|
||||
args: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "name", Value: "Report.txt", Exact: true},
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{query.NewQueryStringQuery(`Name.wildcard:report.txt`)}),
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: `type:File`,
|
||||
args: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "type", Value: "File"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "type", Value: "FOLDER"},
|
||||
},
|
||||
},
|
||||
want: query.NewDisjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Type:1`),
|
||||
query.NewQueryStringQuery(`Type:2`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:john\ smith`),
|
||||
query.NewQueryStringQuery(`Hidden:T`),
|
||||
query.NewQueryStringQuery(`Hidden:T`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -368,7 +329,7 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewBooleanQuery(nil, nil, []query.Query{query.NewQueryStringQuery(`Tags:physik`)}),
|
||||
query.NewBooleanQuery(nil, nil, []query.Query{query.NewQueryStringQuery(`Tags_lowercase:physik`)}),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -484,7 +445,7 @@ func Test_compile(t *testing.T) {
|
||||
query.NewQueryStringQuery(`MimeType:application/rtf`),
|
||||
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
|
||||
}),
|
||||
wildcardQuery("Name", `*tdd*`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:*tdd*`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -510,7 +471,7 @@ func Test_compile(t *testing.T) {
|
||||
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
|
||||
query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`MimeType:application/pdf`),
|
||||
wildcardQuery("Name", `*tdd*`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:*tdd*`),
|
||||
}),
|
||||
}),
|
||||
wantErr: false,
|
||||
@@ -540,7 +501,7 @@ func Test_compile(t *testing.T) {
|
||||
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
|
||||
query.NewQueryStringQuery(`MimeType:application/pdf`),
|
||||
}),
|
||||
wildcardQuery("Name", `*tdd*`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:*tdd*`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -569,7 +530,7 @@ func Test_compile(t *testing.T) {
|
||||
query.NewQueryStringQuery(`MimeType:application/rtf`),
|
||||
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
|
||||
}),
|
||||
wildcardQuery("Name", `*tdd*`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:*tdd*`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -581,7 +542,7 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
phraseQuery("Name", "John Smith +-=&|><!(){}[]^\"~: "),
|
||||
query.NewQueryStringQuery(`Name_lowercase:john\ smith\ \+\-\=\&\|\>\<\!\(\)\{\}\[\]\^\"\~\:\ `),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
|
||||
@@ -2,6 +2,7 @@ package query
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/query/mimetype"
|
||||
@@ -31,10 +32,14 @@ func normalizeNodes(nodes []ast.Node, resolve func(string) string, defaultKey st
|
||||
switch node := n.(type) {
|
||||
case *ast.StringNode:
|
||||
node.Key = resolveKey(node.Key)
|
||||
if FieldValueIsNormalized(node.Key) {
|
||||
node.Value = strings.ToLower(node.Value)
|
||||
}
|
||||
if exp := mimetype.Expand(node.Key, node.Value); exp != nil {
|
||||
out = append(out, normalizeNodes(exp, resolve, defaultKey)...)
|
||||
continue
|
||||
}
|
||||
node.CaseInsensitive = FieldIsCaseInsensitive(node.Key)
|
||||
out = append(out, node)
|
||||
case *ast.DateTimeNode:
|
||||
node.Key = resolveKey(node.Key)
|
||||
|
||||
@@ -20,13 +20,24 @@ func norm(nodes ...ast.Node) []ast.Node {
|
||||
|
||||
func TestResolveField(t *testing.T) {
|
||||
require.Equal(t, "Name", query.ResolveField("")) // empty -> free-text default
|
||||
require.Equal(t, "Name", query.ResolveField("NAME")) // case-insensitive
|
||||
require.Equal(t, "Name", query.ResolveField("NAME")) // canonical, case-insensitive key match
|
||||
require.Equal(t, "Tags", query.ResolveField("tag")) // singular alias
|
||||
require.Equal(t, "MimeType", query.ResolveField("mimetype")) // real field, case-insensitive
|
||||
require.Equal(t, "photo.cameraMake", query.ResolveField("photo.CAMERAMAKE")) // facet, case-insensitive
|
||||
require.Equal(t, "MimeType", query.ResolveField("mimetype")) // real field
|
||||
require.Equal(t, "photo.cameraMake", query.ResolveField("photo.CAMERAMAKE")) // facet, case-insensitive key match
|
||||
require.Equal(t, "unknown.field", query.ResolveField("unknown.field")) // unknown key: unchanged, becomes a dead query
|
||||
}
|
||||
|
||||
func TestFieldIsCaseInsensitive(t *testing.T) {
|
||||
// The four CaseInsensitive override fields (resolved canonical names).
|
||||
for _, f := range []string{"Name", "Path", "Tags", "Favorites"} {
|
||||
require.True(t, query.FieldIsCaseInsensitive(f), f)
|
||||
}
|
||||
// Case-preserved / non-keyword fields are not.
|
||||
for _, f := range []string{"MimeType", "ID", "Content", "unknown"} {
|
||||
require.False(t, query.FieldIsCaseInsensitive(f), f)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) {
|
||||
got := norm(
|
||||
&ast.StringNode{Key: "", Value: "free"},
|
||||
@@ -40,9 +51,9 @@ func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) {
|
||||
ast.NumberNode{Key: "size", Value: 100},
|
||||
)
|
||||
require.Equal(t, []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "free"},
|
||||
&ast.StringNode{Key: "Name", Value: "free", CaseInsensitive: true},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Tags", Value: "x"},
|
||||
&ast.StringNode{Key: "Tags", Value: "x", CaseInsensitive: true},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "photo.cameraMake", Value: "canon"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
@@ -71,11 +82,11 @@ func TestNormalize_GroupKeyDefaulting(t *testing.T) {
|
||||
&ast.GroupNode{Key: "author", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "author", Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "Name", Value: "d"},
|
||||
&ast.StringNode{Key: "Name", Value: "d", CaseInsensitive: true},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "e"},
|
||||
&ast.StringNode{Key: "Name", Value: "e", CaseInsensitive: true},
|
||||
}},
|
||||
}, got)
|
||||
}
|
||||
@@ -87,7 +98,7 @@ func TestNormalize_ConvertsValueNodesToPointers(t *testing.T) {
|
||||
ast.DateTimeNode{Key: "mtime"},
|
||||
)
|
||||
require.Equal(t, []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "x"},
|
||||
&ast.StringNode{Key: "Name", Value: "x", CaseInsensitive: true},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.DateTimeNode{Key: "Mtime"},
|
||||
}, got)
|
||||
|
||||
@@ -15,27 +15,72 @@ var aliases = map[string]string{
|
||||
"favorite": "Favorites",
|
||||
}
|
||||
|
||||
// fieldIndex maps a lowercased KQL key to the real field name: derived once from
|
||||
// the resource struct, overlaid with the explicit aliases.
|
||||
// fieldIndex maps a lowercased KQL key to its canonical field name ("" is the
|
||||
// bare-search default).
|
||||
var fieldIndex = sync.OnceValue(func() map[string]string {
|
||||
idx := mapping.FieldNameIndex(
|
||||
reflect.TypeFor[search.Resource](),
|
||||
search.Resource{}.SearchFieldOverrides(),
|
||||
)
|
||||
idx := mapping.FieldNameIndex(reflect.TypeFor[search.Resource](), search.Resource{}.SearchFieldOverrides())
|
||||
for k, v := range aliases {
|
||||
idx[k] = v
|
||||
}
|
||||
idx[""] = idx["name"]
|
||||
return idx
|
||||
})
|
||||
|
||||
// ResolveField maps a KQL key to the index field name: empty -> Name, a known
|
||||
// key (case-insensitive) -> its field, anything else unchanged.
|
||||
func ResolveField(name string) string {
|
||||
if name == "" {
|
||||
return "Name"
|
||||
// caseInsensitiveFields are the fields searched case-insensitively by default,
|
||||
// derived from the CaseInsensitive overrides.
|
||||
var caseInsensitiveFields = sync.OnceValue(func() map[string]struct{} {
|
||||
out := map[string]struct{}{}
|
||||
for field, opts := range (search.Resource{}).SearchFieldOverrides() {
|
||||
if opts.CaseInsensitive != nil && *opts.CaseInsensitive {
|
||||
out[field] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
})
|
||||
|
||||
// pathFields are hierarchical path fields (TypePath), derived from the overrides.
|
||||
var pathFields = sync.OnceValue(func() map[string]struct{} {
|
||||
out := map[string]struct{}{}
|
||||
for field, opts := range (search.Resource{}).SearchFieldOverrides() {
|
||||
if opts.Type == mapping.TypePath {
|
||||
out[field] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
})
|
||||
|
||||
// ResolveField maps a KQL key to its canonical field name; unknown keys pass through.
|
||||
func ResolveField(name string) string {
|
||||
if v, ok := fieldIndex()[strings.ToLower(name)]; ok {
|
||||
return v
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// normalizedValueFields have their stored values normalized to lowercase at
|
||||
// index time, so query values fold to match even though the fields themselves
|
||||
// are case-preserved keywords.
|
||||
var normalizedValueFields = map[string]struct{}{
|
||||
"MimeType": {},
|
||||
"Type": {},
|
||||
"Hidden": {},
|
||||
}
|
||||
|
||||
// FieldValueIsNormalized reports whether a field's stored values are
|
||||
// normalized lowercase.
|
||||
func FieldValueIsNormalized(field string) bool {
|
||||
_, ok := normalizedValueFields[field]
|
||||
return ok
|
||||
}
|
||||
|
||||
// FieldIsCaseInsensitive reports whether a field's default search is case-insensitive.
|
||||
func FieldIsCaseInsensitive(field string) bool {
|
||||
_, ok := caseInsensitiveFields()[field]
|
||||
return ok
|
||||
}
|
||||
|
||||
// FieldIsPath reports whether a field is a hierarchical path field.
|
||||
func FieldIsPath(field string) bool {
|
||||
_, ok := pathFields()[field]
|
||||
return ok
|
||||
}
|
||||
@@ -72,12 +72,13 @@ type Resource struct {
|
||||
// resourceFieldOverrides is built once (it never changes) and reused on hot
|
||||
// paths instead of reallocating per call.
|
||||
var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts {
|
||||
excludeFromAll := false
|
||||
True, False := true, false
|
||||
return map[string]mapping.FieldOpts{
|
||||
"Name": {Analyzer: "lowercaseKeyword"},
|
||||
"Name": {CaseInsensitive: &True},
|
||||
"Path": {Type: mapping.TypePath, CaseInsensitive: &True},
|
||||
"Content": {Type: mapping.TypeFulltext},
|
||||
"Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll},
|
||||
"Favorites": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll},
|
||||
"Tags": {CaseInsensitive: &True, IncludeInAll: &False},
|
||||
"Favorites": {CaseInsensitive: &True, IncludeInAll: &False},
|
||||
"location": {Type: mapping.TypeGeopoint},
|
||||
}
|
||||
})
|
||||
@@ -89,32 +90,6 @@ func (Resource) SearchFieldOverrides() map[string]mapping.FieldOpts {
|
||||
return resourceFieldOverrides()
|
||||
}
|
||||
|
||||
// lowercaseValueFields is the set of index field names whose query values must
|
||||
// be lowercased to match their index-time lowercasing analyzer (lowercaseKeyword
|
||||
// or the fulltext type). Built once from the field overrides.
|
||||
var lowercaseValueFields = sync.OnceValue(func() map[string]struct{} {
|
||||
out := map[string]struct{}{}
|
||||
for key, opts := range resourceFieldOverrides() {
|
||||
if opts.Analyzer == "lowercaseKeyword" || opts.Type == mapping.TypeFulltext {
|
||||
out[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
// stored values are normalized lowercase, so query values must fold too
|
||||
// even though the index fields preserve case
|
||||
for _, key := range []string{"MimeType", "Type", "Hidden"} {
|
||||
out[key] = struct{}{}
|
||||
}
|
||||
return out
|
||||
})
|
||||
|
||||
// LowercaseValueFields returns the set of index field names whose query values
|
||||
// must be lowercased so query-side matching lines up with the index-time
|
||||
// analyzer. Both search backends use it, so value casing stays consistent; every
|
||||
// other (case-preserved) field keeps its original case. Read-only, do not mutate.
|
||||
func LowercaseValueFields() map[string]struct{} {
|
||||
return lowercaseValueFields()
|
||||
}
|
||||
|
||||
// ResolveReference makes sure the path is relative to the space root
|
||||
func ResolveReference(ctx context.Context, ref *provider.Reference, ri *provider.ResourceInfo, gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) (*provider.Reference, error) {
|
||||
if ref.GetResourceId().GetOpaqueId() == ref.GetResourceId().GetSpaceId() {
|
||||
|
||||
Reference in new issue
Block a user