feat(search): per-field case-insensitive search via _lowercase siblings

Keyword and path fields always index their case-preserved base and, when CaseInsensitive is set, an additional <field>_lowercase sibling used only for matching. The KQL lowering marks a restriction case-insensitive; each backend searches the sibling and lowercases the query value the same way the sibling is precomputed at index time (Go strings.ToLower on both sides, so non-ASCII stays consistent).

Search always returns the case-preserved base, so the sibling never has to be read back. In bleve it is indexed but not stored, kept out of _all, and without doc values. In OpenSearch it deliberately stays in _source: excluding it would make every update-by-query script rebuild all siblings from the document via painless toLowerCase, which lowercases differently than Go and would drift from the query side. Keeping it in _source avoids that, and a lowercased copy of a name or path is negligible disk in a cluster.

The OpenSearch move script keeps the base and its sibling in sync by swapping the moved prefix in Path_lowercase and setting Name_lowercase from Go-lowercased params, so case-insensitive search still finds a file after it moves (previously the sibling went stale). bleve re-indexes the whole document on move/delete/restore, so its siblings stay fresh for free.

This also repairs OpenSearch path search (the query value was no longer folded to lowercase, so path:<Foo> returned nothing) and makes bleve path queries match a folder and its descendants like OpenSearch's path_hierarchy. The Path base stays case-preserved so the move/delete descendant update (an exact TermQuery on Path) matches mixed-case folders.
This commit is contained in:
Dominik Schmidt committed 2026-08-31 13:40:42 +02:00
1 parent fb22dd81a4
commit ec58861e4e
24 files changed
+1309 -440

No files matched your search

+3
View File
@@ -44,6 +44,9 @@ type StringNode struct {
Key string
Value string
Exact bool
// CaseInsensitive marks a case-insensitive restriction; set by the search
// lowering pass, a backend routes it to the field's lowercased form.
CaseInsensitive bool
}
// BooleanNode represents a bool value
+842
View File
@@ -0,0 +1,842 @@
package bleve_test
import (
"context"
"fmt"
bleveSearch "github.com/blevesearch/bleve/v2"
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/opencloud/pkg/log"
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func hiddenByID(idx bleveSearch.Index, id string) bool {
GinkgoHelper()
req := bleveSearch.NewSearchRequest(bleveSearch.NewDocIDQuery([]string{id}))
req.Fields = []string{"Hidden"}
res, err := idx.Search(req)
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits).To(HaveLen(1), "no record for %s", id)
hidden, _ := res.Hits[0].Fields["Hidden"].(bool)
return hidden
}
var _ = Describe("Bleve", func() {
var (
eng *bleve.Backend
idx bleveSearch.Index
doSearch = func(id string, query, path string) (*searchsvc.SearchIndexResponse, error) {
rID, err := storagespace.ParseID(id)
if err != nil {
return nil, err
}
return eng.Search(context.Background(), &searchsvc.SearchIndexRequest{
Query: query,
Ref: &searchmsg.Reference{
ResourceId: &searchmsg.ResourceID{
StorageId: rID.StorageId,
SpaceId: rID.SpaceId,
OpaqueId: rID.OpaqueId,
},
Path: path,
},
})
}
assertDocCount = func(id string, query string, expectedCount int) []*searchmsg.Match {
res, err := doSearch(id, query, "")
ExpectWithOffset(1, err).ToNot(HaveOccurred())
ExpectWithOffset(1, len(res.Matches)).To(Equal(expectedCount), "query returned unexpected number of results: "+query)
return res.Matches
}
rootResource search.Resource
parentResource search.Resource
childResource search.Resource
childResource2 search.Resource
)
BeforeEach(func() {
mapping, err := bleve.NewMapping()
Expect(err).ToNot(HaveOccurred())
idx, err = bleveSearch.NewMemOnly(mapping)
Expect(err).ToNot(HaveOccurred())
eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{})
Expect(err).ToNot(HaveOccurred())
rootResource = search.Resource{
ID: "1$2!2",
RootID: "1$2!2",
Path: ".",
Document: content.Document{},
}
parentResource = search.Resource{
ID: "1$2!3",
ParentID: rootResource.ID,
RootID: rootResource.ID,
Path: "./parent d!r",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER),
Document: content.Document{Name: "parent d!r"},
}
childResource = search.Resource{
ID: "1$2!4",
ParentID: parentResource.ID,
RootID: rootResource.ID,
Path: "./parent d!r/child.pdf",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{Name: "child.pdf"},
}
childResource2 = search.Resource{
ID: "1$2!5",
ParentID: parentResource.ID,
RootID: rootResource.ID,
Path: "./parent d!r/child2.pdf",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{Name: "child2.pdf"},
}
})
Describe("PurgeSpace", func() {
It("takes every record of that space out of the index", func() {
otherSpace := search.Resource{
ID: "1$9!9",
RootID: "1$9!9",
Path: ".",
Document: content.Document{Name: "other"},
}
for _, resource := range []search.Resource{rootResource, parentResource, childResource, otherSpace} {
Expect(eng.Upsert(resource.ID, resource)).To(Succeed())
}
Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed())
count, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(1)), "only the records of that space are gone")
})
It("takes a space out that holds more records than one round", func() {
otherSpace := search.Resource{
ID: "1$9!9",
RootID: "1$9!9",
Path: ".",
Document: content.Document{Name: "other"},
}
Expect(eng.Upsert(otherSpace.ID, otherSpace)).To(Succeed())
for i := range 120 {
resource := search.Resource{
ID: fmt.Sprintf("%s!file-%d", rootResource.RootID, i),
RootID: rootResource.RootID,
Path: fmt.Sprintf("./file-%d", i),
Document: content.Document{Name: fmt.Sprintf("file-%d", i)},
}
Expect(eng.Upsert(resource.ID, resource)).To(Succeed())
}
Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed())
count, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(1)), "only the record of the other space is left")
})
})
Describe("New", func() {
It("returns a new index instance", func() {
b := bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{})
Expect(b).ToNot(BeNil())
})
})
Describe("Search", func() {
Context("by other fields than filename", func() {
It("finds files by tags", func() {
parentResource.Document.Tags = []string{"foo", "bar"}
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Tags:foo", 1)
assertDocCount(rootResource.ID, "Tags:bar", 1)
assertDocCount(rootResource.ID, "Tags:foo Tags:bar", 1)
assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1)
assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1)
assertDocCount(rootResource.ID, "Tags:baz", 0)
})
It("finds files by size", func() {
parentResource.Document.Size = 12345
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Size:12345", 1)
assertDocCount(rootResource.ID, "Size:>1000", 1)
assertDocCount(rootResource.ID, "Size:<100000", 1)
assertDocCount(rootResource.ID, "Size:12344", 0)
assertDocCount(rootResource.ID, "Size:<1000", 0)
assertDocCount(rootResource.ID, "Size:>100000", 0)
})
It("preserves value case for fields not explicitly marked lowercase", func() {
parentResource.Document.Audio = &libregraph.Audio{
Artist: libregraph.PtrString("Some Artist"),
}
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `audio.artist:"Some Artist"`, 1)
assertDocCount(rootResource.ID, `audio.artist:"some artist"`, 0)
})
})
Context("by filename", func() {
It("finds files with spaces in the filename", func() {
parentResource.Document.Name = "Foo oo.pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `name:"foo o*"`, 1)
})
It("finds files by digits in the filename", func() {
parentResource.Document.Name = "12345.pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:1234*", 1)
})
It("filters hidden files", func() {
childResource.Hidden = true
err := eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Hidden:T", 1)
assertDocCount(rootResource.ID, "Hidden:F", 0)
})
Context("with a file in the root of the space", func() {
It("scopes the search to the specified space", func() {
parentResource.Document.Name = "foo.pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:foo.pdf", 1)
assertDocCount("9$8!7", "Name:foo.pdf", 0)
})
})
It("limits the search to the specified fields", func() {
parentResource.Document.Name = "bar.pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:bar.pdf", 1)
assertDocCount(rootResource.ID, "Unknown:field", 0)
})
It("returns the total number of hits", func() {
parentResource.Document.Name = "bar.pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
res, err := doSearch(rootResource.ID, "Name:bar*", "")
Expect(err).ToNot(HaveOccurred())
Expect(res.TotalMatches).To(Equal(int32(1)))
})
It("returns all desired fields", func() {
parentResource.Document.Name = "bar.pdf"
parentResource.Type = 3
parentResource.MimeType = "application/pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
matches := assertDocCount(rootResource.ID, fmt.Sprintf("Name:%s", parentResource.Name), 1)
match := matches[0]
Expect(match.Entity.Ref.Path).To(Equal(parentResource.Path))
Expect(match.Entity.Name).To(Equal(parentResource.Name))
Expect(match.Entity.Size).To(Equal(parentResource.Size))
Expect(match.Entity.Type).To(Equal(parentResource.Type))
Expect(match.Entity.MimeType).To(Equal(parentResource.MimeType))
Expect(match.Entity.Deleted).To(BeFalse())
Expect(match.Score > 0).To(BeTrue())
})
It("finds files by name, prefix or substring match", func() {
parentResource.Document.Name = "foo.pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
queries := []string{"foo.pdf", "foo*", "*oo.p*"}
for _, query := range queries {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, query, 1)
}
})
It("does a case-insensitive search", func() {
parentResource.Document.Name = "foo.pdf"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:foo*", 1)
assertDocCount(rootResource.ID, "Name:Foo*", 1)
})
Context("and an additional file in a subdirectory", func() {
BeforeEach(func() {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
})
It("finds files living deeper in the tree by filename, prefix or substring match", func() {
queries := []string{"child.pdf", "child*", "*ld.*"}
for _, query := range queries {
assertDocCount(rootResource.ID, query, 1)
}
})
})
})
Context("by path", func() {
BeforeEach(func() {
for _, r := range []search.Resource{parentResource, childResource, childResource2} {
Expect(eng.Upsert(r.ID, r)).To(Succeed())
}
})
It("matches a folder and its descendants", func() {
assertDocCount(rootResource.ID, `path:"./parent d!r"`, 3)
})
It("matches a descendant path only itself", func() {
assertDocCount(rootResource.ID, `path:"./parent d!r/child.pdf"`, 1)
})
It("matches case-insensitively", func() {
assertDocCount(rootResource.ID, `path:"./PARENT D!R"`, 3)
})
})
Context("Highlights", func() {
It("highlights only for content searches", func() {
parentResource.Document.Name = "baz.pdf"
parentResource.Document.Content = "foo bar baz"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
res, err := doSearch(rootResource.ID, "Name:baz*", "")
Expect(err).ToNot(HaveOccurred())
Expect(res.TotalMatches).To(Equal(int32(1)))
Expect(res.Matches[0].Entity.Highlights).To(Equal(""))
})
It("highlights search terms", func() {
parentResource.Document.Name = "baz.pdf"
parentResource.Document.Content = "foo bar baz"
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
res, err := doSearch(rootResource.ID, "Content:bar", "")
Expect(err).ToNot(HaveOccurred())
Expect(res.TotalMatches).To(Equal(int32(1)))
Expect(res.Matches[0].Entity.Highlights).To(Equal("foo <mark>bar</mark> baz"))
})
})
Context("with a file in the root of the space and folder with a file. all of them have the same name", func() {
BeforeEach(func() {
parentResource := search.Resource{
ID: "1$2!3",
ParentID: rootResource.ID,
RootID: rootResource.ID,
Path: "./doc",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER),
Document: content.Document{Name: "doc"},
}
childResource := search.Resource{
ID: "1$2!4",
ParentID: parentResource.ID,
RootID: rootResource.ID,
Path: "./doc/doc.pdf",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{Name: "doc.pdf"},
}
childResource2 := search.Resource{
ID: "1$2!7",
ParentID: parentResource.ID,
RootID: rootResource.ID,
Path: "./doc/file.pdf",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{Name: "file.pdf"},
}
rootChildResource := search.Resource{
ID: "1$2!5",
ParentID: rootResource.ID,
RootID: rootResource.ID,
Path: "./doc.pdf",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{Name: "doc.pdf"},
}
rootChildResource2 := search.Resource{
ID: "1$2!6",
ParentID: rootResource.ID,
RootID: rootResource.ID,
Path: "./file.pdf",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{Name: "file.pdf"},
}
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(rootChildResource.ID, rootChildResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(rootChildResource2.ID, rootChildResource2)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource2.ID, childResource2)
Expect(err).ToNot(HaveOccurred())
})
It("search *doc* in a root", func() {
res, err := doSearch(rootResource.ID, "Name:*doc*", "")
Expect(err).ToNot(HaveOccurred())
Expect(res.TotalMatches).To(Equal(int32(3)))
})
It("search *doc* in a subfolder", func() {
res, err := doSearch(rootResource.ID, "Name:*doc*", "./doc")
Expect(err).ToNot(HaveOccurred())
Expect(res.TotalMatches).To(Equal(int32(2)))
})
It("search *file* in a root", func() {
res, err := doSearch(rootResource.ID, "Name:*file*", "")
Expect(err).ToNot(HaveOccurred())
Expect(res.TotalMatches).To(Equal(int32(2)))
})
It("search *file* in a subfolder", func() {
res, err := doSearch(rootResource.ID, "Name:*file*", "./doc")
Expect(err).ToNot(HaveOccurred())
Expect(res.TotalMatches).To(Equal(int32(1)))
})
})
})
Describe("Upsert", func() {
It("adds a resourceInfo to the index", func() {
err := eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
count, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(1)))
query := bleveSearch.NewMatchQuery("child.pdf")
res, err := idx.Search(bleveSearch.NewSearchRequest(query))
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits.Len()).To(Equal(1))
})
It("updates an existing resource in the index", func() {
err := eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
countA, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(countA).To(Equal(uint64(1)))
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
countB, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(countB).To(Equal(uint64(1)))
})
})
Describe("Delete", func() {
It("marks a resource as deleted", func() {
err := eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:*child*", 1)
err = eng.Delete(childResource.ID)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:*child*", 0)
})
It("marks a child resources as deleted", func() {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1)
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
err = eng.Delete(parentResource.ID)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0)
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0)
})
})
Describe("Restore", func() {
It("also marks child resources as restored", func() {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Delete(parentResource.ID)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 0)
assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 0)
err = eng.Restore(parentResource.ID)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 1)
assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 1)
})
})
Describe("Purge", func() {
It("removes a resource from the index", func() {
err := eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:child.pdf", 1)
err = eng.Purge(childResource.ID, false)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, "Name:child.pdf", 0)
})
It("removes a resource and its children from the index", func() {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1)
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
err = eng.Purge(parentResource.ID, false)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0)
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0)
})
It("removes a resource and ignores its children from the index", func() {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1)
err = eng.Delete(parentResource.ID)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
err = eng.Purge(parentResource.ID, true)
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0)
assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1)
})
})
Describe("Move", func() {
It("renames the parent and its child resources", func() {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
parentResource.Path = "newname"
err = eng.Move(parentResource.ID, parentResource.ParentID, "./my/newname")
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, parentResource.Name, 0)
matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1)
Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3"))
Expect(matches[0].Entity.Ref.Path).To(Equal("./my/newname/child.pdf"))
})
DescribeTable("keeps the flag in step with the path",
func(from, target string, hidden bool) {
parentResource.Path = from
parentResource.Hidden = search.IsHidden(from)
childResource.Path = from + "/child.pdf"
childResource.Hidden = parentResource.Hidden
Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed())
Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed())
Expect(eng.Move(parentResource.ID, parentResource.ParentID, target)).To(Succeed())
for _, id := range []string{parentResource.ID, childResource.ID} {
Expect(hiddenByID(idx, id)).
To(Equal(hidden), "%s after moving from %s to %s", id, from, target)
}
},
Entry("into a dot folder", "./parent", "./.trash/parent", true),
Entry("into a plain folder", "./parent", "./archive/parent", false),
Entry("renamed with a leading dot", "./parent", "./.parent", true),
Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false),
Entry("renamed without the leading dot", "./.parent", "./parent", false),
Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true),
)
// the trash leaves the path alone, so the flag has to come through untouched
It("carries the flag through the trash and back", func() {
childResource.Path = "./.secret/file.txt"
childResource.Hidden = true
Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed())
Expect(eng.Delete(childResource.ID)).To(Succeed())
Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after trashing")
Expect(eng.Restore(childResource.ID)).To(Succeed())
Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after restoring")
})
It("moves the parent and its child resources", func() {
err := eng.Upsert(parentResource.ID, parentResource)
Expect(err).ToNot(HaveOccurred())
err = eng.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
parentResource.Path = " "
parentResource.ParentID = "1$2!somewhereopaqueid"
err = eng.Move(parentResource.ID, parentResource.ParentID, "./somewhere/else/newname")
Expect(err).ToNot(HaveOccurred())
assertDocCount(rootResource.ID, `parent d!r`, 0)
matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1)
Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3"))
Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname/child.pdf"))
matches = assertDocCount(rootResource.ID, `newname`, 1)
Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("somewhereopaqueid"))
Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname"))
})
It("keeps case-insensitive search working after a move", func() {
Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed())
Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed())
Expect(eng.Move(parentResource.ID, parentResource.ParentID, "./my/NewName")).To(Succeed())
// the lowercased siblings are rebuilt at the new path, so a
// case-insensitive query finds the folder under its new name and path,
// including the descendant, and no longer under the old path.
assertDocCount(rootResource.ID, "name:NEWNAME", 1)
assertDocCount(rootResource.ID, `path:"./MY/NEWNAME"`, 2)
assertDocCount(rootResource.ID, `path:"./parent d!r"`, 0)
})
})
Describe("StartBatch", func() {
It("starts a new batch", func() {
b, err := eng.NewBatch(100)
Expect(err).ToNot(HaveOccurred())
err = b.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
count, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(0)))
err = b.Push()
Expect(err).ToNot(HaveOccurred())
count, err = idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(1)))
query := bleveSearch.NewMatchQuery("child.pdf")
res, err := idx.Search(bleveSearch.NewSearchRequest(query))
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits.Len()).To(Equal(1))
})
It("doesn't intertwine different batches", func() {
b, err := eng.NewBatch(100)
Expect(err).ToNot(HaveOccurred())
err = b.Upsert(childResource.ID, childResource)
Expect(err).ToNot(HaveOccurred())
count, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(0)))
b2, err := eng.NewBatch(100)
Expect(err).ToNot(HaveOccurred())
err = b2.Upsert(childResource2.ID, childResource2)
Expect(err).ToNot(HaveOccurred())
Expect(b.Push()).To(Succeed())
count, err = idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(1)))
Expect(b2.Push()).To(Succeed())
count, err = idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(2)))
})
})
Describe("File type specific metadata", func() {
Context("with audio metadata", func() {
BeforeEach(func() {
resource := search.Resource{
ID: "1$2!7",
ParentID: rootResource.ID,
RootID: rootResource.ID,
Path: "./some_song.mp3",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{
Name: "some_song.mp3",
MimeType: "audio/mpeg",
Audio: &libregraph.Audio{
Album: libregraph.PtrString("Some Album"),
AlbumArtist: libregraph.PtrString("Some AlbumArtist"),
Artist: libregraph.PtrString("Some Artist"),
Bitrate: libregraph.PtrInt64(192),
Composers: libregraph.PtrString("Some Composers"),
Copyright: libregraph.PtrString(""),
Disc: libregraph.PtrInt32(2),
DiscCount: libregraph.PtrInt32(5),
Duration: libregraph.PtrInt64(225000),
Genre: libregraph.PtrString("Some Genre"),
HasDrm: libregraph.PtrBool(false),
IsVariableBitrate: libregraph.PtrBool(true),
Title: libregraph.PtrString("Some Title"),
Track: libregraph.PtrInt32(34),
TrackCount: libregraph.PtrInt32(99),
Year: libregraph.PtrInt32(2004),
},
},
}
err := eng.Upsert(resource.ID, resource)
Expect(err).ToNot(HaveOccurred())
})
It("returns audio metadata for search", func() {
matches := assertDocCount(rootResource.ID, `*song*`, 1)
audio := matches[0].Entity.Audio
Expect(audio).ToNot(BeNil())
Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album")))
Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist")))
Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist")))
Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192)))
Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers")))
Expect(audio.Copyright).To(Equal(libregraph.PtrString("")))
Expect(audio.Disc).To(Equal(libregraph.PtrInt32(2)))
Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5)))
Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225000)))
Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre")))
Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false)))
Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true)))
Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title")))
Expect(audio.Track).To(Equal(libregraph.PtrInt32(34)))
Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(99)))
Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004)))
})
})
Context("with location metadata", func() {
BeforeEach(func() {
resource := search.Resource{
ID: "1$2!7",
ParentID: rootResource.ID,
RootID: rootResource.ID,
Path: "./team.jpg",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{
Name: "team.jpg",
MimeType: "image/jpeg",
Location: &libregraph.GeoCoordinates{
Altitude: libregraph.PtrFloat64(1047.7),
Latitude: libregraph.PtrFloat64(49.48675890884328),
Longitude: libregraph.PtrFloat64(11.103870357204285),
},
},
}
err := eng.Upsert(resource.ID, resource)
Expect(err).ToNot(HaveOccurred())
})
It("returns audio metadata for search", func() {
matches := assertDocCount(rootResource.ID, `*team*`, 1)
location := matches[0].Entity.Location
Expect(location).ToNot(BeNil())
Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(1047.7)))
Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328)))
Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285)))
})
})
})
})
-40
View File
@@ -10,10 +10,8 @@ import (
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
regexpCharFilter "github.com/blevesearch/bleve/v2/analysis/char/regexp"
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
"github.com/blevesearch/bleve/v2/analysis/token/porter"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/single"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
"github.com/blevesearch/bleve/v2/mapping"
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
@@ -60,44 +58,6 @@ func NewMapping() (mapping.IndexMapping, error) {
indexMapping := bleve.NewIndexMapping()
indexMapping.DefaultAnalyzer = keyword.Name
indexMapping.DefaultMapping = docMapping
err = indexMapping.AddCustomCharFilter("dotToSpace",
map[string]any{
"type": regexpCharFilter.Name,
"regexp": `\.`,
"replace": " ",
},
)
if err != nil {
return nil, err
}
err = indexMapping.AddCustomAnalyzer("lowercaseWords",
map[string]any{
"type": custom.Name,
"char_filters": []string{"dotToSpace"},
"tokenizer": unicode.Name,
"token_filters": []string{
lowercase.Name,
},
},
)
if err != nil {
return nil, err
}
err = indexMapping.AddCustomAnalyzer("lowercaseKeyword",
map[string]any{
"type": custom.Name,
"tokenizer": single.Name,
"token_filters": []string{
lowercase.Name,
},
},
)
if err != nil {
return nil, err
}
err = indexMapping.AddCustomAnalyzer("fulltext",
map[string]any{
"type": custom.Name,
+40 -12
View File
@@ -12,10 +12,8 @@ import (
// struct via reflection. Field names come from json tags; overrides are
// keyed by those names (or dotted paths for nested fields).
//
// The returned mapping references analyzer names (Analyzer field on the
// FieldOpts, plus "fulltext" / "path_hierarchy" for the corresponding Types);
// the caller is responsible for registering those analyzers on the enclosing
// IndexMapping.
// The returned mapping references the "fulltext" analyzer for Fulltext fields;
// the caller registers it on the enclosing IndexMapping.
func BleveBuildMapping(t reflect.Type, overrides map[string]FieldOpts) (*bleveMapping.DocumentMapping, error) {
return buildBleveDocMapping(t, overrides, "")
}
@@ -61,6 +59,16 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix
return nil
}
if fieldType == TypeKeyword || fieldType == TypePath {
// bleve has no path tokenizer, so a path is a plain keyword here.
base := bleveKeywordMapping(fieldType, opts)
doc.AddFieldMappingsAt(fi.Name, base)
if opts.caseInsensitive() {
doc.AddFieldMappingsAt(fi.Name+LowercaseSuffix, lowercaseSibling(base))
}
return nil
}
fm, err := bleveFieldMapping(fieldType, opts)
if err != nil {
return fmt.Errorf("mapping: field %q: %w", key, err)
@@ -71,26 +79,46 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix
return doc, err
}
// bleveKeywordMapping is a case-preserving keyword field; path fields stay out
// of _all by default.
func bleveKeywordMapping(fieldType string, opts FieldOpts) *bleveMapping.FieldMapping {
fm := bleve.NewKeywordFieldMapping()
switch {
case opts.IncludeInAll != nil:
fm.IncludeInAll = *opts.IncludeInAll
case fieldType == TypePath:
fm.IncludeInAll = false
}
return fm
}
// lowercaseSibling derives the lowercased shadow of a keyword/path field from its
// base mapping: used only for case-insensitive matching, so indexed but never
// stored, kept out of _all, and without doc values, since the case-preserved base
// field is what we return and aggregate on.
func lowercaseSibling(base *bleveMapping.FieldMapping) *bleveMapping.FieldMapping {
fm := *base
fm.Store = false
fm.IncludeInAll = false
fm.DocValues = false
return &fm
}
func bleveFieldMapping(fieldType string, opts FieldOpts) (*bleveMapping.FieldMapping, error) {
switch fieldType {
case TypeWildcard:
// bleve has no wildcard type; fall back to keyword-ish text.
fieldType = TypeKeyword
fallthrough
case TypeKeyword, TypeFulltext, TypePath:
case TypeKeyword, TypeFulltext:
fm := bleve.NewTextFieldMapping()
switch {
case opts.Analyzer != "":
fm.Analyzer = opts.Analyzer
case fieldType == TypeFulltext:
if fieldType == TypeFulltext {
fm.Analyzer = "fulltext"
case fieldType == TypePath:
fm.Analyzer = "path_hierarchy"
}
switch {
case opts.IncludeInAll != nil:
fm.IncludeInAll = *opts.IncludeInAll
case fieldType == TypeFulltext, fieldType == TypePath:
case fieldType == TypeFulltext:
fm.IncludeInAll = false
}
return fm, nil
+18 -8
View File
@@ -65,21 +65,31 @@ var _ = Describe("BleveBuildMapping", func() {
})
It("applies field overrides", func() {
includeInAllFalse := false
True, False := true, false
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{
"Name": {Analyzer: "lowercaseKeyword"},
"Name": {CaseInsensitive: &True},
"Content": {Type: TypeFulltext},
"Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &includeInAllFalse},
"Tags": {CaseInsensitive: &True, IncludeInAll: &False},
})
Expect(err).ToNot(HaveOccurred())
nameField := dm.Properties["Name"].Fields[0]
Expect(nameField.Analyzer).To(Equal("lowercaseKeyword"), "Name analyzer")
Expect(nameField.IncludeInAll).To(BeTrue(), "Name IncludeInAll should stay default-true when not overridden")
// Name: case-preserved base keyword + lowercased sibling.
Expect(dm.Properties["Name"]).ToNot(BeNil(), "Name base field")
Expect(dm.Properties["Name"].Fields[0].Analyzer).To(Equal("keyword"), "Name base is a keyword")
Expect(dm.Properties["Name"].Fields[0].Store).To(BeTrue(), "Name base is stored (returned)")
Expect(dm.Properties["Name_lowercase"]).ToNot(BeNil(), "Name_lowercase sibling")
// The sibling is a search-only shadow: indexed but never stored, kept out
// of _all, no doc values (the base is what we return).
sibling := dm.Properties["Name_lowercase"].Fields[0]
Expect(sibling.Index).To(BeTrue(), "Name_lowercase is indexed")
Expect(sibling.Store).To(BeFalse(), "Name_lowercase is not stored")
Expect(sibling.IncludeInAll).To(BeFalse(), "Name_lowercase is out of _all")
Expect(sibling.DocValues).To(BeFalse(), "Name_lowercase has no doc values")
contentField := dm.Properties["Content"].Fields[0]
Expect(contentField.Analyzer).To(Equal("fulltext"), "Content analyzer")
Expect(contentField.IncludeInAll).To(BeFalse(), "Content IncludeInAll should default to false for fulltext type")
tagsField := dm.Properties["Tags"].Fields[0]
Expect(tagsField.IncludeInAll).To(BeFalse(), "Tags IncludeInAll should honor the explicit false override")
// Tags: base + lowercased sibling, both honoring the IncludeInAll override.
Expect(dm.Properties["Tags"].Fields[0].IncludeInAll).To(BeFalse(), "Tags base IncludeInAll honored")
Expect(dm.Properties["Tags_lowercase"].Fields[0].IncludeInAll).To(BeFalse(), "Tags sibling IncludeInAll honored")
})
It("builds an object sub-document plus a geopoint sibling", func() {
+57
View File
@@ -0,0 +1,57 @@
package mapping
import "strings"
func addLowercaseSiblings(m map[string]any, overrides map[string]FieldOpts) {
for key, opts := range overrides {
if !opts.caseInsensitive() || !isCasedType(opts) {
continue
}
parent, leaf, ok := resolveLeaf(m, key)
if !ok {
continue
}
addLowercaseSibling(parent, leaf)
}
}
func isCasedType(opts FieldOpts) bool {
return opts.Type == "" || opts.Type == TypeKeyword || opts.Type == TypePath
}
func resolveLeaf(m map[string]any, dottedPath string) (map[string]any, string, bool) {
parts := strings.Split(dottedPath, ".")
parent := m
for _, p := range parts[:len(parts)-1] {
next, ok := parent[p].(map[string]any)
if !ok {
return nil, "", false
}
parent = next
}
return parent, parts[len(parts)-1], true
}
// addLowercaseSibling writes a <leaf>_lowercase sibling; no-op for non-strings.
func addLowercaseSibling(parent map[string]any, leaf string) {
switch v := parent[leaf].(type) {
case string:
parent[leaf+LowercaseSuffix] = strings.ToLower(v)
case []any:
out := make([]any, 0, len(v))
for _, item := range v {
if s, ok := item.(string); ok {
out = append(out, strings.ToLower(s))
}
}
if len(out) > 0 {
parent[leaf+LowercaseSuffix] = out
}
case []string:
out := make([]string, len(v))
for i, s := range v {
out[i] = strings.ToLower(s)
}
parent[leaf+LowercaseSuffix] = out
}
}
@@ -0,0 +1,41 @@
package mapping
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("PrepareForIndex casing", func() {
It("adds lowercased siblings for CaseInsensitive keyword and path fields", func() {
True := true
type doc struct {
Name string `json:"Name"`
Path string `json:"Path"`
Tags []string `json:"Tags"`
}
d := doc{Name: "Report FINAL", Path: "/Foo/Bar", Tags: []string{"Work", "Urgent"}}
m, err := PrepareForIndex(d, map[string]FieldOpts{
"Name": {CaseInsensitive: &True},
"Path": {Type: TypePath, CaseInsensitive: &True},
"Tags": {CaseInsensitive: &True},
})
Expect(err).ToNot(HaveOccurred())
// Originals stay for the case-preserved base fields and the cascade.
Expect(m["Name"]).To(Equal("Report FINAL"))
Expect(m["Path"]).To(Equal("/Foo/Bar"))
Expect(m["Name_lowercase"]).To(Equal("report final"))
Expect(m["Path_lowercase"]).To(Equal("/foo/bar"))
Expect(m["Tags_lowercase"]).To(Equal([]any{"work", "urgent"}))
})
It("writes no sibling without CaseInsensitive", func() {
type doc struct {
ID string `json:"ID"`
}
m, err := PrepareForIndex(doc{ID: "ABC"}, nil)
Expect(err).ToNot(HaveOccurred())
Expect(m).ToNot(HaveKey("ID" + LowercaseSuffix))
})
})
+19 -23
View File
@@ -56,7 +56,20 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p
return nil
}
fm, err := openSearchFieldMapping(fieldType, opts, fi.GoField.Type)
if fieldType == TypeKeyword || fieldType == TypePath {
// path_hierarchy is case-preserving here; casing lives in the value.
m := map[string]any{"type": "keyword"}
if fieldType == TypePath {
m = map[string]any{"type": "text", "analyzer": "path_hierarchy"}
}
props[fi.Name] = m
if opts.caseInsensitive() {
props[fi.Name+LowercaseSuffix] = m
}
return nil
}
fm, err := openSearchFieldMapping(fieldType, fi.GoField.Type)
if err != nil {
return fmt.Errorf("mapping: field %q: %w", key, err)
}
@@ -66,32 +79,15 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p
return props, err
}
func openSearchFieldMapping(fieldType string, opts FieldOpts, goType reflect.Type) (map[string]any, error) {
// openSearchFieldMapping handles the non-keyword/path types; keyword and path
// are emitted (with their cased forms) by buildOpenSearchProperties directly.
func openSearchFieldMapping(fieldType string, goType reflect.Type) (map[string]any, error) {
switch fieldType {
case TypeKeyword:
m := map[string]any{"type": "keyword"}
if opts.Analyzer != "" {
m["type"] = "text"
m["analyzer"] = opts.Analyzer
}
return m, nil
case TypeFulltext:
m := map[string]any{
return map[string]any{
"type": "text",
"term_vector": "with_positions_offsets",
}
if opts.Analyzer != "" {
m["analyzer"] = opts.Analyzer
}
return m, nil
case TypePath:
m := map[string]any{"type": "text"}
if opts.Analyzer != "" {
m["analyzer"] = opts.Analyzer
} else {
m["analyzer"] = "path_hierarchy"
}
return m, nil
}, nil
case TypeWildcard:
// OpenSearch stores wildcard fields with doc_values=false by
// default, so emit it explicitly to keep local and remote
@@ -78,6 +78,7 @@ var _ = Describe("OpenSearchBuildMapping", func() {
})
It("applies field overrides", func() {
True := true
type doc struct {
Name string `json:"Name"`
Content string `json:"Content"`
@@ -85,23 +86,23 @@ var _ = Describe("OpenSearchBuildMapping", func() {
MimeType string `json:"MimeType"`
}
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{
"Name": {Analyzer: "lowercaseKeyword"},
"Name": {CaseInsensitive: &True},
"Content": {Type: TypeFulltext},
"Path": {Type: TypePath},
"Path": {Type: TypePath, CaseInsensitive: &True},
"MimeType": {Type: TypeWildcard},
})
Expect(err).ToNot(HaveOccurred())
name := props["Name"].(map[string]any)
Expect(name["type"]).To(Equal("text"), "Name: %#v", name)
Expect(name["analyzer"]).To(Equal("lowercaseKeyword"), "Name: %#v", name)
// Name: case-preserved keyword base + lowercased keyword sibling.
Expect(props["Name"]).To(Equal(map[string]any{"type": "keyword"}))
Expect(props["Name_lowercase"]).To(Equal(map[string]any{"type": "keyword"}))
content := props["Content"].(map[string]any)
Expect(content["type"]).To(Equal("text"), "Content: %#v", content)
Expect(content["term_vector"]).To(Equal("with_positions_offsets"), "Content: %#v", content)
_, ok := content["analyzer"]
Expect(ok).To(BeFalse(), "Content should leave analyzer unset (use OpenSearch default)")
path := props["Path"].(map[string]any)
Expect(path["type"]).To(Equal("text"), "Path: %#v", path)
Expect(path["analyzer"]).To(Equal("path_hierarchy"), "Path: %#v", path)
// Path: path_hierarchy base + lowercased sibling, both case-preserving.
Expect(props["Path"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"}))
Expect(props["Path_lowercase"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"}))
mime := props["MimeType"].(map[string]any)
Expect(mime["type"]).To(Equal("wildcard"), "MimeType: %#v", mime)
})
+9 -4
View File
@@ -17,6 +17,9 @@ const (
TypeGeopoint = "geopoint"
)
// LowercaseSuffix names the lowercased sibling of a keyword/path field.
const LowercaseSuffix = "_lowercase"
// FieldOpts overrides the default type inference for a struct field. Keys in
// the override map are json-tag names (e.g. "Name", "location", "audio.artist"),
// not Go field names.
@@ -24,12 +27,14 @@ type FieldOpts struct {
// Type is one of the Type* constants. Empty means "infer from Go type".
Type string
// Analyzer is the name of a custom analyzer registered on the bleve
// IndexMapping (e.g. "lowercaseKeyword", "fulltext"). For OpenSearch it
// becomes the analyzer attribute on the field.
Analyzer string
// CaseInsensitive additionally indexes a lowercased <name>_lowercase sibling
// for case-insensitive search; the case-preserved base is always indexed.
// Nil/false means off. Keyword/path only.
CaseInsensitive *bool
// IncludeInAll controls bleve's _all field inclusion. Nil means "use the
// bleve default for this field type". Has no effect on OpenSearch.
IncludeInAll *bool
}
func (o FieldOpts) caseInsensitive() bool { return o.CaseInsensitive != nil && *o.CaseInsensitive }
+1
View File
@@ -19,5 +19,6 @@ func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, err
return out, nil
}
addGeopointSiblings(out, overrides)
addLowercaseSiblings(out, overrides)
return out, nil
}
+2 -2
View File
@@ -23,9 +23,9 @@ type sample struct {
var _ = Describe("Validate", func() {
It("accepts known override keys", func() {
err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{
"Name": {Analyzer: "lowercaseKeyword"},
"Name": {},
"audio": {Type: TypeObject},
"audio.artist": {Analyzer: "lowercaseKeyword"},
"audio.artist": {},
"location": {Type: TypeGeopoint},
})
Expect(err).ToNot(HaveOccurred())
+33 -15
View File
@@ -64,27 +64,45 @@ func (b *Batch) Upsert(id string, r search.Resource) error {
})
}
func (b *Batch) Move(id string, parentID string, targetPath string) error {
func (b *Batch) Move(id string, parentID string, location string) error {
return b.withSizeLimit(func() error {
op := func() error {
return updateSelfAndDescendants(context.Background(), b.client, b.index, id, func(rootResource search.Resource) *osu.BodyParamScript {
newPath := utils.MakeRelativePath(location)
newName := path.Base(newPath)
return &osu.BodyParamScript{
Source: `
if (ctx._source.ID == params.id ) { ctx._source.Name = params.newName; ctx._source.ParentID = params.parentID; }
ctx._source.Path = ctx._source.Path.replace(params.oldPath, params.newPath);
boolean hidden = false;
for (String name : ctx._source.Path.splitOnToken('/')) {
if (!name.equals('.') && !name.equals('..') && name.startsWith('.')) { hidden = true; break; }
}
ctx._source.Hidden = hidden;
`,
// Keep the case-preserved base fields and their lowercased
// search siblings in sync: swap the moved prefix in both. The
// lowercased new values come from Go's strings.ToLower via
// params, so the sibling stays byte-identical to what
// PrepareForIndex writes on upsert (painless toLowerCase would
// lowercase differently than Go).
Source: fmt.Sprintf(`
if (ctx._source.ID == params.id) {
ctx._source.Name = params.newName;
ctx._source.ParentID = params.parentID;
if (ctx._source.Name%[1]s != null) { ctx._source.Name%[1]s = params.newNameLower; }
}
ctx._source.Path = ctx._source.Path.replace(params.oldPath, params.newPath);
if (ctx._source.Path%[1]s != null) {
ctx._source.Path%[1]s = ctx._source.Path%[1]s.replace(params.oldPathLower, params.newPathLower);
}
boolean hidden = false;
for (String name : ctx._source.Path.splitOnToken('/')) {
if (!name.equals('.') && !name.equals('..') && name.startsWith('.')) { hidden = true; break; }
}
ctx._source.Hidden = hidden;
`, mapping.LowercaseSuffix),
Lang: "painless",
Params: map[string]any{
"id": id,
"parentID": parentID,
"oldPath": rootResource.Path,
"newPath": utils.MakeRelativePath(targetPath),
"newName": path.Base(utils.MakeRelativePath(targetPath)),
"id": id,
"parentID": parentID,
"oldPath": rootResource.Path,
"newPath": newPath,
"newName": newName,
"oldPathLower": strings.ToLower(rootResource.Path),
"newPathLower": strings.ToLower(newPath),
"newNameLower": strings.ToLower(newName),
},
}
})
+1 -7
View File
@@ -61,7 +61,6 @@ func buildResourceMapping() ([]byte, error) {
resourceType := reflect.TypeFor[search.Resource]()
overrides := maps.Clone(search.Resource{}.SearchFieldOverrides())
overrides["MimeType"] = searchmapping.FieldOpts{Type: searchmapping.TypeWildcard}
overrides["Path"] = searchmapping.FieldOpts{Type: searchmapping.TypePath}
if err := searchmapping.Validate(resourceType, overrides); err != nil {
return nil, err
}
@@ -75,16 +74,11 @@ func buildResourceMapping() ([]byte, error) {
"number_of_shards": "1",
"number_of_replicas": "1",
"analysis": map[string]any{
// path_hierarchy is case-preserving; casing lives in the value.
"analyzer": map[string]any{
"path_hierarchy": map[string]any{
"type": "custom",
"tokenizer": "path_hierarchy",
"filter": []string{"lowercase"},
},
"lowercaseKeyword": map[string]any{
"type": "custom",
"tokenizer": "keyword",
"filter": []string{"lowercase"},
},
},
"tokenizer": map[string]any{
@@ -1,25 +0,0 @@
package convert
import (
"strings"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
// LowerValues folds restriction values for fields whose index analyzer
// lowercases (search.LowercaseValueFields, shared with bleve); case-preserved
// fields keep their casing. Runs after query.Normalize, so keys are resolved.
func LowerValues(nodes []ast.Node) []ast.Node {
for _, n := range nodes {
switch node := n.(type) {
case *ast.StringNode:
if _, ok := search.LowercaseValueFields()[node.Key]; ok {
node.Value = strings.ToLower(node.Value)
}
case *ast.GroupNode:
LowerValues(node.Nodes)
}
}
return nodes
}
@@ -18,11 +18,10 @@ func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) {
return nil, err
}
// shared lowering (field resolution + media-type), then value lowercasing.
// shared lowering: field resolution, media-type expansion, value lowercasing.
kqlAst = query.Normalize(kqlAst, query.ResolveField)
kqlNodes := LowerValues(kqlAst.Nodes)
builder, err := TranspileKQLToOpenSearch(kqlNodes)
builder, err := TranspileKQLToOpenSearch(kqlAst.Nodes)
if err != nil {
return nil, fmt.Errorf("failed to compile query: %w", err)
}
@@ -4,12 +4,12 @@ import (
"errors"
"fmt"
"slices"
"strconv"
"strings"
"time"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/pkg/kql"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
)
@@ -99,7 +99,28 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
case *ast.BooleanNode:
return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil
case *ast.StringNode:
return stringNodeQuery(node), nil
field, value := node.Key, node.Value
if node.CaseInsensitive {
field += mapping.LowercaseSuffix
value = strings.ToLower(value)
}
isWildcard := strings.Contains(value, "*")
if isWildcard {
return osu.NewWildcardQuery(field).Value(value), nil
}
totalTerms := strings.Split(value, " ")
isSingleTerm := len(totalTerms) == 1
isMultiTerm := len(totalTerms) >= 1
switch {
case isSingleTerm:
return osu.NewTermQuery[string](field).Value(value), nil
case isMultiTerm:
return osu.NewMatchPhraseQuery(field).Query(value), nil
}
return nil, fmt.Errorf("unsupported string node value: %s", value)
case *ast.DateTimeNode:
return dateTimeNodeQuery(node)
case *ast.NumberNode:
@@ -116,67 +137,26 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
return nil, fmt.Errorf("%w: %T", ErrUnsupportedNodeType, node)
}
// stringNodeQuery picks the query a string node turns into.
func stringNodeQuery(node *ast.StringNode) osu.Builder {
isWildcard := strings.ContainsAny(node.Value, "*?")
switch {
// Name: "*oo-bar", "*oo ba*", "*OO*"
// Title: "*rterly rep*"
// Tags: "*spaced tag*"
case isWildcard && slices.Contains([]string{"Name", "Title"}, node.Key):
patterns := []osu.Builder{wildcardOn(node.Key+".wildcard", node.Value)}
if !strings.HasSuffix(node.Value, "*") {
patterns = append(patterns, wildcardOn(node.Key+".wildcard", node.Value+".*"))
}
return osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(patterns...)
// Tags: "*foo*", "*paced ta*"
case isWildcard && node.Key == "Tags":
return wildcardOn(node.Key+".wildcard", node.Value)
// Path: "./foo*", MimeType: "*plain"
case isWildcard:
return osu.NewWildcardQuery(node.Key).Value(node.Value)
// Name: =new, Title: ="quarterly report"
case node.Exact && slices.Contains([]string{"Name", "Title"}, node.Key):
return osu.NewTermQuery[string](node.Key + ".wildcard").
Value(node.Value).
Params(&osu.TermQueryParams{CaseInsensitive: true})
// Tags: "foo-bar", "spaced tag", "FOO-BAR"
case node.Key == "Tags":
return osu.NewTermQuery[string](node.Key + ".wildcard").
Value(node.Value).
Params(&osu.TermQueryParams{CaseInsensitive: true})
// Name: "foo-bar", "foo bar"
// Title: "quarterly report"
// Content: "foo bar"
case slices.Contains([]string{"Name", "Title", "Content"}, node.Key):
return osu.NewMatchPhraseQuery(node.Key).Query(node.Value)
// Size: "42", Type: "1"
case slices.Contains([]string{"Size", "Type"}, node.Key):
number, err := strconv.ParseInt(node.Value, 10, 64)
if err != nil {
return osu.NewMatchNoneQuery()
}
return osu.NewTermQuery[int64](node.Key).Value(number)
// Path: "./foo bar/", the hierarchy tokens carry no trailing slash
case node.Key == "Path":
return osu.NewTermQuery[string](node.Key).Value(strings.TrimSuffix(node.Value, "/"))
// Hidden: "TRUE" arrives lowered, anything that is no bool matches nothing
case node.Key == "Hidden":
value, err := strconv.ParseBool(node.Value)
if err != nil {
return osu.NewMatchNoneQuery()
}
return osu.NewTermQuery[bool](node.Key).Value(value)
// MimeType: "text/plain"
default:
return osu.NewTermQuery[string](node.Key).Value(node.Value)
// dateTimeNodeQuery turns a date time node into a range query.
func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) {
if node.Operator == nil {
return nil, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
}
query := osu.NewRangeQuery[time.Time](node.Key)
switch node.Operator.Value {
case ">":
return query.Gt(node.Value), nil
case ">=":
return query.Gte(node.Value), nil
case "<":
return query.Lt(node.Value), nil
case "<=":
return query.Lte(node.Value), nil
}
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
}
func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) {
@@ -203,31 +183,3 @@ func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) {
return nil, fmt.Errorf("unsupported operator %s for number node: %w", node.Operator.Value, ErrUnsupportedNodeType)
}
func wildcardOn(field, value string) osu.Builder {
return osu.NewWildcardQuery(field).
Value(value).
Params(&osu.WildcardQueryParams{CaseInsensitive: true})
}
// dateTimeNodeQuery turns a date time node into a range query.
func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) {
if node.Operator == nil {
return nil, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
}
query := osu.NewRangeQuery[time.Time](node.Key)
switch node.Operator.Value {
case ">":
return query.Gt(node.Value), nil
case ">=":
return query.Gte(node.Value), nil
case "<":
return query.Lt(node.Value), nil
case "<=":
return query.Lte(node.Value), nil
}
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
}
@@ -16,13 +16,31 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{
// kql to os dsl - type tests
{
Name: "match phrase query - string node on an analyzed field",
Name: "term query - string node",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "openCloud"},
},
},
Want: osu.NewMatchPhraseQuery("Name").Query("openCloud"),
Want: osu.NewTermQuery[string]("Name").Value("openCloud"),
},
{
Name: "case-insensitive term routes to the lowercased sibling",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "openCloud", CaseInsensitive: true},
},
},
Want: osu.NewTermQuery[string]("Name_lowercase").Value("opencloud"),
},
{
Name: "case-insensitive wildcard routes to the lowercased sibling",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "Open*", CaseInsensitive: true},
},
},
Want: osu.NewWildcardQuery("Name_lowercase").Value("open*"),
},
{
Name: "term query - boolean node - true",
@@ -58,16 +76,10 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
&ast.StringNode{Key: "Name", Value: "open*"},
},
},
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewWildcardQuery("Name.wildcard").
Value("open*").
Params(&osu.WildcardQueryParams{CaseInsensitive: true}),
),
Want: osu.NewWildcardQuery("Name").Value("open*"),
},
{
Name: "wildcard query - string node without an unanalyzed sub field",
Name: "wildcard query - fulltext field",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Content", Value: "open*"},
@@ -142,8 +154,8 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
},
},
Want: osu.NewBoolQuery().Must(
osu.NewMatchPhraseQuery("Name").Query("a"),
osu.NewMatchPhraseQuery("Name").Query("b"),
osu.NewTermQuery[string]("Name").Value("a"),
osu.NewTermQuery[string]("Name").Value("b"),
),
},
{
@@ -155,7 +167,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
}},
},
},
Want: osu.NewMatchPhraseQuery("Name").Query("any"),
Want: osu.NewTermQuery[string]("Name").Value("any"),
},
{
Name: "range query >",
@@ -217,7 +229,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
&ast.StringNode{Key: "Name", Value: "openCloud"},
},
},
Want: osu.NewMatchPhraseQuery("Name").Query("openCloud"),
Want: osu.NewTermQuery[string]("Name").Value("openCloud"),
},
{
Name: "[* *]",
@@ -229,7 +241,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
},
Want: osu.NewBoolQuery().
Must(
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
osu.NewTermQuery[string]("Name").Value("openCloud"),
osu.NewTermQuery[string]("age").Value("32"),
),
},
@@ -244,7 +256,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
},
Want: osu.NewBoolQuery().
Must(
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
osu.NewTermQuery[string]("Name").Value("openCloud"),
osu.NewTermQuery[string]("age").Value("32"),
),
},
@@ -260,7 +272,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
osu.NewTermQuery[string]("Name").Value("openCloud"),
osu.NewTermQuery[string]("age").Value("32"),
),
},
@@ -288,7 +300,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
},
Want: osu.NewBoolQuery().
Must(
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
osu.NewTermQuery[string]("Name").Value("openCloud"),
).
MustNot(
osu.NewTermQuery[string]("age").Value("32"),
@@ -308,7 +320,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
osu.NewTermQuery[string]("Name").Value("openCloud"),
osu.NewTermQuery[string]("age").Value("32"),
osu.NewTermQuery[string]("age").Value("44"),
),
+16 -38
View File
@@ -12,31 +12,10 @@ import (
bleveQuery "github.com/blevesearch/bleve/v2/search/query"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/pkg/kql"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
searchQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query"
)
// lowercaseFields holds the fields whose query-side value is pre-lowercased so
// it matches the index-time lowercasing analyzer. Shared with the OpenSearch
// backend via search.LowercaseValueFields; every other field keeps its casing.
var lowercaseFields = search.LowercaseValueFields()
var _fields = map[string]string{
"rootid": "RootID",
"path": "Path",
"id": "ID",
"name": "Name",
"size": "Size",
"mtime": "Mtime",
"mediatype": "MimeType",
"type": "Type",
"tag": "Tags",
"tags": "Tags",
"content": "Content",
"title": "Title",
"hidden": "Hidden",
"favorite": "Favorites",
}
// The following quoted string enumerates the characters which may be escaped: "+-=&|><!(){}[]^\"~*?:\\/ "
// based on bleve docs https://blevesearch.com/docs/Query-String-Query/
// Wildcards * and ? are excluded
@@ -102,12 +81,21 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
if k != "ID" && k != "Size" && k != "MimeType" {
v = bleveEscaper.Replace(n.Value)
}
if _, ok := lowercaseFields[k]; ok {
if n.CaseInsensitive {
k += mapping.LowercaseSuffix
v = strings.ToLower(v)
}
q := bleveQuery.NewQueryStringQuery(k + ":" + v)
var q bleveQuery.Query = bleveQuery.NewQueryStringQuery(k + ":" + v)
if searchQuery.FieldIsPath(n.Key) {
// bleve has no path hierarchy analyzer, unlike OpenSearch: match
// the folder itself and its descendants (`\/*`, a trailing
// wildcard on the value).
q = bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{
q,
bleveQuery.NewQueryStringQuery(k + ":" + v + `\/*`),
})
}
if prev == nil {
prev = q
@@ -151,7 +139,7 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
}
case *ast.NumberNode:
var q bleveQuery.Query
if field := getField(n.Key); slices.Contains([]string{"Size", "Type"}, field) {
if field := n.Key; slices.Contains([]string{"Size", "Type"}, field) {
q = numberRange(field, n.Operator, n.Value)
} else {
// same answer as the OpenSearch backend: unknown numeric keys
@@ -169,7 +157,7 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
}
case *ast.BooleanNode:
q := bleveQuery.NewBoolFieldQuery(n.Value)
q.SetField(getField(n.Key))
q.SetField(n.Key)
if prev == nil {
prev = q
} else {
@@ -340,16 +328,6 @@ func phrase(field, value string) bleveQuery.Query {
return q
}
func getField(name string) string {
if name == "" {
return "Name"
}
if _, ok := _fields[strings.ToLower(name)]; ok {
return _fields[strings.ToLower(name)]
}
return name
}
func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode {
for _, n := range group.Nodes {
if onode, ok := n.(*ast.StringNode); ok {
@@ -1,7 +1,6 @@
package bleve
import (
"strings"
"testing"
"time"
@@ -20,29 +19,11 @@ var timeMustParse = func(t *testing.T, ts string) time.Time {
return tp
}
func wildcardQuery(field, value string) query.Query {
patterns := []query.Query{query.NewQueryStringQuery(field + ".wildcard:" + value)}
if !strings.HasSuffix(value, "*") {
patterns = append(patterns, query.NewQueryStringQuery(field+".wildcard:"+value+".*"))
}
return query.NewConjunctionQuery([]query.Query{query.NewDisjunctionQuery(patterns)})
}
func phraseQuery(field, value string) query.Query {
q := query.NewMatchPhraseQuery(value)
q.SetField(field)
return q
}
func boolFieldQuery(field string, value bool) query.Query {
q := query.NewBoolFieldQuery(value)
q.SetField(field)
return q
}
// TODO(followup): make this a pure compiler test. Field resolution and
// media-type expansion live in query.Normalize, so this test could feed
// canonical ASTs (real field names, media-type already expanded) and call
// compile() directly, dropping the query.Normalize wrapper and the mediatype
// cases.
func Test_compile(t *testing.T) {
tests := []struct {
name string
@@ -58,7 +39,22 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
phraseQuery("Name", "federated"),
query.NewQueryStringQuery(`Name_lowercase:federated`),
}),
wantErr: false,
},
{
// path fields expand to match the folder itself and its descendants,
// since bleve has no path hierarchy analyzer.
name: `path:/Foo`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "path", Value: "/Foo"},
},
},
want: query.NewDisjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Path_lowercase:\/foo`),
query.NewQueryStringQuery(`Path_lowercase:\/foo\/*`),
}),
wantErr: false,
},
@@ -70,7 +66,7 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
phraseQuery("Name", "John Smith"),
query.NewQueryStringQuery(`Name_lowercase:john\ smith`),
}),
wantErr: false,
},
@@ -84,8 +80,8 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
phraseQuery("Name", "John Smith"),
phraseQuery("Name", "Jane"),
query.NewQueryStringQuery(`Name_lowercase:john\ smith`),
query.NewQueryStringQuery(`Name_lowercase:jane`),
}),
wantErr: false,
},
@@ -99,8 +95,8 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Tags:bestseller`),
query.NewQueryStringQuery(`Tags:book`),
query.NewQueryStringQuery(`Tags_lowercase:bestseller`),
query.NewQueryStringQuery(`Tags_lowercase:book`),
}),
wantErr: false,
},
@@ -116,10 +112,10 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewDisjunctionQuery([]query.Query{
wildcardQuery("Name", `moby\ di*`),
query.NewQueryStringQuery(`Name_lowercase:moby\ di*`),
query.NewConjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Tags:bestseller`),
query.NewQueryStringQuery(`Tags:book`),
query.NewQueryStringQuery(`Tags_lowercase:bestseller`),
query.NewQueryStringQuery(`Tags_lowercase:book`),
}),
}),
wantErr: false,
@@ -137,10 +133,10 @@ func Test_compile(t *testing.T) {
},
want: query.NewDisjunctionQuery([]query.Query{
query.NewConjunctionQuery([]query.Query{
phraseQuery("Name", "a"),
phraseQuery("Name", "b"),
query.NewQueryStringQuery(`Name_lowercase:a`),
query.NewQueryStringQuery(`Name_lowercase:b`),
}),
phraseQuery("Name", "c"),
query.NewQueryStringQuery(`Name_lowercase:c`),
}),
wantErr: false,
},
@@ -156,10 +152,10 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewDisjunctionQuery([]query.Query{
phraseQuery("Name", "a"),
query.NewQueryStringQuery(`Name_lowercase:a`),
query.NewConjunctionQuery([]query.Query{
phraseQuery("Name", "b"),
phraseQuery("Name", "c"),
query.NewQueryStringQuery(`Name_lowercase:b`),
query.NewQueryStringQuery(`Name_lowercase:c`),
}),
}),
wantErr: false,
@@ -181,11 +177,11 @@ func Test_compile(t *testing.T) {
},
want: query.NewConjunctionQuery([]query.Query{
query.NewDisjunctionQuery([]query.Query{
phraseQuery("Name", "a"),
phraseQuery("Name", "b"),
phraseQuery("Name", "c"),
query.NewQueryStringQuery(`Name_lowercase:a`),
query.NewQueryStringQuery(`Name_lowercase:b`),
query.NewQueryStringQuery(`Name_lowercase:c`),
}),
phraseQuery("Name", "d"),
query.NewQueryStringQuery(`Name_lowercase:d`),
}),
wantErr: false,
},
@@ -204,10 +200,10 @@ func Test_compile(t *testing.T) {
},
want: query.NewConjunctionQuery([]query.Query{
query.NewDisjunctionQuery([]query.Query{
wildcardQuery("Name", `moby\ di*`),
query.NewQueryStringQuery(`Tags:bestseller`),
query.NewQueryStringQuery(`Name_lowercase:moby\ di*`),
query.NewQueryStringQuery(`Tags_lowercase:bestseller`),
}),
query.NewQueryStringQuery(`Tags:book`),
query.NewQueryStringQuery(`Tags_lowercase:book`),
}),
wantErr: false,
},
@@ -229,11 +225,11 @@ func Test_compile(t *testing.T) {
},
want: query.NewConjunctionQuery([]query.Query{
query.NewDisjunctionQuery([]query.Query{
wildcardQuery("Name", `moby\ di*`),
query.NewQueryStringQuery(`Tags:bestseller`),
query.NewQueryStringQuery(`Name_lowercase:moby\ di*`),
query.NewQueryStringQuery(`Tags_lowercase:bestseller`),
}),
query.NewQueryStringQuery(`Tags:book`),
query.NewBooleanQuery(nil, nil, []query.Query{query.NewQueryStringQuery(`Tags:read`)}),
query.NewQueryStringQuery(`Tags_lowercase:book`),
query.NewBooleanQuery(nil, nil, []query.Query{query.NewQueryStringQuery(`Tags_lowercase:read`)}),
}),
wantErr: false,
},
@@ -252,7 +248,7 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
phraseQuery("author", "John Smith"),
query.NewQueryStringQuery(`author:John\ Smith`),
query.NewQueryStringQuery(`author:Jane`),
}),
wantErr: false,
@@ -274,9 +270,9 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
phraseQuery("author", "John Smith"),
query.NewQueryStringQuery(`author:John\ Smith`),
query.NewQueryStringQuery(`author:Jane`),
query.NewQueryStringQuery(`Tags:bestseller`),
query.NewQueryStringQuery(`Tags_lowercase:bestseller`),
}),
wantErr: false,
},
@@ -318,44 +314,9 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
phraseQuery("Name", "John Smith"),
boolFieldQuery("Hidden", true),
boolFieldQuery("Hidden", true),
}),
wantErr: false,
},
{
name: `hidden:banana`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "hidden", Value: "banana"},
},
},
want: query.NewConjunctionQuery([]query.Query{query.NewMatchNoneQuery()}),
wantErr: false,
},
{
name: `name="Report.txt"`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "name", Value: "Report.txt", Exact: true},
},
},
want: query.NewConjunctionQuery([]query.Query{query.NewQueryStringQuery(`Name.wildcard:report.txt`)}),
wantErr: false,
},
{
name: `type:File`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "type", Value: "File"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "type", Value: "FOLDER"},
},
},
want: query.NewDisjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Type:1`),
query.NewQueryStringQuery(`Type:2`),
query.NewQueryStringQuery(`Name_lowercase:john\ smith`),
query.NewQueryStringQuery(`Hidden:T`),
query.NewQueryStringQuery(`Hidden:T`),
}),
wantErr: false,
},
@@ -368,7 +329,7 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
query.NewBooleanQuery(nil, nil, []query.Query{query.NewQueryStringQuery(`Tags:physik`)}),
query.NewBooleanQuery(nil, nil, []query.Query{query.NewQueryStringQuery(`Tags_lowercase:physik`)}),
}),
wantErr: false,
},
@@ -484,7 +445,7 @@ func Test_compile(t *testing.T) {
query.NewQueryStringQuery(`MimeType:application/rtf`),
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
}),
wildcardQuery("Name", `*tdd*`),
query.NewQueryStringQuery(`Name_lowercase:*tdd*`),
}),
wantErr: false,
},
@@ -510,7 +471,7 @@ func Test_compile(t *testing.T) {
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
query.NewConjunctionQuery([]query.Query{
query.NewQueryStringQuery(`MimeType:application/pdf`),
wildcardQuery("Name", `*tdd*`),
query.NewQueryStringQuery(`Name_lowercase:*tdd*`),
}),
}),
wantErr: false,
@@ -540,7 +501,7 @@ func Test_compile(t *testing.T) {
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
query.NewQueryStringQuery(`MimeType:application/pdf`),
}),
wildcardQuery("Name", `*tdd*`),
query.NewQueryStringQuery(`Name_lowercase:*tdd*`),
}),
wantErr: false,
},
@@ -569,7 +530,7 @@ func Test_compile(t *testing.T) {
query.NewQueryStringQuery(`MimeType:application/rtf`),
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
}),
wildcardQuery("Name", `*tdd*`),
query.NewQueryStringQuery(`Name_lowercase:*tdd*`),
}),
wantErr: false,
},
@@ -581,7 +542,7 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
phraseQuery("Name", "John Smith +-=&|><!(){}[]^\"~: "),
query.NewQueryStringQuery(`Name_lowercase:john\ smith\ \+\-\=\&\|\>\<\!\(\)\{\}\[\]\^\"\~\:\ `),
}),
wantErr: false,
},
+5
View File
@@ -2,6 +2,7 @@ package query
import (
"reflect"
"strings"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/pkg/query/mimetype"
@@ -31,10 +32,14 @@ func normalizeNodes(nodes []ast.Node, resolve func(string) string, defaultKey st
switch node := n.(type) {
case *ast.StringNode:
node.Key = resolveKey(node.Key)
if FieldValueIsNormalized(node.Key) {
node.Value = strings.ToLower(node.Value)
}
if exp := mimetype.Expand(node.Key, node.Value); exp != nil {
out = append(out, normalizeNodes(exp, resolve, defaultKey)...)
continue
}
node.CaseInsensitive = FieldIsCaseInsensitive(node.Key)
out = append(out, node)
case *ast.DateTimeNode:
node.Key = resolveKey(node.Key)
+19 -8
View File
@@ -20,13 +20,24 @@ func norm(nodes ...ast.Node) []ast.Node {
func TestResolveField(t *testing.T) {
require.Equal(t, "Name", query.ResolveField("")) // empty -> free-text default
require.Equal(t, "Name", query.ResolveField("NAME")) // case-insensitive
require.Equal(t, "Name", query.ResolveField("NAME")) // canonical, case-insensitive key match
require.Equal(t, "Tags", query.ResolveField("tag")) // singular alias
require.Equal(t, "MimeType", query.ResolveField("mimetype")) // real field, case-insensitive
require.Equal(t, "photo.cameraMake", query.ResolveField("photo.CAMERAMAKE")) // facet, case-insensitive
require.Equal(t, "MimeType", query.ResolveField("mimetype")) // real field
require.Equal(t, "photo.cameraMake", query.ResolveField("photo.CAMERAMAKE")) // facet, case-insensitive key match
require.Equal(t, "unknown.field", query.ResolveField("unknown.field")) // unknown key: unchanged, becomes a dead query
}
func TestFieldIsCaseInsensitive(t *testing.T) {
// The four CaseInsensitive override fields (resolved canonical names).
for _, f := range []string{"Name", "Path", "Tags", "Favorites"} {
require.True(t, query.FieldIsCaseInsensitive(f), f)
}
// Case-preserved / non-keyword fields are not.
for _, f := range []string{"MimeType", "ID", "Content", "unknown"} {
require.False(t, query.FieldIsCaseInsensitive(f), f)
}
}
func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) {
got := norm(
&ast.StringNode{Key: "", Value: "free"},
@@ -40,9 +51,9 @@ func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) {
ast.NumberNode{Key: "size", Value: 100},
)
require.Equal(t, []ast.Node{
&ast.StringNode{Key: "Name", Value: "free"},
&ast.StringNode{Key: "Name", Value: "free", CaseInsensitive: true},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Tags", Value: "x"},
&ast.StringNode{Key: "Tags", Value: "x", CaseInsensitive: true},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "photo.cameraMake", Value: "canon"},
&ast.OperatorNode{Value: "AND"},
@@ -71,11 +82,11 @@ func TestNormalize_GroupKeyDefaulting(t *testing.T) {
&ast.GroupNode{Key: "author", Nodes: []ast.Node{
&ast.StringNode{Key: "author", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "Name", Value: "d"},
&ast.StringNode{Key: "Name", Value: "d", CaseInsensitive: true},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "e"},
&ast.StringNode{Key: "Name", Value: "e", CaseInsensitive: true},
}},
}, got)
}
@@ -87,7 +98,7 @@ func TestNormalize_ConvertsValueNodesToPointers(t *testing.T) {
ast.DateTimeNode{Key: "mtime"},
)
require.Equal(t, []ast.Node{
&ast.StringNode{Key: "Name", Value: "x"},
&ast.StringNode{Key: "Name", Value: "x", CaseInsensitive: true},
&ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: "Mtime"},
}, got)
+56 -11
View File
@@ -15,27 +15,72 @@ var aliases = map[string]string{
"favorite": "Favorites",
}
// fieldIndex maps a lowercased KQL key to the real field name: derived once from
// the resource struct, overlaid with the explicit aliases.
// fieldIndex maps a lowercased KQL key to its canonical field name ("" is the
// bare-search default).
var fieldIndex = sync.OnceValue(func() map[string]string {
idx := mapping.FieldNameIndex(
reflect.TypeFor[search.Resource](),
search.Resource{}.SearchFieldOverrides(),
)
idx := mapping.FieldNameIndex(reflect.TypeFor[search.Resource](), search.Resource{}.SearchFieldOverrides())
for k, v := range aliases {
idx[k] = v
}
idx[""] = idx["name"]
return idx
})
// ResolveField maps a KQL key to the index field name: empty -> Name, a known
// key (case-insensitive) -> its field, anything else unchanged.
func ResolveField(name string) string {
if name == "" {
return "Name"
// caseInsensitiveFields are the fields searched case-insensitively by default,
// derived from the CaseInsensitive overrides.
var caseInsensitiveFields = sync.OnceValue(func() map[string]struct{} {
out := map[string]struct{}{}
for field, opts := range (search.Resource{}).SearchFieldOverrides() {
if opts.CaseInsensitive != nil && *opts.CaseInsensitive {
out[field] = struct{}{}
}
}
return out
})
// pathFields are hierarchical path fields (TypePath), derived from the overrides.
var pathFields = sync.OnceValue(func() map[string]struct{} {
out := map[string]struct{}{}
for field, opts := range (search.Resource{}).SearchFieldOverrides() {
if opts.Type == mapping.TypePath {
out[field] = struct{}{}
}
}
return out
})
// ResolveField maps a KQL key to its canonical field name; unknown keys pass through.
func ResolveField(name string) string {
if v, ok := fieldIndex()[strings.ToLower(name)]; ok {
return v
}
return name
}
// normalizedValueFields have their stored values normalized to lowercase at
// index time, so query values fold to match even though the fields themselves
// are case-preserved keywords.
var normalizedValueFields = map[string]struct{}{
"MimeType": {},
"Type": {},
"Hidden": {},
}
// FieldValueIsNormalized reports whether a field's stored values are
// normalized lowercase.
func FieldValueIsNormalized(field string) bool {
_, ok := normalizedValueFields[field]
return ok
}
// FieldIsCaseInsensitive reports whether a field's default search is case-insensitive.
func FieldIsCaseInsensitive(field string) bool {
_, ok := caseInsensitiveFields()[field]
return ok
}
// FieldIsPath reports whether a field is a hierarchical path field.
func FieldIsPath(field string) bool {
_, ok := pathFields()[field]
return ok
}
+5 -30
View File
@@ -72,12 +72,13 @@ type Resource struct {
// resourceFieldOverrides is built once (it never changes) and reused on hot
// paths instead of reallocating per call.
var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts {
excludeFromAll := false
True, False := true, false
return map[string]mapping.FieldOpts{
"Name": {Analyzer: "lowercaseKeyword"},
"Name": {CaseInsensitive: &True},
"Path": {Type: mapping.TypePath, CaseInsensitive: &True},
"Content": {Type: mapping.TypeFulltext},
"Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll},
"Favorites": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll},
"Tags": {CaseInsensitive: &True, IncludeInAll: &False},
"Favorites": {CaseInsensitive: &True, IncludeInAll: &False},
"location": {Type: mapping.TypeGeopoint},
}
})
@@ -89,32 +90,6 @@ func (Resource) SearchFieldOverrides() map[string]mapping.FieldOpts {
return resourceFieldOverrides()
}
// lowercaseValueFields is the set of index field names whose query values must
// be lowercased to match their index-time lowercasing analyzer (lowercaseKeyword
// or the fulltext type). Built once from the field overrides.
var lowercaseValueFields = sync.OnceValue(func() map[string]struct{} {
out := map[string]struct{}{}
for key, opts := range resourceFieldOverrides() {
if opts.Analyzer == "lowercaseKeyword" || opts.Type == mapping.TypeFulltext {
out[key] = struct{}{}
}
}
// stored values are normalized lowercase, so query values must fold too
// even though the index fields preserve case
for _, key := range []string{"MimeType", "Type", "Hidden"} {
out[key] = struct{}{}
}
return out
})
// LowercaseValueFields returns the set of index field names whose query values
// must be lowercased so query-side matching lines up with the index-time
// analyzer. Both search backends use it, so value casing stays consistent; every
// other (case-preserved) field keeps its original case. Read-only, do not mutate.
func LowercaseValueFields() map[string]struct{} {
return lowercaseValueFields()
}
// ResolveReference makes sure the path is relative to the space root
func ResolveReference(ctx context.Context, ref *provider.Reference, ri *provider.ResourceInfo, gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) (*provider.Reference, error) {
if ref.GetResourceId().GetOpaqueId() == ref.GetResourceId().GetSpaceId() {