mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-10-06 10:52:08 -04:00
Keyword and path fields always index their case-preserved base and, when CaseInsensitive is set, an additional <field>_lowercase sibling used only for matching. The KQL lowering marks a restriction case-insensitive; each backend searches the sibling and lowercases the query value the same way the sibling is precomputed at index time (Go strings.ToLower on both sides, so non-ASCII stays consistent). Search always returns the case-preserved base, so the sibling never has to be read back. In bleve it is indexed but not stored, kept out of _all, and without doc values. In OpenSearch it deliberately stays in _source: excluding it would make every update-by-query script rebuild all siblings from the document via painless toLowerCase, which lowercases differently than Go and would drift from the query side. Keeping it in _source avoids that, and a lowercased copy of a name or path is negligible disk in a cluster. The OpenSearch move script keeps the base and its sibling in sync by swapping the moved prefix in Path_lowercase and setting Name_lowercase from Go-lowercased params, so case-insensitive search still finds a file after it moves (previously the sibling went stale). bleve re-indexes the whole document on move/delete/restore, so its siblings stay fresh for free. This also repairs OpenSearch path search (the query value was no longer folded to lowercase, so path:<Foo> returned nothing) and makes bleve path queries match a folder and its descendants like OpenSearch's path_hierarchy. The Path base stays case-preserved so the move/delete descendant update (an exact TermQuery on Path) matches mixed-case folders.
126 lines
5.3 KiB
Go
126 lines
5.3 KiB
Go
package mapping
|
|
|
|
import (
|
|
"reflect"
|
|
"time"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
)
|
|
|
|
type bleveDoc struct {
|
|
Name string `json:"Name"`
|
|
Content string `json:"Content"`
|
|
Tags []string `json:"Tags"`
|
|
Size uint64 `json:"Size"`
|
|
Deleted bool `json:"Deleted"`
|
|
CreatedAt time.Time `json:"CreatedAt"`
|
|
Nested *nested `json:"nested,omitempty"`
|
|
}
|
|
|
|
type nested struct {
|
|
Artist string `json:"artist"`
|
|
Year int `json:"year"`
|
|
}
|
|
|
|
var _ = Describe("BleveBuildMapping", func() {
|
|
It("falls back to text for wildcard fields", func() {
|
|
// bleve wildcard falls back to keyword-ish text (bleve has no wildcard type).
|
|
type doc struct {
|
|
Mime string `json:"mime"`
|
|
}
|
|
dm, err := BleveBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"mime": {Type: TypeWildcard}})
|
|
Expect(err).ToNot(HaveOccurred())
|
|
fms := dm.Properties["mime"].Fields
|
|
Expect(fms).To(HaveLen(1))
|
|
Expect(fms[0].Type).To(Equal("text"), "wildcard should map to text")
|
|
})
|
|
|
|
DescribeTable("infers field types",
|
|
func(field, wantType string) {
|
|
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
prop := dm.Properties[field]
|
|
Expect(prop).ToNot(BeNil(), "missing property %q", field)
|
|
Expect(prop.Fields).ToNot(BeEmpty(), "%q: no field mappings", field)
|
|
Expect(prop.Fields[0].Type).To(Equal(wantType), "%q type", field)
|
|
},
|
|
Entry("Name", "Name", "text"),
|
|
Entry("Content", "Content", "text"),
|
|
Entry("Tags", "Tags", "text"),
|
|
Entry("Size", "Size", "number"),
|
|
Entry("Deleted", "Deleted", "boolean"),
|
|
Entry("CreatedAt", "CreatedAt", "datetime"),
|
|
)
|
|
|
|
It("maps a nested struct as a sub-document", func() {
|
|
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
sub := dm.Properties["nested"]
|
|
Expect(sub).ToNot(BeNil(), "missing nested sub-document")
|
|
Expect(sub.Properties["artist"]).ToNot(BeNil())
|
|
Expect(sub.Properties["year"]).ToNot(BeNil())
|
|
Expect(sub.Properties["artist"].Fields[0].Type).To(Equal("text"), "nested.artist")
|
|
Expect(sub.Properties["year"].Fields[0].Type).To(Equal("number"), "nested.year")
|
|
})
|
|
|
|
It("applies field overrides", func() {
|
|
True, False := true, false
|
|
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{
|
|
"Name": {CaseInsensitive: &True},
|
|
"Content": {Type: TypeFulltext},
|
|
"Tags": {CaseInsensitive: &True, IncludeInAll: &False},
|
|
})
|
|
Expect(err).ToNot(HaveOccurred())
|
|
// Name: case-preserved base keyword + lowercased sibling.
|
|
Expect(dm.Properties["Name"]).ToNot(BeNil(), "Name base field")
|
|
Expect(dm.Properties["Name"].Fields[0].Analyzer).To(Equal("keyword"), "Name base is a keyword")
|
|
Expect(dm.Properties["Name"].Fields[0].Store).To(BeTrue(), "Name base is stored (returned)")
|
|
Expect(dm.Properties["Name_lowercase"]).ToNot(BeNil(), "Name_lowercase sibling")
|
|
// The sibling is a search-only shadow: indexed but never stored, kept out
|
|
// of _all, no doc values (the base is what we return).
|
|
sibling := dm.Properties["Name_lowercase"].Fields[0]
|
|
Expect(sibling.Index).To(BeTrue(), "Name_lowercase is indexed")
|
|
Expect(sibling.Store).To(BeFalse(), "Name_lowercase is not stored")
|
|
Expect(sibling.IncludeInAll).To(BeFalse(), "Name_lowercase is out of _all")
|
|
Expect(sibling.DocValues).To(BeFalse(), "Name_lowercase has no doc values")
|
|
contentField := dm.Properties["Content"].Fields[0]
|
|
Expect(contentField.Analyzer).To(Equal("fulltext"), "Content analyzer")
|
|
Expect(contentField.IncludeInAll).To(BeFalse(), "Content IncludeInAll should default to false for fulltext type")
|
|
// Tags: base + lowercased sibling, both honoring the IncludeInAll override.
|
|
Expect(dm.Properties["Tags"].Fields[0].IncludeInAll).To(BeFalse(), "Tags base IncludeInAll honored")
|
|
Expect(dm.Properties["Tags_lowercase"].Fields[0].IncludeInAll).To(BeFalse(), "Tags sibling IncludeInAll honored")
|
|
})
|
|
|
|
It("builds an object sub-document plus a geopoint sibling", func() {
|
|
type geoDoc struct {
|
|
Location *struct {
|
|
Lon *float64 `json:"longitude,omitempty"`
|
|
Lat *float64 `json:"latitude,omitempty"`
|
|
Alt *float64 `json:"altitude,omitempty"`
|
|
} `json:"location,omitempty"`
|
|
}
|
|
dm, err := BleveBuildMapping(reflect.TypeFor[geoDoc](), map[string]FieldOpts{
|
|
"location": {Type: TypeGeopoint},
|
|
})
|
|
Expect(err).ToNot(HaveOccurred())
|
|
// Original facet stays as an object sub-document with numeric
|
|
// sub-properties - for data retrieval via hit.Fields and ordinary
|
|
// numeric queries.
|
|
loc := dm.Properties["location"]
|
|
Expect(loc).ToNot(BeNil(), "location sub-document missing")
|
|
Expect(loc.Fields).To(BeEmpty(), "location should not carry field mappings directly")
|
|
for _, sub := range []string{"longitude", "latitude", "altitude"} {
|
|
prop, ok := loc.Properties[sub]
|
|
Expect(ok).To(BeTrue(), "missing sub-field %q under location", sub)
|
|
Expect(prop.Fields).ToNot(BeEmpty(), "location.%s Fields", sub)
|
|
Expect(prop.Fields[0].Type).To(Equal("number"), "location.%s type", sub)
|
|
}
|
|
// Sibling geopoint at "<name>_geopoint" for geo-distance queries.
|
|
sibling := dm.Properties["location"+GeopointSuffix]
|
|
Expect(sibling).ToNot(BeNil(), "location%s missing", GeopointSuffix)
|
|
Expect(sibling.Fields).ToNot(BeEmpty(), "location%s Fields", GeopointSuffix)
|
|
Expect(sibling.Fields[0].Type).To(Equal("geopoint"), "location%s type", GeopointSuffix)
|
|
})
|
|
})
|