mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-09-12 21:58:58 -04:00
feat(search): split names and titles into words on both engines
A single word finds the names and titles that contain it, `report` finds Report.txt, on bleve as well as on OpenSearch, which so far only did it by accident of its dynamic mapping. Modelled like SharePoint's NoWordBreaker: a keyword field is one whole value unless the override switches that off, which adds a search-only _words sibling next to _lowercase, analyzed into lowercased words (a dot is a word boundary, no stemming). The base stays the whole value for returning and aggregating, wildcards and whole values keep using _lowercase. Quotes do not change the meaning, a phrase is a phrase either way, and there is no exact-match operator yet.
This commit is contained in:
1 parent
e64692ce9e
commit
6b07ea745c
23 files changed
+249
-48
No files matched your search
@@ -602,6 +602,7 @@ var _ = Describe("Bleve", func() {
|
||||
Expect(count).To(Equal(uint64(1)))
|
||||
|
||||
query := bleveSearch.NewMatchQuery("child.pdf")
|
||||
query.SetField("Name")
|
||||
res, err := idx.Search(bleveSearch.NewSearchRequest(query))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits.Len()).To(Equal(1))
|
||||
@@ -843,6 +844,7 @@ var _ = Describe("Bleve", func() {
|
||||
Expect(count).To(Equal(uint64(1)))
|
||||
|
||||
query := bleveSearch.NewMatchQuery("child.pdf")
|
||||
query.SetField("Name")
|
||||
res, err := idx.Search(bleveSearch.NewSearchRequest(query))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits.Len()).To(Equal(1))
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
|
||||
"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
|
||||
"github.com/blevesearch/bleve/v2/analysis/char/regexp"
|
||||
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
|
||||
"github.com/blevesearch/bleve/v2/analysis/token/porter"
|
||||
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
|
||||
@@ -72,6 +73,28 @@ func NewMapping() (mapping.IndexMapping, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// words: split into lowercased words, a dot is a word boundary too so that
|
||||
// "report" finds "Report.txt"; no stemming, a name is not prose
|
||||
err = indexMapping.AddCustomCharFilter("dot_to_space", map[string]any{
|
||||
"type": regexp.Name,
|
||||
"regexp": `\.`,
|
||||
"replace": " ",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = indexMapping.AddCustomAnalyzer(searchmapping.WordsAnalyzer,
|
||||
map[string]any{
|
||||
"type": custom.Name,
|
||||
"char_filters": []string{"dot_to_space"},
|
||||
"tokenizer": unicode.Name,
|
||||
"token_filters": []string{lowercase.Name},
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return indexMapping, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -64,7 +64,12 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix
|
||||
base := bleveKeywordMapping(fieldType, opts)
|
||||
doc.AddFieldMappingsAt(fi.Name, base)
|
||||
if opts.caseInsensitive() {
|
||||
doc.AddFieldMappingsAt(fi.Name+LowercaseSuffix, lowercaseSibling(base))
|
||||
doc.AddFieldMappingsAt(fi.Name+LowercaseSuffix, searchSibling(base))
|
||||
}
|
||||
if opts.wordBroken() {
|
||||
words := searchSibling(base)
|
||||
words.Analyzer = WordsAnalyzer
|
||||
doc.AddFieldMappingsAt(fi.Name+WordsSuffix, words)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -92,11 +97,11 @@ func bleveKeywordMapping(fieldType string, opts FieldOpts) *bleveMapping.FieldMa
|
||||
return fm
|
||||
}
|
||||
|
||||
// lowercaseSibling derives the lowercased shadow of a keyword/path field from its
|
||||
// base mapping: used only for case-insensitive matching, so indexed but never
|
||||
// stored, kept out of _all, and without doc values, since the case-preserved base
|
||||
// field is what we return and aggregate on.
|
||||
func lowercaseSibling(base *bleveMapping.FieldMapping) *bleveMapping.FieldMapping {
|
||||
// searchSibling derives a search-only shadow of a keyword/path field from its
|
||||
// base mapping (the _lowercase and _words siblings): indexed but never stored,
|
||||
// kept out of _all, and without doc values, since the case-preserved base field
|
||||
// is what we return and aggregate on.
|
||||
func searchSibling(base *bleveMapping.FieldMapping) *bleveMapping.FieldMapping {
|
||||
fm := *base
|
||||
fm.Store = false
|
||||
fm.IncludeInAll = false
|
||||
|
||||
@@ -92,6 +92,22 @@ var _ = Describe("BleveBuildMapping", func() {
|
||||
Expect(dm.Properties["Tags_lowercase"].Fields[0].IncludeInAll).To(BeFalse(), "Tags sibling IncludeInAll honored")
|
||||
})
|
||||
|
||||
It("splits a keyword into words when NoWordBreaker is false", func() {
|
||||
True, False := true, false
|
||||
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{
|
||||
"Name": {NoWordBreaker: &False, CaseInsensitive: &True},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// the base stays a keyword, the words go to a search-only sibling
|
||||
Expect(dm.Properties["Name"].Fields[0].Analyzer).To(Equal("keyword"), "Name base stays a keyword")
|
||||
Expect(dm.Properties["Name"].Fields[0].Store).To(BeTrue(), "Name base is stored (returned)")
|
||||
Expect(dm.Properties["Name_lowercase"].Fields[0].Analyzer).To(Equal("keyword"), "Name_lowercase stays a keyword")
|
||||
words := dm.Properties["Name_words"].Fields[0]
|
||||
Expect(words.Analyzer).To(Equal(WordsAnalyzer), "Name_words is split into words")
|
||||
Expect(words.Store).To(BeFalse(), "Name_words is not stored")
|
||||
Expect(words.IncludeInAll).To(BeFalse(), "Name_words is out of _all")
|
||||
})
|
||||
|
||||
It("builds an object sub-document plus a geopoint sibling", func() {
|
||||
type geoDoc struct {
|
||||
Location *struct {
|
||||
|
||||
@@ -2,16 +2,32 @@ package mapping
|
||||
|
||||
import "strings"
|
||||
|
||||
func addLowercaseSiblings(m map[string]any, overrides map[string]FieldOpts) {
|
||||
// addSearchSiblings writes the _lowercase and _words siblings the overrides ask
|
||||
// for next to their base values.
|
||||
func addSearchSiblings(m map[string]any, overrides map[string]FieldOpts) {
|
||||
for key, opts := range overrides {
|
||||
if !opts.caseInsensitive() || !isCasedType(opts) {
|
||||
if !isCasedType(opts) || (!opts.caseInsensitive() && !opts.wordBroken()) {
|
||||
continue
|
||||
}
|
||||
parent, leaf, ok := resolveLeaf(m, key)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
addLowercaseSibling(parent, leaf)
|
||||
if opts.caseInsensitive() {
|
||||
addLowercaseSibling(parent, leaf)
|
||||
}
|
||||
if opts.wordBroken() {
|
||||
addWordsSibling(parent, leaf)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// addWordsSibling copies the value to a <leaf>_words sibling; the words
|
||||
// analyzer does the splitting and lowercasing. No-op for non-strings.
|
||||
func addWordsSibling(parent map[string]any, leaf string) {
|
||||
switch v := parent[leaf].(type) {
|
||||
case string, []any, []string:
|
||||
parent[leaf+WordsSuffix] = v
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -30,6 +30,21 @@ var _ = Describe("PrepareForIndex casing", func() {
|
||||
Expect(m["Tags_lowercase"]).To(Equal([]any{"work", "urgent"}))
|
||||
})
|
||||
|
||||
It("copies the value to a words sibling when NoWordBreaker is off", func() {
|
||||
True, False := true, false
|
||||
type doc struct {
|
||||
Name string `json:"Name"`
|
||||
}
|
||||
m, err := PrepareForIndex(doc{Name: "Report FINAL"}, map[string]FieldOpts{
|
||||
"Name": {NoWordBreaker: &False, CaseInsensitive: &True},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// the analyzer splits and lowercases, the value goes over as is
|
||||
Expect(m["Name"]).To(Equal("Report FINAL"))
|
||||
Expect(m["Name_lowercase"]).To(Equal("report final"))
|
||||
Expect(m["Name_words"]).To(Equal("Report FINAL"))
|
||||
})
|
||||
|
||||
It("writes no sibling without CaseInsensitive", func() {
|
||||
type doc struct {
|
||||
ID string `json:"ID"`
|
||||
|
||||
@@ -66,6 +66,9 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p
|
||||
if opts.caseInsensitive() {
|
||||
props[fi.Name+LowercaseSuffix] = m
|
||||
}
|
||||
if opts.wordBroken() {
|
||||
props[fi.Name+WordsSuffix] = map[string]any{"type": "text", "analyzer": WordsAnalyzer}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -106,6 +106,21 @@ var _ = Describe("OpenSearchBuildMapping", func() {
|
||||
Expect(mime["type"]).To(Equal("wildcard"), "MimeType: %#v", mime)
|
||||
})
|
||||
|
||||
It("splits a keyword into words when NoWordBreaker is false", func() {
|
||||
True, False := true, false
|
||||
type doc struct {
|
||||
Name string `json:"Name"`
|
||||
}
|
||||
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{
|
||||
"Name": {NoWordBreaker: &False, CaseInsensitive: &True},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// the base stays a keyword, the words go to their own sibling
|
||||
Expect(props["Name"]).To(Equal(map[string]any{"type": "keyword"}))
|
||||
Expect(props["Name_lowercase"]).To(Equal(map[string]any{"type": "keyword"}))
|
||||
Expect(props["Name_words"]).To(Equal(map[string]any{"type": "text", "analyzer": WordsAnalyzer}))
|
||||
})
|
||||
|
||||
It("builds an object plus a geo_point sibling for geopoints", func() {
|
||||
type doc struct {
|
||||
Location *struct {
|
||||
|
||||
@@ -20,6 +20,12 @@ const (
|
||||
// LowercaseSuffix names the lowercased sibling of a keyword/path field.
|
||||
const LowercaseSuffix = "_lowercase"
|
||||
|
||||
// WordsSuffix names the word-broken sibling of a keyword field.
|
||||
const WordsSuffix = "_words"
|
||||
|
||||
// WordsAnalyzer names the analyzer both engines register for the words sibling.
|
||||
const WordsAnalyzer = "words"
|
||||
|
||||
// FieldOpts overrides the default type inference for a struct field. Keys in
|
||||
// the override map are json-tag names (e.g. "Name", "location", "audio.artist"),
|
||||
// not Go field names.
|
||||
@@ -32,9 +38,19 @@ type FieldOpts struct {
|
||||
// Nil/false means off. Keyword/path only.
|
||||
CaseInsensitive *bool
|
||||
|
||||
// NoWordBreaker is SharePoint's switch: nil or true leaves a keyword field
|
||||
// one whole value, false additionally indexes a <name>_words sibling split
|
||||
// into lowercased words (no stemming), so a single word matches a value
|
||||
// that contains it: "report" finds "Report.txt". The base stays the whole
|
||||
// value for returning and aggregating; wildcards and whole-value matches
|
||||
// use the _lowercase sibling, so it wants CaseInsensitive alongside.
|
||||
// Keyword only.
|
||||
NoWordBreaker *bool
|
||||
|
||||
// IncludeInAll controls bleve's _all field inclusion. Nil means "use the
|
||||
// bleve default for this field type". Has no effect on OpenSearch.
|
||||
IncludeInAll *bool
|
||||
}
|
||||
|
||||
func (o FieldOpts) caseInsensitive() bool { return o.CaseInsensitive != nil && *o.CaseInsensitive }
|
||||
func (o FieldOpts) wordBroken() bool { return o.NoWordBreaker != nil && !*o.NoWordBreaker }
|
||||
@@ -19,6 +19,6 @@ func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, err
|
||||
return out, nil
|
||||
}
|
||||
addGeopointSiblings(out, overrides)
|
||||
addLowercaseSiblings(out, overrides)
|
||||
addSearchSiblings(out, overrides)
|
||||
return out, nil
|
||||
}
|
||||
@@ -17,13 +17,16 @@ func Validate(t reflect.Type, overrides map[string]FieldOpts) error {
|
||||
return nil
|
||||
}
|
||||
fields := collectFields(t, "")
|
||||
var unknown, miscased []string
|
||||
var unknown, miscased, unbroken []string
|
||||
for k, opts := range overrides {
|
||||
goType, ok := fields[k]
|
||||
if !ok {
|
||||
unknown = append(unknown, k)
|
||||
continue
|
||||
}
|
||||
if opts.wordBroken() && !effectivelyKeyword(opts, goType) {
|
||||
unbroken = append(unbroken, k)
|
||||
}
|
||||
// CaseInsensitive routes queries to a <field>_lowercase sibling, which is
|
||||
// only generated for keyword/path fields; on any other type the query
|
||||
// would target a non-existent field and silently match nothing. Use the
|
||||
@@ -41,6 +44,10 @@ func Validate(t reflect.Type, overrides map[string]FieldOpts) error {
|
||||
sort.Strings(miscased)
|
||||
return fmt.Errorf("mapping: CaseInsensitive is only valid on keyword/path fields: %s", strings.Join(miscased, ", "))
|
||||
}
|
||||
if len(unbroken) > 0 {
|
||||
sort.Strings(unbroken)
|
||||
return fmt.Errorf("mapping: NoWordBreaker is only valid on keyword fields: %s", strings.Join(unbroken, ", "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -54,6 +61,16 @@ func effectivelyCased(opts FieldOpts, goType reflect.Type) bool {
|
||||
return eff == TypeKeyword || eff == TypePath
|
||||
}
|
||||
|
||||
// effectivelyKeyword reports whether a field is a keyword, the only type
|
||||
// NoWordBreaker applies to.
|
||||
func effectivelyKeyword(opts FieldOpts, goType reflect.Type) bool {
|
||||
eff := opts.Type
|
||||
if eff == "" && goType != nil {
|
||||
eff = inferType(goType)
|
||||
}
|
||||
return eff == TypeKeyword
|
||||
}
|
||||
|
||||
// collectFields maps every known field name (nested as "parent.child") to its Go
|
||||
// type. Embedded structs are flattened, matching encoding/json.
|
||||
func collectFields(t reflect.Type, prefix string) map[string]reflect.Type {
|
||||
|
||||
@@ -55,6 +55,16 @@ var _ = Describe("Validate", func() {
|
||||
Expect(err.Error()).To(ContainSubstring("CaseInsensitive"))
|
||||
})
|
||||
|
||||
It("rejects NoWordBreaker on a non-keyword field", func() {
|
||||
False := false
|
||||
err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{
|
||||
"Name": {Type: TypeFulltext, NoWordBreaker: &False},
|
||||
})
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("Name"))
|
||||
Expect(err.Error()).To(ContainSubstring("NoWordBreaker"))
|
||||
})
|
||||
|
||||
It("rejects CaseInsensitive on an inferred non-keyword field (empty Type)", func() {
|
||||
True := true
|
||||
type doc struct {
|
||||
|
||||
@@ -71,7 +71,7 @@ func (b *Batch) Move(id, parentID, location string) error {
|
||||
newPath := utils.MakeRelativePath(location)
|
||||
newName := path.Base(newPath)
|
||||
return &osu.BodyParamScript{
|
||||
// Keep Name and its lowercased search sibling in sync; Path has
|
||||
// Keep Name and its search siblings in sync; Path has
|
||||
// no sibling (case-sensitive by design). Only the leading
|
||||
// oldPath is replaced (startsWith + substring, not
|
||||
// String.replace, which would also rewrite a repeated segment
|
||||
@@ -85,6 +85,7 @@ func (b *Batch) Move(id, parentID, location string) error {
|
||||
ctx._source.Name = params.newName;
|
||||
ctx._source.ParentID = params.parentID;
|
||||
if (ctx._source.Name%[1]s != null) { ctx._source.Name%[1]s = params.newNameLower; }
|
||||
if (ctx._source.Name%[2]s != null) { ctx._source.Name%[2]s = params.newName; }
|
||||
}
|
||||
if (ctx._source.Path != null && ctx._source.Path.startsWith(params.oldPath)) {
|
||||
ctx._source.Path = params.newPath + ctx._source.Path.substring(params.oldPath.length());
|
||||
@@ -94,7 +95,7 @@ func (b *Batch) Move(id, parentID, location string) error {
|
||||
if (!name.equals('.') && !name.equals('..') && name.startsWith('.')) { hidden = true; break; }
|
||||
}
|
||||
ctx._source.Hidden = hidden;
|
||||
`, mapping.LowercaseSuffix),
|
||||
`, mapping.LowercaseSuffix, mapping.WordsSuffix),
|
||||
Lang: "painless",
|
||||
Params: map[string]any{
|
||||
"id": id,
|
||||
|
||||
@@ -87,6 +87,21 @@ func buildResourceMapping() ([]byte, error) {
|
||||
"tokenizer": "standard",
|
||||
"filter": []string{"lowercase", "porter_stem"},
|
||||
},
|
||||
// words: split into lowercased words, a dot is a word boundary
|
||||
// too so that "report" finds "Report.txt"; no stemming, a name
|
||||
// is not prose
|
||||
searchmapping.WordsAnalyzer: map[string]any{
|
||||
"type": "custom",
|
||||
"char_filter": []string{"dot_to_space"},
|
||||
"tokenizer": "standard",
|
||||
"filter": []string{"lowercase"},
|
||||
},
|
||||
},
|
||||
"char_filter": map[string]any{
|
||||
"dot_to_space": map[string]any{
|
||||
"type": "mapping",
|
||||
"mappings": []string{`. => \u0020`},
|
||||
},
|
||||
},
|
||||
"tokenizer": map[string]any{
|
||||
"path_hierarchy": map[string]any{"type": "path_hierarchy"},
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
|
||||
@@ -119,6 +119,12 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
|
||||
return osu.NewWildcardQuery(field).Value(value), nil
|
||||
}
|
||||
|
||||
// a word-broken field matches the value as a phrase of its words on the
|
||||
// _words sibling, whose analyzer lowercases; wildcards stay on _lowercase
|
||||
if query.FieldIsWordBroken(node.Key) {
|
||||
return osu.NewMatchPhraseQuery(node.Key + mapping.WordsSuffix).Query(node.Value), nil
|
||||
}
|
||||
|
||||
if query.FieldIsFulltext(node.Key) {
|
||||
return osu.NewMatchPhraseQuery(field).Query(value), nil
|
||||
}
|
||||
|
||||
@@ -16,22 +16,22 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{
|
||||
// kql to os dsl - type tests
|
||||
{
|
||||
Name: "term query - string node",
|
||||
Name: "word-broken field matches the value as a phrase on its words sibling",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "openCloud"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
Want: osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
|
||||
},
|
||||
{
|
||||
Name: "case-insensitive term routes to the lowercased sibling",
|
||||
Got: &ast.Ast{
|
||||
Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "openCloud", CaseInsensitive: true},
|
||||
&ast.StringNode{Key: "Tags", Value: "openCloud", CaseInsensitive: true},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[string]("Name_lowercase").Value("opencloud"),
|
||||
Want: osu.NewTermQuery[string]("Tags_lowercase").Value("opencloud"),
|
||||
},
|
||||
{
|
||||
Name: "case-insensitive wildcard routes to the lowercased sibling",
|
||||
@@ -76,7 +76,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
&ast.StringNode{Key: "Name", Value: "open cloud"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewMatchPhraseQuery("Name").Query(`open cloud`),
|
||||
Want: osu.NewMatchPhraseQuery("Name_words").Query(`open cloud`),
|
||||
},
|
||||
{
|
||||
Name: "wildcard query - string node",
|
||||
@@ -127,8 +127,8 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Want: osu.NewBoolQuery().Must(
|
||||
osu.NewTermQuery[string]("Name").Value("a"),
|
||||
osu.NewTermQuery[string]("Name").Value("b"),
|
||||
osu.NewMatchPhraseQuery("Name_words").Query("a"),
|
||||
osu.NewMatchPhraseQuery("Name_words").Query("b"),
|
||||
),
|
||||
},
|
||||
{
|
||||
@@ -140,7 +140,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
}},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[string]("Name").Value("any"),
|
||||
Want: osu.NewMatchPhraseQuery("Name_words").Query("any"),
|
||||
},
|
||||
{
|
||||
Name: "range query >",
|
||||
@@ -202,7 +202,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
&ast.StringNode{Key: "Name", Value: "openCloud"},
|
||||
},
|
||||
},
|
||||
Want: osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
Want: osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
|
||||
},
|
||||
{
|
||||
Name: "[* *]",
|
||||
@@ -214,7 +214,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
@@ -229,7 +229,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
@@ -245,7 +245,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
Want: osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
),
|
||||
},
|
||||
@@ -273,7 +273,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
},
|
||||
Want: osu.NewBoolQuery().
|
||||
Must(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
|
||||
).
|
||||
MustNot(
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
@@ -296,7 +296,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
).
|
||||
Must(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
|
||||
),
|
||||
},
|
||||
{
|
||||
@@ -313,7 +313,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
|
||||
Want: osu.NewBoolQuery().
|
||||
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
|
||||
Should(
|
||||
osu.NewTermQuery[string]("Name").Value("openCloud"),
|
||||
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
|
||||
osu.NewTermQuery[string]("age").Value("32"),
|
||||
osu.NewTermQuery[string]("age").Value("44"),
|
||||
),
|
||||
|
||||
@@ -88,6 +88,13 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
|
||||
v = strings.ToLower(v)
|
||||
}
|
||||
|
||||
// a word-broken field matches the value as a phrase of its words on the
|
||||
// _words sibling (a quoted query string term is a match phrase query
|
||||
// run through the field's analyzer); wildcards stay on _lowercase
|
||||
if searchQuery.FieldIsWordBroken(n.Key) && !strings.Contains(n.Value, "*") {
|
||||
k, v = n.Key+mapping.WordsSuffix, `"`+strings.ReplaceAll(n.Value, `"`, `\"`)+`"`
|
||||
}
|
||||
|
||||
var q bleveQuery.Query = bleveQuery.NewQueryStringQuery(k + ":" + v)
|
||||
if searchQuery.FieldIsPath(n.Key) {
|
||||
// bleve has no path hierarchy analyzer, unlike OpenSearch: match the
|
||||
|
||||
@@ -39,7 +39,7 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name_lowercase:federated`),
|
||||
query.NewQueryStringQuery(`Name_words:"federated"`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -72,7 +72,7 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name_lowercase:john\ smith`),
|
||||
query.NewQueryStringQuery(`Name_words:"John Smith"`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -86,8 +86,8 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name_lowercase:john\ smith`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:jane`),
|
||||
query.NewQueryStringQuery(`Name_words:"John Smith"`),
|
||||
query.NewQueryStringQuery(`Name_words:"Jane"`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -139,10 +139,10 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
want: query.NewDisjunctionQuery([]query.Query{
|
||||
query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name_lowercase:a`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:b`),
|
||||
query.NewQueryStringQuery(`Name_words:"a"`),
|
||||
query.NewQueryStringQuery(`Name_words:"b"`),
|
||||
}),
|
||||
query.NewQueryStringQuery(`Name_lowercase:c`),
|
||||
query.NewQueryStringQuery(`Name_words:"c"`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -158,10 +158,10 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewDisjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name_lowercase:a`),
|
||||
query.NewQueryStringQuery(`Name_words:"a"`),
|
||||
query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name_lowercase:b`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:c`),
|
||||
query.NewQueryStringQuery(`Name_words:"b"`),
|
||||
query.NewQueryStringQuery(`Name_words:"c"`),
|
||||
}),
|
||||
}),
|
||||
wantErr: false,
|
||||
@@ -183,11 +183,11 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewDisjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name_lowercase:a`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:b`),
|
||||
query.NewQueryStringQuery(`Name_lowercase:c`),
|
||||
query.NewQueryStringQuery(`Name_words:"a"`),
|
||||
query.NewQueryStringQuery(`Name_words:"b"`),
|
||||
query.NewQueryStringQuery(`Name_words:"c"`),
|
||||
}),
|
||||
query.NewQueryStringQuery(`Name_lowercase:d`),
|
||||
query.NewQueryStringQuery(`Name_words:"d"`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
@@ -320,7 +320,7 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name_lowercase:john\ smith`),
|
||||
query.NewQueryStringQuery(`Name_words:"John Smith"`),
|
||||
query.NewQueryStringQuery(`Hidden:t`),
|
||||
query.NewQueryStringQuery(`Hidden:t`),
|
||||
}),
|
||||
@@ -548,7 +548,7 @@ func Test_compile(t *testing.T) {
|
||||
},
|
||||
},
|
||||
want: query.NewConjunctionQuery([]query.Query{
|
||||
query.NewQueryStringQuery(`Name_lowercase:john\ smith\ \+\-\=\&\|\>\<\!\(\)\{\}\[\]\^\"\~\:\ `),
|
||||
query.NewQueryStringQuery(`Name_words:"John Smith +-=&|><!(){}[]^\"~: "`),
|
||||
}),
|
||||
wantErr: false,
|
||||
},
|
||||
|
||||
@@ -30,8 +30,8 @@ var _ = Describe("ResolveField", func() {
|
||||
|
||||
var _ = Describe("FieldIsCaseInsensitive", func() {
|
||||
It("reports the CaseInsensitive override fields", func() {
|
||||
// The three CaseInsensitive override fields (resolved canonical names).
|
||||
for _, f := range []string{"Name", "Tags", "Favorites"} {
|
||||
// The CaseInsensitive override fields (resolved canonical names).
|
||||
for _, f := range []string{"Name", "Title", "Tags", "Favorites"} {
|
||||
Expect(query.FieldIsCaseInsensitive(f)).To(BeTrue(), f)
|
||||
}
|
||||
// Case-preserved / non-keyword fields are not.
|
||||
@@ -41,6 +41,18 @@ var _ = Describe("FieldIsCaseInsensitive", func() {
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("FieldIsWordBroken", func() {
|
||||
It("reports the fields with NoWordBreaker switched off", func() {
|
||||
for _, f := range []string{"Name", "Title"} {
|
||||
Expect(query.FieldIsWordBroken(f)).To(BeTrue(), f)
|
||||
}
|
||||
// whole-value keywords, paths and full text are not
|
||||
for _, f := range []string{"Tags", "Favorites", "MimeType", "ID", "Content", "Path", "unknown"} {
|
||||
Expect(query.FieldIsWordBroken(f)).To(BeFalse(), f)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("Normalize", func() {
|
||||
It("resolves fields and expands mediatype", func() {
|
||||
got := norm(
|
||||
|
||||
@@ -102,3 +102,22 @@ func FieldIsFulltext(field string) bool {
|
||||
_, ok := fulltextFields()[field]
|
||||
return ok
|
||||
}
|
||||
|
||||
// wordBrokenFields are the keyword fields split into words (NoWordBreaker set
|
||||
// to false), derived from the overrides.
|
||||
var wordBrokenFields = sync.OnceValue(func() map[string]struct{} {
|
||||
out := map[string]struct{}{}
|
||||
for field, opts := range (search.Resource{}).SearchFieldOverrides() {
|
||||
if opts.NoWordBreaker != nil && !*opts.NoWordBreaker {
|
||||
out[field] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
})
|
||||
|
||||
// FieldIsWordBroken reports whether a field is split into words, so a value
|
||||
// without a wildcard matches it as a phrase of those words instead of as a whole.
|
||||
func FieldIsWordBroken(field string) bool {
|
||||
_, ok := wordBrokenFields()[field]
|
||||
return ok
|
||||
}
|
||||
@@ -74,7 +74,10 @@ type Resource struct {
|
||||
var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts {
|
||||
True, False := true, false
|
||||
return map[string]mapping.FieldOpts{
|
||||
"Name": {CaseInsensitive: &True},
|
||||
// a single word finds names and titles that contain it; wildcards and
|
||||
// whole values go through the lowercased sibling
|
||||
"Name": {NoWordBreaker: &False, CaseInsensitive: &True},
|
||||
"Title": {NoWordBreaker: &False, CaseInsensitive: &True},
|
||||
"Path": {Type: mapping.TypePath},
|
||||
"Content": {Type: mapping.TypeFulltext},
|
||||
"Tags": {CaseInsensitive: &True, IncludeInAll: &False},
|
||||
|
||||
Vendored
+1
@@ -125,6 +125,7 @@ github.com/blevesearch/bleve/v2/analysis
|
||||
github.com/blevesearch/bleve/v2/analysis/analyzer/custom
|
||||
github.com/blevesearch/bleve/v2/analysis/analyzer/keyword
|
||||
github.com/blevesearch/bleve/v2/analysis/analyzer/standard
|
||||
github.com/blevesearch/bleve/v2/analysis/char/regexp
|
||||
github.com/blevesearch/bleve/v2/analysis/datetime/flexible
|
||||
github.com/blevesearch/bleve/v2/analysis/char/regexp
|
||||
github.com/blevesearch/bleve/v2/analysis/datetime/optional
|
||||
|
||||
Reference in new issue
Block a user