feat(search): split names and titles into words on both engines

A single word finds the names and titles that contain it, `report` finds
Report.txt, on bleve as well as on OpenSearch, which so far only did it by
accident of its dynamic mapping. Modelled like SharePoint's NoWordBreaker:
a keyword field is one whole value unless the override switches that off,
which adds a search-only _words sibling next to _lowercase, analyzed into
lowercased words (a dot is a word boundary, no stemming). The base stays the
whole value for returning and aggregating, wildcards and whole values keep
using _lowercase. Quotes do not change the meaning, a phrase is a phrase
either way, and there is no exact-match operator yet.
This commit is contained in:
Dominik Schmidt committed 2026-08-31 13:40:42 +02:00
1 parent e64692ce9e
commit 6b07ea745c
23 files changed
+249 -48

No files matched your search

@@ -602,6 +602,7 @@ var _ = Describe("Bleve", func() {
Expect(count).To(Equal(uint64(1)))
query := bleveSearch.NewMatchQuery("child.pdf")
query.SetField("Name")
res, err := idx.Search(bleveSearch.NewSearchRequest(query))
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits.Len()).To(Equal(1))
@@ -843,6 +844,7 @@ var _ = Describe("Bleve", func() {
Expect(count).To(Equal(uint64(1)))
query := bleveSearch.NewMatchQuery("child.pdf")
query.SetField("Name")
res, err := idx.Search(bleveSearch.NewSearchRequest(query))
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits.Len()).To(Equal(1))
+23
View File
@@ -10,6 +10,7 @@ import (
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
"github.com/blevesearch/bleve/v2/analysis/char/regexp"
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
"github.com/blevesearch/bleve/v2/analysis/token/porter"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
@@ -72,6 +73,28 @@ func NewMapping() (mapping.IndexMapping, error) {
return nil, err
}
// words: split into lowercased words, a dot is a word boundary too so that
// "report" finds "Report.txt"; no stemming, a name is not prose
err = indexMapping.AddCustomCharFilter("dot_to_space", map[string]any{
"type": regexp.Name,
"regexp": `\.`,
"replace": " ",
})
if err != nil {
return nil, err
}
err = indexMapping.AddCustomAnalyzer(searchmapping.WordsAnalyzer,
map[string]any{
"type": custom.Name,
"char_filters": []string{"dot_to_space"},
"tokenizer": unicode.Name,
"token_filters": []string{lowercase.Name},
},
)
if err != nil {
return nil, err
}
return indexMapping, nil
}
+11 -6
View File
@@ -64,7 +64,12 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix
base := bleveKeywordMapping(fieldType, opts)
doc.AddFieldMappingsAt(fi.Name, base)
if opts.caseInsensitive() {
doc.AddFieldMappingsAt(fi.Name+LowercaseSuffix, lowercaseSibling(base))
doc.AddFieldMappingsAt(fi.Name+LowercaseSuffix, searchSibling(base))
}
if opts.wordBroken() {
words := searchSibling(base)
words.Analyzer = WordsAnalyzer
doc.AddFieldMappingsAt(fi.Name+WordsSuffix, words)
}
return nil
}
@@ -92,11 +97,11 @@ func bleveKeywordMapping(fieldType string, opts FieldOpts) *bleveMapping.FieldMa
return fm
}
// lowercaseSibling derives the lowercased shadow of a keyword/path field from its
// base mapping: used only for case-insensitive matching, so indexed but never
// stored, kept out of _all, and without doc values, since the case-preserved base
// field is what we return and aggregate on.
func lowercaseSibling(base *bleveMapping.FieldMapping) *bleveMapping.FieldMapping {
// searchSibling derives a search-only shadow of a keyword/path field from its
// base mapping (the _lowercase and _words siblings): indexed but never stored,
// kept out of _all, and without doc values, since the case-preserved base field
// is what we return and aggregate on.
func searchSibling(base *bleveMapping.FieldMapping) *bleveMapping.FieldMapping {
fm := *base
fm.Store = false
fm.IncludeInAll = false
+16
View File
@@ -92,6 +92,22 @@ var _ = Describe("BleveBuildMapping", func() {
Expect(dm.Properties["Tags_lowercase"].Fields[0].IncludeInAll).To(BeFalse(), "Tags sibling IncludeInAll honored")
})
It("splits a keyword into words when NoWordBreaker is false", func() {
True, False := true, false
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{
"Name": {NoWordBreaker: &False, CaseInsensitive: &True},
})
Expect(err).ToNot(HaveOccurred())
// the base stays a keyword, the words go to a search-only sibling
Expect(dm.Properties["Name"].Fields[0].Analyzer).To(Equal("keyword"), "Name base stays a keyword")
Expect(dm.Properties["Name"].Fields[0].Store).To(BeTrue(), "Name base is stored (returned)")
Expect(dm.Properties["Name_lowercase"].Fields[0].Analyzer).To(Equal("keyword"), "Name_lowercase stays a keyword")
words := dm.Properties["Name_words"].Fields[0]
Expect(words.Analyzer).To(Equal(WordsAnalyzer), "Name_words is split into words")
Expect(words.Store).To(BeFalse(), "Name_words is not stored")
Expect(words.IncludeInAll).To(BeFalse(), "Name_words is out of _all")
})
It("builds an object sub-document plus a geopoint sibling", func() {
type geoDoc struct {
Location *struct {
+19 -3
View File
@@ -2,16 +2,32 @@ package mapping
import "strings"
func addLowercaseSiblings(m map[string]any, overrides map[string]FieldOpts) {
// addSearchSiblings writes the _lowercase and _words siblings the overrides ask
// for next to their base values.
func addSearchSiblings(m map[string]any, overrides map[string]FieldOpts) {
for key, opts := range overrides {
if !opts.caseInsensitive() || !isCasedType(opts) {
if !isCasedType(opts) || (!opts.caseInsensitive() && !opts.wordBroken()) {
continue
}
parent, leaf, ok := resolveLeaf(m, key)
if !ok {
continue
}
addLowercaseSibling(parent, leaf)
if opts.caseInsensitive() {
addLowercaseSibling(parent, leaf)
}
if opts.wordBroken() {
addWordsSibling(parent, leaf)
}
}
}
// addWordsSibling copies the value to a <leaf>_words sibling; the words
// analyzer does the splitting and lowercasing. No-op for non-strings.
func addWordsSibling(parent map[string]any, leaf string) {
switch v := parent[leaf].(type) {
case string, []any, []string:
parent[leaf+WordsSuffix] = v
}
}
@@ -30,6 +30,21 @@ var _ = Describe("PrepareForIndex casing", func() {
Expect(m["Tags_lowercase"]).To(Equal([]any{"work", "urgent"}))
})
It("copies the value to a words sibling when NoWordBreaker is off", func() {
True, False := true, false
type doc struct {
Name string `json:"Name"`
}
m, err := PrepareForIndex(doc{Name: "Report FINAL"}, map[string]FieldOpts{
"Name": {NoWordBreaker: &False, CaseInsensitive: &True},
})
Expect(err).ToNot(HaveOccurred())
// the analyzer splits and lowercases, the value goes over as is
Expect(m["Name"]).To(Equal("Report FINAL"))
Expect(m["Name_lowercase"]).To(Equal("report final"))
Expect(m["Name_words"]).To(Equal("Report FINAL"))
})
It("writes no sibling without CaseInsensitive", func() {
type doc struct {
ID string `json:"ID"`
@@ -66,6 +66,9 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p
if opts.caseInsensitive() {
props[fi.Name+LowercaseSuffix] = m
}
if opts.wordBroken() {
props[fi.Name+WordsSuffix] = map[string]any{"type": "text", "analyzer": WordsAnalyzer}
}
return nil
}
@@ -106,6 +106,21 @@ var _ = Describe("OpenSearchBuildMapping", func() {
Expect(mime["type"]).To(Equal("wildcard"), "MimeType: %#v", mime)
})
It("splits a keyword into words when NoWordBreaker is false", func() {
True, False := true, false
type doc struct {
Name string `json:"Name"`
}
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{
"Name": {NoWordBreaker: &False, CaseInsensitive: &True},
})
Expect(err).ToNot(HaveOccurred())
// the base stays a keyword, the words go to their own sibling
Expect(props["Name"]).To(Equal(map[string]any{"type": "keyword"}))
Expect(props["Name_lowercase"]).To(Equal(map[string]any{"type": "keyword"}))
Expect(props["Name_words"]).To(Equal(map[string]any{"type": "text", "analyzer": WordsAnalyzer}))
})
It("builds an object plus a geo_point sibling for geopoints", func() {
type doc struct {
Location *struct {
+16
View File
@@ -20,6 +20,12 @@ const (
// LowercaseSuffix names the lowercased sibling of a keyword/path field.
const LowercaseSuffix = "_lowercase"
// WordsSuffix names the word-broken sibling of a keyword field.
const WordsSuffix = "_words"
// WordsAnalyzer names the analyzer both engines register for the words sibling.
const WordsAnalyzer = "words"
// FieldOpts overrides the default type inference for a struct field. Keys in
// the override map are json-tag names (e.g. "Name", "location", "audio.artist"),
// not Go field names.
@@ -32,9 +38,19 @@ type FieldOpts struct {
// Nil/false means off. Keyword/path only.
CaseInsensitive *bool
// NoWordBreaker is SharePoint's switch: nil or true leaves a keyword field
// one whole value, false additionally indexes a <name>_words sibling split
// into lowercased words (no stemming), so a single word matches a value
// that contains it: "report" finds "Report.txt". The base stays the whole
// value for returning and aggregating; wildcards and whole-value matches
// use the _lowercase sibling, so it wants CaseInsensitive alongside.
// Keyword only.
NoWordBreaker *bool
// IncludeInAll controls bleve's _all field inclusion. Nil means "use the
// bleve default for this field type". Has no effect on OpenSearch.
IncludeInAll *bool
}
func (o FieldOpts) caseInsensitive() bool { return o.CaseInsensitive != nil && *o.CaseInsensitive }
func (o FieldOpts) wordBroken() bool { return o.NoWordBreaker != nil && !*o.NoWordBreaker }
+1 -1
View File
@@ -19,6 +19,6 @@ func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, err
return out, nil
}
addGeopointSiblings(out, overrides)
addLowercaseSiblings(out, overrides)
addSearchSiblings(out, overrides)
return out, nil
}
+18 -1
View File
@@ -17,13 +17,16 @@ func Validate(t reflect.Type, overrides map[string]FieldOpts) error {
return nil
}
fields := collectFields(t, "")
var unknown, miscased []string
var unknown, miscased, unbroken []string
for k, opts := range overrides {
goType, ok := fields[k]
if !ok {
unknown = append(unknown, k)
continue
}
if opts.wordBroken() && !effectivelyKeyword(opts, goType) {
unbroken = append(unbroken, k)
}
// CaseInsensitive routes queries to a <field>_lowercase sibling, which is
// only generated for keyword/path fields; on any other type the query
// would target a non-existent field and silently match nothing. Use the
@@ -41,6 +44,10 @@ func Validate(t reflect.Type, overrides map[string]FieldOpts) error {
sort.Strings(miscased)
return fmt.Errorf("mapping: CaseInsensitive is only valid on keyword/path fields: %s", strings.Join(miscased, ", "))
}
if len(unbroken) > 0 {
sort.Strings(unbroken)
return fmt.Errorf("mapping: NoWordBreaker is only valid on keyword fields: %s", strings.Join(unbroken, ", "))
}
return nil
}
@@ -54,6 +61,16 @@ func effectivelyCased(opts FieldOpts, goType reflect.Type) bool {
return eff == TypeKeyword || eff == TypePath
}
// effectivelyKeyword reports whether a field is a keyword, the only type
// NoWordBreaker applies to.
func effectivelyKeyword(opts FieldOpts, goType reflect.Type) bool {
eff := opts.Type
if eff == "" && goType != nil {
eff = inferType(goType)
}
return eff == TypeKeyword
}
// collectFields maps every known field name (nested as "parent.child") to its Go
// type. Embedded structs are flattened, matching encoding/json.
func collectFields(t reflect.Type, prefix string) map[string]reflect.Type {
@@ -55,6 +55,16 @@ var _ = Describe("Validate", func() {
Expect(err.Error()).To(ContainSubstring("CaseInsensitive"))
})
It("rejects NoWordBreaker on a non-keyword field", func() {
False := false
err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{
"Name": {Type: TypeFulltext, NoWordBreaker: &False},
})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("Name"))
Expect(err.Error()).To(ContainSubstring("NoWordBreaker"))
})
It("rejects CaseInsensitive on an inferred non-keyword field (empty Type)", func() {
True := true
type doc struct {
+3 -2
View File
@@ -71,7 +71,7 @@ func (b *Batch) Move(id, parentID, location string) error {
newPath := utils.MakeRelativePath(location)
newName := path.Base(newPath)
return &osu.BodyParamScript{
// Keep Name and its lowercased search sibling in sync; Path has
// Keep Name and its search siblings in sync; Path has
// no sibling (case-sensitive by design). Only the leading
// oldPath is replaced (startsWith + substring, not
// String.replace, which would also rewrite a repeated segment
@@ -85,6 +85,7 @@ func (b *Batch) Move(id, parentID, location string) error {
ctx._source.Name = params.newName;
ctx._source.ParentID = params.parentID;
if (ctx._source.Name%[1]s != null) { ctx._source.Name%[1]s = params.newNameLower; }
if (ctx._source.Name%[2]s != null) { ctx._source.Name%[2]s = params.newName; }
}
if (ctx._source.Path != null && ctx._source.Path.startsWith(params.oldPath)) {
ctx._source.Path = params.newPath + ctx._source.Path.substring(params.oldPath.length());
@@ -94,7 +95,7 @@ func (b *Batch) Move(id, parentID, location string) error {
if (!name.equals('.') && !name.equals('..') && name.startsWith('.')) { hidden = true; break; }
}
ctx._source.Hidden = hidden;
`, mapping.LowercaseSuffix),
`, mapping.LowercaseSuffix, mapping.WordsSuffix),
Lang: "painless",
Params: map[string]any{
"id": id,
+15
View File
@@ -87,6 +87,21 @@ func buildResourceMapping() ([]byte, error) {
"tokenizer": "standard",
"filter": []string{"lowercase", "porter_stem"},
},
// words: split into lowercased words, a dot is a word boundary
// too so that "report" finds "Report.txt"; no stemming, a name
// is not prose
searchmapping.WordsAnalyzer: map[string]any{
"type": "custom",
"char_filter": []string{"dot_to_space"},
"tokenizer": "standard",
"filter": []string{"lowercase"},
},
},
"char_filter": map[string]any{
"dot_to_space": map[string]any{
"type": "mapping",
"mappings": []string{`. => \u0020`},
},
},
"tokenizer": map[string]any{
"path_hierarchy": map[string]any{"type": "path_hierarchy"},
@@ -10,7 +10,6 @@ import (
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
@@ -119,6 +119,12 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
return osu.NewWildcardQuery(field).Value(value), nil
}
// a word-broken field matches the value as a phrase of its words on the
// _words sibling, whose analyzer lowercases; wildcards stay on _lowercase
if query.FieldIsWordBroken(node.Key) {
return osu.NewMatchPhraseQuery(node.Key + mapping.WordsSuffix).Query(node.Value), nil
}
if query.FieldIsFulltext(node.Key) {
return osu.NewMatchPhraseQuery(field).Query(value), nil
}
@@ -16,22 +16,22 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{
// kql to os dsl - type tests
{
Name: "term query - string node",
Name: "word-broken field matches the value as a phrase on its words sibling",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "openCloud"},
},
},
Want: osu.NewTermQuery[string]("Name").Value("openCloud"),
Want: osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
},
{
Name: "case-insensitive term routes to the lowercased sibling",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "openCloud", CaseInsensitive: true},
&ast.StringNode{Key: "Tags", Value: "openCloud", CaseInsensitive: true},
},
},
Want: osu.NewTermQuery[string]("Name_lowercase").Value("opencloud"),
Want: osu.NewTermQuery[string]("Tags_lowercase").Value("opencloud"),
},
{
Name: "case-insensitive wildcard routes to the lowercased sibling",
@@ -76,7 +76,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
&ast.StringNode{Key: "Name", Value: "open cloud"},
},
},
Want: osu.NewMatchPhraseQuery("Name").Query(`open cloud`),
Want: osu.NewMatchPhraseQuery("Name_words").Query(`open cloud`),
},
{
Name: "wildcard query - string node",
@@ -127,8 +127,8 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
},
},
Want: osu.NewBoolQuery().Must(
osu.NewTermQuery[string]("Name").Value("a"),
osu.NewTermQuery[string]("Name").Value("b"),
osu.NewMatchPhraseQuery("Name_words").Query("a"),
osu.NewMatchPhraseQuery("Name_words").Query("b"),
),
},
{
@@ -140,7 +140,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
}},
},
},
Want: osu.NewTermQuery[string]("Name").Value("any"),
Want: osu.NewMatchPhraseQuery("Name_words").Query("any"),
},
{
Name: "range query >",
@@ -202,7 +202,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
&ast.StringNode{Key: "Name", Value: "openCloud"},
},
},
Want: osu.NewTermQuery[string]("Name").Value("openCloud"),
Want: osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
},
{
Name: "[* *]",
@@ -214,7 +214,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
},
Want: osu.NewBoolQuery().
Must(
osu.NewTermQuery[string]("Name").Value("openCloud"),
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
osu.NewTermQuery[string]("age").Value("32"),
),
},
@@ -229,7 +229,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
},
Want: osu.NewBoolQuery().
Must(
osu.NewTermQuery[string]("Name").Value("openCloud"),
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
osu.NewTermQuery[string]("age").Value("32"),
),
},
@@ -245,7 +245,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewTermQuery[string]("Name").Value("openCloud"),
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
osu.NewTermQuery[string]("age").Value("32"),
),
},
@@ -273,7 +273,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
},
Want: osu.NewBoolQuery().
Must(
osu.NewTermQuery[string]("Name").Value("openCloud"),
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
).
MustNot(
osu.NewTermQuery[string]("age").Value("32"),
@@ -296,7 +296,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
osu.NewTermQuery[string]("age").Value("32"),
).
Must(
osu.NewTermQuery[string]("Name").Value("openCloud"),
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
),
},
{
@@ -313,7 +313,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewTermQuery[string]("Name").Value("openCloud"),
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
osu.NewTermQuery[string]("age").Value("32"),
osu.NewTermQuery[string]("age").Value("44"),
),
@@ -88,6 +88,13 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
v = strings.ToLower(v)
}
// a word-broken field matches the value as a phrase of its words on the
// _words sibling (a quoted query string term is a match phrase query
// run through the field's analyzer); wildcards stay on _lowercase
if searchQuery.FieldIsWordBroken(n.Key) && !strings.Contains(n.Value, "*") {
k, v = n.Key+mapping.WordsSuffix, `"`+strings.ReplaceAll(n.Value, `"`, `\"`)+`"`
}
var q bleveQuery.Query = bleveQuery.NewQueryStringQuery(k + ":" + v)
if searchQuery.FieldIsPath(n.Key) {
// bleve has no path hierarchy analyzer, unlike OpenSearch: match the
@@ -39,7 +39,7 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Name_lowercase:federated`),
query.NewQueryStringQuery(`Name_words:"federated"`),
}),
wantErr: false,
},
@@ -72,7 +72,7 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Name_lowercase:john\ smith`),
query.NewQueryStringQuery(`Name_words:"John Smith"`),
}),
wantErr: false,
},
@@ -86,8 +86,8 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Name_lowercase:john\ smith`),
query.NewQueryStringQuery(`Name_lowercase:jane`),
query.NewQueryStringQuery(`Name_words:"John Smith"`),
query.NewQueryStringQuery(`Name_words:"Jane"`),
}),
wantErr: false,
},
@@ -139,10 +139,10 @@ func Test_compile(t *testing.T) {
},
want: query.NewDisjunctionQuery([]query.Query{
query.NewConjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Name_lowercase:a`),
query.NewQueryStringQuery(`Name_lowercase:b`),
query.NewQueryStringQuery(`Name_words:"a"`),
query.NewQueryStringQuery(`Name_words:"b"`),
}),
query.NewQueryStringQuery(`Name_lowercase:c`),
query.NewQueryStringQuery(`Name_words:"c"`),
}),
wantErr: false,
},
@@ -158,10 +158,10 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewDisjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Name_lowercase:a`),
query.NewQueryStringQuery(`Name_words:"a"`),
query.NewConjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Name_lowercase:b`),
query.NewQueryStringQuery(`Name_lowercase:c`),
query.NewQueryStringQuery(`Name_words:"b"`),
query.NewQueryStringQuery(`Name_words:"c"`),
}),
}),
wantErr: false,
@@ -183,11 +183,11 @@ func Test_compile(t *testing.T) {
},
want: query.NewConjunctionQuery([]query.Query{
query.NewDisjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Name_lowercase:a`),
query.NewQueryStringQuery(`Name_lowercase:b`),
query.NewQueryStringQuery(`Name_lowercase:c`),
query.NewQueryStringQuery(`Name_words:"a"`),
query.NewQueryStringQuery(`Name_words:"b"`),
query.NewQueryStringQuery(`Name_words:"c"`),
}),
query.NewQueryStringQuery(`Name_lowercase:d`),
query.NewQueryStringQuery(`Name_words:"d"`),
}),
wantErr: false,
},
@@ -320,7 +320,7 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Name_lowercase:john\ smith`),
query.NewQueryStringQuery(`Name_words:"John Smith"`),
query.NewQueryStringQuery(`Hidden:t`),
query.NewQueryStringQuery(`Hidden:t`),
}),
@@ -548,7 +548,7 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Name_lowercase:john\ smith\ \+\-\=\&\|\>\<\!\(\)\{\}\[\]\^\"\~\:\ `),
query.NewQueryStringQuery(`Name_words:"John Smith +-=&|><!(){}[]^\"~: "`),
}),
wantErr: false,
},
+14 -2
View File
@@ -30,8 +30,8 @@ var _ = Describe("ResolveField", func() {
var _ = Describe("FieldIsCaseInsensitive", func() {
It("reports the CaseInsensitive override fields", func() {
// The three CaseInsensitive override fields (resolved canonical names).
for _, f := range []string{"Name", "Tags", "Favorites"} {
// The CaseInsensitive override fields (resolved canonical names).
for _, f := range []string{"Name", "Title", "Tags", "Favorites"} {
Expect(query.FieldIsCaseInsensitive(f)).To(BeTrue(), f)
}
// Case-preserved / non-keyword fields are not.
@@ -41,6 +41,18 @@ var _ = Describe("FieldIsCaseInsensitive", func() {
})
})
var _ = Describe("FieldIsWordBroken", func() {
It("reports the fields with NoWordBreaker switched off", func() {
for _, f := range []string{"Name", "Title"} {
Expect(query.FieldIsWordBroken(f)).To(BeTrue(), f)
}
// whole-value keywords, paths and full text are not
for _, f := range []string{"Tags", "Favorites", "MimeType", "ID", "Content", "Path", "unknown"} {
Expect(query.FieldIsWordBroken(f)).To(BeFalse(), f)
}
})
})
var _ = Describe("Normalize", func() {
It("resolves fields and expands mediatype", func() {
got := norm(
+19
View File
@@ -102,3 +102,22 @@ func FieldIsFulltext(field string) bool {
_, ok := fulltextFields()[field]
return ok
}
// wordBrokenFields are the keyword fields split into words (NoWordBreaker set
// to false), derived from the overrides.
var wordBrokenFields = sync.OnceValue(func() map[string]struct{} {
out := map[string]struct{}{}
for field, opts := range (search.Resource{}).SearchFieldOverrides() {
if opts.NoWordBreaker != nil && !*opts.NoWordBreaker {
out[field] = struct{}{}
}
}
return out
})
// FieldIsWordBroken reports whether a field is split into words, so a value
// without a wildcard matches it as a phrase of those words instead of as a whole.
func FieldIsWordBroken(field string) bool {
_, ok := wordBrokenFields()[field]
return ok
}
+4 -1
View File
@@ -74,7 +74,10 @@ type Resource struct {
var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts {
True, False := true, false
return map[string]mapping.FieldOpts{
"Name": {CaseInsensitive: &True},
// a single word finds names and titles that contain it; wildcards and
// whole values go through the lowercased sibling
"Name": {NoWordBreaker: &False, CaseInsensitive: &True},
"Title": {NoWordBreaker: &False, CaseInsensitive: &True},
"Path": {Type: mapping.TypePath},
"Content": {Type: mapping.TypeFulltext},
"Tags": {CaseInsensitive: &True, IncludeInAll: &False},
+1
View File
@@ -125,6 +125,7 @@ github.com/blevesearch/bleve/v2/analysis
github.com/blevesearch/bleve/v2/analysis/analyzer/custom
github.com/blevesearch/bleve/v2/analysis/analyzer/keyword
github.com/blevesearch/bleve/v2/analysis/analyzer/standard
github.com/blevesearch/bleve/v2/analysis/char/regexp
github.com/blevesearch/bleve/v2/analysis/datetime/flexible
github.com/blevesearch/bleve/v2/analysis/char/regexp
github.com/blevesearch/bleve/v2/analysis/datetime/optional