diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go index a40572c615..6df0975a9d 100644 --- a/services/search/pkg/bleve/backend_test.go +++ b/services/search/pkg/bleve/backend_test.go @@ -602,6 +602,7 @@ var _ = Describe("Bleve", func() { Expect(count).To(Equal(uint64(1))) query := bleveSearch.NewMatchQuery("child.pdf") + query.SetField("Name") res, err := idx.Search(bleveSearch.NewSearchRequest(query)) Expect(err).ToNot(HaveOccurred()) Expect(res.Hits.Len()).To(Equal(1)) @@ -843,6 +844,7 @@ var _ = Describe("Bleve", func() { Expect(count).To(Equal(uint64(1))) query := bleveSearch.NewMatchQuery("child.pdf") + query.SetField("Name") res, err := idx.Search(bleveSearch.NewSearchRequest(query)) Expect(err).ToNot(HaveOccurred()) Expect(res.Hits.Len()).To(Equal(1)) diff --git a/services/search/pkg/bleve/index.go b/services/search/pkg/bleve/index.go index 76a92625ce..c65373a55f 100644 --- a/services/search/pkg/bleve/index.go +++ b/services/search/pkg/bleve/index.go @@ -10,6 +10,7 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2/analysis/char/regexp" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" "github.com/blevesearch/bleve/v2/analysis/token/porter" "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" @@ -72,6 +73,28 @@ func NewMapping() (mapping.IndexMapping, error) { return nil, err } + // words: split into lowercased words, a dot is a word boundary too so that + // "report" finds "Report.txt"; no stemming, a name is not prose + err = indexMapping.AddCustomCharFilter("dot_to_space", map[string]any{ + "type": regexp.Name, + "regexp": `\.`, + "replace": " ", + }) + if err != nil { + return nil, err + } + err = indexMapping.AddCustomAnalyzer(searchmapping.WordsAnalyzer, + map[string]any{ + "type": custom.Name, + "char_filters": []string{"dot_to_space"}, + "tokenizer": unicode.Name, + "token_filters": []string{lowercase.Name}, + }, + ) + if err != nil { + return nil, err + } + return indexMapping, nil } diff --git a/services/search/pkg/mapping/bleve.go b/services/search/pkg/mapping/bleve.go index 70e1f0949e..150a5a427a 100644 --- a/services/search/pkg/mapping/bleve.go +++ b/services/search/pkg/mapping/bleve.go @@ -64,7 +64,12 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix base := bleveKeywordMapping(fieldType, opts) doc.AddFieldMappingsAt(fi.Name, base) if opts.caseInsensitive() { - doc.AddFieldMappingsAt(fi.Name+LowercaseSuffix, lowercaseSibling(base)) + doc.AddFieldMappingsAt(fi.Name+LowercaseSuffix, searchSibling(base)) + } + if opts.wordBroken() { + words := searchSibling(base) + words.Analyzer = WordsAnalyzer + doc.AddFieldMappingsAt(fi.Name+WordsSuffix, words) } return nil } @@ -92,11 +97,11 @@ func bleveKeywordMapping(fieldType string, opts FieldOpts) *bleveMapping.FieldMa return fm } -// lowercaseSibling derives the lowercased shadow of a keyword/path field from its -// base mapping: used only for case-insensitive matching, so indexed but never -// stored, kept out of _all, and without doc values, since the case-preserved base -// field is what we return and aggregate on. -func lowercaseSibling(base *bleveMapping.FieldMapping) *bleveMapping.FieldMapping { +// searchSibling derives a search-only shadow of a keyword/path field from its +// base mapping (the _lowercase and _words siblings): indexed but never stored, +// kept out of _all, and without doc values, since the case-preserved base field +// is what we return and aggregate on. +func searchSibling(base *bleveMapping.FieldMapping) *bleveMapping.FieldMapping { fm := *base fm.Store = false fm.IncludeInAll = false diff --git a/services/search/pkg/mapping/bleve_test.go b/services/search/pkg/mapping/bleve_test.go index 4fcb09f7ab..dc76c6bd90 100644 --- a/services/search/pkg/mapping/bleve_test.go +++ b/services/search/pkg/mapping/bleve_test.go @@ -92,6 +92,22 @@ var _ = Describe("BleveBuildMapping", func() { Expect(dm.Properties["Tags_lowercase"].Fields[0].IncludeInAll).To(BeFalse(), "Tags sibling IncludeInAll honored") }) + It("splits a keyword into words when NoWordBreaker is false", func() { + True, False := true, false + dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{ + "Name": {NoWordBreaker: &False, CaseInsensitive: &True}, + }) + Expect(err).ToNot(HaveOccurred()) + // the base stays a keyword, the words go to a search-only sibling + Expect(dm.Properties["Name"].Fields[0].Analyzer).To(Equal("keyword"), "Name base stays a keyword") + Expect(dm.Properties["Name"].Fields[0].Store).To(BeTrue(), "Name base is stored (returned)") + Expect(dm.Properties["Name_lowercase"].Fields[0].Analyzer).To(Equal("keyword"), "Name_lowercase stays a keyword") + words := dm.Properties["Name_words"].Fields[0] + Expect(words.Analyzer).To(Equal(WordsAnalyzer), "Name_words is split into words") + Expect(words.Store).To(BeFalse(), "Name_words is not stored") + Expect(words.IncludeInAll).To(BeFalse(), "Name_words is out of _all") + }) + It("builds an object sub-document plus a geopoint sibling", func() { type geoDoc struct { Location *struct { diff --git a/services/search/pkg/mapping/casing.go b/services/search/pkg/mapping/casing.go index f05413533e..abe9348477 100644 --- a/services/search/pkg/mapping/casing.go +++ b/services/search/pkg/mapping/casing.go @@ -2,16 +2,32 @@ package mapping import "strings" -func addLowercaseSiblings(m map[string]any, overrides map[string]FieldOpts) { +// addSearchSiblings writes the _lowercase and _words siblings the overrides ask +// for next to their base values. +func addSearchSiblings(m map[string]any, overrides map[string]FieldOpts) { for key, opts := range overrides { - if !opts.caseInsensitive() || !isCasedType(opts) { + if !isCasedType(opts) || (!opts.caseInsensitive() && !opts.wordBroken()) { continue } parent, leaf, ok := resolveLeaf(m, key) if !ok { continue } - addLowercaseSibling(parent, leaf) + if opts.caseInsensitive() { + addLowercaseSibling(parent, leaf) + } + if opts.wordBroken() { + addWordsSibling(parent, leaf) + } + } +} + +// addWordsSibling copies the value to a _words sibling; the words +// analyzer does the splitting and lowercasing. No-op for non-strings. +func addWordsSibling(parent map[string]any, leaf string) { + switch v := parent[leaf].(type) { + case string, []any, []string: + parent[leaf+WordsSuffix] = v } } diff --git a/services/search/pkg/mapping/casing_test.go b/services/search/pkg/mapping/casing_test.go index 5fba3e9eae..255eb36ef4 100644 --- a/services/search/pkg/mapping/casing_test.go +++ b/services/search/pkg/mapping/casing_test.go @@ -30,6 +30,21 @@ var _ = Describe("PrepareForIndex casing", func() { Expect(m["Tags_lowercase"]).To(Equal([]any{"work", "urgent"})) }) + It("copies the value to a words sibling when NoWordBreaker is off", func() { + True, False := true, false + type doc struct { + Name string `json:"Name"` + } + m, err := PrepareForIndex(doc{Name: "Report FINAL"}, map[string]FieldOpts{ + "Name": {NoWordBreaker: &False, CaseInsensitive: &True}, + }) + Expect(err).ToNot(HaveOccurred()) + // the analyzer splits and lowercases, the value goes over as is + Expect(m["Name"]).To(Equal("Report FINAL")) + Expect(m["Name_lowercase"]).To(Equal("report final")) + Expect(m["Name_words"]).To(Equal("Report FINAL")) + }) + It("writes no sibling without CaseInsensitive", func() { type doc struct { ID string `json:"ID"` diff --git a/services/search/pkg/mapping/opensearch.go b/services/search/pkg/mapping/opensearch.go index 22af1a9831..5254e48423 100644 --- a/services/search/pkg/mapping/opensearch.go +++ b/services/search/pkg/mapping/opensearch.go @@ -66,6 +66,9 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p if opts.caseInsensitive() { props[fi.Name+LowercaseSuffix] = m } + if opts.wordBroken() { + props[fi.Name+WordsSuffix] = map[string]any{"type": "text", "analyzer": WordsAnalyzer} + } return nil } diff --git a/services/search/pkg/mapping/opensearch_test.go b/services/search/pkg/mapping/opensearch_test.go index 1eab516763..4ad05548bb 100644 --- a/services/search/pkg/mapping/opensearch_test.go +++ b/services/search/pkg/mapping/opensearch_test.go @@ -106,6 +106,21 @@ var _ = Describe("OpenSearchBuildMapping", func() { Expect(mime["type"]).To(Equal("wildcard"), "MimeType: %#v", mime) }) + It("splits a keyword into words when NoWordBreaker is false", func() { + True, False := true, false + type doc struct { + Name string `json:"Name"` + } + props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{ + "Name": {NoWordBreaker: &False, CaseInsensitive: &True}, + }) + Expect(err).ToNot(HaveOccurred()) + // the base stays a keyword, the words go to their own sibling + Expect(props["Name"]).To(Equal(map[string]any{"type": "keyword"})) + Expect(props["Name_lowercase"]).To(Equal(map[string]any{"type": "keyword"})) + Expect(props["Name_words"]).To(Equal(map[string]any{"type": "text", "analyzer": WordsAnalyzer})) + }) + It("builds an object plus a geo_point sibling for geopoints", func() { type doc struct { Location *struct { diff --git a/services/search/pkg/mapping/opts.go b/services/search/pkg/mapping/opts.go index 9ae7d295a0..f2589fc865 100644 --- a/services/search/pkg/mapping/opts.go +++ b/services/search/pkg/mapping/opts.go @@ -20,6 +20,12 @@ const ( // LowercaseSuffix names the lowercased sibling of a keyword/path field. const LowercaseSuffix = "_lowercase" +// WordsSuffix names the word-broken sibling of a keyword field. +const WordsSuffix = "_words" + +// WordsAnalyzer names the analyzer both engines register for the words sibling. +const WordsAnalyzer = "words" + // FieldOpts overrides the default type inference for a struct field. Keys in // the override map are json-tag names (e.g. "Name", "location", "audio.artist"), // not Go field names. @@ -32,9 +38,19 @@ type FieldOpts struct { // Nil/false means off. Keyword/path only. CaseInsensitive *bool + // NoWordBreaker is SharePoint's switch: nil or true leaves a keyword field + // one whole value, false additionally indexes a _words sibling split + // into lowercased words (no stemming), so a single word matches a value + // that contains it: "report" finds "Report.txt". The base stays the whole + // value for returning and aggregating; wildcards and whole-value matches + // use the _lowercase sibling, so it wants CaseInsensitive alongside. + // Keyword only. + NoWordBreaker *bool + // IncludeInAll controls bleve's _all field inclusion. Nil means "use the // bleve default for this field type". Has no effect on OpenSearch. IncludeInAll *bool } func (o FieldOpts) caseInsensitive() bool { return o.CaseInsensitive != nil && *o.CaseInsensitive } +func (o FieldOpts) wordBroken() bool { return o.NoWordBreaker != nil && !*o.NoWordBreaker } diff --git a/services/search/pkg/mapping/serialize.go b/services/search/pkg/mapping/serialize.go index ec5420784a..b4e5310989 100644 --- a/services/search/pkg/mapping/serialize.go +++ b/services/search/pkg/mapping/serialize.go @@ -19,6 +19,6 @@ func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, err return out, nil } addGeopointSiblings(out, overrides) - addLowercaseSiblings(out, overrides) + addSearchSiblings(out, overrides) return out, nil } diff --git a/services/search/pkg/mapping/validate.go b/services/search/pkg/mapping/validate.go index fa6a8b222e..3147eb20ed 100644 --- a/services/search/pkg/mapping/validate.go +++ b/services/search/pkg/mapping/validate.go @@ -17,13 +17,16 @@ func Validate(t reflect.Type, overrides map[string]FieldOpts) error { return nil } fields := collectFields(t, "") - var unknown, miscased []string + var unknown, miscased, unbroken []string for k, opts := range overrides { goType, ok := fields[k] if !ok { unknown = append(unknown, k) continue } + if opts.wordBroken() && !effectivelyKeyword(opts, goType) { + unbroken = append(unbroken, k) + } // CaseInsensitive routes queries to a _lowercase sibling, which is // only generated for keyword/path fields; on any other type the query // would target a non-existent field and silently match nothing. Use the @@ -41,6 +44,10 @@ func Validate(t reflect.Type, overrides map[string]FieldOpts) error { sort.Strings(miscased) return fmt.Errorf("mapping: CaseInsensitive is only valid on keyword/path fields: %s", strings.Join(miscased, ", ")) } + if len(unbroken) > 0 { + sort.Strings(unbroken) + return fmt.Errorf("mapping: NoWordBreaker is only valid on keyword fields: %s", strings.Join(unbroken, ", ")) + } return nil } @@ -54,6 +61,16 @@ func effectivelyCased(opts FieldOpts, goType reflect.Type) bool { return eff == TypeKeyword || eff == TypePath } +// effectivelyKeyword reports whether a field is a keyword, the only type +// NoWordBreaker applies to. +func effectivelyKeyword(opts FieldOpts, goType reflect.Type) bool { + eff := opts.Type + if eff == "" && goType != nil { + eff = inferType(goType) + } + return eff == TypeKeyword +} + // collectFields maps every known field name (nested as "parent.child") to its Go // type. Embedded structs are flattened, matching encoding/json. func collectFields(t reflect.Type, prefix string) map[string]reflect.Type { diff --git a/services/search/pkg/mapping/validate_test.go b/services/search/pkg/mapping/validate_test.go index 7a34183a4b..39bf5034d5 100644 --- a/services/search/pkg/mapping/validate_test.go +++ b/services/search/pkg/mapping/validate_test.go @@ -55,6 +55,16 @@ var _ = Describe("Validate", func() { Expect(err.Error()).To(ContainSubstring("CaseInsensitive")) }) + It("rejects NoWordBreaker on a non-keyword field", func() { + False := false + err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{ + "Name": {Type: TypeFulltext, NoWordBreaker: &False}, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Name")) + Expect(err.Error()).To(ContainSubstring("NoWordBreaker")) + }) + It("rejects CaseInsensitive on an inferred non-keyword field (empty Type)", func() { True := true type doc struct { diff --git a/services/search/pkg/opensearch/batch.go b/services/search/pkg/opensearch/batch.go index 637764bff2..9cc9fb852a 100644 --- a/services/search/pkg/opensearch/batch.go +++ b/services/search/pkg/opensearch/batch.go @@ -71,7 +71,7 @@ func (b *Batch) Move(id, parentID, location string) error { newPath := utils.MakeRelativePath(location) newName := path.Base(newPath) return &osu.BodyParamScript{ - // Keep Name and its lowercased search sibling in sync; Path has + // Keep Name and its search siblings in sync; Path has // no sibling (case-sensitive by design). Only the leading // oldPath is replaced (startsWith + substring, not // String.replace, which would also rewrite a repeated segment @@ -85,6 +85,7 @@ func (b *Batch) Move(id, parentID, location string) error { ctx._source.Name = params.newName; ctx._source.ParentID = params.parentID; if (ctx._source.Name%[1]s != null) { ctx._source.Name%[1]s = params.newNameLower; } + if (ctx._source.Name%[2]s != null) { ctx._source.Name%[2]s = params.newName; } } if (ctx._source.Path != null && ctx._source.Path.startsWith(params.oldPath)) { ctx._source.Path = params.newPath + ctx._source.Path.substring(params.oldPath.length()); @@ -94,7 +95,7 @@ func (b *Batch) Move(id, parentID, location string) error { if (!name.equals('.') && !name.equals('..') && name.startsWith('.')) { hidden = true; break; } } ctx._source.Hidden = hidden; - `, mapping.LowercaseSuffix), + `, mapping.LowercaseSuffix, mapping.WordsSuffix), Lang: "painless", Params: map[string]any{ "id": id, diff --git a/services/search/pkg/opensearch/index.go b/services/search/pkg/opensearch/index.go index 744dbb8a12..8049cc4bd0 100644 --- a/services/search/pkg/opensearch/index.go +++ b/services/search/pkg/opensearch/index.go @@ -87,6 +87,21 @@ func buildResourceMapping() ([]byte, error) { "tokenizer": "standard", "filter": []string{"lowercase", "porter_stem"}, }, + // words: split into lowercased words, a dot is a word boundary + // too so that "report" finds "Report.txt"; no stemming, a name + // is not prose + searchmapping.WordsAnalyzer: map[string]any{ + "type": "custom", + "char_filter": []string{"dot_to_space"}, + "tokenizer": "standard", + "filter": []string{"lowercase"}, + }, + }, + "char_filter": map[string]any{ + "dot_to_space": map[string]any{ + "type": "mapping", + "mappings": []string{`. => \u0020`}, + }, }, "tokenizer": map[string]any{ "path_hierarchy": map[string]any{"type": "path_hierarchy"}, diff --git a/services/search/pkg/opensearch/index_test.go b/services/search/pkg/opensearch/index_test.go index 3ba5a38cec..5837189e82 100644 --- a/services/search/pkg/opensearch/index_test.go +++ b/services/search/pkg/opensearch/index_test.go @@ -10,7 +10,6 @@ import ( "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index 0e74dd96c0..6ff8016001 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -119,6 +119,12 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { return osu.NewWildcardQuery(field).Value(value), nil } + // a word-broken field matches the value as a phrase of its words on the + // _words sibling, whose analyzer lowercases; wildcards stay on _lowercase + if query.FieldIsWordBroken(node.Key) { + return osu.NewMatchPhraseQuery(node.Key + mapping.WordsSuffix).Query(node.Value), nil + } + if query.FieldIsFulltext(node.Key) { return osu.NewMatchPhraseQuery(field).Query(value), nil } diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go index 85996976d7..b16085a9b4 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go @@ -16,22 +16,22 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{ // kql to os dsl - type tests { - Name: "term query - string node", + Name: "word-broken field matches the value as a phrase on its words sibling", Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "Name", Value: "openCloud"}, }, }, - Want: osu.NewTermQuery[string]("Name").Value("openCloud"), + Want: osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), }, { Name: "case-insensitive term routes to the lowercased sibling", Got: &ast.Ast{ Nodes: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "openCloud", CaseInsensitive: true}, + &ast.StringNode{Key: "Tags", Value: "openCloud", CaseInsensitive: true}, }, }, - Want: osu.NewTermQuery[string]("Name_lowercase").Value("opencloud"), + Want: osu.NewTermQuery[string]("Tags_lowercase").Value("opencloud"), }, { Name: "case-insensitive wildcard routes to the lowercased sibling", @@ -76,7 +76,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { &ast.StringNode{Key: "Name", Value: "open cloud"}, }, }, - Want: osu.NewMatchPhraseQuery("Name").Query(`open cloud`), + Want: osu.NewMatchPhraseQuery("Name_words").Query(`open cloud`), }, { Name: "wildcard query - string node", @@ -127,8 +127,8 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, }, Want: osu.NewBoolQuery().Must( - osu.NewTermQuery[string]("Name").Value("a"), - osu.NewTermQuery[string]("Name").Value("b"), + osu.NewMatchPhraseQuery("Name_words").Query("a"), + osu.NewMatchPhraseQuery("Name_words").Query("b"), ), }, { @@ -140,7 +140,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }}, }, }, - Want: osu.NewTermQuery[string]("Name").Value("any"), + Want: osu.NewMatchPhraseQuery("Name_words").Query("any"), }, { Name: "range query >", @@ -202,7 +202,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { &ast.StringNode{Key: "Name", Value: "openCloud"}, }, }, - Want: osu.NewTermQuery[string]("Name").Value("openCloud"), + Want: osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), }, { Name: "[* *]", @@ -214,7 +214,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewBoolQuery(). Must( - osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), osu.NewTermQuery[string]("age").Value("32"), ), }, @@ -229,7 +229,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewBoolQuery(). Must( - osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), osu.NewTermQuery[string]("age").Value("32"), ), }, @@ -245,7 +245,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { Want: osu.NewBoolQuery(). Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). Should( - osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), osu.NewTermQuery[string]("age").Value("32"), ), }, @@ -273,7 +273,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewBoolQuery(). Must( - osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), ). MustNot( osu.NewTermQuery[string]("age").Value("32"), @@ -296,7 +296,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { osu.NewTermQuery[string]("age").Value("32"), ). Must( - osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), ), }, { @@ -313,7 +313,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { Want: osu.NewBoolQuery(). Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). Should( - osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), osu.NewTermQuery[string]("age").Value("32"), osu.NewTermQuery[string]("age").Value("44"), ), diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index 4a536d3776..8536acc78b 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -88,6 +88,13 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { v = strings.ToLower(v) } + // a word-broken field matches the value as a phrase of its words on the + // _words sibling (a quoted query string term is a match phrase query + // run through the field's analyzer); wildcards stay on _lowercase + if searchQuery.FieldIsWordBroken(n.Key) && !strings.Contains(n.Value, "*") { + k, v = n.Key+mapping.WordsSuffix, `"`+strings.ReplaceAll(n.Value, `"`, `\"`)+`"` + } + var q bleveQuery.Query = bleveQuery.NewQueryStringQuery(k + ":" + v) if searchQuery.FieldIsPath(n.Key) { // bleve has no path hierarchy analyzer, unlike OpenSearch: match the diff --git a/services/search/pkg/query/bleve/compiler_test.go b/services/search/pkg/query/bleve/compiler_test.go index c1662bbf19..a7ff7ab9dd 100644 --- a/services/search/pkg/query/bleve/compiler_test.go +++ b/services/search/pkg/query/bleve/compiler_test.go @@ -39,7 +39,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:federated`), + query.NewQueryStringQuery(`Name_words:"federated"`), }), wantErr: false, }, @@ -72,7 +72,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:john\ smith`), + query.NewQueryStringQuery(`Name_words:"John Smith"`), }), wantErr: false, }, @@ -86,8 +86,8 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:john\ smith`), - query.NewQueryStringQuery(`Name_lowercase:jane`), + query.NewQueryStringQuery(`Name_words:"John Smith"`), + query.NewQueryStringQuery(`Name_words:"Jane"`), }), wantErr: false, }, @@ -139,10 +139,10 @@ func Test_compile(t *testing.T) { }, want: query.NewDisjunctionQuery([]query.Query{ query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:a`), - query.NewQueryStringQuery(`Name_lowercase:b`), + query.NewQueryStringQuery(`Name_words:"a"`), + query.NewQueryStringQuery(`Name_words:"b"`), }), - query.NewQueryStringQuery(`Name_lowercase:c`), + query.NewQueryStringQuery(`Name_words:"c"`), }), wantErr: false, }, @@ -158,10 +158,10 @@ func Test_compile(t *testing.T) { }, }, want: query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:a`), + query.NewQueryStringQuery(`Name_words:"a"`), query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:b`), - query.NewQueryStringQuery(`Name_lowercase:c`), + query.NewQueryStringQuery(`Name_words:"b"`), + query.NewQueryStringQuery(`Name_words:"c"`), }), }), wantErr: false, @@ -183,11 +183,11 @@ func Test_compile(t *testing.T) { }, want: query.NewConjunctionQuery([]query.Query{ query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:a`), - query.NewQueryStringQuery(`Name_lowercase:b`), - query.NewQueryStringQuery(`Name_lowercase:c`), + query.NewQueryStringQuery(`Name_words:"a"`), + query.NewQueryStringQuery(`Name_words:"b"`), + query.NewQueryStringQuery(`Name_words:"c"`), }), - query.NewQueryStringQuery(`Name_lowercase:d`), + query.NewQueryStringQuery(`Name_words:"d"`), }), wantErr: false, }, @@ -320,7 +320,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:john\ smith`), + query.NewQueryStringQuery(`Name_words:"John Smith"`), query.NewQueryStringQuery(`Hidden:t`), query.NewQueryStringQuery(`Hidden:t`), }), @@ -548,7 +548,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:john\ smith\ \+\-\=\&\|\>\<\!\(\)\{\}\[\]\^\"\~\:\ `), + query.NewQueryStringQuery(`Name_words:"John Smith +-=&|>