diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index 6ff8016001..9747c7e2b3 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "slices" + "strconv" "strings" "time" @@ -108,17 +109,44 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { case *ast.BooleanNode: return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil case *ast.StringNode: + // hidden takes bool words only; anything else matches nothing + if node.Key == "Hidden" { + b, err := strconv.ParseBool(node.Value) + if err != nil { + return osu.NewMatchNoneQuery(), nil + } + return osu.NewTermQuery[bool](node.Key).Value(b), nil + } + field, value := node.Key, node.Value + if query.FieldIsPath(node.Key) { + value = strings.TrimSuffix(value, "/") + } if node.CaseInsensitive { field += mapping.LowercaseSuffix value = strings.ToLower(value) } - isWildcard := strings.Contains(value, "*") - if isWildcard { + if isWildcard := strings.ContainsAny(value, "*?"); isWildcard { + // a wildcard on a word-broken field forgives a missing extension: + // *report also matches Report.txt + if query.FieldIsWordBroken(node.Key) && !strings.HasSuffix(value, "*") { + return osu.NewBoolQuery(). + Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). + Should( + osu.NewWildcardQuery(field).Value(value), + osu.NewWildcardQuery(field).Value(value+".*"), + ), nil + } return osu.NewWildcardQuery(field).Value(value), nil } + // = matches the whole value, on the lowercased sibling for + // case-insensitive fields + if node.Exact { + return osu.NewTermQuery[string](field).Value(value), nil + } + // a word-broken field matches the value as a phrase of its words on the // _words sibling, whose analyzer lowercases; wildcards stay on _lowercase if query.FieldIsWordBroken(node.Key) { diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index 8536acc78b..d2f049724a 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -6,8 +6,6 @@ import ( "strconv" "strings" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/blevesearch/bleve/v2" bleveQuery "github.com/blevesearch/bleve/v2/search/query" "github.com/opencloud-eu/opencloud/pkg/ast" @@ -74,28 +72,71 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { for i := offset; i < len(nodes); i++ { switch n := nodes[i].(type) { case *ast.StringNode: + // hidden takes bool words only; anything else matches nothing + if n.Key == "Hidden" { + var q bleveQuery.Query + if b, err := strconv.ParseBool(n.Value); err == nil { + bq := bleveQuery.NewBoolFieldQuery(b) + bq.SetField(n.Key) + q = bq + } else { + q = bleveQuery.NewMatchNoneQuery() + } + if prev == nil { + prev = q + } else { + next = q + } + break + } + // keys are resolved and media-type expanded by normalize. MimeType // skips the escaper so the category wildcards (image/*) keep their `*`; // bleve treats `/` and `+` as literals mid-term, so a literal MIME like // image/svg+xml still matches exactly. + val := n.Value + if searchQuery.FieldIsPath(n.Key) { + val = strings.TrimSuffix(val, "/") + } k := n.Key - v := n.Value + v := val if k != "ID" && k != "Size" && k != "MimeType" { - v = bleveEscaper.Replace(n.Value) + v = bleveEscaper.Replace(val) } if n.CaseInsensitive { k += mapping.LowercaseSuffix v = strings.ToLower(v) + val = strings.ToLower(val) } + isWildcard := strings.ContainsAny(val, "*?") + // a word-broken field matches the value as a phrase of its words on the // _words sibling (a quoted query string term is a match phrase query // run through the field's analyzer); wildcards stay on _lowercase - if searchQuery.FieldIsWordBroken(n.Key) && !strings.Contains(n.Value, "*") { - k, v = n.Key+mapping.WordsSuffix, `"`+strings.ReplaceAll(n.Value, `"`, `\"`)+`"` + if searchQuery.FieldIsWordBroken(n.Key) && !isWildcard && !n.Exact { + k, v = n.Key+mapping.WordsSuffix, `"`+strings.ReplaceAll(val, `"`, `\"`)+`"` } var q bleveQuery.Query = bleveQuery.NewQueryStringQuery(k + ":" + v) + switch { + case n.Exact && !isWildcard: + // = matches the whole value, on the lowercased sibling for + // case-insensitive fields + tq := bleveQuery.NewTermQuery(val) + tq.SetField(k) + q = tq + case isWildcard && searchQuery.FieldIsWordBroken(n.Key) && !strings.HasSuffix(val, "*"): + // a wildcard on a word-broken field forgives a missing extension: + // *report also matches Report.txt + bq := bleve.NewBooleanQuery() + bq.AddShould( + bleveQuery.NewQueryStringQuery(k+":"+v), + bleveQuery.NewQueryStringQuery(k+":"+v+".*"), + ) + bq.SetMinShould(1) + q = bq + } if searchQuery.FieldIsPath(n.Key) { // bleve has no path hierarchy analyzer, unlike OpenSearch: match the // folder itself and its descendants (`\/*`). A BooleanQuery keeps @@ -316,31 +357,6 @@ func numberRange(field string, operator *ast.OperatorNode, value float64) bleveQ return q } -func pathAndBelow(field, path string) bleveQuery.Query { - path = strings.TrimSuffix(path, "/") - - self := bleveQuery.NewTermQuery(path) - self.SetField(field) - - below := bleveQuery.NewPrefixQuery(path + "/") - below.SetField(field) - - return closed(bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{self, below})) -} - -func closed(q bleveQuery.Query) bleveQuery.Query { - // a bare disjunction reads as an open OR chain to mapBinary, a later OR - // would merge into it and widen the group - return bleveQuery.NewConjunctionQuery([]bleveQuery.Query{q}) -} - -func phrase(field, value string) bleveQuery.Query { - q := bleveQuery.NewMatchPhraseQuery(value) - q.SetField(field) - - return q -} - func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode { for _, n := range group.Nodes { if onode, ok := n.(*ast.StringNode); ok { @@ -349,81 +365,3 @@ func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode { } return group } - -func resourceType(value string) string { - switch strings.ToLower(value) { - case "file": - return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10) - case "folder": - return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10) - default: - return value - } -} - -func mimeType(k, v string) (bleveQuery.Query, bool) { - switch v { - case "file": - q := bleve.NewBooleanQuery() - q.AddMustNot(bleveQuery.NewQueryStringQuery(k + ":httpd/unix-directory")) - return q, false - case "folder": - return bleveQuery.NewQueryStringQuery(k + ":httpd/unix-directory"), false - case "document": - return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k, - "application/msword", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "application/vnd.openxmlformats-officedocument.wordprocessingml.form", - "application/vnd.oasis.opendocument.text", - "text/plain", - "text/markdown", - "application/rtf", - "application/vnd.apple.pages", - )), true - case "spreadsheet": - return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k, - "application/vnd.ms-excel", - "application/vnd.oasis.opendocument.spreadsheet", - "text/csv", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "application/vnd.apple.numbers", - )), true - case "presentation": - return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k, - "application/vnd.openxmlformats-officedocument.presentationml.presentation", - "application/vnd.oasis.opendocument.presentation", - "application/vnd.ms-powerpoint", - "application/vnd.apple.keynote", - )), true - case "pdf": - return bleveQuery.NewQueryStringQuery(k + ":application/pdf"), false - case "image": - return bleveQuery.NewQueryStringQuery(k + ":image/*"), false - case "video": - return bleveQuery.NewQueryStringQuery(k + ":video/*"), false - case "audio": - return bleveQuery.NewQueryStringQuery(k + ":audio/*"), false - case "archive": - return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k, - "application/zip", - "application/gzip", - "application/x-gzip", - "application/x-7z-compressed", - "application/x-rar-compressed", - "application/x-tar", - "application/x-bzip2", - "application/x-bzip", - "application/x-tgz", - )), true - default: - return bleveQuery.NewQueryStringQuery(k + ":" + v), false - } -} - -func newQueryStringQueryList(k string, v ...string) []bleveQuery.Query { - list := make([]bleveQuery.Query, len(v)) - for i := 0; i < len(v); i++ { - list[i] = bleveQuery.NewQueryStringQuery(k + ":" + v[i]) - } - return list -} diff --git a/services/search/pkg/query/bleve/compiler_test.go b/services/search/pkg/query/bleve/compiler_test.go index a7ff7ab9dd..c4becff38f 100644 --- a/services/search/pkg/query/bleve/compiler_test.go +++ b/services/search/pkg/query/bleve/compiler_test.go @@ -24,6 +24,13 @@ var timeMustParse = func(t *testing.T, ts string) time.Time { // canonical ASTs (real field names, media-type already expanded) and call // compile() directly, dropping the query.Normalize wrapper and the mediatype // cases. +func boolFieldQuery(field string, value bool) query.Query { + q := query.NewBoolFieldQuery(value) + q.SetField(field) + + return q +} + func Test_compile(t *testing.T) { tests := []struct { name string @@ -72,7 +79,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_words:"John Smith"`), + query.NewQueryStringQuery(`Name_words:"john smith"`), }), wantErr: false, }, @@ -86,8 +93,8 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_words:"John Smith"`), - query.NewQueryStringQuery(`Name_words:"Jane"`), + query.NewQueryStringQuery(`Name_words:"john smith"`), + query.NewQueryStringQuery(`Name_words:"jane"`), }), wantErr: false, }, @@ -320,9 +327,9 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_words:"John Smith"`), - query.NewQueryStringQuery(`Hidden:t`), - query.NewQueryStringQuery(`Hidden:t`), + query.NewQueryStringQuery(`Name_words:"john smith"`), + boolFieldQuery("Hidden", true), + boolFieldQuery("Hidden", true), }), wantErr: false, }, @@ -548,7 +555,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_words:"John Smith +-=&|>