feat(search): adopt the parity-pinned query semantics on the sibling routing

From #3408: hidden takes bool words only, type categories map to the stored value in the shared pass, ? counts as a wildcard, a non-suffix wildcard on a word-broken field forgives the extension, = matches the whole value on the lowercased sibling, paths lose their trailing slash. Dead compiler helpers removed.

services/search/pkg/opensearch/internal/convert/kql_transpile.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"slices"
+	"strconv"
 	"strings"
 	"time"
 
@@ -108,17 +109,44 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
 	case *ast.BooleanNode:
 		return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil
 	case *ast.StringNode:
+		// hidden takes bool words only; anything else matches nothing
+		if node.Key == "Hidden" {
+			b, err := strconv.ParseBool(node.Value)
+			if err != nil {
+				return osu.NewMatchNoneQuery(), nil
+			}
+			return osu.NewTermQuery[bool](node.Key).Value(b), nil
+		}
+
 		field, value := node.Key, node.Value
+		if query.FieldIsPath(node.Key) {
+			value = strings.TrimSuffix(value, "/")
+		}
 		if node.CaseInsensitive {
 			field += mapping.LowercaseSuffix
 			value = strings.ToLower(value)
 		}
 
-		isWildcard := strings.Contains(value, "*")
-		if isWildcard {
+		if isWildcard := strings.ContainsAny(value, "*?"); isWildcard {
+			// a wildcard on a word-broken field forgives a missing extension:
+			// *report also matches Report.txt
+			if query.FieldIsWordBroken(node.Key) && !strings.HasSuffix(value, "*") {
+				return osu.NewBoolQuery().
+					Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
+					Should(
+						osu.NewWildcardQuery(field).Value(value),
+						osu.NewWildcardQuery(field).Value(value+".*"),
+					), nil
+			}
 			return osu.NewWildcardQuery(field).Value(value), nil
 		}
 
+		// = matches the whole value, on the lowercased sibling for
+		// case-insensitive fields
+		if node.Exact {
+			return osu.NewTermQuery[string](field).Value(value), nil
+		}
+
 		// a word-broken field matches the value as a phrase of its words on the
 		// _words sibling, whose analyzer lowercases; wildcards stay on _lowercase
 		if query.FieldIsWordBroken(node.Key) {

services/search/pkg/query/bleve/compiler.go

@@ -6,8 +6,6 @@ import (
 	"strconv"
 	"strings"
 
-	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
-
 	"github.com/blevesearch/bleve/v2"
 	bleveQuery "github.com/blevesearch/bleve/v2/search/query"
 	"github.com/opencloud-eu/opencloud/pkg/ast"
@@ -74,28 +72,71 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
 	for i := offset; i < len(nodes); i++ {
 		switch n := nodes[i].(type) {
 		case *ast.StringNode:
+			// hidden takes bool words only; anything else matches nothing
+			if n.Key == "Hidden" {
+				var q bleveQuery.Query
+				if b, err := strconv.ParseBool(n.Value); err == nil {
+					bq := bleveQuery.NewBoolFieldQuery(b)
+					bq.SetField(n.Key)
+					q = bq
+				} else {
+					q = bleveQuery.NewMatchNoneQuery()
+				}
+				if prev == nil {
+					prev = q
+				} else {
+					next = q
+				}
+				break
+			}
+
 			// keys are resolved and media-type expanded by normalize. MimeType
 			// skips the escaper so the category wildcards (image/*) keep their `*`;
 			// bleve treats `/` and `+` as literals mid-term, so a literal MIME like
 			// image/svg+xml still matches exactly.
+			val := n.Value
+			if searchQuery.FieldIsPath(n.Key) {
+				val = strings.TrimSuffix(val, "/")
+			}
 			k := n.Key
-			v := n.Value
+			v := val
 			if k != "ID" && k != "Size" && k != "MimeType" {
-				v = bleveEscaper.Replace(n.Value)
+				v = bleveEscaper.Replace(val)
 			}
 			if n.CaseInsensitive {
 				k += mapping.LowercaseSuffix
 				v = strings.ToLower(v)
+				val = strings.ToLower(val)
 			}
 
+			isWildcard := strings.ContainsAny(val, "*?")
+
 			// a word-broken field matches the value as a phrase of its words on the
 			// _words sibling (a quoted query string term is a match phrase query
 			// run through the field's analyzer); wildcards stay on _lowercase
-			if searchQuery.FieldIsWordBroken(n.Key) && !strings.Contains(n.Value, "*") {
-				k, v = n.Key+mapping.WordsSuffix, `"`+strings.ReplaceAll(n.Value, `"`, `\"`)+`"`
+			if searchQuery.FieldIsWordBroken(n.Key) && !isWildcard && !n.Exact {
+				k, v = n.Key+mapping.WordsSuffix, `"`+strings.ReplaceAll(val, `"`, `\"`)+`"`
 			}
 
 			var q bleveQuery.Query = bleveQuery.NewQueryStringQuery(k + ":" + v)
+			switch {
+			case n.Exact && !isWildcard:
+				// = matches the whole value, on the lowercased sibling for
+				// case-insensitive fields
+				tq := bleveQuery.NewTermQuery(val)
+				tq.SetField(k)
+				q = tq
+			case isWildcard && searchQuery.FieldIsWordBroken(n.Key) && !strings.HasSuffix(val, "*"):
+				// a wildcard on a word-broken field forgives a missing extension:
+				// *report also matches Report.txt
+				bq := bleve.NewBooleanQuery()
+				bq.AddShould(
+					bleveQuery.NewQueryStringQuery(k+":"+v),
+					bleveQuery.NewQueryStringQuery(k+":"+v+".*"),
+				)
+				bq.SetMinShould(1)
+				q = bq
+			}
 			if searchQuery.FieldIsPath(n.Key) {
 				// bleve has no path hierarchy analyzer, unlike OpenSearch: match the
 				// folder itself and its descendants (`\/*`). A BooleanQuery keeps
@@ -316,31 +357,6 @@ func numberRange(field string, operator *ast.OperatorNode, value float64) bleveQ
 	return q
 }
 
-func pathAndBelow(field, path string) bleveQuery.Query {
-	path = strings.TrimSuffix(path, "/")
-
-	self := bleveQuery.NewTermQuery(path)
-	self.SetField(field)
-
-	below := bleveQuery.NewPrefixQuery(path + "/")
-	below.SetField(field)
-
-	return closed(bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{self, below}))
-}
-
-func closed(q bleveQuery.Query) bleveQuery.Query {
-	// a bare disjunction reads as an open OR chain to mapBinary, a later OR
-	// would merge into it and widen the group
-	return bleveQuery.NewConjunctionQuery([]bleveQuery.Query{q})
-}
-
-func phrase(field, value string) bleveQuery.Query {
-	q := bleveQuery.NewMatchPhraseQuery(value)
-	q.SetField(field)
-
-	return q
-}
-
 func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode {
 	for _, n := range group.Nodes {
 		if onode, ok := n.(*ast.StringNode); ok {
@@ -349,81 +365,3 @@ func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode {
 	}
 	return group
 }
-
-func resourceType(value string) string {
-	switch strings.ToLower(value) {
-	case "file":
-		return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10)
-	case "folder":
-		return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10)
-	default:
-		return value
-	}
-}
-
-func mimeType(k, v string) (bleveQuery.Query, bool) {
-	switch v {
-	case "file":
-		q := bleve.NewBooleanQuery()
-		q.AddMustNot(bleveQuery.NewQueryStringQuery(k + ":httpd/unix-directory"))
-		return q, false
-	case "folder":
-		return bleveQuery.NewQueryStringQuery(k + ":httpd/unix-directory"), false
-	case "document":
-		return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k,
-			"application/msword",
-			"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-			"application/vnd.openxmlformats-officedocument.wordprocessingml.form",
-			"application/vnd.oasis.opendocument.text",
-			"text/plain",
-			"text/markdown",
-			"application/rtf",
-			"application/vnd.apple.pages",
-		)), true
-	case "spreadsheet":
-		return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k,
-			"application/vnd.ms-excel",
-			"application/vnd.oasis.opendocument.spreadsheet",
-			"text/csv",
-			"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-			"application/vnd.apple.numbers",
-		)), true
-	case "presentation":
-		return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k,
-			"application/vnd.openxmlformats-officedocument.presentationml.presentation",
-			"application/vnd.oasis.opendocument.presentation",
-			"application/vnd.ms-powerpoint",
-			"application/vnd.apple.keynote",
-		)), true
-	case "pdf":
-		return bleveQuery.NewQueryStringQuery(k + ":application/pdf"), false
-	case "image":
-		return bleveQuery.NewQueryStringQuery(k + ":image/*"), false
-	case "video":
-		return bleveQuery.NewQueryStringQuery(k + ":video/*"), false
-	case "audio":
-		return bleveQuery.NewQueryStringQuery(k + ":audio/*"), false
-	case "archive":
-		return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k,
-			"application/zip",
-			"application/gzip",
-			"application/x-gzip",
-			"application/x-7z-compressed",
-			"application/x-rar-compressed",
-			"application/x-tar",
-			"application/x-bzip2",
-			"application/x-bzip",
-			"application/x-tgz",
-		)), true
-	default:
-		return bleveQuery.NewQueryStringQuery(k + ":" + v), false
-	}
-}
-
-func newQueryStringQueryList(k string, v ...string) []bleveQuery.Query {
-	list := make([]bleveQuery.Query, len(v))
-	for i := 0; i < len(v); i++ {
-		list[i] = bleveQuery.NewQueryStringQuery(k + ":" + v[i])
-	}
-	return list
-}

services/search/pkg/query/bleve/compiler_test.go

@@ -24,6 +24,13 @@ var timeMustParse = func(t *testing.T, ts string) time.Time {
 // canonical ASTs (real field names, media-type already expanded) and call
 // compile() directly, dropping the query.Normalize wrapper and the mediatype
 // cases.
+func boolFieldQuery(field string, value bool) query.Query {
+	q := query.NewBoolFieldQuery(value)
+	q.SetField(field)
+
+	return q
+}
+
 func Test_compile(t *testing.T) {
 	tests := []struct {
 		name    string
@@ -72,7 +79,7 @@ func Test_compile(t *testing.T) {
 				},
 			},
 			want: query.NewConjunctionQuery([]query.Query{
-				query.NewQueryStringQuery(`Name_words:"John Smith"`),
+				query.NewQueryStringQuery(`Name_words:"john smith"`),
 			}),
 			wantErr: false,
 		},
@@ -86,8 +93,8 @@ func Test_compile(t *testing.T) {
 				},
 			},
 			want: query.NewConjunctionQuery([]query.Query{
-				query.NewQueryStringQuery(`Name_words:"John Smith"`),
-				query.NewQueryStringQuery(`Name_words:"Jane"`),
+				query.NewQueryStringQuery(`Name_words:"john smith"`),
+				query.NewQueryStringQuery(`Name_words:"jane"`),
 			}),
 			wantErr: false,
 		},
@@ -320,9 +327,9 @@ func Test_compile(t *testing.T) {
 				},
 			},
 			want: query.NewConjunctionQuery([]query.Query{
-				query.NewQueryStringQuery(`Name_words:"John Smith"`),
-				query.NewQueryStringQuery(`Hidden:t`),
-				query.NewQueryStringQuery(`Hidden:t`),
+				query.NewQueryStringQuery(`Name_words:"john smith"`),
+				boolFieldQuery("Hidden", true),
+				boolFieldQuery("Hidden", true),
 			}),
 			wantErr: false,
 		},
@@ -548,7 +555,7 @@ func Test_compile(t *testing.T) {
 				},
 			},
 			want: query.NewConjunctionQuery([]query.Query{
-				query.NewQueryStringQuery(`Name_words:"John Smith +-=&|><!(){}[]^\"~: "`),
+				query.NewQueryStringQuery(`Name_words:"john smith +-=&|><!(){}[]^\"~: "`),
 			}),
 			wantErr: false,
 		},

services/search/pkg/query/normalize.go

@@ -1,6 +1,9 @@
 package query
 
 import (
+	"strconv"
+
+	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
 	"reflect"
 	"strings"
 
@@ -35,6 +38,9 @@ func normalizeNodes(nodes []ast.Node, resolve func(string) string, defaultKey st
 			if FieldValueIsNormalized(node.Key) {
 				node.Value = strings.ToLower(node.Value)
 			}
+			if node.Key == "Type" {
+				node.Value = resourceType(node.Value)
+			}
 			if exp := mimetype.Expand(node.Key, node.Value); exp != nil {
 				out = append(out, normalizeNodes(exp, resolve, defaultKey)...)
 				continue
@@ -79,3 +85,16 @@ func toPointer(n ast.Node) ast.Node {
 	}
 	return n
 }
+
+// resourceType maps the type categories to the stored resource type value;
+// unknown values pass through and become dead term queries.
+func resourceType(value string) string {
+	switch strings.ToLower(value) {
+	case "file":
+		return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10)
+	case "folder":
+		return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10)
+	default:
+		return value
+	}
+}