From fb22dd81a4bc955bfc059198a698ac52bf39507b Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Wed, 29 Jul 2026 17:39:23 +0200 Subject: [PATCH] refactor(search): run the OpenSearch backend on the shared lowering pass KQLToOpenSearchBoolQuery runs query.Normalize, then only value lowercasing stays backend-specific; remapKey and unfoldValue are gone. --- .../opensearch/internal/convert/kql_expand.go | 209 +----- .../internal/convert/kql_expand_test.go | 664 +----------------- .../opensearch/internal/convert/kql_query.go | 8 +- services/search/pkg/query/normalize.go | 3 + services/search/pkg/query/normalize_test.go | 10 +- 5 files changed, 65 insertions(+), 829 deletions(-) diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand.go b/services/search/pkg/opensearch/internal/convert/kql_expand.go index 5c63a3072f..8d382c8745 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand.go @@ -1,210 +1,25 @@ package convert import ( - "fmt" - "reflect" - "strconv" "strings" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) -func ExpandKQL(nodes []ast.Node) ([]ast.Node, error) { - return kqlExpander{}.expand(nodes, "") -} - -type kqlExpander struct{} - -func (e kqlExpander) expand(nodes []ast.Node, defaultKey string) ([]ast.Node, error) { - for i, node := range nodes { - rnode := reflect.ValueOf(node) - - // we need to ensure that the node is a pointer to an ast.Node in every case - if rnode.Kind() != reflect.Ptr { - ptr := reflect.New(rnode.Type()) - ptr.Elem().Set(rnode) - rnode = ptr - cnode, ok := rnode.Interface().(ast.Node) - if !ok { - return nil, fmt.Errorf("expected node to be of type ast.Node, got %T", rnode.Interface()) - } - - node = cnode // Update the original node to the pointer - nodes[i] = node // Update the original slice with the pointer - } - - var unfoldedNodes []ast.Node - switch cnode := node.(type) { - case *ast.GroupNode: - if cnode.Key != "" { // group nodes should not get a default key - cnode.Key = e.remapKey(cnode.Key, defaultKey) - } - - groupNodes, err := e.expand(cnode.Nodes, cnode.Key) - if err != nil { - return nil, err - } - cnode.Nodes = groupNodes +// LowerValues folds restriction values for fields whose index analyzer +// lowercases (search.LowercaseValueFields, shared with bleve); case-preserved +// fields keep their casing. Runs after query.Normalize, so keys are resolved. +func LowerValues(nodes []ast.Node) []ast.Node { + for _, n := range nodes { + switch node := n.(type) { case *ast.StringNode: - cnode.Key = e.remapKey(cnode.Key, defaultKey) - cnode.Value = e.lowerValue(cnode.Key, cnode.Value) - unfoldedNodes = e.unfoldValue(cnode.Key, cnode.Value) - case *ast.DateTimeNode: - cnode.Key = e.remapKey(cnode.Key, defaultKey) - case *ast.BooleanNode: - cnode.Key = e.remapKey(cnode.Key, defaultKey) - case *ast.NumberNode: - cnode.Key = e.remapKey(cnode.Key, defaultKey) - } - - if unfoldedNodes != nil { - // Insert unfolded nodes at the current index - nodes = append(nodes[:i], append(unfoldedNodes, nodes[i+1:]...)...) - // Adjust index to account for new nodes - i += len(unfoldedNodes) - 1 + if _, ok := search.LowercaseValueFields()[node.Key]; ok { + node.Value = strings.ToLower(node.Value) + } + case *ast.GroupNode: + LowerValues(node.Nodes) } } - - return nodes, nil -} - -func (_ kqlExpander) remapKey(current string, defaultKey string) string { - if defaultKey == "" { - defaultKey = "Name" // Set a default key if none is provided - } - - key, ok := map[string]string{ - "": defaultKey, // Default case if current is empty - "title": "Title", - "rootid": "RootID", - "path": "Path", - "id": "ID", - "name": "Name", - "size": "Size", - "mtime": "Mtime", - "mediatype": "MimeType", - "type": "Type", - "tag": "Tags", - "tags": "Tags", - "content": "Content", - "hidden": "Hidden", - "favorite": "Favorites", - }[strings.ToLower(current)] - if !ok { - return current // Return the original key if not found - } - - return key -} - -func (_ kqlExpander) lowerValue(key, value string) string { - // only fold the value for fields whose index analyzer lowercases too; - // case-preserved (keyword) fields must keep their casing or they never match - // their stored token. Shared with bleve via search.LowercaseValueFields. - if _, ok := search.LowercaseValueFields()[key]; !ok { - return value - } - return strings.ToLower(value) -} - -func (_ kqlExpander) unfoldValue(key, value string) []ast.Node { - result, ok := map[string][]ast.Node{ - "Type:file": { - &ast.StringNode{Key: key, Value: strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10)}, - }, - "Type:folder": { - &ast.StringNode{Key: key, Value: strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10)}, - }, - "MimeType:file": { - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, - }, - "MimeType:folder": { - &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, - }, - "MimeType:document": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/msword"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.text"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "text/plain"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "text/markdown"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/rtf"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.apple.pages"}, - }}, - }, - "MimeType:spreadsheet": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/vnd.ms-excel"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadsheet"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "text/csv"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"}, - }}, - }, - "MimeType:presentation": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.presentation"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.ms-powerpoint"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.apple.keynote"}, - }}, - }, - "MimeType:pdf": { - &ast.StringNode{Key: key, Value: "application/pdf"}, - }, - "MimeType:image": { - &ast.StringNode{Key: key, Value: "image/*"}, - }, - "MimeType:video": { - &ast.StringNode{Key: key, Value: "video/*"}, - }, - "MimeType:audio": { - &ast.StringNode{Key: key, Value: "audio/*"}, - }, - "MimeType:archive": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/zip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/gzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-gzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-7z-compressed"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-rar-compressed"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-tar"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-bzip2"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-bzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-tgz"}, - }}, - }, - }[fmt.Sprintf("%s:%s", key, value)] - if !ok { - return nil - } - - return result + return nodes } diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go index 44534d7bcd..ef1927b48d 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go @@ -1,643 +1,57 @@ package convert_test import ( - "fmt" "testing" "github.com/stretchr/testify/require" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" - "github.com/opencloud-eu/opencloud/pkg/ast" - "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" ) -func TestExpandKQLAST(t *testing.T) { - t.Run("always converts a value node to a pointer node", func(t *testing.T) { - tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - { - Name: "ast.node.V -> ast.node.PTR", - Got: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - ast.OperatorNode{Value: "AND"}, - &ast.DateTimeNode{Key: "c"}, - &ast.OperatorNode{Value: "OR"}, - ast.DateTimeNode{Key: "d"}, - ast.OperatorNode{Value: "OR"}, - &ast.BooleanNode{Key: "f"}, - &ast.OperatorNode{Value: "NOT"}, - ast.BooleanNode{Key: "g"}, - ast.OperatorNode{Value: "NOT"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - }}, - }}, - }}, - ast.GroupNode{Key: "i", Nodes: []ast.Node{ - ast.StringNode{Key: "a"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - ast.OperatorNode{Value: "OR"}, - ast.GroupNode{Key: "h", Nodes: []ast.Node{ - ast.StringNode{Key: "a"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - ast.OperatorNode{Value: "OR"}, - ast.GroupNode{Key: "h", Nodes: []ast.Node{ - ast.StringNode{Key: "a"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - }}, - }}, - }}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "AND"}, - &ast.DateTimeNode{Key: "c"}, - &ast.OperatorNode{Value: "OR"}, - &ast.DateTimeNode{Key: "d"}, - &ast.OperatorNode{Value: "OR"}, - &ast.BooleanNode{Key: "f"}, - &ast.OperatorNode{Value: "NOT"}, - &ast.BooleanNode{Key: "g"}, - &ast.OperatorNode{Value: "NOT"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - }}, - }}, - }}, - &ast.GroupNode{Key: "i", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - }}, - }}, - }}, - }, +// LowerValues runs after the shared query.Normalize pass, so it operates on +// already-resolved pointer nodes. Field resolution, media-type expansion and +// group-key defaulting are tested once at the query.Normalize level (see +// pkg/query normalize_test), not here. Only lowercase-analyzed fields get their +// value folded; case-preserved fields keep their casing. +func TestLowerValues(t *testing.T) { + tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ + { + Name: "lowercase-analyzed field: value is folded, recursing into groups", + Got: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "StringNode"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "StringNode"}, + }}, }, - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - result, err := convert.ExpandKQL(test.Got) - require.NoError(t, err) - require.Equal(t, test.Want, result) - }) - } - }) - - t.Run("remaps some keys", func(t *testing.T) { - var tests []opensearchtest.TableTest[[]ast.Node, []ast.Node] - - for k, v := range map[string]string{ - "": "Name", // Default to "Name" if no key is provided - "rootid": "RootID", - "path": "Path", - "id": "ID", - "name": "Name", - "size": "Size", - "mtime": "Mtime", - "mediatype": "MimeType", - "type": "Type", - "tag": "Tags", - "tags": "Tags", - "content": "Content", - "hidden": "Hidden", - "favorite": "Favorites", - "any": "any", // Example of an unknown key that should remain unchanged - } { - tests = append(tests, opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - Name: fmt.Sprintf("%s -> %s", k, v), - Got: []ast.Node{ - &ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: k}, - ast.OperatorNode{Value: "AND"}, - &ast.DateTimeNode{Key: k}, - &ast.OperatorNode{Value: "OR"}, - ast.DateTimeNode{Key: k}, - ast.OperatorNode{Value: "OR"}, - &ast.BooleanNode{Key: k}, - &ast.OperatorNode{Value: "NOT"}, - ast.BooleanNode{Key: k}, - ast.OperatorNode{Value: "NOT"}, - &ast.GroupNode{Key: k, Nodes: []ast.Node{ - &ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: k, Nodes: []ast.Node{ - &ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: k, Nodes: []ast.Node{ - &ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: k}, - }}, - }}, - }}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "AND"}, - &ast.DateTimeNode{Key: v}, - &ast.OperatorNode{Value: "OR"}, - &ast.DateTimeNode{Key: v}, - &ast.OperatorNode{Value: "OR"}, - &ast.BooleanNode{Key: v}, - &ast.OperatorNode{Value: "NOT"}, - &ast.BooleanNode{Key: v}, - &ast.OperatorNode{Value: "NOT"}, - &ast.GroupNode{Key: func() string { - switch { - case k == "": - return k - default: - return v - } - }(), Nodes: []ast.Node{ - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: func() string { - switch { - case k == "": - return k - default: - return v - } - }(), Nodes: []ast.Node{ - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: func() string { - switch { - case k == "": - return k - default: - return v - } - }(), Nodes: []ast.Node{ - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: v}, - }}, - }}, - }}, - }, - }) - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - result, err := convert.ExpandKQL(test.Got) - require.NoError(t, err) - require.Equal(t, test.Want, result) - }) - } - }) - - t.Run("lowercases some values", func(t *testing.T) { - tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - { - Name: "Name: StringNode -> stringnode", - Got: []ast.Node{ - ast.StringNode{Key: "Name", Value: "StringNode"}, - ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "Name", Value: "StringNode"}, - }}, - }, - Want: []ast.Node{ + Want: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "stringnode"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ &ast.StringNode{Key: "Name", Value: "stringnode"}, - &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "stringnode"}, - }}, - }, + }}, }, - { - Name: "aBc: StringNode -> StringNode", - Got: []ast.Node{ - ast.StringNode{Key: "aBc", Value: "StringNode"}, - }, - Want: []ast.Node{ + }, + { + Name: "case-preserved field: value keeps its casing", + Got: []ast.Node{ + &ast.StringNode{Key: "aBc", Value: "StringNode"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ &ast.StringNode{Key: "aBc", Value: "StringNode"}, - }, + }}, }, - { - Name: "Path: ./Documents -> ./Documents", - Got: []ast.Node{ - ast.StringNode{Key: "Path", Value: "./Documents"}, - ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "Path", Value: "./Documents"}, - }}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: "Path", Value: "./Documents"}, - &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "Path", Value: "./Documents"}, - }}, - }, + Want: []ast.Node{ + &ast.StringNode{Key: "aBc", Value: "StringNode"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + &ast.StringNode{Key: "aBc", Value: "StringNode"}, + }}, }, - { - Name: "Hidden: TRUE -> true", - Got: []ast.Node{ - ast.StringNode{Key: "Hidden", Value: "TRUE"}, - ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "Hidden", Value: "TRUE"}, - }}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: "Hidden", Value: "true"}, - &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "Hidden", Value: "true"}, - }}, - }, - }, - { - Name: "ID: 1$1!AB23 -> 1$1!AB23", - Got: []ast.Node{ - ast.StringNode{Key: "ID", Value: "1$1!AB23"}, - ast.StringNode{Key: "RootID", Value: "1$1!AB23"}, - ast.StringNode{Key: "ParentID", Value: "1$1!AB23"}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: "ID", Value: "1$1!AB23"}, - &ast.StringNode{Key: "RootID", Value: "1$1!AB23"}, - &ast.StringNode{Key: "ParentID", Value: "1$1!AB23"}, - }, - }, - } + }, + } - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - result, err := convert.ExpandKQL(test.Got) - require.NoError(t, err) - require.Equal(t, test.Want, result) - }) - } - }) - - t.Run("unfolds some values", func(t *testing.T) { - tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - { - Name: "MimeType:unknown", - Got: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "unknown"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "unknown"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:file", - Got: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "file"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:folder", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "folder"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:document", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "document"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "application/msword"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.text"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "text/plain"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "text/markdown"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/rtf"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.pages"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:spreadsheet", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "spreadsheet"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-excel"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadsheet"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "text/csv"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.numbers"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:presentation", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "presentation"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.presentation"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-powerpoint"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.keynote"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:pdf", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "pdf"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "MimeType", Value: "application/pdf"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:image", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "image"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "MimeType", Value: "image/*"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:video", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "video"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "MimeType", Value: "video/*"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:audio", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "audio"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "MimeType", Value: "audio/*"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:archive", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "archive"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "application/zip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/gzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-gzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-7z-compressed"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-rar-compressed"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-tar"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-bzip2"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-bzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-tgz"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - if test.Skip { - t.Skip("Skipping test due to known issue") - } - result, err := convert.ExpandKQL(test.Got) - require.NoError(t, err) - require.EqualValues(t, test.Want, result) - }) - } - }) - - t.Run("different cases", func(t *testing.T) { - tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - { - Name: "use the group node key as default key", - Got: []ast.Node{ - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Value: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Key: "a", Nodes: []ast.Node{ - &ast.StringNode{Value: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Key: "mediatype", Nodes: []ast.Node{ - &ast.StringNode{Value: "file"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "mediatype", Value: "file"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - }, - Want: []ast.Node{ - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Key: "a", Nodes: []ast.Node{ - &ast.StringNode{Key: "a", Value: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Key: "MimeType", Nodes: []ast.Node{ - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - }, - }, - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - if test.Skip { - t.Skip("Skipping test due to known issue") - } - result, err := convert.ExpandKQL(test.Got) - require.NoError(t, err) - require.EqualValues(t, test.Want, result) - }) - } - }) + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + require.Equal(t, test.Want, convert.LowerValues(test.Got)) + }) + } } diff --git a/services/search/pkg/opensearch/internal/convert/kql_query.go b/services/search/pkg/opensearch/internal/convert/kql_query.go index f824990e6d..18687dda74 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_query.go +++ b/services/search/pkg/opensearch/internal/convert/kql_query.go @@ -5,6 +5,7 @@ import ( "github.com/opencloud-eu/opencloud/pkg/kql" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" + "github.com/opencloud-eu/opencloud/services/search/pkg/query" ) var ( @@ -17,10 +18,9 @@ func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) { return nil, err } - kqlNodes, err := ExpandKQL(kqlAst.Nodes) - if err != nil { - return nil, fmt.Errorf("failed to expand KQL AST nodes: %w", err) - } + // shared lowering (field resolution + media-type), then value lowercasing. + kqlAst = query.Normalize(kqlAst, query.ResolveField) + kqlNodes := LowerValues(kqlAst.Nodes) builder, err := TranspileKQLToOpenSearch(kqlNodes) if err != nil { diff --git a/services/search/pkg/query/normalize.go b/services/search/pkg/query/normalize.go index 8c1269604b..78a4a55fe9 100644 --- a/services/search/pkg/query/normalize.go +++ b/services/search/pkg/query/normalize.go @@ -42,6 +42,9 @@ func normalizeNodes(nodes []ast.Node, resolve func(string) string, defaultKey st case *ast.BooleanNode: node.Key = resolveKey(node.Key) out = append(out, node) + case *ast.NumberNode: + node.Key = resolveKey(node.Key) + out = append(out, node) case *ast.GroupNode: groupKey := defaultKey if node.Key != "" { diff --git a/services/search/pkg/query/normalize_test.go b/services/search/pkg/query/normalize_test.go index 49cea2c153..66ba3243f0 100644 --- a/services/search/pkg/query/normalize_test.go +++ b/services/search/pkg/query/normalize_test.go @@ -19,9 +19,9 @@ func norm(nodes ...ast.Node) []ast.Node { } func TestResolveField(t *testing.T) { - require.Equal(t, "Name", query.ResolveField("")) // empty -> free-text default - require.Equal(t, "Name", query.ResolveField("NAME")) // case-insensitive - require.Equal(t, "Tags", query.ResolveField("tag")) // singular alias + require.Equal(t, "Name", query.ResolveField("")) // empty -> free-text default + require.Equal(t, "Name", query.ResolveField("NAME")) // case-insensitive + require.Equal(t, "Tags", query.ResolveField("tag")) // singular alias require.Equal(t, "MimeType", query.ResolveField("mimetype")) // real field, case-insensitive require.Equal(t, "photo.cameraMake", query.ResolveField("photo.CAMERAMAKE")) // facet, case-insensitive require.Equal(t, "unknown.field", query.ResolveField("unknown.field")) // unknown key: unchanged, becomes a dead query @@ -36,6 +36,8 @@ func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) { &ast.StringNode{Key: "photo.cameramake", Value: "canon"}, &ast.OperatorNode{Value: "AND"}, &ast.StringNode{Key: "mediatype", Value: "file"}, + &ast.OperatorNode{Value: "AND"}, + ast.NumberNode{Key: "size", Value: 100}, ) require.Equal(t, []ast.Node{ &ast.StringNode{Key: "Name", Value: "free"}, @@ -46,6 +48,8 @@ func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) { &ast.OperatorNode{Value: "AND"}, &ast.OperatorNode{Value: "NOT"}, &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + &ast.OperatorNode{Value: "AND"}, + &ast.NumberNode{Key: "Size", Value: 100}, }, got) }