diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand.go b/services/search/pkg/opensearch/internal/convert/kql_expand.go index 5fd149f81e..8d382c8745 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand.go @@ -1,199 +1,25 @@ package convert import ( - "fmt" - "reflect" "strings" "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) -func ExpandKQL(nodes []ast.Node) ([]ast.Node, error) { - return kqlExpander{}.expand(nodes, "") -} - -type kqlExpander struct{} - -func (e kqlExpander) expand(nodes []ast.Node, defaultKey string) ([]ast.Node, error) { - for i, node := range nodes { - rnode := reflect.ValueOf(node) - - // we need to ensure that the node is a pointer to an ast.Node in every case - if rnode.Kind() != reflect.Ptr { - ptr := reflect.New(rnode.Type()) - ptr.Elem().Set(rnode) - rnode = ptr - cnode, ok := rnode.Interface().(ast.Node) - if !ok { - return nil, fmt.Errorf("expected node to be of type ast.Node, got %T", rnode.Interface()) - } - - node = cnode // Update the original node to the pointer - nodes[i] = node // Update the original slice with the pointer - } - - var unfoldedNodes []ast.Node - switch cnode := node.(type) { - case *ast.GroupNode: - if cnode.Key != "" { // group nodes should not get a default key - cnode.Key = e.remapKey(cnode.Key, defaultKey) - } - - groupNodes, err := e.expand(cnode.Nodes, cnode.Key) - if err != nil { - return nil, err - } - cnode.Nodes = groupNodes +// LowerValues folds restriction values for fields whose index analyzer +// lowercases (search.LowercaseValueFields, shared with bleve); case-preserved +// fields keep their casing. Runs after query.Normalize, so keys are resolved. +func LowerValues(nodes []ast.Node) []ast.Node { + for _, n := range nodes { + switch node := n.(type) { case *ast.StringNode: - cnode.Key = e.remapKey(cnode.Key, defaultKey) - cnode.Value = e.lowerValue(cnode.Key, cnode.Value) - unfoldedNodes = e.unfoldValue(cnode.Key, cnode.Value) - case *ast.DateTimeNode: - cnode.Key = e.remapKey(cnode.Key, defaultKey) - case *ast.BooleanNode: - cnode.Key = e.remapKey(cnode.Key, defaultKey) - } - - if unfoldedNodes != nil { - // Insert unfolded nodes at the current index - nodes = append(nodes[:i], append(unfoldedNodes, nodes[i+1:]...)...) - // Adjust index to account for new nodes - i += len(unfoldedNodes) - 1 + if _, ok := search.LowercaseValueFields()[node.Key]; ok { + node.Value = strings.ToLower(node.Value) + } + case *ast.GroupNode: + LowerValues(node.Nodes) } } - - return nodes, nil -} - -func (_ kqlExpander) remapKey(current string, defaultKey string) string { - if defaultKey == "" { - defaultKey = "Name" // Set a default key if none is provided - } - - key, ok := map[string]string{ - "": defaultKey, // Default case if current is empty - "rootid": "RootID", - "path": "Path", - "id": "ID", - "name": "Name", - "size": "Size", - "mtime": "Mtime", - "mediatype": "MimeType", - "type": "Type", - "tag": "Tags", - "tags": "Tags", - "content": "Content", - "hidden": "Hidden", - }[current] - if !ok { - return current // Return the original key if not found - } - - return key -} - -func (_ kqlExpander) lowerValue(key, value string) string { - // only fold the value for fields whose index analyzer lowercases too; - // case-preserved (keyword) fields must keep their casing or they never match - // their stored token. Shared with bleve via search.LowercaseValueFields. - if _, ok := search.LowercaseValueFields()[key]; !ok { - return value - } - return strings.ToLower(value) -} - -func (_ kqlExpander) unfoldValue(key, value string) []ast.Node { - result, ok := map[string][]ast.Node{ - "MimeType:file": { - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, - }, - "MimeType:folder": { - &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, - }, - "MimeType:document": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/msword"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.text"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "text/plain"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "text/markdown"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/rtf"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.apple.pages"}, - }}, - }, - "MimeType:spreadsheet": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/vnd.ms-excel"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadsheet"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "text/csv"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"}, - }}, - }, - "MimeType:presentation": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.presentation"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.ms-powerpoint"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.apple.keynote"}, - }}, - }, - "MimeType:pdf": { - &ast.StringNode{Key: key, Value: "application/pdf"}, - }, - "MimeType:image": { - &ast.StringNode{Key: key, Value: "image/*"}, - }, - "MimeType:video": { - &ast.StringNode{Key: key, Value: "video/*"}, - }, - "MimeType:audio": { - &ast.StringNode{Key: key, Value: "audio/*"}, - }, - "MimeType:archive": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/zip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/gzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-gzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-7z-compressed"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-rar-compressed"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-tar"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-bzip2"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-bzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-tgz"}, - }}, - }, - }[fmt.Sprintf("%s:%s", key, value)] - if !ok { - return nil - } - - return result + return nodes } diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go index 6a62b151a9..b511e68270 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go @@ -1,607 +1,57 @@ package convert_test import ( - "fmt" "testing" "github.com/stretchr/testify/require" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" - "github.com/opencloud-eu/opencloud/pkg/ast" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" + opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) -func TestExpandKQLAST(t *testing.T) { - t.Run("always converts a value node to a pointer node", func(t *testing.T) { - tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - { - Name: "ast.node.V -> ast.node.PTR", - Got: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - ast.OperatorNode{Value: "AND"}, - &ast.DateTimeNode{Key: "c"}, - &ast.OperatorNode{Value: "OR"}, - ast.DateTimeNode{Key: "d"}, - ast.OperatorNode{Value: "OR"}, - &ast.BooleanNode{Key: "f"}, - &ast.OperatorNode{Value: "NOT"}, - ast.BooleanNode{Key: "g"}, - ast.OperatorNode{Value: "NOT"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - }}, - }}, - }}, - ast.GroupNode{Key: "i", Nodes: []ast.Node{ - ast.StringNode{Key: "a"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - ast.OperatorNode{Value: "OR"}, - ast.GroupNode{Key: "h", Nodes: []ast.Node{ - ast.StringNode{Key: "a"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - ast.OperatorNode{Value: "OR"}, - ast.GroupNode{Key: "h", Nodes: []ast.Node{ - ast.StringNode{Key: "a"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - }}, - }}, - }}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "AND"}, - &ast.DateTimeNode{Key: "c"}, - &ast.OperatorNode{Value: "OR"}, - &ast.DateTimeNode{Key: "d"}, - &ast.OperatorNode{Value: "OR"}, - &ast.BooleanNode{Key: "f"}, - &ast.OperatorNode{Value: "NOT"}, - &ast.BooleanNode{Key: "g"}, - &ast.OperatorNode{Value: "NOT"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - }}, - }}, - }}, - &ast.GroupNode{Key: "i", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - }}, - }}, - }}, - }, +// LowerValues runs after the shared query.Normalize pass, so it operates on +// already-resolved pointer nodes. Field resolution, media-type expansion and +// group-key defaulting are tested once at the query.Normalize level (see +// pkg/query normalize_test), not here. Only lowercase-analyzed fields get their +// value folded; case-preserved fields keep their casing. +func TestLowerValues(t *testing.T) { + tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ + { + Name: "lowercase-analyzed field: value is folded, recursing into groups", + Got: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "StringNode"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "StringNode"}, + }}, }, - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - result, err := convert.ExpandKQL(test.Got) - require.NoError(t, err) - require.Equal(t, test.Want, result) - }) - } - }) - - t.Run("remaps some keys", func(t *testing.T) { - var tests []opensearchtest.TableTest[[]ast.Node, []ast.Node] - - for k, v := range map[string]string{ - "": "Name", // Default to "Name" if no key is provided - "rootid": "RootID", - "path": "Path", - "id": "ID", - "name": "Name", - "size": "Size", - "mtime": "Mtime", - "mediatype": "MimeType", - "type": "Type", - "tag": "Tags", - "tags": "Tags", - "content": "Content", - "hidden": "Hidden", - "any": "any", // Example of an unknown key that should remain unchanged - } { - tests = append(tests, opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - Name: fmt.Sprintf("%s -> %s", k, v), - Got: []ast.Node{ - &ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: k}, - ast.OperatorNode{Value: "AND"}, - &ast.DateTimeNode{Key: k}, - &ast.OperatorNode{Value: "OR"}, - ast.DateTimeNode{Key: k}, - ast.OperatorNode{Value: "OR"}, - &ast.BooleanNode{Key: k}, - &ast.OperatorNode{Value: "NOT"}, - ast.BooleanNode{Key: k}, - ast.OperatorNode{Value: "NOT"}, - &ast.GroupNode{Key: k, Nodes: []ast.Node{ - &ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: k, Nodes: []ast.Node{ - &ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: k, Nodes: []ast.Node{ - &ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: k}, - }}, - }}, - }}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "AND"}, - &ast.DateTimeNode{Key: v}, - &ast.OperatorNode{Value: "OR"}, - &ast.DateTimeNode{Key: v}, - &ast.OperatorNode{Value: "OR"}, - &ast.BooleanNode{Key: v}, - &ast.OperatorNode{Value: "NOT"}, - &ast.BooleanNode{Key: v}, - &ast.OperatorNode{Value: "NOT"}, - &ast.GroupNode{Key: func() string { - switch { - case k == "": - return k - default: - return v - } - }(), Nodes: []ast.Node{ - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: func() string { - switch { - case k == "": - return k - default: - return v - } - }(), Nodes: []ast.Node{ - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: func() string { - switch { - case k == "": - return k - default: - return v - } - }(), Nodes: []ast.Node{ - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: v}, - }}, - }}, - }}, - }, - }) - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - result, err := convert.ExpandKQL(test.Got) - require.NoError(t, err) - require.Equal(t, test.Want, result) - }) - } - }) - - t.Run("lowercases some values", func(t *testing.T) { - tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - { - Name: "lowercase-analyzed field: value is folded", - Got: []ast.Node{ - ast.StringNode{Key: "Name", Value: "StringNode"}, - ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "Name", Value: "StringNode"}, - }}, - }, - Want: []ast.Node{ + Want: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "stringnode"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ &ast.StringNode{Key: "Name", Value: "stringnode"}, - &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "stringnode"}, - }}, - }, + }}, }, - { - Name: "case-preserved field: value keeps its casing", - Got: []ast.Node{ - ast.StringNode{Key: "aBc", Value: "StringNode"}, - ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "aBc", Value: "StringNode"}, - }}, - }, - Want: []ast.Node{ + }, + { + Name: "case-preserved field: value keeps its casing", + Got: []ast.Node{ + &ast.StringNode{Key: "aBc", Value: "StringNode"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ &ast.StringNode{Key: "aBc", Value: "StringNode"}, - &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "aBc", Value: "StringNode"}, - }}, - }, + }}, }, - } + Want: []ast.Node{ + &ast.StringNode{Key: "aBc", Value: "StringNode"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + &ast.StringNode{Key: "aBc", Value: "StringNode"}, + }}, + }, + }, + } - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - result, err := convert.ExpandKQL(test.Got) - require.NoError(t, err) - require.Equal(t, test.Want, result) - }) - } - }) - - t.Run("unfolds some values", func(t *testing.T) { - tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - { - Name: "MimeType:unknown", - Got: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "unknown"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "unknown"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:file", - Got: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "file"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:folder", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "folder"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:document", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "document"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "application/msword"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.text"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "text/plain"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "text/markdown"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/rtf"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.pages"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:spreadsheet", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "spreadsheet"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-excel"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadsheet"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "text/csv"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadshee"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.numbers"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:presentation", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "presentation"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.presentation"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-powerpoint"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.keynote"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:pdf", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "pdf"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "MimeType", Value: "application/pdf"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:image", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "image"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "MimeType", Value: "image/*"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:video", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "video"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "MimeType", Value: "video/*"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:audio", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "audio"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "MimeType", Value: "audio/*"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:archive", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "archive"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "application/zip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/gzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-gzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-7z-compressed"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-rar-compressed"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-tar"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-bzip2"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-bzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-tgz"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - if test.Skip { - t.Skip("Skipping test due to known issue") - } - result, err := convert.ExpandKQL(test.Got) - require.NoError(t, err) - require.EqualValues(t, test.Want, result) - }) - } - }) - - t.Run("different cases", func(t *testing.T) { - tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - { - Name: "use the group node key as default key", - Got: []ast.Node{ - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Value: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Key: "a", Nodes: []ast.Node{ - &ast.StringNode{Value: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Key: "mediatype", Nodes: []ast.Node{ - &ast.StringNode{Value: "file"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "mediatype", Value: "file"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - }, - Want: []ast.Node{ - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Key: "a", Nodes: []ast.Node{ - &ast.StringNode{Key: "a", Value: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Key: "MimeType", Nodes: []ast.Node{ - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - }, - }, - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - if test.Skip { - t.Skip("Skipping test due to known issue") - } - result, err := convert.ExpandKQL(test.Got) - require.NoError(t, err) - require.EqualValues(t, test.Want, result) - }) - } - }) + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + require.Equal(t, test.Want, convert.LowerValues(test.Got)) + }) + } } diff --git a/services/search/pkg/opensearch/internal/convert/kql_query.go b/services/search/pkg/opensearch/internal/convert/kql_query.go index f04e577077..297a0fcbec 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_query.go +++ b/services/search/pkg/opensearch/internal/convert/kql_query.go @@ -5,6 +5,7 @@ import ( "github.com/opencloud-eu/opencloud/pkg/kql" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" + "github.com/opencloud-eu/opencloud/services/search/pkg/query" ) var ( @@ -17,10 +18,9 @@ func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) { return nil, fmt.Errorf("failed to build query: %w", err) } - kqlNodes, err := ExpandKQL(kqlAst.Nodes) - if err != nil { - return nil, fmt.Errorf("failed to expand KQL AST nodes: %w", err) - } + // shared lowering (field resolution + media-type), then value lowercasing. + kqlAst = query.Normalize(kqlAst, query.ResolveField) + kqlNodes := LowerValues(kqlAst.Nodes) builder, err := TranspileKQLToOpenSearch(kqlNodes) if err != nil {