refactor(search): run the OpenSearch backend on the shared lowering pass

KQLToOpenSearchBoolQuery runs query.Normalize, then only value lowercasing stays
backend-specific; remapKey and unfoldValue are gone.
This commit is contained in:
Dominik Schmidt
2026-07-29 17:39:23 +02:00
parent 330d2f3a31
commit 1b697e4001
3 changed files with 56 additions and 780 deletions

View File

@@ -1,199 +1,25 @@
package convert
import (
"fmt"
"reflect"
"strings"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func ExpandKQL(nodes []ast.Node) ([]ast.Node, error) {
return kqlExpander{}.expand(nodes, "")
}
type kqlExpander struct{}
func (e kqlExpander) expand(nodes []ast.Node, defaultKey string) ([]ast.Node, error) {
for i, node := range nodes {
rnode := reflect.ValueOf(node)
// we need to ensure that the node is a pointer to an ast.Node in every case
if rnode.Kind() != reflect.Ptr {
ptr := reflect.New(rnode.Type())
ptr.Elem().Set(rnode)
rnode = ptr
cnode, ok := rnode.Interface().(ast.Node)
if !ok {
return nil, fmt.Errorf("expected node to be of type ast.Node, got %T", rnode.Interface())
}
node = cnode // Update the original node to the pointer
nodes[i] = node // Update the original slice with the pointer
}
var unfoldedNodes []ast.Node
switch cnode := node.(type) {
case *ast.GroupNode:
if cnode.Key != "" { // group nodes should not get a default key
cnode.Key = e.remapKey(cnode.Key, defaultKey)
}
groupNodes, err := e.expand(cnode.Nodes, cnode.Key)
if err != nil {
return nil, err
}
cnode.Nodes = groupNodes
// LowerValues folds restriction values for fields whose index analyzer
// lowercases (search.LowercaseValueFields, shared with bleve); case-preserved
// fields keep their casing. Runs after query.Normalize, so keys are resolved.
func LowerValues(nodes []ast.Node) []ast.Node {
for _, n := range nodes {
switch node := n.(type) {
case *ast.StringNode:
cnode.Key = e.remapKey(cnode.Key, defaultKey)
cnode.Value = e.lowerValue(cnode.Key, cnode.Value)
unfoldedNodes = e.unfoldValue(cnode.Key, cnode.Value)
case *ast.DateTimeNode:
cnode.Key = e.remapKey(cnode.Key, defaultKey)
case *ast.BooleanNode:
cnode.Key = e.remapKey(cnode.Key, defaultKey)
}
if unfoldedNodes != nil {
// Insert unfolded nodes at the current index
nodes = append(nodes[:i], append(unfoldedNodes, nodes[i+1:]...)...)
// Adjust index to account for new nodes
i += len(unfoldedNodes) - 1
if _, ok := search.LowercaseValueFields()[node.Key]; ok {
node.Value = strings.ToLower(node.Value)
}
case *ast.GroupNode:
LowerValues(node.Nodes)
}
}
return nodes, nil
}
func (_ kqlExpander) remapKey(current string, defaultKey string) string {
if defaultKey == "" {
defaultKey = "Name" // Set a default key if none is provided
}
key, ok := map[string]string{
"": defaultKey, // Default case if current is empty
"rootid": "RootID",
"path": "Path",
"id": "ID",
"name": "Name",
"size": "Size",
"mtime": "Mtime",
"mediatype": "MimeType",
"type": "Type",
"tag": "Tags",
"tags": "Tags",
"content": "Content",
"hidden": "Hidden",
}[current]
if !ok {
return current // Return the original key if not found
}
return key
}
func (_ kqlExpander) lowerValue(key, value string) string {
// only fold the value for fields whose index analyzer lowercases too;
// case-preserved (keyword) fields must keep their casing or they never match
// their stored token. Shared with bleve via search.LowercaseValueFields.
if _, ok := search.LowercaseValueFields()[key]; !ok {
return value
}
return strings.ToLower(value)
}
func (_ kqlExpander) unfoldValue(key, value string) []ast.Node {
result, ok := map[string][]ast.Node{
"MimeType:file": {
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: key, Value: "httpd/unix-directory"},
},
"MimeType:folder": {
&ast.StringNode{Key: key, Value: "httpd/unix-directory"},
},
"MimeType:document": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/msword"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.text"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "text/plain"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "text/markdown"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/rtf"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.apple.pages"},
}},
},
"MimeType:spreadsheet": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/vnd.ms-excel"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadsheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "text/csv"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"},
}},
},
"MimeType:presentation": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.ms-powerpoint"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.apple.keynote"},
}},
},
"MimeType:pdf": {
&ast.StringNode{Key: key, Value: "application/pdf"},
},
"MimeType:image": {
&ast.StringNode{Key: key, Value: "image/*"},
},
"MimeType:video": {
&ast.StringNode{Key: key, Value: "video/*"},
},
"MimeType:audio": {
&ast.StringNode{Key: key, Value: "audio/*"},
},
"MimeType:archive": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/zip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-7z-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-rar-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-tar"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-bzip2"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-bzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-tgz"},
}},
},
}[fmt.Sprintf("%s:%s", key, value)]
if !ok {
return nil
}
return result
return nodes
}

View File

@@ -1,607 +1,57 @@
package convert_test
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestExpandKQLAST(t *testing.T) {
t.Run("always converts a value node to a pointer node", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "ast.node.V -> ast.node.PTR",
Got: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: "c"},
&ast.OperatorNode{Value: "OR"},
ast.DateTimeNode{Key: "d"},
ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: "f"},
&ast.OperatorNode{Value: "NOT"},
ast.BooleanNode{Key: "g"},
ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
}},
}},
}},
ast.GroupNode{Key: "i", Nodes: []ast.Node{
ast.StringNode{Key: "a"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
ast.OperatorNode{Value: "OR"},
ast.GroupNode{Key: "h", Nodes: []ast.Node{
ast.StringNode{Key: "a"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
ast.OperatorNode{Value: "OR"},
ast.GroupNode{Key: "h", Nodes: []ast.Node{
ast.StringNode{Key: "a"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
}},
}},
}},
},
Want: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: "c"},
&ast.OperatorNode{Value: "OR"},
&ast.DateTimeNode{Key: "d"},
&ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: "f"},
&ast.OperatorNode{Value: "NOT"},
&ast.BooleanNode{Key: "g"},
&ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
}},
}},
}},
&ast.GroupNode{Key: "i", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
}},
}},
}},
},
// LowerValues runs after the shared query.Normalize pass, so it operates on
// already-resolved pointer nodes. Field resolution, media-type expansion and
// group-key defaulting are tested once at the query.Normalize level (see
// pkg/query normalize_test), not here. Only lowercase-analyzed fields get their
// value folded; case-preserved fields keep their casing.
func TestLowerValues(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "lowercase-analyzed field: value is folded, recursing into groups",
Got: []ast.Node{
&ast.StringNode{Key: "Name", Value: "StringNode"},
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "StringNode"},
}},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.Equal(t, test.Want, result)
})
}
})
t.Run("remaps some keys", func(t *testing.T) {
var tests []opensearchtest.TableTest[[]ast.Node, []ast.Node]
for k, v := range map[string]string{
"": "Name", // Default to "Name" if no key is provided
"rootid": "RootID",
"path": "Path",
"id": "ID",
"name": "Name",
"size": "Size",
"mtime": "Mtime",
"mediatype": "MimeType",
"type": "Type",
"tag": "Tags",
"tags": "Tags",
"content": "Content",
"hidden": "Hidden",
"any": "any", // Example of an unknown key that should remain unchanged
} {
tests = append(tests, opensearchtest.TableTest[[]ast.Node, []ast.Node]{
Name: fmt.Sprintf("%s -> %s", k, v),
Got: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: k},
&ast.OperatorNode{Value: "OR"},
ast.DateTimeNode{Key: k},
ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: k},
&ast.OperatorNode{Value: "NOT"},
ast.BooleanNode{Key: k},
ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: k, Nodes: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: k, Nodes: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: k, Nodes: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
}},
}},
}},
},
Want: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.DateTimeNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: v},
&ast.OperatorNode{Value: "NOT"},
&ast.BooleanNode{Key: v},
&ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: func() string {
switch {
case k == "":
return k
default:
return v
}
}(), Nodes: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: func() string {
switch {
case k == "":
return k
default:
return v
}
}(), Nodes: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: func() string {
switch {
case k == "":
return k
default:
return v
}
}(), Nodes: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
}},
}},
}},
},
})
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.Equal(t, test.Want, result)
})
}
})
t.Run("lowercases some values", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "lowercase-analyzed field: value is folded",
Got: []ast.Node{
ast.StringNode{Key: "Name", Value: "StringNode"},
ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
ast.StringNode{Key: "Name", Value: "StringNode"},
}},
},
Want: []ast.Node{
Want: []ast.Node{
&ast.StringNode{Key: "Name", Value: "stringnode"},
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "stringnode"},
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "stringnode"},
}},
},
}},
},
{
Name: "case-preserved field: value keeps its casing",
Got: []ast.Node{
ast.StringNode{Key: "aBc", Value: "StringNode"},
ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
ast.StringNode{Key: "aBc", Value: "StringNode"},
}},
},
Want: []ast.Node{
},
{
Name: "case-preserved field: value keeps its casing",
Got: []ast.Node{
&ast.StringNode{Key: "aBc", Value: "StringNode"},
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
&ast.StringNode{Key: "aBc", Value: "StringNode"},
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
&ast.StringNode{Key: "aBc", Value: "StringNode"},
}},
},
}},
},
}
Want: []ast.Node{
&ast.StringNode{Key: "aBc", Value: "StringNode"},
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
&ast.StringNode{Key: "aBc", Value: "StringNode"},
}},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.Equal(t, test.Want, result)
})
}
})
t.Run("unfolds some values", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "MimeType:unknown",
Got: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "unknown"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: "some-name"},
},
Want: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "unknown"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:file",
Got: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "file"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: "some-name"},
},
Want: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:folder",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "folder"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:document",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "document"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/msword"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.text"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "text/plain"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "text/markdown"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/rtf"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.pages"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:spreadsheet",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "spreadsheet"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-excel"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadsheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "text/csv"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadshee"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.numbers"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:presentation",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "presentation"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-powerpoint"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.keynote"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:pdf",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "pdf"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "application/pdf"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:image",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "image"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "image/*"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:video",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "video"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "video/*"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:audio",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "audio"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "audio/*"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:archive",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "archive"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/zip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-7z-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-rar-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-tar"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-bzip2"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-bzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-tgz"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
if test.Skip {
t.Skip("Skipping test due to known issue")
}
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.EqualValues(t, test.Want, result)
})
}
})
t.Run("different cases", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "use the group node key as default key",
Got: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "a", Nodes: []ast.Node{
&ast.StringNode{Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "mediatype", Nodes: []ast.Node{
&ast.StringNode{Value: "file"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "mediatype", Value: "file"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
},
Want: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "a", Nodes: []ast.Node{
&ast.StringNode{Key: "a", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "MimeType", Nodes: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
if test.Skip {
t.Skip("Skipping test due to known issue")
}
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.EqualValues(t, test.Want, result)
})
}
})
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
require.Equal(t, test.Want, convert.LowerValues(test.Got))
})
}
}

View File

@@ -5,6 +5,7 @@ import (
"github.com/opencloud-eu/opencloud/pkg/kql"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
)
var (
@@ -17,10 +18,9 @@ func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) {
return nil, fmt.Errorf("failed to build query: %w", err)
}
kqlNodes, err := ExpandKQL(kqlAst.Nodes)
if err != nil {
return nil, fmt.Errorf("failed to expand KQL AST nodes: %w", err)
}
// shared lowering (field resolution + media-type), then value lowercasing.
kqlAst = query.Normalize(kqlAst, query.ResolveField)
kqlNodes := LowerValues(kqlAst.Nodes)
builder, err := TranspileKQLToOpenSearch(kqlNodes)
if err != nil {