mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-08-01 02:11:15 -04:00
refactor(search): run the OpenSearch backend on the shared lowering pass
KQLToOpenSearchBoolQuery runs query.Normalize, then only value lowercasing stays backend-specific; remapKey and unfoldValue are gone.
This commit is contained in:
@@ -1,199 +1,25 @@
|
||||
package convert
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
func ExpandKQL(nodes []ast.Node) ([]ast.Node, error) {
|
||||
return kqlExpander{}.expand(nodes, "")
|
||||
}
|
||||
|
||||
type kqlExpander struct{}
|
||||
|
||||
func (e kqlExpander) expand(nodes []ast.Node, defaultKey string) ([]ast.Node, error) {
|
||||
for i, node := range nodes {
|
||||
rnode := reflect.ValueOf(node)
|
||||
|
||||
// we need to ensure that the node is a pointer to an ast.Node in every case
|
||||
if rnode.Kind() != reflect.Ptr {
|
||||
ptr := reflect.New(rnode.Type())
|
||||
ptr.Elem().Set(rnode)
|
||||
rnode = ptr
|
||||
cnode, ok := rnode.Interface().(ast.Node)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("expected node to be of type ast.Node, got %T", rnode.Interface())
|
||||
}
|
||||
|
||||
node = cnode // Update the original node to the pointer
|
||||
nodes[i] = node // Update the original slice with the pointer
|
||||
}
|
||||
|
||||
var unfoldedNodes []ast.Node
|
||||
switch cnode := node.(type) {
|
||||
case *ast.GroupNode:
|
||||
if cnode.Key != "" { // group nodes should not get a default key
|
||||
cnode.Key = e.remapKey(cnode.Key, defaultKey)
|
||||
}
|
||||
|
||||
groupNodes, err := e.expand(cnode.Nodes, cnode.Key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cnode.Nodes = groupNodes
|
||||
// LowerValues folds restriction values for fields whose index analyzer
|
||||
// lowercases (search.LowercaseValueFields, shared with bleve); case-preserved
|
||||
// fields keep their casing. Runs after query.Normalize, so keys are resolved.
|
||||
func LowerValues(nodes []ast.Node) []ast.Node {
|
||||
for _, n := range nodes {
|
||||
switch node := n.(type) {
|
||||
case *ast.StringNode:
|
||||
cnode.Key = e.remapKey(cnode.Key, defaultKey)
|
||||
cnode.Value = e.lowerValue(cnode.Key, cnode.Value)
|
||||
unfoldedNodes = e.unfoldValue(cnode.Key, cnode.Value)
|
||||
case *ast.DateTimeNode:
|
||||
cnode.Key = e.remapKey(cnode.Key, defaultKey)
|
||||
case *ast.BooleanNode:
|
||||
cnode.Key = e.remapKey(cnode.Key, defaultKey)
|
||||
}
|
||||
|
||||
if unfoldedNodes != nil {
|
||||
// Insert unfolded nodes at the current index
|
||||
nodes = append(nodes[:i], append(unfoldedNodes, nodes[i+1:]...)...)
|
||||
// Adjust index to account for new nodes
|
||||
i += len(unfoldedNodes) - 1
|
||||
if _, ok := search.LowercaseValueFields()[node.Key]; ok {
|
||||
node.Value = strings.ToLower(node.Value)
|
||||
}
|
||||
case *ast.GroupNode:
|
||||
LowerValues(node.Nodes)
|
||||
}
|
||||
}
|
||||
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
func (_ kqlExpander) remapKey(current string, defaultKey string) string {
|
||||
if defaultKey == "" {
|
||||
defaultKey = "Name" // Set a default key if none is provided
|
||||
}
|
||||
|
||||
key, ok := map[string]string{
|
||||
"": defaultKey, // Default case if current is empty
|
||||
"rootid": "RootID",
|
||||
"path": "Path",
|
||||
"id": "ID",
|
||||
"name": "Name",
|
||||
"size": "Size",
|
||||
"mtime": "Mtime",
|
||||
"mediatype": "MimeType",
|
||||
"type": "Type",
|
||||
"tag": "Tags",
|
||||
"tags": "Tags",
|
||||
"content": "Content",
|
||||
"hidden": "Hidden",
|
||||
}[current]
|
||||
if !ok {
|
||||
return current // Return the original key if not found
|
||||
}
|
||||
|
||||
return key
|
||||
}
|
||||
|
||||
func (_ kqlExpander) lowerValue(key, value string) string {
|
||||
// only fold the value for fields whose index analyzer lowercases too;
|
||||
// case-preserved (keyword) fields must keep their casing or they never match
|
||||
// their stored token. Shared with bleve via search.LowercaseValueFields.
|
||||
if _, ok := search.LowercaseValueFields()[key]; !ok {
|
||||
return value
|
||||
}
|
||||
return strings.ToLower(value)
|
||||
}
|
||||
|
||||
func (_ kqlExpander) unfoldValue(key, value string) []ast.Node {
|
||||
result, ok := map[string][]ast.Node{
|
||||
"MimeType:file": {
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: key, Value: "httpd/unix-directory"},
|
||||
},
|
||||
"MimeType:folder": {
|
||||
&ast.StringNode{Key: key, Value: "httpd/unix-directory"},
|
||||
},
|
||||
"MimeType:document": {
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: key, Value: "application/msword"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.text"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "text/plain"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "text/markdown"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/rtf"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.apple.pages"},
|
||||
}},
|
||||
},
|
||||
"MimeType:spreadsheet": {
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.ms-excel"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadsheet"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "text/csv"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"},
|
||||
}},
|
||||
},
|
||||
"MimeType:presentation": {
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.presentation"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.ms-powerpoint"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/vnd.apple.keynote"},
|
||||
}},
|
||||
},
|
||||
"MimeType:pdf": {
|
||||
&ast.StringNode{Key: key, Value: "application/pdf"},
|
||||
},
|
||||
"MimeType:image": {
|
||||
&ast.StringNode{Key: key, Value: "image/*"},
|
||||
},
|
||||
"MimeType:video": {
|
||||
&ast.StringNode{Key: key, Value: "video/*"},
|
||||
},
|
||||
"MimeType:audio": {
|
||||
&ast.StringNode{Key: key, Value: "audio/*"},
|
||||
},
|
||||
"MimeType:archive": {
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: key, Value: "application/zip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/gzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-gzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-7z-compressed"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-rar-compressed"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-tar"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-bzip2"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-bzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: key, Value: "application/x-tgz"},
|
||||
}},
|
||||
},
|
||||
}[fmt.Sprintf("%s:%s", key, value)]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
return result
|
||||
return nodes
|
||||
}
|
||||
|
||||
@@ -1,607 +1,57 @@
|
||||
package convert_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
|
||||
opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
|
||||
)
|
||||
|
||||
func TestExpandKQLAST(t *testing.T) {
|
||||
t.Run("always converts a value node to a pointer node", func(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
{
|
||||
Name: "ast.node.V -> ast.node.PTR",
|
||||
Got: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
&ast.DateTimeNode{Key: "c"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
ast.DateTimeNode{Key: "d"},
|
||||
ast.OperatorNode{Value: "OR"},
|
||||
&ast.BooleanNode{Key: "f"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
ast.BooleanNode{Key: "g"},
|
||||
ast.OperatorNode{Value: "NOT"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
ast.GroupNode{Key: "i", Nodes: []ast.Node{
|
||||
ast.StringNode{Key: "a"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
ast.OperatorNode{Value: "OR"},
|
||||
ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
ast.StringNode{Key: "a"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
ast.OperatorNode{Value: "OR"},
|
||||
ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
ast.StringNode{Key: "a"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "b"},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.DateTimeNode{Key: "c"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.DateTimeNode{Key: "d"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.BooleanNode{Key: "f"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.BooleanNode{Key: "g"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
&ast.GroupNode{Key: "i", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "b"},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
},
|
||||
// LowerValues runs after the shared query.Normalize pass, so it operates on
|
||||
// already-resolved pointer nodes. Field resolution, media-type expansion and
|
||||
// group-key defaulting are tested once at the query.Normalize level (see
|
||||
// pkg/query normalize_test), not here. Only lowercase-analyzed fields get their
|
||||
// value folded; case-preserved fields keep their casing.
|
||||
func TestLowerValues(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
{
|
||||
Name: "lowercase-analyzed field: value is folded, recursing into groups",
|
||||
Got: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "StringNode"},
|
||||
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "StringNode"},
|
||||
}},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
result, err := convert.ExpandKQL(test.Got)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, test.Want, result)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("remaps some keys", func(t *testing.T) {
|
||||
var tests []opensearchtest.TableTest[[]ast.Node, []ast.Node]
|
||||
|
||||
for k, v := range map[string]string{
|
||||
"": "Name", // Default to "Name" if no key is provided
|
||||
"rootid": "RootID",
|
||||
"path": "Path",
|
||||
"id": "ID",
|
||||
"name": "Name",
|
||||
"size": "Size",
|
||||
"mtime": "Mtime",
|
||||
"mediatype": "MimeType",
|
||||
"type": "Type",
|
||||
"tag": "Tags",
|
||||
"tags": "Tags",
|
||||
"content": "Content",
|
||||
"hidden": "Hidden",
|
||||
"any": "any", // Example of an unknown key that should remain unchanged
|
||||
} {
|
||||
tests = append(tests, opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
Name: fmt.Sprintf("%s -> %s", k, v),
|
||||
Got: []ast.Node{
|
||||
&ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: k},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
&ast.DateTimeNode{Key: k},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
ast.DateTimeNode{Key: k},
|
||||
ast.OperatorNode{Value: "OR"},
|
||||
&ast.BooleanNode{Key: k},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
ast.BooleanNode{Key: k},
|
||||
ast.OperatorNode{Value: "NOT"},
|
||||
&ast.GroupNode{Key: k, Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: k, Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: k, Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: k},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: k},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.DateTimeNode{Key: v},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.DateTimeNode{Key: v},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.BooleanNode{Key: v},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.BooleanNode{Key: v},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.GroupNode{Key: func() string {
|
||||
switch {
|
||||
case k == "":
|
||||
return k
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}(), Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: func() string {
|
||||
switch {
|
||||
case k == "":
|
||||
return k
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}(), Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.GroupNode{Key: func() string {
|
||||
switch {
|
||||
case k == "":
|
||||
return k
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}(), Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: v},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: v},
|
||||
}},
|
||||
}},
|
||||
}},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
result, err := convert.ExpandKQL(test.Got)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, test.Want, result)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("lowercases some values", func(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
{
|
||||
Name: "lowercase-analyzed field: value is folded",
|
||||
Got: []ast.Node{
|
||||
ast.StringNode{Key: "Name", Value: "StringNode"},
|
||||
ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
ast.StringNode{Key: "Name", Value: "StringNode"},
|
||||
}},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
Want: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "stringnode"},
|
||||
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "stringnode"},
|
||||
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "stringnode"},
|
||||
}},
|
||||
},
|
||||
}},
|
||||
},
|
||||
{
|
||||
Name: "case-preserved field: value keeps its casing",
|
||||
Got: []ast.Node{
|
||||
ast.StringNode{Key: "aBc", Value: "StringNode"},
|
||||
ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
ast.StringNode{Key: "aBc", Value: "StringNode"},
|
||||
}},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
},
|
||||
{
|
||||
Name: "case-preserved field: value keeps its casing",
|
||||
Got: []ast.Node{
|
||||
&ast.StringNode{Key: "aBc", Value: "StringNode"},
|
||||
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "aBc", Value: "StringNode"},
|
||||
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "aBc", Value: "StringNode"},
|
||||
}},
|
||||
},
|
||||
}},
|
||||
},
|
||||
}
|
||||
Want: []ast.Node{
|
||||
&ast.StringNode{Key: "aBc", Value: "StringNode"},
|
||||
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "aBc", Value: "StringNode"},
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
result, err := convert.ExpandKQL(test.Got)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, test.Want, result)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unfolds some values", func(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
{
|
||||
Name: "MimeType:unknown",
|
||||
Got: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "unknown"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "unknown"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:file",
|
||||
Got: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "file"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:folder",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "folder"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:document",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "document"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/msword"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.text"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "text/plain"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "text/markdown"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/rtf"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.pages"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:spreadsheet",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "spreadsheet"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-excel"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadsheet"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "text/csv"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadshee"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.numbers"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:presentation",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "presentation"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.presentation"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-powerpoint"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.keynote"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:pdf",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "pdf"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/pdf"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:image",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "image"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "image/*"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:video",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "video"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "video/*"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:audio",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "audio"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "audio/*"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "MimeType:archive",
|
||||
Got: []ast.Node{
|
||||
ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Key: "MimeType", Value: "archive"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.StringNode{Value: "some-name"},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.BooleanNode{Key: "Deleted", Value: false},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/zip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/gzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-gzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-7z-compressed"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-rar-compressed"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-tar"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-bzip2"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-bzip"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/x-tgz"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Name", Value: `some-name`},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
if test.Skip {
|
||||
t.Skip("Skipping test due to known issue")
|
||||
}
|
||||
result, err := convert.ExpandKQL(test.Got)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, test.Want, result)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("different cases", func(t *testing.T) {
|
||||
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
|
||||
{
|
||||
Name: "use the group node key as default key",
|
||||
Got: []ast.Node{
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Key: "a", Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Key: "mediatype", Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "file"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "mediatype", Value: "file"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
},
|
||||
Want: []ast.Node{
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Key: "a", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "a", Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Key: "MimeType", Nodes: []ast.Node{
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "c", Value: "d"},
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
if test.Skip {
|
||||
t.Skip("Skipping test due to known issue")
|
||||
}
|
||||
result, err := convert.ExpandKQL(test.Got)
|
||||
require.NoError(t, err)
|
||||
require.EqualValues(t, test.Want, result)
|
||||
})
|
||||
}
|
||||
})
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
require.Equal(t, test.Want, convert.LowerValues(test.Got))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/kql"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -17,10 +18,9 @@ func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) {
|
||||
return nil, fmt.Errorf("failed to build query: %w", err)
|
||||
}
|
||||
|
||||
kqlNodes, err := ExpandKQL(kqlAst.Nodes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to expand KQL AST nodes: %w", err)
|
||||
}
|
||||
// shared lowering (field resolution + media-type), then value lowercasing.
|
||||
kqlAst = query.Normalize(kqlAst, query.ResolveField)
|
||||
kqlNodes := LowerValues(kqlAst.Nodes)
|
||||
|
||||
builder, err := TranspileKQLToOpenSearch(kqlNodes)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user