From f118e0d2c384694e618b24aae66407fa77aee955 Mon Sep 17 00:00:00 2001 From: fschade Date: Thu, 31 Jul 2025 10:09:41 +0200 Subject: [PATCH] enhancement(search): implement kql to os dsl wildcard-query --- services/search/pkg/opensearch/kql.go | 98 ++++++++++++++-------- services/search/pkg/opensearch/kql_test.go | 69 +++++++++++++-- 2 files changed, 124 insertions(+), 43 deletions(-) diff --git a/services/search/pkg/opensearch/kql.go b/services/search/pkg/opensearch/kql.go index 6253c1125c..e9a4c80aca 100644 --- a/services/search/pkg/opensearch/kql.go +++ b/services/search/pkg/opensearch/kql.go @@ -1,6 +1,7 @@ package opensearch import ( + "errors" "fmt" "strings" @@ -8,6 +9,10 @@ import ( "github.com/opencloud-eu/opencloud/pkg/kql" ) +var ( + ErrUnsupportedNodeType = fmt.Errorf("unsupported node type") +) + type KQL struct{} func NewKQL() (*KQL, error) { @@ -23,6 +28,56 @@ func (k *KQL) Compile(tree *ast.Ast) (Builder, error) { return q, nil } +func (k *KQL) compile(nodes []ast.Node) (Builder, error) { + if len(nodes) == 0 { + return nil, fmt.Errorf("no nodes to compile") + } + + if len(nodes) == 1 { + builder, err := k.getBuilder(nodes[0]) + if err != nil { + return nil, fmt.Errorf("failed to get builder for single node: %w", err) + } + return builder, nil + } + + boolQuery := NewBoolQuery() + add := boolQuery.Must + + for i, node := range nodes { + nextOp := k.getOperatorValueAt(nodes, i+1) + + switch { + case nextOp == kql.BoolOR: + add = boolQuery.Should + case nextOp == kql.BoolAND: + add = boolQuery.Must + } + + builder, err := k.getBuilder(node) + switch { + // if the node is not known, we skip it, such as an operator node + case errors.Is(err, ErrUnsupportedNodeType): + continue + case err != nil: + return nil, fmt.Errorf("failed to get builder for node %T: %w", node, err) + } + + if _, ok := node.(*ast.OperatorNode); ok { + // operatorNodes are not builders, so we skip them + continue + } + + add(builder) + } + + if len(boolQuery.should) != 0 { + boolQuery.options.MinimumShouldMatch = 1 + } + + return boolQuery, nil +} + func (k *KQL) getFieldName(name string) string { if name == "" { return "Name" @@ -67,6 +122,11 @@ func (k *KQL) getBuilder(node ast.Node) (Builder, error) { var builder Builder switch node := node.(type) { case *ast.StringNode: + if strings.Contains(node.Value, "*") { + builder = NewWildcardQuery(k.getFieldName(node.Key)).Value(node.Value) + break + } + switch len(strings.Split(node.Value, " ")) { case 1: builder = NewTermQuery[string](k.getFieldName(node.Key)).Value(node.Value) @@ -79,43 +139,9 @@ func (k *KQL) getBuilder(node ast.Node) (Builder, error) { return nil, fmt.Errorf("failed to build group: %w", err) } builder = group + default: + return nil, fmt.Errorf("%w: %T", ErrUnsupportedNodeType, node) } return builder, nil } - -func (k *KQL) compile(nodes []ast.Node) (Builder, error) { - boolQuery := NewBoolQuery() - add := boolQuery.Must - - for i, node := range nodes { - prevOp := k.getOperatorValueAt(nodes, i-1) - nextOp := k.getOperatorValueAt(nodes, i+1) - - switch { - case nextOp == kql.BoolOR || prevOp == kql.BoolOR: - add = boolQuery.Should - case nextOp == kql.BoolAND || prevOp == kql.BoolAND: - add = boolQuery.Must - } - - if _, ok := node.(*ast.OperatorNode); ok { - // operatorNodes are not builders, so we skip them - continue - } - - builder, err := k.getBuilder(node) - if err != nil { - return nil, fmt.Errorf("failed to get builder for node %T: %w", node, err) - } - - switch { - case len(nodes) == 1: - return builder, nil - default: - add(builder) - } - } - - return boolQuery, nil -} diff --git a/services/search/pkg/opensearch/kql_test.go b/services/search/pkg/opensearch/kql_test.go index e55b25f806..8dfb6a2c61 100644 --- a/services/search/pkg/opensearch/kql_test.go +++ b/services/search/pkg/opensearch/kql_test.go @@ -32,7 +32,7 @@ func TestKQL_Compile(t *testing.T) { }, // kql to os dsl - type tests { - name: "remaps known field names", + name: "term query", got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "Name", Value: "openCloud"}, @@ -41,7 +41,7 @@ func TestKQL_Compile(t *testing.T) { want: opensearch.NewTermQuery[string]("Name").Value("openCloud"), }, { - name: "remaps known field names", + name: "match-phrase query", got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "Name", Value: "open cloud"}, @@ -49,6 +49,41 @@ func TestKQL_Compile(t *testing.T) { }, want: opensearch.NewMatchPhraseQuery("Name").Query("open cloud"), }, + { + name: "wildcard query", + got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "open*"}, + }, + }, + want: opensearch.NewWildcardQuery("Name").Value("open*"), + }, + { + name: "bool query", + got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Value: "a"}, + &ast.StringNode{Value: "b"}, + }}, + }, + }, + want: opensearch.NewBoolQuery().Must( + opensearch.NewTermQuery[string]("Name").Value("a"), + opensearch.NewTermQuery[string]("Name").Value("b"), + ), + }, + { + name: "no bool query for single term", + got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Value: "any"}, + }}, + }, + }, + want: opensearch.NewWildcardQuery("Name").Value("open*"), + }, // kql to os dsl - structure tests { name: "[*]", @@ -97,7 +132,7 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "age", Value: "32"}, }, }, - want: opensearch.NewBoolQuery(). + want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). Should( opensearch.NewTermQuery[string]("Name").Value("openCloud"), opensearch.NewTermQuery[string]("age").Value("32"), @@ -114,7 +149,7 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "age", Value: "44"}, }, }, - want: opensearch.NewBoolQuery(). + want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). Should( opensearch.NewTermQuery[string]("Name").Value("openCloud"), opensearch.NewTermQuery[string]("age").Value("32"), @@ -132,7 +167,7 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "c", Value: "c"}, }, }, - want: opensearch.NewBoolQuery(). + want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). Must( opensearch.NewTermQuery[string]("a").Value("a"), ). @@ -152,13 +187,33 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "c", Value: "c"}, }, }, - want: opensearch.NewBoolQuery(). + want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). Must( + opensearch.NewTermQuery[string]("b").Value("b"), opensearch.NewTermQuery[string]("c").Value("c"), ). Should( opensearch.NewTermQuery[string]("a").Value("a"), + ), + }, + { + name: "NEW[* OR * AND *]", + got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "a", Value: "a"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "b", Value: "b"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "c", Value: "c"}, + }, + }, + want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). + Should( + opensearch.NewTermQuery[string]("a").Value("a"), + ). + Must( opensearch.NewTermQuery[string]("b").Value("b"), + opensearch.NewTermQuery[string]("c").Value("c"), ), }, { @@ -178,7 +233,7 @@ func TestKQL_Compile(t *testing.T) { }, want: opensearch.NewBoolQuery(). Must( - opensearch.NewBoolQuery(). + opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). Should( opensearch.NewTermQuery[string]("a").Value("a"), opensearch.NewTermQuery[string]("b").Value("b"),