diff --git a/pkg/ast/ast.go b/pkg/ast/ast.go index 9aafe853aa..f1a7e3263e 100644 --- a/pkg/ast/ast.go +++ b/pkg/ast/ast.go @@ -43,6 +43,7 @@ type StringNode struct { *Base Key string Value string + Exact bool } // BooleanNode represents a bool value @@ -60,6 +61,14 @@ type DateTimeNode struct { Value time.Time } +// NumberNode represents a numeric value +type NumberNode struct { + *Base + Key string + Operator *OperatorNode + Value float64 +} + // OperatorNode represents an operator value like // AND, OR, NOT, =, <= ... and so on type OperatorNode struct { diff --git a/pkg/ast/test/test.go b/pkg/ast/test/test.go index d7fb985226..8a39b75022 100644 --- a/pkg/ast/test/test.go +++ b/pkg/ast/test/test.go @@ -21,6 +21,7 @@ func DiffAst(x, y any, opts ...cmp.Option) string { cmpopts.IgnoreFields(ast.GroupNode{}, "Base"), cmpopts.IgnoreFields(ast.BooleanNode{}, "Base"), cmpopts.IgnoreFields(ast.DateTimeNode{}, "Base"), + cmpopts.IgnoreFields(ast.NumberNode{}, "Base"), )..., ) } diff --git a/pkg/kql/cast.go b/pkg/kql/cast.go index c1fb185623..ecd0cac27f 100644 --- a/pkg/kql/cast.go +++ b/pkg/kql/cast.go @@ -2,6 +2,7 @@ package kql import ( "fmt" + "strconv" "time" "github.com/jinzhu/now" @@ -136,3 +137,12 @@ func toTimeRange(in any) (*time.Time, *time.Time, error) { return &from, &to, nil } + +func toFloat(v any) (float64, error) { + value, err := toString(v) + if err != nil { + return 0, err + } + + return strconv.ParseFloat(value, 64) +} diff --git a/pkg/kql/dictionary.peg b/pkg/kql/dictionary.peg index 5f59e78afb..c21f46f4c4 100644 --- a/pkg/kql/dictionary.peg +++ b/pkg/kql/dictionary.peg @@ -40,6 +40,7 @@ GroupNode <- PropertyRestrictionNodes <- YesNoPropertyRestrictionNode / DateTimeRestrictionNode / + NumberRestrictionNode / TextPropertyRestrictionNode YesNoPropertyRestrictionNode <- @@ -69,9 +70,22 @@ DateTimeRestrictionNode <- return buildNaturalLanguageDateTimeNodes(k, v, c.text, c.pos) } +NumberRestrictionNode <- + k:Key o:( + OperatorGreaterOrEqualNode / + OperatorLessOrEqualNode / + OperatorGreaterNode / + OperatorLessNode + ) '"'? v:Number '"'? { + return buildNumberNode(k, o, v, c.text, c.pos) + } + TextPropertyRestrictionNode <- - k:Key (OperatorColonNode / OperatorEqualNode) v:(String / [^ ()]+){ - return buildStringNode(k, v, c.text, c.pos) + k:Key OperatorEqualNode v:(String / [^ ()]+) { + return buildStringNode(k, v, true, c.text, c.pos) + } / + k:Key OperatorColonNode v:(String / [^ ()]+) { + return buildStringNode(k, v, false, c.text, c.pos) } //////////////////////////////////////////////////////// @@ -84,12 +98,12 @@ FreeTextKeywordNodes <- PhraseNode <- OperatorColonNode? _ v:String _ OperatorColonNode? { - return buildStringNode("", v, c.text, c.pos) + return buildStringNode("", v, false, c.text, c.pos) } WordNode <- OperatorColonNode? _ v:[^ :()]+ _ OperatorColonNode? { - return buildStringNode("", v, c.text, c.pos) + return buildStringNode("", v, false, c.text, c.pos) } //////////////////////////////////////////////////////// @@ -229,6 +243,11 @@ String <- return v, nil } +Number <- + [0-9]+ ("." [0-9]+)? { + return string(c.text), nil + } + Digit <- [0-9] { return c.text, nil diff --git a/pkg/kql/dictionary_gen.go b/pkg/kql/dictionary_gen.go index aec6f49d1d..9777ece435 100644 --- a/pkg/kql/dictionary_gen.go +++ b/pkg/kql/dictionary_gen.go @@ -43,12 +43,12 @@ var g = &grammar{ pos: position{line: 19, col: 6, offset: 351}, exprs: []any{ &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, run: (*parser).callonNodes3, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -75,27 +75,27 @@ var g = &grammar{ name: "GroupNode", }, &actionExpr{ - pos: position{line: 46, col: 5, offset: 1038}, + pos: position{line: 47, col: 5, offset: 1066}, run: (*parser).callonNode3, expr: &seqExpr{ - pos: position{line: 46, col: 5, offset: 1038}, + pos: position{line: 47, col: 5, offset: 1066}, exprs: []any{ &labeledExpr{ - pos: position{line: 46, col: 5, offset: 1038}, + pos: position{line: 47, col: 5, offset: 1066}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode6, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode9, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -104,23 +104,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode15, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -136,23 +136,23 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 46, col: 12, offset: 1045}, + pos: position{line: 47, col: 12, offset: 1073}, alternatives: []any{ &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonNode18, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", }, }, &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonNode20, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", @@ -161,19 +161,19 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 46, col: 51, offset: 1084}, + pos: position{line: 47, col: 51, offset: 1112}, label: "v", expr: &choiceExpr{ - pos: position{line: 46, col: 54, offset: 1087}, + pos: position{line: 47, col: 54, offset: 1115}, alternatives: []any{ &litMatcher{ - pos: position{line: 46, col: 54, offset: 1087}, + pos: position{line: 47, col: 54, offset: 1115}, val: "true", ignoreCase: false, want: "\"true\"", }, &litMatcher{ - pos: position{line: 46, col: 63, offset: 1096}, + pos: position{line: 47, col: 63, offset: 1124}, val: "false", ignoreCase: false, want: "\"false\"", @@ -185,27 +185,27 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 51, col: 5, offset: 1197}, + pos: position{line: 52, col: 5, offset: 1225}, run: (*parser).callonNode26, expr: &seqExpr{ - pos: position{line: 51, col: 5, offset: 1197}, + pos: position{line: 52, col: 5, offset: 1225}, exprs: []any{ &labeledExpr{ - pos: position{line: 51, col: 5, offset: 1197}, + pos: position{line: 52, col: 5, offset: 1225}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode29, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode32, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -214,23 +214,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode38, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -246,66 +246,66 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 51, col: 11, offset: 1203}, + pos: position{line: 52, col: 11, offset: 1231}, label: "o", expr: &choiceExpr{ - pos: position{line: 52, col: 9, offset: 1215}, + pos: position{line: 53, col: 9, offset: 1243}, alternatives: []any{ &actionExpr{ - pos: position{line: 145, col: 5, offset: 3346}, + pos: position{line: 159, col: 5, offset: 3745}, run: (*parser).callonNode42, expr: &litMatcher{ - pos: position{line: 145, col: 5, offset: 3346}, + pos: position{line: 159, col: 5, offset: 3745}, val: ">=", ignoreCase: false, want: "\">=\"", }, }, &actionExpr{ - pos: position{line: 135, col: 5, offset: 3162}, + pos: position{line: 149, col: 5, offset: 3561}, run: (*parser).callonNode44, expr: &litMatcher{ - pos: position{line: 135, col: 5, offset: 3162}, + pos: position{line: 149, col: 5, offset: 3561}, val: "<=", ignoreCase: false, want: "\"<=\"", }, }, &actionExpr{ - pos: position{line: 140, col: 5, offset: 3251}, + pos: position{line: 154, col: 5, offset: 3650}, run: (*parser).callonNode46, expr: &litMatcher{ - pos: position{line: 140, col: 5, offset: 3251}, + pos: position{line: 154, col: 5, offset: 3650}, val: ">", ignoreCase: false, want: "\">\"", }, }, &actionExpr{ - pos: position{line: 130, col: 5, offset: 3070}, + pos: position{line: 144, col: 5, offset: 3469}, run: (*parser).callonNode48, expr: &litMatcher{ - pos: position{line: 130, col: 5, offset: 3070}, + pos: position{line: 144, col: 5, offset: 3469}, val: "<", ignoreCase: false, want: "\"<\"", }, }, &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonNode50, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", }, }, &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonNode52, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -315,43 +315,43 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 58, col: 7, offset: 1395}, + pos: position{line: 59, col: 7, offset: 1423}, expr: &litMatcher{ - pos: position{line: 58, col: 7, offset: 1395}, + pos: position{line: 59, col: 7, offset: 1423}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, }, &labeledExpr{ - pos: position{line: 58, col: 12, offset: 1400}, + pos: position{line: 59, col: 12, offset: 1428}, label: "v", expr: &choiceExpr{ - pos: position{line: 59, col: 9, offset: 1412}, + pos: position{line: 60, col: 9, offset: 1440}, alternatives: []any{ &actionExpr{ - pos: position{line: 195, col: 5, offset: 4185}, + pos: position{line: 209, col: 5, offset: 4584}, run: (*parser).callonNode58, expr: &seqExpr{ - pos: position{line: 195, col: 5, offset: 4185}, + pos: position{line: 209, col: 5, offset: 4584}, exprs: []any{ &actionExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, run: (*parser).callonNode60, expr: &seqExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, exprs: []any{ &actionExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, run: (*parser).callonNode62, expr: &seqExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode64, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -359,10 +359,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode66, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -370,10 +370,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode68, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -381,10 +381,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode70, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -395,22 +395,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 14, offset: 3957}, + pos: position{line: 199, col: 14, offset: 4356}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, run: (*parser).callonNode73, expr: &seqExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode75, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -418,10 +418,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode77, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -432,22 +432,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 28, offset: 3971}, + pos: position{line: 199, col: 28, offset: 4370}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, run: (*parser).callonNode80, expr: &seqExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode82, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -455,10 +455,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode84, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -472,28 +472,28 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 195, col: 14, offset: 4194}, + pos: position{line: 209, col: 14, offset: 4593}, val: "T", ignoreCase: false, want: "\"T\"", }, &actionExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, run: (*parser).callonNode87, expr: &seqExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, exprs: []any{ &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode89, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode91, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -501,10 +501,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode93, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -515,22 +515,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 14, offset: 4044}, + pos: position{line: 204, col: 14, offset: 4443}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode96, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode98, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -538,10 +538,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode100, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -552,22 +552,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 29, offset: 4059}, + pos: position{line: 204, col: 29, offset: 4458}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, run: (*parser).callonNode103, expr: &seqExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode105, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -575,10 +575,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode107, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -589,23 +589,23 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 190, col: 44, offset: 4074}, + pos: position{line: 204, col: 44, offset: 4473}, expr: &seqExpr{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, exprs: []any{ &litMatcher{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 190, col: 49, offset: 4079}, + pos: position{line: 204, col: 49, offset: 4478}, expr: &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode113, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -617,35 +617,35 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, alternatives: []any{ &litMatcher{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, val: "Z", ignoreCase: false, want: "\"Z\"", }, &seqExpr{ - pos: position{line: 190, col: 65, offset: 4095}, + pos: position{line: 204, col: 65, offset: 4494}, exprs: []any{ &charClassMatcher{ - pos: position{line: 190, col: 66, offset: 4096}, + pos: position{line: 204, col: 66, offset: 4495}, val: "[+-]", chars: []rune{'+', '-'}, ignoreCase: false, inverted: false, }, &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode119, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode121, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -653,10 +653,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode123, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -667,22 +667,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 86, offset: 4116}, + pos: position{line: 204, col: 86, offset: 4515}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode126, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode128, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -690,10 +690,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode130, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -714,22 +714,22 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, run: (*parser).callonNode132, expr: &seqExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, exprs: []any{ &actionExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, run: (*parser).callonNode134, expr: &seqExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode136, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -737,10 +737,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode138, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -748,10 +748,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode140, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -759,10 +759,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode142, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -773,22 +773,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 14, offset: 3957}, + pos: position{line: 199, col: 14, offset: 4356}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, run: (*parser).callonNode145, expr: &seqExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode147, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -796,10 +796,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode149, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -810,22 +810,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 28, offset: 3971}, + pos: position{line: 199, col: 28, offset: 4370}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, run: (*parser).callonNode152, expr: &seqExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode154, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -833,10 +833,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode156, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -850,22 +850,22 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, run: (*parser).callonNode158, expr: &seqExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, exprs: []any{ &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode160, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode162, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -873,10 +873,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode164, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -887,22 +887,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 14, offset: 4044}, + pos: position{line: 204, col: 14, offset: 4443}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode167, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode169, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -910,10 +910,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode171, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -924,22 +924,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 29, offset: 4059}, + pos: position{line: 204, col: 29, offset: 4458}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, run: (*parser).callonNode174, expr: &seqExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode176, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -947,10 +947,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode178, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -961,23 +961,23 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 190, col: 44, offset: 4074}, + pos: position{line: 204, col: 44, offset: 4473}, expr: &seqExpr{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, exprs: []any{ &litMatcher{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 190, col: 49, offset: 4079}, + pos: position{line: 204, col: 49, offset: 4478}, expr: &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode184, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -989,35 +989,35 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, alternatives: []any{ &litMatcher{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, val: "Z", ignoreCase: false, want: "\"Z\"", }, &seqExpr{ - pos: position{line: 190, col: 65, offset: 4095}, + pos: position{line: 204, col: 65, offset: 4494}, exprs: []any{ &charClassMatcher{ - pos: position{line: 190, col: 66, offset: 4096}, + pos: position{line: 204, col: 66, offset: 4495}, val: "[+-]", chars: []rune{'+', '-'}, ignoreCase: false, inverted: false, }, &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode190, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode192, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1025,10 +1025,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode194, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1039,22 +1039,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 86, offset: 4116}, + pos: position{line: 204, col: 86, offset: 4515}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode197, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode199, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1062,10 +1062,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode201, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1086,9 +1086,9 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 62, col: 7, offset: 1465}, + pos: position{line: 63, col: 7, offset: 1493}, expr: &litMatcher{ - pos: position{line: 62, col: 7, offset: 1465}, + pos: position{line: 63, col: 7, offset: 1493}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1098,27 +1098,27 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 65, col: 5, offset: 1541}, + pos: position{line: 66, col: 5, offset: 1569}, run: (*parser).callonNode205, expr: &seqExpr{ - pos: position{line: 65, col: 5, offset: 1541}, + pos: position{line: 66, col: 5, offset: 1569}, exprs: []any{ &labeledExpr{ - pos: position{line: 65, col: 5, offset: 1541}, + pos: position{line: 66, col: 5, offset: 1569}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode208, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode211, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1127,23 +1127,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode217, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1159,23 +1159,23 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 66, col: 9, offset: 1557}, + pos: position{line: 67, col: 9, offset: 1585}, alternatives: []any{ &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonNode220, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", }, }, &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonNode222, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1184,79 +1184,79 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 68, col: 7, offset: 1609}, + pos: position{line: 69, col: 7, offset: 1637}, expr: &litMatcher{ - pos: position{line: 68, col: 7, offset: 1609}, + pos: position{line: 69, col: 7, offset: 1637}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, }, &labeledExpr{ - pos: position{line: 68, col: 12, offset: 1614}, + pos: position{line: 69, col: 12, offset: 1642}, label: "v", expr: &choiceExpr{ - pos: position{line: 200, col: 5, offset: 4273}, + pos: position{line: 214, col: 5, offset: 4672}, alternatives: []any{ &litMatcher{ - pos: position{line: 200, col: 5, offset: 4273}, + pos: position{line: 214, col: 5, offset: 4672}, val: "today", ignoreCase: false, want: "\"today\"", }, &litMatcher{ - pos: position{line: 201, col: 5, offset: 4287}, + pos: position{line: 215, col: 5, offset: 4686}, val: "yesterday", ignoreCase: false, want: "\"yesterday\"", }, &litMatcher{ - pos: position{line: 202, col: 5, offset: 4305}, + pos: position{line: 216, col: 5, offset: 4704}, val: "this week", ignoreCase: false, want: "\"this week\"", }, &litMatcher{ - pos: position{line: 203, col: 5, offset: 4323}, + pos: position{line: 217, col: 5, offset: 4722}, val: "last week", ignoreCase: false, want: "\"last week\"", }, &litMatcher{ - pos: position{line: 204, col: 5, offset: 4341}, + pos: position{line: 218, col: 5, offset: 4740}, val: "last 7 days", ignoreCase: false, want: "\"last 7 days\"", }, &litMatcher{ - pos: position{line: 205, col: 5, offset: 4361}, + pos: position{line: 219, col: 5, offset: 4760}, val: "this month", ignoreCase: false, want: "\"this month\"", }, &litMatcher{ - pos: position{line: 206, col: 5, offset: 4380}, + pos: position{line: 220, col: 5, offset: 4779}, val: "last month", ignoreCase: false, want: "\"last month\"", }, &litMatcher{ - pos: position{line: 207, col: 5, offset: 4399}, + pos: position{line: 221, col: 5, offset: 4798}, val: "last 30 days", ignoreCase: false, want: "\"last 30 days\"", }, &litMatcher{ - pos: position{line: 208, col: 5, offset: 4420}, + pos: position{line: 222, col: 5, offset: 4819}, val: "this year", ignoreCase: false, want: "\"this year\"", }, &actionExpr{ - pos: position{line: 209, col: 5, offset: 4438}, + pos: position{line: 223, col: 5, offset: 4837}, run: (*parser).callonNode237, expr: &litMatcher{ - pos: position{line: 209, col: 5, offset: 4438}, + pos: position{line: 223, col: 5, offset: 4837}, val: "last year", ignoreCase: false, want: "\"last year\"", @@ -1266,9 +1266,9 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 68, col: 38, offset: 1640}, + pos: position{line: 69, col: 38, offset: 1668}, expr: &litMatcher{ - pos: position{line: 68, col: 38, offset: 1640}, + pos: position{line: 69, col: 38, offset: 1668}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1278,27 +1278,27 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 73, col: 5, offset: 1759}, + pos: position{line: 74, col: 5, offset: 1781}, run: (*parser).callonNode241, expr: &seqExpr{ - pos: position{line: 73, col: 5, offset: 1759}, + pos: position{line: 74, col: 5, offset: 1781}, exprs: []any{ &labeledExpr{ - pos: position{line: 73, col: 5, offset: 1759}, + pos: position{line: 74, col: 5, offset: 1781}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode244, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode247, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1307,23 +1307,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode253, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1338,56 +1338,219 @@ var g = &grammar{ }, }, }, - &choiceExpr{ - pos: position{line: 73, col: 12, offset: 1766}, - alternatives: []any{ - &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode256, - expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, - val: ":", - ignoreCase: false, - want: "\":\"", + &labeledExpr{ + pos: position{line: 74, col: 11, offset: 1787}, + label: "o", + expr: &choiceExpr{ + pos: position{line: 75, col: 9, offset: 1799}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 159, col: 5, offset: 3745}, + run: (*parser).callonNode257, + expr: &litMatcher{ + pos: position{line: 159, col: 5, offset: 3745}, + val: ">=", + ignoreCase: false, + want: "\">=\"", + }, }, - }, - &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, - run: (*parser).callonNode258, - expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, - val: "=", - ignoreCase: false, - want: "\"=\"", + &actionExpr{ + pos: position{line: 149, col: 5, offset: 3561}, + run: (*parser).callonNode259, + expr: &litMatcher{ + pos: position{line: 149, col: 5, offset: 3561}, + val: "<=", + ignoreCase: false, + want: "\"<=\"", + }, + }, + &actionExpr{ + pos: position{line: 154, col: 5, offset: 3650}, + run: (*parser).callonNode261, + expr: &litMatcher{ + pos: position{line: 154, col: 5, offset: 3650}, + val: ">", + ignoreCase: false, + want: "\">\"", + }, + }, + &actionExpr{ + pos: position{line: 144, col: 5, offset: 3469}, + run: (*parser).callonNode263, + expr: &litMatcher{ + pos: position{line: 144, col: 5, offset: 3469}, + val: "<", + ignoreCase: false, + want: "\"<\"", + }, }, }, }, }, + &zeroOrOneExpr{ + pos: position{line: 79, col: 7, offset: 1923}, + expr: &litMatcher{ + pos: position{line: 79, col: 7, offset: 1923}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + }, &labeledExpr{ - pos: position{line: 73, col: 51, offset: 1805}, + pos: position{line: 79, col: 12, offset: 1928}, + label: "v", + expr: &actionExpr{ + pos: position{line: 247, col: 5, offset: 5207}, + run: (*parser).callonNode268, + expr: &seqExpr{ + pos: position{line: 247, col: 5, offset: 5207}, + exprs: []any{ + &oneOrMoreExpr{ + pos: position{line: 247, col: 5, offset: 5207}, + expr: &charClassMatcher{ + pos: position{line: 247, col: 5, offset: 5207}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + &zeroOrOneExpr{ + pos: position{line: 247, col: 12, offset: 5214}, + expr: &seqExpr{ + pos: position{line: 247, col: 13, offset: 5215}, + exprs: []any{ + &litMatcher{ + pos: position{line: 247, col: 13, offset: 5215}, + val: ".", + ignoreCase: false, + want: "\".\"", + }, + &oneOrMoreExpr{ + pos: position{line: 247, col: 17, offset: 5219}, + expr: &charClassMatcher{ + pos: position{line: 247, col: 17, offset: 5219}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + }, + }, + }, + }, + }, + &zeroOrOneExpr{ + pos: position{line: 79, col: 21, offset: 1937}, + expr: &litMatcher{ + pos: position{line: 79, col: 21, offset: 1937}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 84, col: 5, offset: 2041}, + run: (*parser).callonNode279, + expr: &seqExpr{ + pos: position{line: 84, col: 5, offset: 2041}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 84, col: 5, offset: 2041}, + label: "k", + expr: &actionExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + run: (*parser).callonNode282, + expr: &seqExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + exprs: []any{ + &oneOrMoreExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode285, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + &zeroOrMoreExpr{ + pos: position{line: 237, col: 11, offset: 5083}, + expr: &seqExpr{ + pos: position{line: 237, col: 12, offset: 5084}, + exprs: []any{ + &litMatcher{ + pos: position{line: 237, col: 12, offset: 5084}, + val: ".", + ignoreCase: false, + want: "\".\"", + }, + &oneOrMoreExpr{ + pos: position{line: 237, col: 16, offset: 5088}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode291, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 139, col: 5, offset: 3384}, + run: (*parser).callonNode293, + expr: &litMatcher{ + pos: position{line: 139, col: 5, offset: 3384}, + val: "=", + ignoreCase: false, + want: "\"=\"", + }, + }, + &labeledExpr{ + pos: position{line: 84, col: 29, offset: 2065}, label: "v", expr: &choiceExpr{ - pos: position{line: 73, col: 54, offset: 1808}, + pos: position{line: 84, col: 32, offset: 2068}, alternatives: []any{ &actionExpr{ - pos: position{line: 228, col: 5, offset: 4747}, - run: (*parser).callonNode262, + pos: position{line: 242, col: 5, offset: 5146}, + run: (*parser).callonNode297, expr: &seqExpr{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, exprs: []any{ &litMatcher{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, &labeledExpr{ - pos: position{line: 228, col: 9, offset: 4751}, + pos: position{line: 242, col: 9, offset: 5150}, label: "v", expr: &zeroOrMoreExpr{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, expr: &charClassMatcher{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, val: "[^\"]", chars: []rune{'"'}, ignoreCase: false, @@ -1396,7 +1559,7 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 228, col: 17, offset: 4759}, + pos: position{line: 242, col: 17, offset: 5158}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1405,9 +1568,9 @@ var g = &grammar{ }, }, &oneOrMoreExpr{ - pos: position{line: 73, col: 63, offset: 1817}, + pos: position{line: 84, col: 41, offset: 2077}, expr: &charClassMatcher{ - pos: position{line: 73, col: 63, offset: 1817}, + pos: position{line: 84, col: 41, offset: 2077}, val: "[^ ()]", chars: []rune{' ', '(', ')'}, ignoreCase: false, @@ -1421,19 +1584,147 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 105, col: 5, offset: 2609}, - run: (*parser).callonNode271, + pos: position{line: 87, col: 5, offset: 2158}, + run: (*parser).callonNode306, + expr: &seqExpr{ + pos: position{line: 87, col: 5, offset: 2158}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 87, col: 5, offset: 2158}, + label: "k", + expr: &actionExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + run: (*parser).callonNode309, + expr: &seqExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + exprs: []any{ + &oneOrMoreExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode312, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + &zeroOrMoreExpr{ + pos: position{line: 237, col: 11, offset: 5083}, + expr: &seqExpr{ + pos: position{line: 237, col: 12, offset: 5084}, + exprs: []any{ + &litMatcher{ + pos: position{line: 237, col: 12, offset: 5084}, + val: ".", + ignoreCase: false, + want: "\".\"", + }, + &oneOrMoreExpr{ + pos: position{line: 237, col: 16, offset: 5088}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode318, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode320, + expr: &litMatcher{ + pos: position{line: 134, col: 5, offset: 3298}, + val: ":", + ignoreCase: false, + want: "\":\"", + }, + }, + &labeledExpr{ + pos: position{line: 87, col: 29, offset: 2182}, + label: "v", + expr: &choiceExpr{ + pos: position{line: 87, col: 32, offset: 2185}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 242, col: 5, offset: 5146}, + run: (*parser).callonNode324, + expr: &seqExpr{ + pos: position{line: 242, col: 5, offset: 5146}, + exprs: []any{ + &litMatcher{ + pos: position{line: 242, col: 5, offset: 5146}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + &labeledExpr{ + pos: position{line: 242, col: 9, offset: 5150}, + label: "v", + expr: &zeroOrMoreExpr{ + pos: position{line: 242, col: 11, offset: 5152}, + expr: &charClassMatcher{ + pos: position{line: 242, col: 11, offset: 5152}, + val: "[^\"]", + chars: []rune{'"'}, + ignoreCase: false, + inverted: true, + }, + }, + }, + &litMatcher{ + pos: position{line: 242, col: 17, offset: 5158}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + }, + }, + }, + &oneOrMoreExpr{ + pos: position{line: 87, col: 41, offset: 2194}, + expr: &charClassMatcher{ + pos: position{line: 87, col: 41, offset: 2194}, + val: "[^ ()]", + chars: []rune{' ', '(', ')'}, + ignoreCase: false, + inverted: true, + }, + }, + }, + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 119, col: 5, offset: 3008}, + run: (*parser).callonNode333, expr: &choiceExpr{ - pos: position{line: 105, col: 6, offset: 2610}, + pos: position{line: 119, col: 6, offset: 3009}, alternatives: []any{ &litMatcher{ - pos: position{line: 105, col: 6, offset: 2610}, + pos: position{line: 119, col: 6, offset: 3009}, val: "AND", ignoreCase: false, want: "\"AND\"", }, &litMatcher{ - pos: position{line: 105, col: 14, offset: 2618}, + pos: position{line: 119, col: 14, offset: 3017}, val: "+", ignoreCase: false, want: "\"+\"", @@ -1442,19 +1733,19 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 110, col: 5, offset: 2710}, - run: (*parser).callonNode275, + pos: position{line: 124, col: 5, offset: 3109}, + run: (*parser).callonNode337, expr: &choiceExpr{ - pos: position{line: 110, col: 6, offset: 2711}, + pos: position{line: 124, col: 6, offset: 3110}, alternatives: []any{ &litMatcher{ - pos: position{line: 110, col: 6, offset: 2711}, + pos: position{line: 124, col: 6, offset: 3110}, val: "NOT", ignoreCase: false, want: "\"NOT\"", }, &litMatcher{ - pos: position{line: 110, col: 14, offset: 2719}, + pos: position{line: 124, col: 14, offset: 3118}, val: "-", ignoreCase: false, want: "\"-\"", @@ -1463,28 +1754,28 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 115, col: 5, offset: 2810}, - run: (*parser).callonNode279, + pos: position{line: 129, col: 5, offset: 3209}, + run: (*parser).callonNode341, expr: &litMatcher{ - pos: position{line: 115, col: 6, offset: 2811}, + pos: position{line: 129, col: 6, offset: 3210}, val: "OR", ignoreCase: false, want: "\"OR\"", }, }, &actionExpr{ - pos: position{line: 86, col: 6, offset: 2097}, - run: (*parser).callonNode281, + pos: position{line: 100, col: 6, offset: 2482}, + run: (*parser).callonNode343, expr: &seqExpr{ - pos: position{line: 86, col: 6, offset: 2097}, + pos: position{line: 100, col: 6, offset: 2482}, exprs: []any{ &zeroOrOneExpr{ - pos: position{line: 86, col: 6, offset: 2097}, + pos: position{line: 100, col: 6, offset: 2482}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode284, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode346, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1492,12 +1783,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode286, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode348, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1506,27 +1797,27 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 86, col: 27, offset: 2118}, + pos: position{line: 100, col: 27, offset: 2503}, label: "v", expr: &actionExpr{ - pos: position{line: 228, col: 5, offset: 4747}, - run: (*parser).callonNode290, + pos: position{line: 242, col: 5, offset: 5146}, + run: (*parser).callonNode352, expr: &seqExpr{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, exprs: []any{ &litMatcher{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, &labeledExpr{ - pos: position{line: 228, col: 9, offset: 4751}, + pos: position{line: 242, col: 9, offset: 5150}, label: "v", expr: &zeroOrMoreExpr{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, expr: &charClassMatcher{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, val: "[^\"]", chars: []rune{'"'}, ignoreCase: false, @@ -1535,7 +1826,7 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 228, col: 17, offset: 4759}, + pos: position{line: 242, col: 17, offset: 5158}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1545,12 +1836,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode297, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode359, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1559,12 +1850,12 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 86, col: 38, offset: 2129}, + pos: position{line: 100, col: 38, offset: 2514}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode301, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode363, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1575,18 +1866,18 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 91, col: 6, offset: 2227}, - run: (*parser).callonNode303, + pos: position{line: 105, col: 6, offset: 2619}, + run: (*parser).callonNode365, expr: &seqExpr{ - pos: position{line: 91, col: 6, offset: 2227}, + pos: position{line: 105, col: 6, offset: 2619}, exprs: []any{ &zeroOrOneExpr{ - pos: position{line: 91, col: 6, offset: 2227}, + pos: position{line: 105, col: 6, offset: 2619}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode306, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode368, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1594,12 +1885,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode308, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode370, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1608,12 +1899,12 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 91, col: 27, offset: 2248}, + pos: position{line: 105, col: 27, offset: 2640}, label: "v", expr: &oneOrMoreExpr{ - pos: position{line: 91, col: 29, offset: 2250}, + pos: position{line: 105, col: 29, offset: 2642}, expr: &charClassMatcher{ - pos: position{line: 91, col: 29, offset: 2250}, + pos: position{line: 105, col: 29, offset: 2642}, val: "[^ :()]", chars: []rune{' ', ':', '(', ')'}, ignoreCase: false, @@ -1622,12 +1913,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode314, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode376, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1636,12 +1927,12 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 91, col: 40, offset: 2261}, + pos: position{line: 105, col: 40, offset: 2653}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode318, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode380, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1669,18 +1960,18 @@ var g = &grammar{ expr: &zeroOrOneExpr{ pos: position{line: 32, col: 7, offset: 614}, expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonGroupNode5, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonGroupNode8, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1689,23 +1980,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonGroupNode14, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1727,20 +2018,20 @@ var g = &grammar{ pos: position{line: 32, col: 13, offset: 620}, alternatives: []any{ &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonGroupNode18, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", }, }, &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonGroupNode20, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", @@ -2733,118 +3024,118 @@ func (p *parser) callonNode244() (any, error) { return p.cur.onNode244() } -func (c *current) onNode256() (any, error) { +func (c *current) onNode257() (any, error) { return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode256() (any, error) { +func (p *parser) callonNode257() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode256() + return p.cur.onNode257() } -func (c *current) onNode258() (any, error) { +func (c *current) onNode259() (any, error) { return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode258() (any, error) { +func (p *parser) callonNode259() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode258() + return p.cur.onNode259() } -func (c *current) onNode262(v any) (any, error) { - return v, nil +func (c *current) onNode261() (any, error) { + return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode262() (any, error) { +func (p *parser) callonNode261() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode262(stack["v"]) + return p.cur.onNode261() } -func (c *current) onNode241(k, v any) (any, error) { - return buildStringNode(k, v, c.text, c.pos) +func (c *current) onNode263() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode263() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode263() +} + +func (c *current) onNode268() (any, error) { + return string(c.text), nil + +} + +func (p *parser) callonNode268() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode268() +} + +func (c *current) onNode241(k, o, v any) (any, error) { + return buildNumberNode(k, o, v, c.text, c.pos) } func (p *parser) callonNode241() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode241(stack["k"], stack["v"]) + return p.cur.onNode241(stack["k"], stack["o"], stack["v"]) } -func (c *current) onNode271() (any, error) { +func (c *current) onNode285() (any, error) { + return c.text, nil + +} + +func (p *parser) callonNode285() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode285() +} + +func (c *current) onNode291() (any, error) { + return c.text, nil + +} + +func (p *parser) callonNode291() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode291() +} + +func (c *current) onNode282() (any, error) { + return c.text, nil + +} + +func (p *parser) callonNode282() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode282() +} + +func (c *current) onNode293(k any) (any, error) { return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode271() (any, error) { +func (p *parser) callonNode293() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode271() -} - -func (c *current) onNode275() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode275() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode275() -} - -func (c *current) onNode279() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode279() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode279() -} - -func (c *current) onNode284() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode284() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode284() -} - -func (c *current) onNode286() (any, error) { - return nil, nil - -} - -func (p *parser) callonNode286() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode286() -} - -func (c *current) onNode290(v any) (any, error) { - return v, nil - -} - -func (p *parser) callonNode290() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode290(stack["v"]) + return p.cur.onNode293(stack["k"]) } func (c *current) onNode297(v any) (any, error) { - return nil, nil + return v, nil } @@ -2854,63 +3145,30 @@ func (p *parser) callonNode297() (any, error) { return p.cur.onNode297(stack["v"]) } -func (c *current) onNode301() (any, error) { - return buildOperatorNode(c.text, c.pos) +func (c *current) onNode279(k, v any) (any, error) { + return buildStringNode(k, v, true, c.text, c.pos) } -func (p *parser) callonNode301() (any, error) { +func (p *parser) callonNode279() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode301() + return p.cur.onNode279(stack["k"], stack["v"]) } -func (c *current) onNode281(v any) (any, error) { - return buildStringNode("", v, c.text, c.pos) +func (c *current) onNode312() (any, error) { + return c.text, nil } -func (p *parser) callonNode281() (any, error) { +func (p *parser) callonNode312() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode281(stack["v"]) -} - -func (c *current) onNode306() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode306() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode306() -} - -func (c *current) onNode308() (any, error) { - return nil, nil - -} - -func (p *parser) callonNode308() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode308() -} - -func (c *current) onNode314(v any) (any, error) { - return nil, nil - -} - -func (p *parser) callonNode314() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode314(stack["v"]) + return p.cur.onNode312() } func (c *current) onNode318() (any, error) { - return buildOperatorNode(c.text, c.pos) + return c.text, nil } @@ -2920,15 +3178,202 @@ func (p *parser) callonNode318() (any, error) { return p.cur.onNode318() } -func (c *current) onNode303(v any) (any, error) { - return buildStringNode("", v, c.text, c.pos) +func (c *current) onNode309() (any, error) { + return c.text, nil } -func (p *parser) callonNode303() (any, error) { +func (p *parser) callonNode309() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode303(stack["v"]) + return p.cur.onNode309() +} + +func (c *current) onNode320(k any) (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode320() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode320(stack["k"]) +} + +func (c *current) onNode324(v any) (any, error) { + return v, nil + +} + +func (p *parser) callonNode324() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode324(stack["v"]) +} + +func (c *current) onNode306(k, v any) (any, error) { + return buildStringNode(k, v, false, c.text, c.pos) + +} + +func (p *parser) callonNode306() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode306(stack["k"], stack["v"]) +} + +func (c *current) onNode333() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode333() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode333() +} + +func (c *current) onNode337() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode337() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode337() +} + +func (c *current) onNode341() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode341() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode341() +} + +func (c *current) onNode346() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode346() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode346() +} + +func (c *current) onNode348() (any, error) { + return nil, nil + +} + +func (p *parser) callonNode348() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode348() +} + +func (c *current) onNode352(v any) (any, error) { + return v, nil + +} + +func (p *parser) callonNode352() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode352(stack["v"]) +} + +func (c *current) onNode359(v any) (any, error) { + return nil, nil + +} + +func (p *parser) callonNode359() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode359(stack["v"]) +} + +func (c *current) onNode363() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode363() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode363() +} + +func (c *current) onNode343(v any) (any, error) { + return buildStringNode("", v, false, c.text, c.pos) + +} + +func (p *parser) callonNode343() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode343(stack["v"]) +} + +func (c *current) onNode368() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode368() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode368() +} + +func (c *current) onNode370() (any, error) { + return nil, nil + +} + +func (p *parser) callonNode370() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode370() +} + +func (c *current) onNode376(v any) (any, error) { + return nil, nil + +} + +func (p *parser) callonNode376() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode376(stack["v"]) +} + +func (c *current) onNode380() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode380() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode380() +} + +func (c *current) onNode365(v any) (any, error) { + return buildStringNode("", v, false, c.text, c.pos) + +} + +func (p *parser) callonNode365() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode365(stack["v"]) } func (c *current) onGroupNode8() (any, error) { diff --git a/pkg/kql/dictionary_test.go b/pkg/kql/dictionary_test.go index 81ace7b811..25f6e2f902 100644 --- a/pkg/kql/dictionary_test.go +++ b/pkg/kql/dictionary_test.go @@ -145,6 +145,22 @@ func TestParse_Spec(t *testing.T) { }, }, }, + { + name: `author="John Smith"`, + ast: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "author", Value: "John Smith", Exact: true}, + }, + }, + }, + { + name: `filename=budget.xlsx`, + ast: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "filename", Value: "budget.xlsx", Exact: true}, + }, + }, + }, // 3.2.3 Implicit Operator for Property Restriction { name: `author:"John Smith" filetype:docx`, @@ -423,6 +439,60 @@ func TestParse_Spec(t *testing.T) { } } +func TestParse_NumberRestrictionNode(t *testing.T) { + tests := []testCase{ + { + name: "format", + query: join([]string{ + `size>100`, + `size>"100"`, + `size>=15.5`, + `size<100`, + `size<=100`, + }), + ast: &ast.Ast{ + Nodes: []ast.Node{ + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: ">"}, + Value: 100, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: ">"}, + Value: 100, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: ">="}, + Value: 15.5, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: "<"}, + Value: 100, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: "<="}, + Value: 100, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + testKQL(t, tc) + }) + } +} + func TestParse_DateTimeRestrictionNode(t *testing.T) { tests := []testCase{ { diff --git a/pkg/kql/engine_suite_test.go b/pkg/kql/engine_suite_test.go deleted file mode 100644 index c4ad289b8b..0000000000 --- a/pkg/kql/engine_suite_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package kql - -import ( - "time" -) - -// PatchTimeNow is here to patch the package time now func, -// which is used in the test suite -func PatchTimeNow(t func() time.Time) { - timeNow = t -} diff --git a/pkg/kql/factory.go b/pkg/kql/factory.go index 2ac49113c4..5f55802d38 100644 --- a/pkg/kql/factory.go +++ b/pkg/kql/factory.go @@ -50,7 +50,7 @@ func buildAST(n any, text []byte, pos position) (*ast.Ast, error) { return a, nil } -func buildStringNode(k, v any, text []byte, pos position) (*ast.StringNode, error) { +func buildStringNode(k, v any, exact bool, text []byte, pos position) (*ast.StringNode, error) { b, err := base(text, pos) if err != nil { return nil, err @@ -70,6 +70,7 @@ func buildStringNode(k, v any, text []byte, pos position) (*ast.StringNode, erro Base: b, Key: key, Value: value, + Exact: exact, }, nil } @@ -101,6 +102,36 @@ func buildDateTimeNode(k, o, v any, text []byte, pos position) (*ast.DateTimeNod Value: value, }, nil } + +func buildNumberNode(k, o, v any, text []byte, pos position) (*ast.NumberNode, error) { + b, err := base(text, pos) + if err != nil { + return nil, err + } + + operator, err := toNode[*ast.OperatorNode](o) + if err != nil { + return nil, err + } + + key, err := toString(k) + if err != nil { + return nil, err + } + + value, err := toFloat(v) + if err != nil { + return nil, err + } + + return &ast.NumberNode{ + Base: b, + Key: key, + Operator: operator, + Value: value, + }, nil +} + func buildNaturalLanguageDateTimeNodes(k, v any, text []byte, pos position) ([]ast.Node, error) { b, err := base(text, pos) if err != nil { diff --git a/pkg/kql/kql.go b/pkg/kql/kql.go index 4cba6c7b91..a4132f4650 100644 --- a/pkg/kql/kql.go +++ b/pkg/kql/kql.go @@ -47,3 +47,9 @@ func (b Builder) Build(q string) (*ast.Ast, error) { // timeNow mirrors time.Now by default, the only reason why this exists // is to monkey patch it from the tests. See PatchTimeNow var timeNow = time.Now + +// PatchTimeNow pins the clock the natural language dates resolve against, +// so a test can hold "today" still while it runs +func PatchTimeNow(t func() time.Time) { + timeNow = t +} diff --git a/services/search/MIGRATION.md b/services/search/MIGRATION.md new file mode 100644 index 0000000000..a0cb646e61 --- /dev/null +++ b/services/search/MIGRATION.md @@ -0,0 +1,55 @@ +# Migrating the search index + +A version that changes how resources are indexed works on a new index and +leaves the old one untouched. The service starts normally, but the new index is +empty: search finds nothing until it is filled. The old index stays around +until you remove it. + +## v7.x.x to %%NEXT%% + +### OpenSearch + +The new index is `opencloud-resource-v3`. Fill it in one of two ways: + +- copy the old index, fast and keeps the extracted file contents, or +- index all spaces again, slower since every file is read once more, but drops + documents that no longer have a resource. + +The address below is the one from `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ADDRESSES`, +`opencloud-resource` the name from +`SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME`. + +```shell +# either copy the old index +curl -X POST "https://opensearch.example.com:9200/_reindex?wait_for_completion=false" \ + -H 'Content-Type: application/json' -d ' + {"source":{"index":"opencloud-resource"},"dest":{"index":"opencloud-resource-v3"}}' + +# the answer carries a task id, watch it while it runs +curl "https://opensearch.example.com:9200/_tasks/" + +# or index all spaces again, the service keeps running while it happens +opencloud search index --all-spaces +``` + +Once the new index is filled, remove the old one: + +```shell +curl -X DELETE "https://opensearch.example.com:9200/opencloud-resource" +``` + +### bleve + +The new index is the `bleve-v2` directory next to the old `bleve` one, both in +`$OC_BASE_DATA_PATH/search` by default (`SEARCH_ENGINE_BLEVE_DATA_PATH`). A +bleve index cannot be copied, index all spaces again: + +```shell +opencloud search index --all-spaces +``` + +Once the new index is filled, remove the old one: + +```shell +rm -r "$OC_BASE_DATA_PATH/search/bleve" +``` diff --git a/services/search/README.md b/services/search/README.md index 03e56c687a..c0e9036aac 100644 --- a/services/search/README.md +++ b/services/search/README.md @@ -55,6 +55,12 @@ Additionally, the following optional settings can be set: * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ENABLE_DEBUG_LOGGER=val`: Enable debug logging. * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_INSECURE=val`: Skip TLS certificate verification. +### Bringing an existing index over + +A version that changes how resources are indexed works on a new index and leaves the old one untouched. +The new index starts empty and has to be filled, see [MIGRATION.md](MIGRATION.md). New installations are +not affected. + ## Query language By default, [KQL](https://learn.microsoft.com/en-us/sharepoint/dev/general-development/keyword-query-language-kql-syntax-reference) is used as the query language. diff --git a/services/search/internal/opensearchtest/helper.go b/services/search/internal/opensearchtest/helper.go index 9ac497a2a3..b900b022c4 100644 --- a/services/search/internal/opensearchtest/helper.go +++ b/services/search/internal/opensearchtest/helper.go @@ -5,10 +5,7 @@ import ( "testing" "time" - opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "github.com/stretchr/testify/require" - - "github.com/opencloud-eu/opencloud/pkg/conversions" ) var TimeMustParse = func(t testing.TB, ts string) time.Time { @@ -23,14 +20,3 @@ func JSONMustMarshal(t testing.TB, data any) string { require.NoError(t, err, "failed to marshal data to JSON") return string(jsonData) } - -func SearchHitsMustBeConverted[T any](t testing.TB, hits []opensearchgoAPI.SearchHit) []T { - ts := make([]T, len(hits)) - for i, hit := range hits { - resource, err := conversions.To[T](hit.Source) - require.NoError(t, err) - ts[i] = resource - } - - return ts -} diff --git a/services/search/internal/opensearchtest/os.go b/services/search/internal/opensearchtest/os.go index 8d4ad649ad..16c3c10025 100644 --- a/services/search/internal/opensearchtest/os.go +++ b/services/search/internal/opensearchtest/os.go @@ -61,7 +61,6 @@ func (tc *TestClient) IndicesReset(ctx context.Context, indices []string) error } if len(indicesToDelete) == 0 { - // If no indices to delete, return nil return nil } @@ -138,76 +137,6 @@ func (tc *TestClient) IndicesCreate(ctx context.Context, index string, body io.R } } -func (tc *TestClient) IndicesCount(ctx context.Context, indices []string, body io.Reader) (int, error) { - if err := tc.IndicesRefresh(ctx, indices, []int{404}); err != nil { - return 0, err - } - - resp, err := tc.c.Indices.Count(ctx, &opensearchgoAPI.IndicesCountReq{ - Indices: indices, - Body: body, - }) - - switch { - case err != nil: - return 0, fmt.Errorf("failed to count documents in indices: %w", err) - default: - return resp.Count, nil - } -} - -func (tc *TestClient) DocumentCreate(ctx context.Context, index, id string, body io.Reader) error { - if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil { - return err - } - - _, err := tc.c.Document.Create(ctx, opensearchgoAPI.DocumentCreateReq{ - Index: index, - DocumentID: id, - Body: body, - }) - switch { - case err != nil: - return fmt.Errorf("failed to create document in index %s: %w", index, err) - default: - return nil - } -} - -func (tc *TestClient) Update(ctx context.Context, index, id string, body io.Reader) error { - if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil { - return err - } - - _, err := tc.c.Update(ctx, opensearchgoAPI.UpdateReq{ - Index: index, - DocumentID: id, - Body: body, - }) - switch { - case err != nil: - return fmt.Errorf("failed to update document in index %s: %w", index, err) - default: - return nil - } -} - -func (tc *TestClient) Search(ctx context.Context, index string, body io.Reader) (opensearchgoAPI.SearchHits, error) { - if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil { - return opensearchgoAPI.SearchHits{}, err - } - - resp, err := tc.c.Search(ctx, &opensearchgoAPI.SearchReq{ - Indices: []string{index}, - Body: body, - }) - if err != nil { - return opensearchgoAPI.SearchHits{}, fmt.Errorf("failed to search in index %s: %w", index, err) - } - - return resp.Hits, nil -} - type testRequireClient struct { tc *TestClient t testing.TB @@ -228,29 +157,3 @@ func (trc *testRequireClient) IndicesCreate(index string, body io.Reader) { func (trc *testRequireClient) IndicesDelete(indices []string) { require.NoError(trc.t, trc.tc.IndicesDelete(trc.t.Context(), indices)) } - -func (trc *testRequireClient) IndicesCount(indices []string, body io.Reader, expected int) { - count, err := trc.tc.IndicesCount(trc.t.Context(), indices, body) - - switch { - case expected <= 0: - require.True(trc.t, count <= 0, "expected indices to have no documents, but got a count of %d", count) - default: - require.Equal(trc.t, expected, count, "expected indices to have %d documents, but got %d", expected, count) - require.NoError(trc.t, err, "expected indices to have documents, but got an error") - } -} - -func (trc *testRequireClient) DocumentCreate(index, id string, body io.Reader) { - require.NoError(trc.t, trc.tc.DocumentCreate(trc.t.Context(), index, id, body)) -} - -func (trc *testRequireClient) Update(index, id string, body io.Reader) { - require.NoError(trc.t, trc.tc.Update(trc.t.Context(), index, id, body)) -} - -func (trc *testRequireClient) Search(index string, body io.Reader) opensearchgoAPI.SearchHits { - hits, err := trc.tc.Search(trc.t.Context(), index, body) - require.NoError(trc.t, err) - return hits -} diff --git a/services/search/internal/opensearchtest/suite.go b/services/search/internal/opensearchtest/suite.go index 433907292a..9b6e41e412 100644 --- a/services/search/internal/opensearchtest/suite.go +++ b/services/search/internal/opensearchtest/suite.go @@ -67,7 +67,6 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func keepContainer := os.Getenv("KEEP_TEST_CONTAINER") == "true" if keepContainer { - // the reaper would take the kept container down with the session if err := os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true"); err != nil { return nil, fmt.Errorf("failed to disable the testcontainers reaper: %w", err) } @@ -83,14 +82,6 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func opensearch.WithPassword(cfg.Engine.OpenSearch.Client.Password), testcontainers.WithName(containerName), testcontainers.WithReuseByName(containerName), - // test indexes are tiny; don't let a full host disk trip the flood-stage - // create-index / read-only blocks mid-run - testcontainers.WithEnv(map[string]string{ - "cluster.routing.allocation.disk.threshold_enabled": "false", - }), - // a health probe answers at once on a reused container, a log wait - // would sit out the log timeout on it; a cold boot takes well over the - // previous 5s testcontainers.WithWaitStrategy( wait.ForHTTP("/_cluster/health?wait_for_status=yellow&timeout=1s"). WithPort(openSearchPort). @@ -106,16 +97,13 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func address, err := container.Address(ctx) if err != nil { - _ = container.Terminate(ctx) // attempt to clean up the container + _ = container.Terminate(ctx) return nil, fmt.Errorf("failed to get OpenSearch container address: %w", err) } - // Ensure the address is set in the default configuration. cfg.Engine.OpenSearch.Client.Addresses = []string{address} return func() { - // KEEP_TEST_CONTAINER=true leaves the container up for the next run, - // which picks it up again by name instead of booting a fresh one if keepContainer { _, _ = fmt.Fprintf(os.Stderr, "keeping OpenSearch container %s\n", containerName) return diff --git a/services/search/internal/opensearchtest/testdata.go b/services/search/internal/opensearchtest/testdata.go index ef41c91190..84ceef3a93 100644 --- a/services/search/internal/opensearchtest/testdata.go +++ b/services/search/internal/opensearchtest/testdata.go @@ -16,16 +16,12 @@ var Testdata = struct { Resources resourceTestdata }{ Resources: resourceTestdata{ - Root: fromTestData[search.Resource]("resource_root.json"), - Folder: fromTestData[search.Resource]("resource_folder.json"), - File: fromTestData[search.Resource]("resource_file.json"), + File: fromTestData[search.Resource]("resource_file.json"), }, } type resourceTestdata struct { - Root search.Resource - File search.Resource - Folder search.Resource + File search.Resource } func fromTestData[D any](name string) D { diff --git a/services/search/pkg/bleve/index.go b/services/search/pkg/bleve/index.go index fea6986ae0..0aa61b4c48 100644 --- a/services/search/pkg/bleve/index.go +++ b/services/search/pkg/bleve/index.go @@ -8,8 +8,8 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + regexpCharFilter "github.com/blevesearch/bleve/v2/analysis/char/regexp" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" - "github.com/blevesearch/bleve/v2/analysis/token/porter" "github.com/blevesearch/bleve/v2/analysis/tokenizer/single" "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" "github.com/blevesearch/bleve/v2/mapping" @@ -18,10 +18,19 @@ import ( "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) +const ( + wildcardSuffix = ".wildcard" + indexVersion = "v2" +) + +func indexPath(root string) string { + return filepath.Join(root, "bleve-"+indexVersion) +} + func NewIndex(root string) (bleve.Index, error) { - destination := filepath.Join(root, "bleve") + destination := indexPath(root) index, err := bleve.Open(destination) - if errors.Is(bleve.ErrorIndexPathDoesNotExist, err) { + if errors.Is(err, bleve.ErrorIndexPathDoesNotExist) { indexMapping, err := NewMapping() if err != nil { return nil, err @@ -38,30 +47,61 @@ func NewIndex(root string) (bleve.Index, error) { } func NewMapping() (mapping.IndexMapping, error) { - nameMapping := bleve.NewTextFieldMapping() - nameMapping.Analyzer = "lowercaseKeyword" + words := func() *mapping.FieldMapping { + fm := bleve.NewTextFieldMapping() + fm.Analyzer = "lowercaseWords" + + return fm + } + + whole := func(field string) *mapping.FieldMapping { + fm := bleve.NewTextFieldMapping() + fm.Analyzer = "lowercaseKeyword" + fm.IncludeInAll = false + fm.Name = field + wildcardSuffix + + return fm + } lowercaseMapping := bleve.NewTextFieldMapping() lowercaseMapping.IncludeInAll = false lowercaseMapping.Analyzer = "lowercaseKeyword" - fulltextFieldMapping := bleve.NewTextFieldMapping() - fulltextFieldMapping.Analyzer = "fulltext" - fulltextFieldMapping.IncludeInAll = false + contentMapping := words() + contentMapping.IncludeInAll = false docMapping := bleve.NewDocumentMapping() - docMapping.AddFieldMappingsAt("Name", nameMapping) + docMapping.AddFieldMappingsAt("Name", + words(), + whole("Name"), + ) + docMapping.AddFieldMappingsAt("Title", + words(), + whole("Title"), + ) docMapping.AddFieldMappingsAt("Tags", lowercaseMapping) docMapping.AddFieldMappingsAt("Favorites", lowercaseMapping) - docMapping.AddFieldMappingsAt("Content", fulltextFieldMapping) + docMapping.AddFieldMappingsAt("Content", contentMapping) indexMapping := bleve.NewIndexMapping() indexMapping.DefaultAnalyzer = keyword.Name indexMapping.DefaultMapping = docMapping - err := indexMapping.AddCustomAnalyzer("lowercaseKeyword", + err := indexMapping.AddCustomCharFilter("dotToSpace", map[string]any{ - "type": custom.Name, - "tokenizer": single.Name, + "type": regexpCharFilter.Name, + "regexp": `\.`, + "replace": " ", + }, + ) + if err != nil { + return nil, err + } + + err = indexMapping.AddCustomAnalyzer("lowercaseWords", + map[string]any{ + "type": custom.Name, + "char_filters": []string{"dotToSpace"}, + "tokenizer": unicode.Name, "token_filters": []string{ lowercase.Name, }, @@ -71,13 +111,12 @@ func NewMapping() (mapping.IndexMapping, error) { return nil, err } - err = indexMapping.AddCustomAnalyzer("fulltext", + err = indexMapping.AddCustomAnalyzer("lowercaseKeyword", map[string]any{ "type": custom.Name, - "tokenizer": unicode.Name, + "tokenizer": single.Name, "token_filters": []string{ lowercase.Name, - porter.Name, }, }, ) diff --git a/services/search/pkg/bleve/index_test.go b/services/search/pkg/bleve/index_test.go new file mode 100644 index 0000000000..a62dfc2c28 --- /dev/null +++ b/services/search/pkg/bleve/index_test.go @@ -0,0 +1,39 @@ +package bleve_test + +import ( + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" +) + +var _ = Describe("Index", func() { + Describe("NewIndex", func() { + It("puts the index into a directory of its own generation", func() { + root := GinkgoT().TempDir() + + index, err := bleve.NewIndex(root) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(index.Close) + + Expect(index.Name()).To(Equal(filepath.Join(root, "bleve-v2"))) + Expect(filepath.Join(root, "bleve-v2")).To(BeADirectory()) + }) + + It("opens the index that is already there", func() { + root := GinkgoT().TempDir() + + index, err := bleve.NewIndex(root) + Expect(err).ToNot(HaveOccurred()) + Expect(index.Close()).To(Succeed()) + + reopened, err := bleve.NewIndex(root) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(reopened.Close) + + Expect(reopened.Name()).To(Equal(filepath.Join(root, "bleve-v2"))) + }) + }) +}) diff --git a/services/search/pkg/content/content.go b/services/search/pkg/content/content.go index e185351678..1c5d914231 100644 --- a/services/search/pkg/content/content.go +++ b/services/search/pkg/content/content.go @@ -18,7 +18,7 @@ type Document struct { Name string Content string Size uint64 - Mtime string + Mtime string `json:"Mtime,omitempty"` MimeType string Tags []string Favorites []string diff --git a/services/search/pkg/content/tika.go b/services/search/pkg/content/tika.go index 8b6ac5e60b..2b475efb17 100644 --- a/services/search/pkg/content/tika.go +++ b/services/search/pkg/content/tika.go @@ -11,8 +11,8 @@ import ( gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/google/go-tika/tika" - "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" libregraph "github.com/opencloud-eu/libre-graph-api-go" + "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" "github.com/opencloud-eu/opencloud/pkg/log" "github.com/opencloud-eu/opencloud/services/search/pkg/config" @@ -83,7 +83,11 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document, } for _, meta := range metas { - if title, err := getFirstValue(meta, "title"); err == nil { + title, err := getFirstValue(meta, "dc:title") + if err != nil { + title, err = getFirstValue(meta, "title") + } + if err == nil { doc.Title = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Title, title)) } diff --git a/services/search/pkg/content/tika_test.go b/services/search/pkg/content/tika_test.go index b6ee193528..286ffa8b9c 100644 --- a/services/search/pkg/content/tika_test.go +++ b/services/search/pkg/content/tika_test.go @@ -91,6 +91,28 @@ var _ = Describe("Tika", func() { Expect(doc.Content).To(Equal(body)) }) + It("adds the title", func() { + fullResponse = `[{"dc:title": "quarterly report", "X-TIKA:content": "some data"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Title).To(Equal("quarterly report")) + }) + + It("adds the title of an older tika", func() { + fullResponse = `[{"title": "quarterly report"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Title).To(Equal("quarterly report")) + }) + It("adds audio content", func() { fullResponse = `[ { diff --git a/services/search/pkg/opensearch/backend.go b/services/search/pkg/opensearch/backend.go index 000b24ebf1..6082643a93 100644 --- a/services/search/pkg/opensearch/backend.go +++ b/services/search/pkg/opensearch/backend.go @@ -33,7 +33,10 @@ type Backend struct { client *opensearchgoAPI.Client } -func NewBackend(index string, client *opensearchgoAPI.Client) (*Backend, error) { +// NewBackend creates a backend on the versioned generation of the named index. +func NewBackend(name string, client *opensearchgoAPI.Client) (*Backend, error) { + index := IndexName(name) + pingResp, err := client.Ping(context.TODO(), &opensearchgoAPI.PingReq{}) switch { case err != nil: @@ -172,7 +175,7 @@ func (b *Backend) DocCount() (uint64, error) { &opensearchgoAPI.IndicesCountReq{ Indices: []string{b.index}, }, - osu.NewTermQuery[bool]("Deleted").Value(false), + osu.NewMatchAllQuery(), ) if err != nil { return 0, fmt.Errorf("failed to build count request: %w", err) diff --git a/services/search/pkg/opensearch/backend_test.go b/services/search/pkg/opensearch/backend_test.go index 71ef3860fc..ecf728fe12 100644 --- a/services/search/pkg/opensearch/backend_test.go +++ b/services/search/pkg/opensearch/backend_test.go @@ -16,8 +16,6 @@ func TestOpenSearchBackend(t *testing.T) { RunSpecs(t, "OpenSearch Backend Suite") } -// what the engine does with its index is covered for both engines by -// services/search/pkg/parity; this is the one thing only OpenSearch can do var _ = Describe("Backend", func() { Describe("NewBackend", func() { It("fails to create if the cluster is not healthy", func() { diff --git a/services/search/pkg/opensearch/index.go b/services/search/pkg/opensearch/index.go index 73a1cf2eb9..277c9f5c5c 100644 --- a/services/search/pkg/opensearch/index.go +++ b/services/search/pkg/opensearch/index.go @@ -8,6 +8,7 @@ import ( "fmt" "path" "reflect" + "strings" "github.com/go-jose/go-jose/v3/json" opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" @@ -16,9 +17,8 @@ import ( var ( ErrManualActionRequired = errors.New("manual action required") - IndexManagerLatest = IndexIndexManagerResourceV2 - IndexIndexManagerResourceV1 IndexManager = "resource_v1.json" - IndexIndexManagerResourceV2 IndexManager = "resource_v2.json" + IndexManagerLatest = IndexIndexManagerResourceV3 + IndexIndexManagerResourceV3 IndexManager = "resource_v3.json" ) //go:embed internal/indexes/*.json @@ -26,6 +26,30 @@ var indexes embed.FS type IndexManager string +// Version is the part of the definition file name that says which generation it +// is, resource_v3.json carries v3. +func (m IndexManager) Version() string { + name := strings.TrimSuffix(string(m), path.Ext(string(m))) + _, version, found := strings.Cut(name, "_") + if !found { + return "" + } + + return version +} + +// IndexName puts the generation of the definition behind the configured name, +// so a new one starts on an index of its own instead of refusing to work with +// the one that is there. +func IndexName(name string) string { + version := IndexManagerLatest.Version() + if version == "" { + return name + } + + return name + "-" + version +} + func (m IndexManager) String() string { b, err := m.MarshalJSON() if err != nil { @@ -48,6 +72,43 @@ func (m IndexManager) MarshalJSON() ([]byte, error) { return body, nil } +func coveredAt(declared, index gjson.Result, declaredPath, indexPath string) (string, string, bool) { + declaredRaw := declared.Get(declaredPath).Raw + indexRaw := index.Get(indexPath).Raw + + var declaredValue, indexValue any + if err := json.Unmarshal([]byte(declaredRaw), &declaredValue); err != nil { + return declaredRaw, indexRaw, false + } + + if err := json.Unmarshal([]byte(indexRaw), &indexValue); err != nil { + return declaredRaw, indexRaw, false + } + + return declaredRaw, indexRaw, covered(declaredValue, indexValue) +} + +func covered(declared, index any) bool { + declaredMap, ok := declared.(map[string]any) + if !ok { + return reflect.DeepEqual(declared, index) + } + + indexMap, ok := index.(map[string]any) + if !ok { + return false + } + + for key, declaredValue := range declaredMap { + indexValue, ok := indexMap[key] + if !ok || !covered(declaredValue, indexValue) { + return false + } + } + + return true +} + func (m IndexManager) Apply(ctx context.Context, name string, client *opensearchgoAPI.Client) error { localIndexB, err := m.MarshalJSON() if err != nil { @@ -86,32 +147,16 @@ func (m IndexManager) Apply(ctx context.Context, name string, client *opensearch localIndexJson := gjson.ParseBytes(localIndexB) remoteIndexJson := gjson.ParseBytes(remoteIndexB) - compare := func(lvPath, rvPath string) (any, any, bool) { - lv := localIndexJson.Get(lvPath).Raw - rv := remoteIndexJson.Get(rvPath).Raw - - var lvv, rvv any - if err := json.Unmarshal([]byte(lv), &lvv); err != nil { - return nil, nil, false - } - - if err := json.Unmarshal([]byte(rv), &rvv); err != nil { - return nil, nil, false - } - - return lv, rv, reflect.DeepEqual(lvv, rvv) - } - var errs []error for k := range localIndexJson.Get("settings").Map() { - if lv, rv, ok := compare("settings."+k, "settings.index."+k); !ok { + if lv, rv, ok := coveredAt(localIndexJson, remoteIndexJson, "settings."+k, "settings.index."+k); !ok { errs = append(errs, fmt.Errorf("settings.%s local %s, remote %s", k, lv, rv)) } } for k := range localIndexJson.Get("mappings.properties").Map() { - if _, _, ok := compare("mappings.properties."+k, "mappings.properties."+k); !ok { + if _, _, ok := coveredAt(localIndexJson, remoteIndexJson, "mappings.properties."+k, "mappings.properties."+k); !ok { errs = append(errs, fmt.Errorf("mappings.properties.%s", k)) } } diff --git a/services/search/pkg/opensearch/index_test.go b/services/search/pkg/opensearch/index_test.go index 6a813572a2..faba72f674 100644 --- a/services/search/pkg/opensearch/index_test.go +++ b/services/search/pkg/opensearch/index_test.go @@ -47,6 +47,34 @@ func TestIndexManager(t *testing.T) { require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client())) }) + t.Run("accepts an index that carries more than the definition declares", func(t *testing.T) { + indexManager := opensearch.IndexManagerLatest + indexName := "opencloud-test-resource" + + tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + + body, err := sjson.Set(indexManager.String(), "mappings.properties.Path.fields.raw.type", "keyword") + require.NoError(t, err) + tc.Require.IndicesCreate(indexName, strings.NewReader(body)) + + require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client())) + }) + + t.Run("fails when the index misses something the definition declares", func(t *testing.T) { + indexManager := opensearch.IndexManagerLatest + indexName := "opencloud-test-resource" + + tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + + body, err := sjson.Delete(indexManager.String(), "mappings.properties.Path.analyzer") + require.NoError(t, err) + tc.Require.IndicesCreate(indexName, strings.NewReader(body)) + + require.ErrorIs(t, indexManager.Apply(t.Context(), indexName, tc.Client()), opensearch.ErrManualActionRequired) + }) + t.Run("fails to create index if it already exists but is not up to date", func(t *testing.T) { indexManager := opensearch.IndexManagerLatest indexName := "opencloud-test-resource" diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand.go b/services/search/pkg/opensearch/internal/convert/kql_expand.go index 5688f6dfe6..17258512c9 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand.go @@ -4,8 +4,11 @@ import ( "fmt" "reflect" "slices" + "strconv" "strings" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/opencloud-eu/opencloud/pkg/ast" ) @@ -53,6 +56,8 @@ func (e kqlExpander) expand(nodes []ast.Node, defaultKey string) ([]ast.Node, er cnode.Key = e.remapKey(cnode.Key, defaultKey) case *ast.BooleanNode: cnode.Key = e.remapKey(cnode.Key, defaultKey) + case *ast.NumberNode: + cnode.Key = e.remapKey(cnode.Key, defaultKey) } if unfoldedNodes != nil { @@ -73,6 +78,7 @@ func (_ kqlExpander) remapKey(current string, defaultKey string) string { key, ok := map[string]string{ "": defaultKey, // Default case if current is empty + "title": "Title", "rootid": "RootID", "path": "Path", "id": "ID", @@ -85,7 +91,8 @@ func (_ kqlExpander) remapKey(current string, defaultKey string) string { "tags": "Tags", "content": "Content", "hidden": "Hidden", - }[current] + "favorite": "Favorites", + }[strings.ToLower(current)] if !ok { return current // Return the original key if not found } @@ -94,15 +101,21 @@ func (_ kqlExpander) remapKey(current string, defaultKey string) string { } func (_ kqlExpander) lowerValue(key, value string) string { - if slices.Contains([]string{"Hidden"}, key) { - return value // ignore certain keys and return the original value + if slices.Contains([]string{"Name", "Title", "Tags", "Content", "MimeType", "Type", "Hidden"}, key) { + return strings.ToLower(value) } - return strings.ToLower(value) + return value } func (_ kqlExpander) unfoldValue(key, value string) []ast.Node { result, ok := map[string][]ast.Node{ + "Type:file": { + &ast.StringNode{Key: key, Value: strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10)}, + }, + "Type:folder": { + &ast.StringNode{Key: key, Value: strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10)}, + }, "MimeType:file": { &ast.OperatorNode{Value: "NOT"}, &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, @@ -139,8 +152,6 @@ func (_ kqlExpander) unfoldValue(key, value string) []ast.Node { &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"}, - &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"}, }}, }, diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go index 1b3cb0b914..44534d7bcd 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go @@ -142,6 +142,7 @@ func TestExpandKQLAST(t *testing.T) { "tags": "Tags", "content": "Content", "hidden": "Hidden", + "favorite": "Favorites", "any": "any", // Example of an unknown key that should remain unchanged } { tests = append(tests, opensearchtest.TableTest[[]ast.Node, []ast.Node]{ @@ -244,35 +245,72 @@ func TestExpandKQLAST(t *testing.T) { t.Run("lowercases some values", func(t *testing.T) { tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ { - Name: "!Hidden: StringNode -> stringnode", + Name: "Name: StringNode -> stringnode", Got: []ast.Node{ - ast.StringNode{Key: "aBc", Value: "StringNode"}, + ast.StringNode{Key: "Name", Value: "StringNode"}, ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "aBc", Value: "StringNode"}, + ast.StringNode{Key: "Name", Value: "StringNode"}, }}, }, Want: []ast.Node{ - &ast.StringNode{Key: "aBc", Value: "stringnode"}, + &ast.StringNode{Key: "Name", Value: "stringnode"}, &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "aBc", Value: "stringnode"}, + &ast.StringNode{Key: "Name", Value: "stringnode"}, }}, }, }, { - Name: "Hidden: StringNode -> StringNode", + Name: "aBc: StringNode -> StringNode", Got: []ast.Node{ - ast.StringNode{Key: "Hidden", Value: "StringNode"}, + ast.StringNode{Key: "aBc", Value: "StringNode"}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "aBc", Value: "StringNode"}, + }, + }, + { + Name: "Path: ./Documents -> ./Documents", + Got: []ast.Node{ + ast.StringNode{Key: "Path", Value: "./Documents"}, ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "Hidden", Value: "StringNode"}, + ast.StringNode{Key: "Path", Value: "./Documents"}, }}, }, Want: []ast.Node{ - &ast.StringNode{Key: "Hidden", Value: "StringNode"}, + &ast.StringNode{Key: "Path", Value: "./Documents"}, &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "Hidden", Value: "StringNode"}, + &ast.StringNode{Key: "Path", Value: "./Documents"}, }}, }, }, + { + Name: "Hidden: TRUE -> true", + Got: []ast.Node{ + ast.StringNode{Key: "Hidden", Value: "TRUE"}, + ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + ast.StringNode{Key: "Hidden", Value: "TRUE"}, + }}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "true"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "true"}, + }}, + }, + }, + { + Name: "ID: 1$1!AB23 -> 1$1!AB23", + Got: []ast.Node{ + ast.StringNode{Key: "ID", Value: "1$1!AB23"}, + ast.StringNode{Key: "RootID", Value: "1$1!AB23"}, + ast.StringNode{Key: "ParentID", Value: "1$1!AB23"}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "ID", Value: "1$1!AB23"}, + &ast.StringNode{Key: "RootID", Value: "1$1!AB23"}, + &ast.StringNode{Key: "ParentID", Value: "1$1!AB23"}, + }, + }, } for _, test := range tests { @@ -384,8 +422,6 @@ func TestExpandKQLAST(t *testing.T) { &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadshee"}, - &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.numbers"}, }}, &ast.OperatorNode{Value: "AND"}, diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index d94c1ac5fd..e3fb25a807 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "slices" + "strconv" "strings" "time" @@ -101,6 +102,8 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { return stringNodeQuery(node), nil case *ast.DateTimeNode: return dateTimeNodeQuery(node) + case *ast.NumberNode: + return numberNodeQuery(node) case *ast.GroupNode: group, err := t.transpile(node.Nodes) if err != nil { @@ -115,22 +118,35 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { // stringNodeQuery picks the query a string node turns into. func stringNodeQuery(node *ast.StringNode) osu.Builder { - isWildcard := strings.Contains(node.Value, "*") + isWildcard := strings.ContainsAny(node.Value, "*?") switch { // Name: "*oo-bar", "*oo ba*", "*OO*" // Title: "*rterly rep*" // Tags: "*spaced tag*" - case isWildcard && slices.Contains([]string{"Name", "Title", "Tags"}, node.Key): - return osu.NewWildcardQuery(node.Key + ".keyword"). - Value(node.Value). - Params(&osu.WildcardQueryParams{CaseInsensitive: true}) + case isWildcard && slices.Contains([]string{"Name", "Title"}, node.Key): + patterns := []osu.Builder{wildcardOn(node.Key+".wildcard", node.Value)} + if !strings.HasSuffix(node.Value, "*") { + patterns = append(patterns, wildcardOn(node.Key+".wildcard", node.Value+".*")) + } + + return osu.NewBoolQuery(). + Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). + Should(patterns...) + // Tags: "*foo*", "*paced ta*" + case isWildcard && node.Key == "Tags": + return wildcardOn(node.Key+".wildcard", node.Value) // Path: "./foo*", MimeType: "*plain" case isWildcard: return osu.NewWildcardQuery(node.Key).Value(node.Value) + // Name: =new, Title: ="quarterly report" + case node.Exact && slices.Contains([]string{"Name", "Title"}, node.Key): + return osu.NewTermQuery[string](node.Key + ".wildcard"). + Value(node.Value). + Params(&osu.TermQueryParams{CaseInsensitive: true}) // Tags: "foo-bar", "spaced tag", "FOO-BAR" case node.Key == "Tags": - return osu.NewTermQuery[string](node.Key + ".keyword"). + return osu.NewTermQuery[string](node.Key + ".wildcard"). Value(node.Value). Params(&osu.TermQueryParams{CaseInsensitive: true}) // Name: "foo-bar", "foo bar" @@ -138,12 +154,62 @@ func stringNodeQuery(node *ast.StringNode) osu.Builder { // Content: "foo bar" case slices.Contains([]string{"Name", "Title", "Content"}, node.Key): return osu.NewMatchPhraseQuery(node.Key).Query(node.Value) - // Path: "./foo bar", MimeType: "text/plain" + // Size: "42", Type: "1" + case slices.Contains([]string{"Size", "Type"}, node.Key): + number, err := strconv.ParseInt(node.Value, 10, 64) + if err != nil { + return osu.NewMatchNoneQuery() + } + + return osu.NewTermQuery[int64](node.Key).Value(number) + // Path: "./foo bar/", the hierarchy tokens carry no trailing slash + case node.Key == "Path": + return osu.NewTermQuery[string](node.Key).Value(strings.TrimSuffix(node.Value, "/")) + // Hidden: "TRUE" arrives lowered, anything that is no bool matches nothing + case node.Key == "Hidden": + value, err := strconv.ParseBool(node.Value) + if err != nil { + return osu.NewMatchNoneQuery() + } + + return osu.NewTermQuery[bool](node.Key).Value(value) + // MimeType: "text/plain" default: return osu.NewTermQuery[string](node.Key).Value(node.Value) } } +func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) { + if node.Operator == nil { + return nil, fmt.Errorf("number node without operator: %w", ErrUnsupportedNodeType) + } + + if !slices.Contains([]string{"Size", "Type"}, node.Key) { + return osu.NewMatchNoneQuery(), nil + } + + query := osu.NewRangeQuery[float64](node.Key) + + switch node.Operator.Value { + case ">": + return query.Gt(node.Value), nil + case ">=": + return query.Gte(node.Value), nil + case "<": + return query.Lt(node.Value), nil + case "<=": + return query.Lte(node.Value), nil + } + + return nil, fmt.Errorf("unsupported operator %s for number node: %w", node.Operator.Value, ErrUnsupportedNodeType) +} + +func wildcardOn(field, value string) osu.Builder { + return osu.NewWildcardQuery(field). + Value(value). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}) +} + // dateTimeNodeQuery turns a date time node into a range query. func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) { if node.Operator == nil { diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go index 1b627c8133..9e03fd23df 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go @@ -58,12 +58,16 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { &ast.StringNode{Key: "Name", Value: "open*"}, }, }, - Want: osu.NewWildcardQuery("Name.keyword"). - Value("open*"). - Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + Want: osu.NewBoolQuery(). + Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). + Should( + osu.NewWildcardQuery("Name.wildcard"). + Value("open*"). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + ), }, { - Name: "wildcard query - string node without a keyword sub field", + Name: "wildcard query - string node without an unanalyzed sub field", Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "Content", Value: "open*"}, @@ -71,6 +75,62 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewWildcardQuery("Content").Value("open*"), }, + { + Name: "wildcard query - a question mark counts as a wildcard", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "fo?o"}, + }, + }, + Want: osu.NewBoolQuery(). + Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). + Should( + osu.NewWildcardQuery("Name.wildcard"). + Value("fo?o"). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + osu.NewWildcardQuery("Name.wildcard"). + Value("fo?o.*"). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + ), + }, + { + Name: "term query - an equals restriction matches the whole name", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "foo bar.txt", Exact: true}, + }, + }, + Want: osu.NewTermQuery[string]("Name.wildcard"). + Value("foo bar.txt"). + Params(&osu.TermQueryParams{CaseInsensitive: true}), + }, + { + Name: "term query - a path loses its trailing slash", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Path", Value: "./Documents/"}, + }, + }, + Want: osu.NewTermQuery[string]("Path").Value("./Documents"), + }, + { + Name: "term query - a hidden string turns into a bool", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "true"}, + }, + }, + Want: osu.NewTermQuery[bool]("Hidden").Value(true), + }, + { + Name: "match-none query - a hidden string that is no bool", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "banana"}, + }, + }, + Want: osu.NewMatchNoneQuery(), + }, { Name: "bool query", Got: &ast.Ast{ diff --git a/services/search/pkg/opensearch/internal/convert/opensearch_test.go b/services/search/pkg/opensearch/internal/convert/opensearch_test.go index d2fba09c70..8f034e327c 100644 --- a/services/search/pkg/opensearch/internal/convert/opensearch_test.go +++ b/services/search/pkg/opensearch/internal/convert/opensearch_test.go @@ -10,7 +10,7 @@ import ( "github.com/opencloud-eu/opencloud/pkg/conversions" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) diff --git a/services/search/pkg/opensearch/internal/indexes/resource_v3.json b/services/search/pkg/opensearch/internal/indexes/resource_v3.json new file mode 100644 index 0000000000..5180f6ad56 --- /dev/null +++ b/services/search/pkg/opensearch/internal/indexes/resource_v3.json @@ -0,0 +1,122 @@ +{ + "settings": { + "number_of_shards": "1", + "number_of_replicas": "1", + "analysis": { + "analyzer": { + "path_hierarchy": { + "tokenizer": "path_hierarchy", + "type": "custom" + }, + "name_words": { + "type": "custom", + "char_filter": [ + "dot_to_space" + ], + "tokenizer": "standard", + "filter": [ + "lowercase" + ] + } + }, + "tokenizer": { + "path_hierarchy": { + "type": "path_hierarchy" + } + }, + "normalizer": { + "lowercase": { + "type": "custom", + "filter": [ + "lowercase" + ] + } + }, + "char_filter": { + "dot_to_space": { + "type": "pattern_replace", + "pattern": "\\.", + "replacement": " " + } + } + } + }, + "mappings": { + "properties": { + "Content": { + "type": "text", + "analyzer": "name_words", + "term_vector": "with_positions_offsets" + }, + "ID": { + "type": "keyword" + }, + "ParentID": { + "type": "keyword" + }, + "RootID": { + "type": "keyword" + }, + "MimeType": { + "type": "wildcard" + }, + "Path": { + "type": "text", + "analyzer": "path_hierarchy" + }, + "Deleted": { + "type": "boolean" + }, + "Hidden": { + "type": "boolean" + }, + "Favorites": { + "type": "keyword" + }, + "Tags": { + "type": "text", + "fields": { + "wildcard": { + "type": "wildcard", + "normalizer": "lowercase" + } + } + }, + "Name": { + "type": "text", + "analyzer": "name_words", + "fields": { + "wildcard": { + "type": "wildcard", + "normalizer": "lowercase" + } + } + }, + "Title": { + "type": "text", + "analyzer": "name_words", + "fields": { + "wildcard": { + "type": "wildcard", + "normalizer": "lowercase" + } + } + }, + "Mtime": { + "type": "date", + "ignore_malformed": true + } + }, + "dynamic_templates": [ + { + "audio_facets": { + "path_match": "audio.*", + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + } + ] + } +} diff --git a/services/search/pkg/opensearch/internal/osu/query_match_none.go b/services/search/pkg/opensearch/internal/osu/query_match_none.go new file mode 100644 index 0000000000..ef071ac57c --- /dev/null +++ b/services/search/pkg/opensearch/internal/osu/query_match_none.go @@ -0,0 +1,55 @@ +package osu + +import ( + "encoding/json" +) + +// MatchNoneQuery matches no document. +type MatchNoneQuery struct{} + +// NewMatchNoneQuery creates a query that matches no document. +func NewMatchNoneQuery() *MatchNoneQuery { + return &MatchNoneQuery{} +} + +// Map returns the query as a map. +func (q *MatchNoneQuery) Map() (map[string]any, error) { + return map[string]any{ + "match_none": map[string]any{}, + }, nil +} + +// MarshalJSON returns the query as JSON. +func (q *MatchNoneQuery) MarshalJSON() ([]byte, error) { + data, err := q.Map() + if err != nil { + return nil, err + } + + return json.Marshal(data) +} + +// MatchAllQuery matches every document. +type MatchAllQuery struct{} + +// NewMatchAllQuery creates a query that matches every document. +func NewMatchAllQuery() *MatchAllQuery { + return &MatchAllQuery{} +} + +// Map returns the query as a map. +func (q *MatchAllQuery) Map() (map[string]any, error) { + return map[string]any{ + "match_all": map[string]any{}, + }, nil +} + +// MarshalJSON returns the query as JSON. +func (q *MatchAllQuery) MarshalJSON() ([]byte, error) { + data, err := q.Map() + if err != nil { + return nil, err + } + + return json.Marshal(data) +} diff --git a/services/search/pkg/opensearch/internal/osu/query_term_level_range.go b/services/search/pkg/opensearch/internal/osu/query_term_level_range.go index 3e81bb00f1..8a38db2317 100644 --- a/services/search/pkg/opensearch/internal/osu/query_term_level_range.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_range.go @@ -6,7 +6,8 @@ import ( "time" ) -type RangeQuery[T time.Time | string] struct { +// RangeQuery matches documents whose field value lies inside the bounds. +type RangeQuery[T time.Time | string | float64] struct { field string gt T gte T @@ -22,7 +23,8 @@ type RangeQueryParams struct { TimeZone string `json:"time_zone,omitempty"` } -func NewRangeQuery[T time.Time | string](field string) *RangeQuery[T] { +// NewRangeQuery creates a range query for the given field. +func NewRangeQuery[T time.Time | string | float64](field string) *RangeQuery[T] { return &RangeQuery[T]{field: field} } diff --git a/services/search/pkg/opensearch/internal/osu/request_test.go b/services/search/pkg/opensearch/internal/osu/request_test.go index f8e0010258..833536ff2e 100644 --- a/services/search/pkg/opensearch/internal/osu/request_test.go +++ b/services/search/pkg/opensearch/internal/osu/request_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" ) diff --git a/services/search/pkg/opensearch/opensearch_test.go b/services/search/pkg/opensearch/opensearch_test.go index cd5fb6fdbc..690c2d70f3 100644 --- a/services/search/pkg/opensearch/opensearch_test.go +++ b/services/search/pkg/opensearch/opensearch_test.go @@ -6,7 +6,7 @@ import ( "os" "testing" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/config" ) diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index 310397cae6..0a84fabfc4 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -2,23 +2,26 @@ package bleve import ( "fmt" + "slices" + "strconv" "strings" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/blevesearch/bleve/v2" bleveQuery "github.com/blevesearch/bleve/v2/search/query" "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/pkg/kql" ) -// lowercaseFields lists the bleve fields whose index mapping uses a -// lowercasing analyzer. Values bound to these fields are pre-lowercased -// so query-side matching stays consistent with the index. -// Keep in sync with services/search/pkg/bleve/index.go NewMapping. var lowercaseFields = map[string]struct{}{ "Name": {}, + "Title": {}, "Tags": {}, "Favorites": {}, "Content": {}, + "MimeType": {}, + "Hidden": {}, } var _fields = map[string]string{ @@ -33,6 +36,7 @@ var _fields = map[string]string{ "tag": "Tags", "tags": "Tags", "content": "Content", + "title": "Title", "hidden": "Hidden", "favorite": "Favorites", } @@ -53,7 +57,6 @@ var bleveEscaper = strings.NewReplacer( `)`, `\)`, `{`, `\{`, `}`, `\}`, - `{`, `\}`, `[`, `\[`, `]`, `\]`, `^`, `\^`, @@ -106,14 +109,43 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { v = strings.ToLower(v) } + if k == "Type" { + v = resourceType(v) + } + var q bleveQuery.Query var group bool - switch k { - case "MimeType": + switch { + case k == "Hidden": + value, err := strconv.ParseBool(v) + if err != nil { + q = bleveQuery.NewMatchNoneQuery() + break + } + + bq := bleveQuery.NewBoolFieldQuery(value) + bq.SetField(k) + q = bq + case k == "MimeType": q, group = mimeType(k, v) if prev == nil { isGroup = group } + case slices.Contains([]string{"Name", "Title"}, k) && strings.ContainsAny(n.Value, "*?"): + patterns := []bleveQuery.Query{bleveQuery.NewQueryStringQuery(k + ".wildcard:" + v)} + if !strings.HasSuffix(v, "*") { + patterns = append(patterns, bleveQuery.NewQueryStringQuery(k+".wildcard:"+v+".*")) + } + + q = closed(bleveQuery.NewDisjunctionQuery(patterns)) + case n.Exact && !strings.ContainsAny(n.Value, "*?") && slices.Contains([]string{"Name", "Title"}, k): + q = bleveQuery.NewQueryStringQuery(k + ".wildcard:" + v) + case k == "Path" && !strings.ContainsAny(n.Value, "*?"): + q = pathAndBelow(k, n.Value) + case slices.Contains([]string{"Name", "Title", "Content"}, k) && !strings.ContainsAny(n.Value, "*?"): + q = phrase(k, n.Value) + case strings.Contains(n.Value, " ") && !strings.ContainsAny(n.Value, "*?"): + q = phrase(k, n.Value) default: q = bleveQuery.NewQueryStringQuery(k + ":" + v) } @@ -153,13 +185,25 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { continue } + if prev == nil { + prev = q + } else { + next = q + } + case *ast.NumberNode: + q := numberRange(getField(n.Key), n.Operator, n.Value) + if q == nil { + continue + } + if prev == nil { prev = q } else { next = q } case *ast.BooleanNode: - q := bleveQuery.NewQueryStringQuery(getField(n.Key) + fmt.Sprintf(":%v", n.Value)) + q := bleveQuery.NewBoolFieldQuery(n.Value) + q.SetField(getField(n.Key)) if prev == nil { prev = q } else { @@ -216,6 +260,10 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { func nextNode(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { if n, ok := nodes[offset].(*ast.GroupNode); ok { + if n.Key != "" { + n = normalizeGroupingProperty(n) + } + gq, _, err := walk(0, n.Nodes) if err != nil { return nil, 0, err @@ -277,6 +325,57 @@ func mapBinary(operator *ast.OperatorNode, ln, rn bleveQuery.Query, leftIsGroup }) } +func numberRange(field string, operator *ast.OperatorNode, value float64) bleveQuery.Query { + if operator == nil { + return nil + } + + inclusive, exclusive := true, false + + var q *bleveQuery.NumericRangeQuery + switch operator.Value { + case ">": + q = bleveQuery.NewNumericRangeInclusiveQuery(&value, nil, &exclusive, nil) + case ">=": + q = bleveQuery.NewNumericRangeInclusiveQuery(&value, nil, &inclusive, nil) + case "<": + q = bleveQuery.NewNumericRangeInclusiveQuery(nil, &value, nil, &exclusive) + case "<=": + q = bleveQuery.NewNumericRangeInclusiveQuery(nil, &value, nil, &inclusive) + default: + return nil + } + + q.SetField(field) + + return q +} + +func pathAndBelow(field, path string) bleveQuery.Query { + path = strings.TrimSuffix(path, "/") + + self := bleveQuery.NewTermQuery(path) + self.SetField(field) + + below := bleveQuery.NewPrefixQuery(path + "/") + below.SetField(field) + + return closed(bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{self, below})) +} + +func closed(q bleveQuery.Query) bleveQuery.Query { + // a bare disjunction reads as an open OR chain to mapBinary, a later OR + // would merge into it and widen the group + return bleveQuery.NewConjunctionQuery([]bleveQuery.Query{q}) +} + +func phrase(field, value string) bleveQuery.Query { + q := bleveQuery.NewMatchPhraseQuery(value) + q.SetField(field) + + return q +} + func getField(name string) string { if name == "" { return "Name" @@ -296,6 +395,17 @@ func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode { return group } +func resourceType(value string) string { + switch strings.ToLower(value) { + case "file": + return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10) + case "folder": + return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10) + default: + return value + } +} + func mimeType(k, v string) (bleveQuery.Query, bool) { switch v { case "file": @@ -321,7 +431,6 @@ func mimeType(k, v string) (bleveQuery.Query, bool) { "application/vnd.oasis.opendocument.spreadsheet", "text/csv", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "application/vnd.oasis.opendocument.spreadsheet", "application/vnd.apple.numbers", )), true case "presentation": diff --git a/services/search/pkg/query/bleve/compiler_test.go b/services/search/pkg/query/bleve/compiler_test.go index ff94e88726..30c283f694 100644 --- a/services/search/pkg/query/bleve/compiler_test.go +++ b/services/search/pkg/query/bleve/compiler_test.go @@ -1,6 +1,7 @@ package bleve import ( + "strings" "testing" "time" @@ -18,6 +19,29 @@ var timeMustParse = func(t *testing.T, ts string) time.Time { return tp } +func wildcardQuery(field, value string) query.Query { + patterns := []query.Query{query.NewQueryStringQuery(field + ".wildcard:" + value)} + if !strings.HasSuffix(value, "*") { + patterns = append(patterns, query.NewQueryStringQuery(field+".wildcard:"+value+".*")) + } + + return query.NewConjunctionQuery([]query.Query{query.NewDisjunctionQuery(patterns)}) +} + +func phraseQuery(field, value string) query.Query { + q := query.NewMatchPhraseQuery(value) + q.SetField(field) + + return q +} + +func boolFieldQuery(field string, value bool) query.Query { + q := query.NewBoolFieldQuery(value) + q.SetField(field) + + return q +} + func Test_compile(t *testing.T) { tests := []struct { name string @@ -33,7 +57,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:federated`), + phraseQuery("Name", "federated"), }), wantErr: false, }, @@ -45,7 +69,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith`), + phraseQuery("Name", "John Smith"), }), wantErr: false, }, @@ -59,8 +83,8 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith`), - query.NewQueryStringQuery(`Name:jane`), + phraseQuery("Name", "John Smith"), + phraseQuery("Name", "Jane"), }), wantErr: false, }, @@ -91,7 +115,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:moby\ di*`), + wildcardQuery("Name", `moby\ di*`), query.NewConjunctionQuery([]query.Query{ query.NewQueryStringQuery(`Tags:bestseller`), query.NewQueryStringQuery(`Tags:book`), @@ -112,10 +136,10 @@ func Test_compile(t *testing.T) { }, want: query.NewDisjunctionQuery([]query.Query{ query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:a`), - query.NewQueryStringQuery(`Name:b`), + phraseQuery("Name", "a"), + phraseQuery("Name", "b"), }), - query.NewQueryStringQuery(`Name:c`), + phraseQuery("Name", "c"), }), wantErr: false, }, @@ -131,10 +155,10 @@ func Test_compile(t *testing.T) { }, }, want: query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:a`), + phraseQuery("Name", "a"), query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:b`), - query.NewQueryStringQuery(`Name:c`), + phraseQuery("Name", "b"), + phraseQuery("Name", "c"), }), }), wantErr: false, @@ -156,11 +180,11 @@ func Test_compile(t *testing.T) { }, want: query.NewConjunctionQuery([]query.Query{ query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:a`), - query.NewQueryStringQuery(`Name:b`), - query.NewQueryStringQuery(`Name:c`), + phraseQuery("Name", "a"), + phraseQuery("Name", "b"), + phraseQuery("Name", "c"), }), - query.NewQueryStringQuery(`Name:d`), + phraseQuery("Name", "d"), }), wantErr: false, }, @@ -179,7 +203,7 @@ func Test_compile(t *testing.T) { }, want: query.NewConjunctionQuery([]query.Query{ query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:moby\ di*`), + wildcardQuery("Name", `moby\ di*`), query.NewQueryStringQuery(`Tags:bestseller`), }), query.NewQueryStringQuery(`Tags:book`), @@ -204,7 +228,7 @@ func Test_compile(t *testing.T) { }, want: query.NewConjunctionQuery([]query.Query{ query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:moby\ di*`), + wildcardQuery("Name", `moby\ di*`), query.NewQueryStringQuery(`Tags:bestseller`), }), query.NewQueryStringQuery(`Tags:book`), @@ -227,7 +251,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`author:John\ Smith`), + phraseQuery("author", "John Smith"), query.NewQueryStringQuery(`author:Jane`), }), wantErr: false, @@ -249,7 +273,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`author:John\ Smith`), + phraseQuery("author", "John Smith"), query.NewQueryStringQuery(`author:Jane`), query.NewQueryStringQuery(`Tags:bestseller`), }), @@ -293,9 +317,44 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith`), - query.NewQueryStringQuery(`Hidden:T`), - query.NewQueryStringQuery(`Hidden:T`), + phraseQuery("Name", "John Smith"), + boolFieldQuery("Hidden", true), + boolFieldQuery("Hidden", true), + }), + wantErr: false, + }, + { + name: `hidden:banana`, + args: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "hidden", Value: "banana"}, + }, + }, + want: query.NewConjunctionQuery([]query.Query{query.NewMatchNoneQuery()}), + wantErr: false, + }, + { + name: `name="Report.txt"`, + args: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "name", Value: "Report.txt", Exact: true}, + }, + }, + want: query.NewConjunctionQuery([]query.Query{query.NewQueryStringQuery(`Name.wildcard:report.txt`)}), + wantErr: false, + }, + { + name: `type:File`, + args: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "type", Value: "File"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "type", Value: "FOLDER"}, + }, + }, + want: query.NewDisjunctionQuery([]query.Query{ + query.NewQueryStringQuery(`Type:1`), + query.NewQueryStringQuery(`Type:2`), }), wantErr: false, }, @@ -424,7 +483,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/rtf`), query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), }), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), wantErr: false, }, @@ -450,7 +509,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), query.NewConjunctionQuery([]query.Query{ query.NewQueryStringQuery(`MimeType:application/pdf`), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), }), wantErr: false, @@ -480,7 +539,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), query.NewQueryStringQuery(`MimeType:application/pdf`), }), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), wantErr: false, }, @@ -509,7 +568,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/rtf`), query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), }), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), wantErr: false, }, @@ -521,7 +580,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith\ \+\-\=\&\|\>\<\!\(\)\{\}\[\]\^\"\~\:\ `), + phraseQuery("Name", "John Smith +-=&|>quarterly reportsome data" to "q1.html" + And user "Alice" has uploaded file with content "notessome data" to "q2.html" + When user "Alice" searches for 'Title:"quarterly report"' using the WebDAV API + Then the HTTP status code should be "207" + And the search result of user "Alice" should contain only these entries: + | q1.html | diff --git a/vendor/github.com/blevesearch/bleve/v2/analysis/char/regexp/regexp.go b/vendor/github.com/blevesearch/bleve/v2/analysis/char/regexp/regexp.go new file mode 100644 index 0000000000..a94236af9a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/char/regexp/regexp.go @@ -0,0 +1,65 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package regexp + +import ( + "fmt" + "regexp" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "regexp" + +type CharFilter struct { + r *regexp.Regexp + replacement []byte +} + +func New(r *regexp.Regexp, replacement []byte) *CharFilter { + return &CharFilter{ + r: r, + replacement: replacement, + } +} + +func (s *CharFilter) Filter(input []byte) []byte { + return s.r.ReplaceAll(input, s.replacement) +} + +func CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) { + regexpStr, ok := config["regexp"].(string) + if !ok { + return nil, fmt.Errorf("must specify regexp") + } + r, err := regexp.Compile(regexpStr) + if err != nil { + return nil, fmt.Errorf("unable to build regexp char filter: %v", err) + } + replaceBytes := []byte(" ") + replaceStr, ok := config["replace"].(string) + if ok { + replaceBytes = []byte(replaceStr) + } + return New(r, replaceBytes), nil +} + +func init() { + err := registry.RegisterCharFilter(Name, CharFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index f13e2c9497..beaceb675a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -126,6 +126,7 @@ github.com/blevesearch/bleve/v2/analysis/analyzer/custom github.com/blevesearch/bleve/v2/analysis/analyzer/keyword github.com/blevesearch/bleve/v2/analysis/analyzer/standard github.com/blevesearch/bleve/v2/analysis/datetime/flexible +github.com/blevesearch/bleve/v2/analysis/char/regexp github.com/blevesearch/bleve/v2/analysis/datetime/optional github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds