diff --git a/pkg/ast/ast.go b/pkg/ast/ast.go index 9aafe853aa..f1a7e3263e 100644 --- a/pkg/ast/ast.go +++ b/pkg/ast/ast.go @@ -43,6 +43,7 @@ type StringNode struct { *Base Key string Value string + Exact bool } // BooleanNode represents a bool value @@ -60,6 +61,14 @@ type DateTimeNode struct { Value time.Time } +// NumberNode represents a numeric value +type NumberNode struct { + *Base + Key string + Operator *OperatorNode + Value float64 +} + // OperatorNode represents an operator value like // AND, OR, NOT, =, <= ... and so on type OperatorNode struct { diff --git a/pkg/ast/test/test.go b/pkg/ast/test/test.go index d7fb985226..8a39b75022 100644 --- a/pkg/ast/test/test.go +++ b/pkg/ast/test/test.go @@ -21,6 +21,7 @@ func DiffAst(x, y any, opts ...cmp.Option) string { cmpopts.IgnoreFields(ast.GroupNode{}, "Base"), cmpopts.IgnoreFields(ast.BooleanNode{}, "Base"), cmpopts.IgnoreFields(ast.DateTimeNode{}, "Base"), + cmpopts.IgnoreFields(ast.NumberNode{}, "Base"), )..., ) } diff --git a/pkg/kql/cast.go b/pkg/kql/cast.go index c1fb185623..ecd0cac27f 100644 --- a/pkg/kql/cast.go +++ b/pkg/kql/cast.go @@ -2,6 +2,7 @@ package kql import ( "fmt" + "strconv" "time" "github.com/jinzhu/now" @@ -136,3 +137,12 @@ func toTimeRange(in any) (*time.Time, *time.Time, error) { return &from, &to, nil } + +func toFloat(v any) (float64, error) { + value, err := toString(v) + if err != nil { + return 0, err + } + + return strconv.ParseFloat(value, 64) +} diff --git a/pkg/kql/dictionary.peg b/pkg/kql/dictionary.peg index 5f59e78afb..c21f46f4c4 100644 --- a/pkg/kql/dictionary.peg +++ b/pkg/kql/dictionary.peg @@ -40,6 +40,7 @@ GroupNode <- PropertyRestrictionNodes <- YesNoPropertyRestrictionNode / DateTimeRestrictionNode / + NumberRestrictionNode / TextPropertyRestrictionNode YesNoPropertyRestrictionNode <- @@ -69,9 +70,22 @@ DateTimeRestrictionNode <- return buildNaturalLanguageDateTimeNodes(k, v, c.text, c.pos) } +NumberRestrictionNode <- + k:Key o:( + OperatorGreaterOrEqualNode / + OperatorLessOrEqualNode / + OperatorGreaterNode / + OperatorLessNode + ) '"'? v:Number '"'? { + return buildNumberNode(k, o, v, c.text, c.pos) + } + TextPropertyRestrictionNode <- - k:Key (OperatorColonNode / OperatorEqualNode) v:(String / [^ ()]+){ - return buildStringNode(k, v, c.text, c.pos) + k:Key OperatorEqualNode v:(String / [^ ()]+) { + return buildStringNode(k, v, true, c.text, c.pos) + } / + k:Key OperatorColonNode v:(String / [^ ()]+) { + return buildStringNode(k, v, false, c.text, c.pos) } //////////////////////////////////////////////////////// @@ -84,12 +98,12 @@ FreeTextKeywordNodes <- PhraseNode <- OperatorColonNode? _ v:String _ OperatorColonNode? { - return buildStringNode("", v, c.text, c.pos) + return buildStringNode("", v, false, c.text, c.pos) } WordNode <- OperatorColonNode? _ v:[^ :()]+ _ OperatorColonNode? { - return buildStringNode("", v, c.text, c.pos) + return buildStringNode("", v, false, c.text, c.pos) } //////////////////////////////////////////////////////// @@ -229,6 +243,11 @@ String <- return v, nil } +Number <- + [0-9]+ ("." [0-9]+)? { + return string(c.text), nil + } + Digit <- [0-9] { return c.text, nil diff --git a/pkg/kql/dictionary_gen.go b/pkg/kql/dictionary_gen.go index aec6f49d1d..9777ece435 100644 --- a/pkg/kql/dictionary_gen.go +++ b/pkg/kql/dictionary_gen.go @@ -43,12 +43,12 @@ var g = &grammar{ pos: position{line: 19, col: 6, offset: 351}, exprs: []any{ &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, run: (*parser).callonNodes3, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -75,27 +75,27 @@ var g = &grammar{ name: "GroupNode", }, &actionExpr{ - pos: position{line: 46, col: 5, offset: 1038}, + pos: position{line: 47, col: 5, offset: 1066}, run: (*parser).callonNode3, expr: &seqExpr{ - pos: position{line: 46, col: 5, offset: 1038}, + pos: position{line: 47, col: 5, offset: 1066}, exprs: []any{ &labeledExpr{ - pos: position{line: 46, col: 5, offset: 1038}, + pos: position{line: 47, col: 5, offset: 1066}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode6, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode9, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -104,23 +104,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode15, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -136,23 +136,23 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 46, col: 12, offset: 1045}, + pos: position{line: 47, col: 12, offset: 1073}, alternatives: []any{ &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonNode18, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", }, }, &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonNode20, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", @@ -161,19 +161,19 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 46, col: 51, offset: 1084}, + pos: position{line: 47, col: 51, offset: 1112}, label: "v", expr: &choiceExpr{ - pos: position{line: 46, col: 54, offset: 1087}, + pos: position{line: 47, col: 54, offset: 1115}, alternatives: []any{ &litMatcher{ - pos: position{line: 46, col: 54, offset: 1087}, + pos: position{line: 47, col: 54, offset: 1115}, val: "true", ignoreCase: false, want: "\"true\"", }, &litMatcher{ - pos: position{line: 46, col: 63, offset: 1096}, + pos: position{line: 47, col: 63, offset: 1124}, val: "false", ignoreCase: false, want: "\"false\"", @@ -185,27 +185,27 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 51, col: 5, offset: 1197}, + pos: position{line: 52, col: 5, offset: 1225}, run: (*parser).callonNode26, expr: &seqExpr{ - pos: position{line: 51, col: 5, offset: 1197}, + pos: position{line: 52, col: 5, offset: 1225}, exprs: []any{ &labeledExpr{ - pos: position{line: 51, col: 5, offset: 1197}, + pos: position{line: 52, col: 5, offset: 1225}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode29, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode32, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -214,23 +214,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode38, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -246,66 +246,66 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 51, col: 11, offset: 1203}, + pos: position{line: 52, col: 11, offset: 1231}, label: "o", expr: &choiceExpr{ - pos: position{line: 52, col: 9, offset: 1215}, + pos: position{line: 53, col: 9, offset: 1243}, alternatives: []any{ &actionExpr{ - pos: position{line: 145, col: 5, offset: 3346}, + pos: position{line: 159, col: 5, offset: 3745}, run: (*parser).callonNode42, expr: &litMatcher{ - pos: position{line: 145, col: 5, offset: 3346}, + pos: position{line: 159, col: 5, offset: 3745}, val: ">=", ignoreCase: false, want: "\">=\"", }, }, &actionExpr{ - pos: position{line: 135, col: 5, offset: 3162}, + pos: position{line: 149, col: 5, offset: 3561}, run: (*parser).callonNode44, expr: &litMatcher{ - pos: position{line: 135, col: 5, offset: 3162}, + pos: position{line: 149, col: 5, offset: 3561}, val: "<=", ignoreCase: false, want: "\"<=\"", }, }, &actionExpr{ - pos: position{line: 140, col: 5, offset: 3251}, + pos: position{line: 154, col: 5, offset: 3650}, run: (*parser).callonNode46, expr: &litMatcher{ - pos: position{line: 140, col: 5, offset: 3251}, + pos: position{line: 154, col: 5, offset: 3650}, val: ">", ignoreCase: false, want: "\">\"", }, }, &actionExpr{ - pos: position{line: 130, col: 5, offset: 3070}, + pos: position{line: 144, col: 5, offset: 3469}, run: (*parser).callonNode48, expr: &litMatcher{ - pos: position{line: 130, col: 5, offset: 3070}, + pos: position{line: 144, col: 5, offset: 3469}, val: "<", ignoreCase: false, want: "\"<\"", }, }, &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonNode50, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", }, }, &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonNode52, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -315,43 +315,43 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 58, col: 7, offset: 1395}, + pos: position{line: 59, col: 7, offset: 1423}, expr: &litMatcher{ - pos: position{line: 58, col: 7, offset: 1395}, + pos: position{line: 59, col: 7, offset: 1423}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, }, &labeledExpr{ - pos: position{line: 58, col: 12, offset: 1400}, + pos: position{line: 59, col: 12, offset: 1428}, label: "v", expr: &choiceExpr{ - pos: position{line: 59, col: 9, offset: 1412}, + pos: position{line: 60, col: 9, offset: 1440}, alternatives: []any{ &actionExpr{ - pos: position{line: 195, col: 5, offset: 4185}, + pos: position{line: 209, col: 5, offset: 4584}, run: (*parser).callonNode58, expr: &seqExpr{ - pos: position{line: 195, col: 5, offset: 4185}, + pos: position{line: 209, col: 5, offset: 4584}, exprs: []any{ &actionExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, run: (*parser).callonNode60, expr: &seqExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, exprs: []any{ &actionExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, run: (*parser).callonNode62, expr: &seqExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode64, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -359,10 +359,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode66, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -370,10 +370,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode68, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -381,10 +381,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode70, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -395,22 +395,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 14, offset: 3957}, + pos: position{line: 199, col: 14, offset: 4356}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, run: (*parser).callonNode73, expr: &seqExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode75, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -418,10 +418,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode77, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -432,22 +432,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 28, offset: 3971}, + pos: position{line: 199, col: 28, offset: 4370}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, run: (*parser).callonNode80, expr: &seqExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode82, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -455,10 +455,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode84, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -472,28 +472,28 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 195, col: 14, offset: 4194}, + pos: position{line: 209, col: 14, offset: 4593}, val: "T", ignoreCase: false, want: "\"T\"", }, &actionExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, run: (*parser).callonNode87, expr: &seqExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, exprs: []any{ &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode89, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode91, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -501,10 +501,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode93, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -515,22 +515,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 14, offset: 4044}, + pos: position{line: 204, col: 14, offset: 4443}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode96, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode98, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -538,10 +538,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode100, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -552,22 +552,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 29, offset: 4059}, + pos: position{line: 204, col: 29, offset: 4458}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, run: (*parser).callonNode103, expr: &seqExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode105, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -575,10 +575,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode107, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -589,23 +589,23 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 190, col: 44, offset: 4074}, + pos: position{line: 204, col: 44, offset: 4473}, expr: &seqExpr{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, exprs: []any{ &litMatcher{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 190, col: 49, offset: 4079}, + pos: position{line: 204, col: 49, offset: 4478}, expr: &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode113, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -617,35 +617,35 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, alternatives: []any{ &litMatcher{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, val: "Z", ignoreCase: false, want: "\"Z\"", }, &seqExpr{ - pos: position{line: 190, col: 65, offset: 4095}, + pos: position{line: 204, col: 65, offset: 4494}, exprs: []any{ &charClassMatcher{ - pos: position{line: 190, col: 66, offset: 4096}, + pos: position{line: 204, col: 66, offset: 4495}, val: "[+-]", chars: []rune{'+', '-'}, ignoreCase: false, inverted: false, }, &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode119, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode121, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -653,10 +653,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode123, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -667,22 +667,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 86, offset: 4116}, + pos: position{line: 204, col: 86, offset: 4515}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode126, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode128, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -690,10 +690,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode130, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -714,22 +714,22 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, run: (*parser).callonNode132, expr: &seqExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, exprs: []any{ &actionExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, run: (*parser).callonNode134, expr: &seqExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode136, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -737,10 +737,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode138, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -748,10 +748,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode140, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -759,10 +759,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode142, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -773,22 +773,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 14, offset: 3957}, + pos: position{line: 199, col: 14, offset: 4356}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, run: (*parser).callonNode145, expr: &seqExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode147, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -796,10 +796,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode149, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -810,22 +810,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 28, offset: 3971}, + pos: position{line: 199, col: 28, offset: 4370}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, run: (*parser).callonNode152, expr: &seqExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode154, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -833,10 +833,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode156, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -850,22 +850,22 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, run: (*parser).callonNode158, expr: &seqExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, exprs: []any{ &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode160, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode162, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -873,10 +873,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode164, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -887,22 +887,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 14, offset: 4044}, + pos: position{line: 204, col: 14, offset: 4443}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode167, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode169, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -910,10 +910,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode171, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -924,22 +924,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 29, offset: 4059}, + pos: position{line: 204, col: 29, offset: 4458}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, run: (*parser).callonNode174, expr: &seqExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode176, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -947,10 +947,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode178, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -961,23 +961,23 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 190, col: 44, offset: 4074}, + pos: position{line: 204, col: 44, offset: 4473}, expr: &seqExpr{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, exprs: []any{ &litMatcher{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 190, col: 49, offset: 4079}, + pos: position{line: 204, col: 49, offset: 4478}, expr: &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode184, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -989,35 +989,35 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, alternatives: []any{ &litMatcher{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, val: "Z", ignoreCase: false, want: "\"Z\"", }, &seqExpr{ - pos: position{line: 190, col: 65, offset: 4095}, + pos: position{line: 204, col: 65, offset: 4494}, exprs: []any{ &charClassMatcher{ - pos: position{line: 190, col: 66, offset: 4096}, + pos: position{line: 204, col: 66, offset: 4495}, val: "[+-]", chars: []rune{'+', '-'}, ignoreCase: false, inverted: false, }, &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode190, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode192, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1025,10 +1025,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode194, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1039,22 +1039,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 86, offset: 4116}, + pos: position{line: 204, col: 86, offset: 4515}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode197, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode199, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1062,10 +1062,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode201, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1086,9 +1086,9 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 62, col: 7, offset: 1465}, + pos: position{line: 63, col: 7, offset: 1493}, expr: &litMatcher{ - pos: position{line: 62, col: 7, offset: 1465}, + pos: position{line: 63, col: 7, offset: 1493}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1098,27 +1098,27 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 65, col: 5, offset: 1541}, + pos: position{line: 66, col: 5, offset: 1569}, run: (*parser).callonNode205, expr: &seqExpr{ - pos: position{line: 65, col: 5, offset: 1541}, + pos: position{line: 66, col: 5, offset: 1569}, exprs: []any{ &labeledExpr{ - pos: position{line: 65, col: 5, offset: 1541}, + pos: position{line: 66, col: 5, offset: 1569}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode208, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode211, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1127,23 +1127,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode217, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1159,23 +1159,23 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 66, col: 9, offset: 1557}, + pos: position{line: 67, col: 9, offset: 1585}, alternatives: []any{ &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonNode220, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", }, }, &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonNode222, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1184,79 +1184,79 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 68, col: 7, offset: 1609}, + pos: position{line: 69, col: 7, offset: 1637}, expr: &litMatcher{ - pos: position{line: 68, col: 7, offset: 1609}, + pos: position{line: 69, col: 7, offset: 1637}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, }, &labeledExpr{ - pos: position{line: 68, col: 12, offset: 1614}, + pos: position{line: 69, col: 12, offset: 1642}, label: "v", expr: &choiceExpr{ - pos: position{line: 200, col: 5, offset: 4273}, + pos: position{line: 214, col: 5, offset: 4672}, alternatives: []any{ &litMatcher{ - pos: position{line: 200, col: 5, offset: 4273}, + pos: position{line: 214, col: 5, offset: 4672}, val: "today", ignoreCase: false, want: "\"today\"", }, &litMatcher{ - pos: position{line: 201, col: 5, offset: 4287}, + pos: position{line: 215, col: 5, offset: 4686}, val: "yesterday", ignoreCase: false, want: "\"yesterday\"", }, &litMatcher{ - pos: position{line: 202, col: 5, offset: 4305}, + pos: position{line: 216, col: 5, offset: 4704}, val: "this week", ignoreCase: false, want: "\"this week\"", }, &litMatcher{ - pos: position{line: 203, col: 5, offset: 4323}, + pos: position{line: 217, col: 5, offset: 4722}, val: "last week", ignoreCase: false, want: "\"last week\"", }, &litMatcher{ - pos: position{line: 204, col: 5, offset: 4341}, + pos: position{line: 218, col: 5, offset: 4740}, val: "last 7 days", ignoreCase: false, want: "\"last 7 days\"", }, &litMatcher{ - pos: position{line: 205, col: 5, offset: 4361}, + pos: position{line: 219, col: 5, offset: 4760}, val: "this month", ignoreCase: false, want: "\"this month\"", }, &litMatcher{ - pos: position{line: 206, col: 5, offset: 4380}, + pos: position{line: 220, col: 5, offset: 4779}, val: "last month", ignoreCase: false, want: "\"last month\"", }, &litMatcher{ - pos: position{line: 207, col: 5, offset: 4399}, + pos: position{line: 221, col: 5, offset: 4798}, val: "last 30 days", ignoreCase: false, want: "\"last 30 days\"", }, &litMatcher{ - pos: position{line: 208, col: 5, offset: 4420}, + pos: position{line: 222, col: 5, offset: 4819}, val: "this year", ignoreCase: false, want: "\"this year\"", }, &actionExpr{ - pos: position{line: 209, col: 5, offset: 4438}, + pos: position{line: 223, col: 5, offset: 4837}, run: (*parser).callonNode237, expr: &litMatcher{ - pos: position{line: 209, col: 5, offset: 4438}, + pos: position{line: 223, col: 5, offset: 4837}, val: "last year", ignoreCase: false, want: "\"last year\"", @@ -1266,9 +1266,9 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 68, col: 38, offset: 1640}, + pos: position{line: 69, col: 38, offset: 1668}, expr: &litMatcher{ - pos: position{line: 68, col: 38, offset: 1640}, + pos: position{line: 69, col: 38, offset: 1668}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1278,27 +1278,27 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 73, col: 5, offset: 1759}, + pos: position{line: 74, col: 5, offset: 1781}, run: (*parser).callonNode241, expr: &seqExpr{ - pos: position{line: 73, col: 5, offset: 1759}, + pos: position{line: 74, col: 5, offset: 1781}, exprs: []any{ &labeledExpr{ - pos: position{line: 73, col: 5, offset: 1759}, + pos: position{line: 74, col: 5, offset: 1781}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode244, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode247, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1307,23 +1307,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode253, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1338,56 +1338,219 @@ var g = &grammar{ }, }, }, - &choiceExpr{ - pos: position{line: 73, col: 12, offset: 1766}, - alternatives: []any{ - &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode256, - expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, - val: ":", - ignoreCase: false, - want: "\":\"", + &labeledExpr{ + pos: position{line: 74, col: 11, offset: 1787}, + label: "o", + expr: &choiceExpr{ + pos: position{line: 75, col: 9, offset: 1799}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 159, col: 5, offset: 3745}, + run: (*parser).callonNode257, + expr: &litMatcher{ + pos: position{line: 159, col: 5, offset: 3745}, + val: ">=", + ignoreCase: false, + want: "\">=\"", + }, }, - }, - &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, - run: (*parser).callonNode258, - expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, - val: "=", - ignoreCase: false, - want: "\"=\"", + &actionExpr{ + pos: position{line: 149, col: 5, offset: 3561}, + run: (*parser).callonNode259, + expr: &litMatcher{ + pos: position{line: 149, col: 5, offset: 3561}, + val: "<=", + ignoreCase: false, + want: "\"<=\"", + }, + }, + &actionExpr{ + pos: position{line: 154, col: 5, offset: 3650}, + run: (*parser).callonNode261, + expr: &litMatcher{ + pos: position{line: 154, col: 5, offset: 3650}, + val: ">", + ignoreCase: false, + want: "\">\"", + }, + }, + &actionExpr{ + pos: position{line: 144, col: 5, offset: 3469}, + run: (*parser).callonNode263, + expr: &litMatcher{ + pos: position{line: 144, col: 5, offset: 3469}, + val: "<", + ignoreCase: false, + want: "\"<\"", + }, }, }, }, }, + &zeroOrOneExpr{ + pos: position{line: 79, col: 7, offset: 1923}, + expr: &litMatcher{ + pos: position{line: 79, col: 7, offset: 1923}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + }, &labeledExpr{ - pos: position{line: 73, col: 51, offset: 1805}, + pos: position{line: 79, col: 12, offset: 1928}, + label: "v", + expr: &actionExpr{ + pos: position{line: 247, col: 5, offset: 5207}, + run: (*parser).callonNode268, + expr: &seqExpr{ + pos: position{line: 247, col: 5, offset: 5207}, + exprs: []any{ + &oneOrMoreExpr{ + pos: position{line: 247, col: 5, offset: 5207}, + expr: &charClassMatcher{ + pos: position{line: 247, col: 5, offset: 5207}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + &zeroOrOneExpr{ + pos: position{line: 247, col: 12, offset: 5214}, + expr: &seqExpr{ + pos: position{line: 247, col: 13, offset: 5215}, + exprs: []any{ + &litMatcher{ + pos: position{line: 247, col: 13, offset: 5215}, + val: ".", + ignoreCase: false, + want: "\".\"", + }, + &oneOrMoreExpr{ + pos: position{line: 247, col: 17, offset: 5219}, + expr: &charClassMatcher{ + pos: position{line: 247, col: 17, offset: 5219}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + }, + }, + }, + }, + }, + &zeroOrOneExpr{ + pos: position{line: 79, col: 21, offset: 1937}, + expr: &litMatcher{ + pos: position{line: 79, col: 21, offset: 1937}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 84, col: 5, offset: 2041}, + run: (*parser).callonNode279, + expr: &seqExpr{ + pos: position{line: 84, col: 5, offset: 2041}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 84, col: 5, offset: 2041}, + label: "k", + expr: &actionExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + run: (*parser).callonNode282, + expr: &seqExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + exprs: []any{ + &oneOrMoreExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode285, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + &zeroOrMoreExpr{ + pos: position{line: 237, col: 11, offset: 5083}, + expr: &seqExpr{ + pos: position{line: 237, col: 12, offset: 5084}, + exprs: []any{ + &litMatcher{ + pos: position{line: 237, col: 12, offset: 5084}, + val: ".", + ignoreCase: false, + want: "\".\"", + }, + &oneOrMoreExpr{ + pos: position{line: 237, col: 16, offset: 5088}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode291, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 139, col: 5, offset: 3384}, + run: (*parser).callonNode293, + expr: &litMatcher{ + pos: position{line: 139, col: 5, offset: 3384}, + val: "=", + ignoreCase: false, + want: "\"=\"", + }, + }, + &labeledExpr{ + pos: position{line: 84, col: 29, offset: 2065}, label: "v", expr: &choiceExpr{ - pos: position{line: 73, col: 54, offset: 1808}, + pos: position{line: 84, col: 32, offset: 2068}, alternatives: []any{ &actionExpr{ - pos: position{line: 228, col: 5, offset: 4747}, - run: (*parser).callonNode262, + pos: position{line: 242, col: 5, offset: 5146}, + run: (*parser).callonNode297, expr: &seqExpr{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, exprs: []any{ &litMatcher{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, &labeledExpr{ - pos: position{line: 228, col: 9, offset: 4751}, + pos: position{line: 242, col: 9, offset: 5150}, label: "v", expr: &zeroOrMoreExpr{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, expr: &charClassMatcher{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, val: "[^\"]", chars: []rune{'"'}, ignoreCase: false, @@ -1396,7 +1559,7 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 228, col: 17, offset: 4759}, + pos: position{line: 242, col: 17, offset: 5158}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1405,9 +1568,9 @@ var g = &grammar{ }, }, &oneOrMoreExpr{ - pos: position{line: 73, col: 63, offset: 1817}, + pos: position{line: 84, col: 41, offset: 2077}, expr: &charClassMatcher{ - pos: position{line: 73, col: 63, offset: 1817}, + pos: position{line: 84, col: 41, offset: 2077}, val: "[^ ()]", chars: []rune{' ', '(', ')'}, ignoreCase: false, @@ -1421,19 +1584,147 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 105, col: 5, offset: 2609}, - run: (*parser).callonNode271, + pos: position{line: 87, col: 5, offset: 2158}, + run: (*parser).callonNode306, + expr: &seqExpr{ + pos: position{line: 87, col: 5, offset: 2158}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 87, col: 5, offset: 2158}, + label: "k", + expr: &actionExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + run: (*parser).callonNode309, + expr: &seqExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + exprs: []any{ + &oneOrMoreExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode312, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + &zeroOrMoreExpr{ + pos: position{line: 237, col: 11, offset: 5083}, + expr: &seqExpr{ + pos: position{line: 237, col: 12, offset: 5084}, + exprs: []any{ + &litMatcher{ + pos: position{line: 237, col: 12, offset: 5084}, + val: ".", + ignoreCase: false, + want: "\".\"", + }, + &oneOrMoreExpr{ + pos: position{line: 237, col: 16, offset: 5088}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode318, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode320, + expr: &litMatcher{ + pos: position{line: 134, col: 5, offset: 3298}, + val: ":", + ignoreCase: false, + want: "\":\"", + }, + }, + &labeledExpr{ + pos: position{line: 87, col: 29, offset: 2182}, + label: "v", + expr: &choiceExpr{ + pos: position{line: 87, col: 32, offset: 2185}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 242, col: 5, offset: 5146}, + run: (*parser).callonNode324, + expr: &seqExpr{ + pos: position{line: 242, col: 5, offset: 5146}, + exprs: []any{ + &litMatcher{ + pos: position{line: 242, col: 5, offset: 5146}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + &labeledExpr{ + pos: position{line: 242, col: 9, offset: 5150}, + label: "v", + expr: &zeroOrMoreExpr{ + pos: position{line: 242, col: 11, offset: 5152}, + expr: &charClassMatcher{ + pos: position{line: 242, col: 11, offset: 5152}, + val: "[^\"]", + chars: []rune{'"'}, + ignoreCase: false, + inverted: true, + }, + }, + }, + &litMatcher{ + pos: position{line: 242, col: 17, offset: 5158}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + }, + }, + }, + &oneOrMoreExpr{ + pos: position{line: 87, col: 41, offset: 2194}, + expr: &charClassMatcher{ + pos: position{line: 87, col: 41, offset: 2194}, + val: "[^ ()]", + chars: []rune{' ', '(', ')'}, + ignoreCase: false, + inverted: true, + }, + }, + }, + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 119, col: 5, offset: 3008}, + run: (*parser).callonNode333, expr: &choiceExpr{ - pos: position{line: 105, col: 6, offset: 2610}, + pos: position{line: 119, col: 6, offset: 3009}, alternatives: []any{ &litMatcher{ - pos: position{line: 105, col: 6, offset: 2610}, + pos: position{line: 119, col: 6, offset: 3009}, val: "AND", ignoreCase: false, want: "\"AND\"", }, &litMatcher{ - pos: position{line: 105, col: 14, offset: 2618}, + pos: position{line: 119, col: 14, offset: 3017}, val: "+", ignoreCase: false, want: "\"+\"", @@ -1442,19 +1733,19 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 110, col: 5, offset: 2710}, - run: (*parser).callonNode275, + pos: position{line: 124, col: 5, offset: 3109}, + run: (*parser).callonNode337, expr: &choiceExpr{ - pos: position{line: 110, col: 6, offset: 2711}, + pos: position{line: 124, col: 6, offset: 3110}, alternatives: []any{ &litMatcher{ - pos: position{line: 110, col: 6, offset: 2711}, + pos: position{line: 124, col: 6, offset: 3110}, val: "NOT", ignoreCase: false, want: "\"NOT\"", }, &litMatcher{ - pos: position{line: 110, col: 14, offset: 2719}, + pos: position{line: 124, col: 14, offset: 3118}, val: "-", ignoreCase: false, want: "\"-\"", @@ -1463,28 +1754,28 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 115, col: 5, offset: 2810}, - run: (*parser).callonNode279, + pos: position{line: 129, col: 5, offset: 3209}, + run: (*parser).callonNode341, expr: &litMatcher{ - pos: position{line: 115, col: 6, offset: 2811}, + pos: position{line: 129, col: 6, offset: 3210}, val: "OR", ignoreCase: false, want: "\"OR\"", }, }, &actionExpr{ - pos: position{line: 86, col: 6, offset: 2097}, - run: (*parser).callonNode281, + pos: position{line: 100, col: 6, offset: 2482}, + run: (*parser).callonNode343, expr: &seqExpr{ - pos: position{line: 86, col: 6, offset: 2097}, + pos: position{line: 100, col: 6, offset: 2482}, exprs: []any{ &zeroOrOneExpr{ - pos: position{line: 86, col: 6, offset: 2097}, + pos: position{line: 100, col: 6, offset: 2482}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode284, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode346, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1492,12 +1783,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode286, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode348, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1506,27 +1797,27 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 86, col: 27, offset: 2118}, + pos: position{line: 100, col: 27, offset: 2503}, label: "v", expr: &actionExpr{ - pos: position{line: 228, col: 5, offset: 4747}, - run: (*parser).callonNode290, + pos: position{line: 242, col: 5, offset: 5146}, + run: (*parser).callonNode352, expr: &seqExpr{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, exprs: []any{ &litMatcher{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, &labeledExpr{ - pos: position{line: 228, col: 9, offset: 4751}, + pos: position{line: 242, col: 9, offset: 5150}, label: "v", expr: &zeroOrMoreExpr{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, expr: &charClassMatcher{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, val: "[^\"]", chars: []rune{'"'}, ignoreCase: false, @@ -1535,7 +1826,7 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 228, col: 17, offset: 4759}, + pos: position{line: 242, col: 17, offset: 5158}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1545,12 +1836,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode297, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode359, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1559,12 +1850,12 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 86, col: 38, offset: 2129}, + pos: position{line: 100, col: 38, offset: 2514}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode301, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode363, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1575,18 +1866,18 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 91, col: 6, offset: 2227}, - run: (*parser).callonNode303, + pos: position{line: 105, col: 6, offset: 2619}, + run: (*parser).callonNode365, expr: &seqExpr{ - pos: position{line: 91, col: 6, offset: 2227}, + pos: position{line: 105, col: 6, offset: 2619}, exprs: []any{ &zeroOrOneExpr{ - pos: position{line: 91, col: 6, offset: 2227}, + pos: position{line: 105, col: 6, offset: 2619}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode306, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode368, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1594,12 +1885,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode308, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode370, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1608,12 +1899,12 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 91, col: 27, offset: 2248}, + pos: position{line: 105, col: 27, offset: 2640}, label: "v", expr: &oneOrMoreExpr{ - pos: position{line: 91, col: 29, offset: 2250}, + pos: position{line: 105, col: 29, offset: 2642}, expr: &charClassMatcher{ - pos: position{line: 91, col: 29, offset: 2250}, + pos: position{line: 105, col: 29, offset: 2642}, val: "[^ :()]", chars: []rune{' ', ':', '(', ')'}, ignoreCase: false, @@ -1622,12 +1913,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode314, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode376, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1636,12 +1927,12 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 91, col: 40, offset: 2261}, + pos: position{line: 105, col: 40, offset: 2653}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode318, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode380, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1669,18 +1960,18 @@ var g = &grammar{ expr: &zeroOrOneExpr{ pos: position{line: 32, col: 7, offset: 614}, expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonGroupNode5, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonGroupNode8, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1689,23 +1980,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonGroupNode14, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1727,20 +2018,20 @@ var g = &grammar{ pos: position{line: 32, col: 13, offset: 620}, alternatives: []any{ &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonGroupNode18, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", }, }, &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonGroupNode20, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", @@ -2733,118 +3024,118 @@ func (p *parser) callonNode244() (any, error) { return p.cur.onNode244() } -func (c *current) onNode256() (any, error) { +func (c *current) onNode257() (any, error) { return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode256() (any, error) { +func (p *parser) callonNode257() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode256() + return p.cur.onNode257() } -func (c *current) onNode258() (any, error) { +func (c *current) onNode259() (any, error) { return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode258() (any, error) { +func (p *parser) callonNode259() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode258() + return p.cur.onNode259() } -func (c *current) onNode262(v any) (any, error) { - return v, nil +func (c *current) onNode261() (any, error) { + return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode262() (any, error) { +func (p *parser) callonNode261() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode262(stack["v"]) + return p.cur.onNode261() } -func (c *current) onNode241(k, v any) (any, error) { - return buildStringNode(k, v, c.text, c.pos) +func (c *current) onNode263() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode263() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode263() +} + +func (c *current) onNode268() (any, error) { + return string(c.text), nil + +} + +func (p *parser) callonNode268() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode268() +} + +func (c *current) onNode241(k, o, v any) (any, error) { + return buildNumberNode(k, o, v, c.text, c.pos) } func (p *parser) callonNode241() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode241(stack["k"], stack["v"]) + return p.cur.onNode241(stack["k"], stack["o"], stack["v"]) } -func (c *current) onNode271() (any, error) { +func (c *current) onNode285() (any, error) { + return c.text, nil + +} + +func (p *parser) callonNode285() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode285() +} + +func (c *current) onNode291() (any, error) { + return c.text, nil + +} + +func (p *parser) callonNode291() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode291() +} + +func (c *current) onNode282() (any, error) { + return c.text, nil + +} + +func (p *parser) callonNode282() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode282() +} + +func (c *current) onNode293(k any) (any, error) { return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode271() (any, error) { +func (p *parser) callonNode293() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode271() -} - -func (c *current) onNode275() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode275() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode275() -} - -func (c *current) onNode279() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode279() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode279() -} - -func (c *current) onNode284() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode284() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode284() -} - -func (c *current) onNode286() (any, error) { - return nil, nil - -} - -func (p *parser) callonNode286() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode286() -} - -func (c *current) onNode290(v any) (any, error) { - return v, nil - -} - -func (p *parser) callonNode290() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode290(stack["v"]) + return p.cur.onNode293(stack["k"]) } func (c *current) onNode297(v any) (any, error) { - return nil, nil + return v, nil } @@ -2854,63 +3145,30 @@ func (p *parser) callonNode297() (any, error) { return p.cur.onNode297(stack["v"]) } -func (c *current) onNode301() (any, error) { - return buildOperatorNode(c.text, c.pos) +func (c *current) onNode279(k, v any) (any, error) { + return buildStringNode(k, v, true, c.text, c.pos) } -func (p *parser) callonNode301() (any, error) { +func (p *parser) callonNode279() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode301() + return p.cur.onNode279(stack["k"], stack["v"]) } -func (c *current) onNode281(v any) (any, error) { - return buildStringNode("", v, c.text, c.pos) +func (c *current) onNode312() (any, error) { + return c.text, nil } -func (p *parser) callonNode281() (any, error) { +func (p *parser) callonNode312() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode281(stack["v"]) -} - -func (c *current) onNode306() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode306() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode306() -} - -func (c *current) onNode308() (any, error) { - return nil, nil - -} - -func (p *parser) callonNode308() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode308() -} - -func (c *current) onNode314(v any) (any, error) { - return nil, nil - -} - -func (p *parser) callonNode314() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode314(stack["v"]) + return p.cur.onNode312() } func (c *current) onNode318() (any, error) { - return buildOperatorNode(c.text, c.pos) + return c.text, nil } @@ -2920,15 +3178,202 @@ func (p *parser) callonNode318() (any, error) { return p.cur.onNode318() } -func (c *current) onNode303(v any) (any, error) { - return buildStringNode("", v, c.text, c.pos) +func (c *current) onNode309() (any, error) { + return c.text, nil } -func (p *parser) callonNode303() (any, error) { +func (p *parser) callonNode309() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode303(stack["v"]) + return p.cur.onNode309() +} + +func (c *current) onNode320(k any) (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode320() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode320(stack["k"]) +} + +func (c *current) onNode324(v any) (any, error) { + return v, nil + +} + +func (p *parser) callonNode324() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode324(stack["v"]) +} + +func (c *current) onNode306(k, v any) (any, error) { + return buildStringNode(k, v, false, c.text, c.pos) + +} + +func (p *parser) callonNode306() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode306(stack["k"], stack["v"]) +} + +func (c *current) onNode333() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode333() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode333() +} + +func (c *current) onNode337() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode337() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode337() +} + +func (c *current) onNode341() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode341() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode341() +} + +func (c *current) onNode346() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode346() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode346() +} + +func (c *current) onNode348() (any, error) { + return nil, nil + +} + +func (p *parser) callonNode348() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode348() +} + +func (c *current) onNode352(v any) (any, error) { + return v, nil + +} + +func (p *parser) callonNode352() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode352(stack["v"]) +} + +func (c *current) onNode359(v any) (any, error) { + return nil, nil + +} + +func (p *parser) callonNode359() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode359(stack["v"]) +} + +func (c *current) onNode363() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode363() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode363() +} + +func (c *current) onNode343(v any) (any, error) { + return buildStringNode("", v, false, c.text, c.pos) + +} + +func (p *parser) callonNode343() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode343(stack["v"]) +} + +func (c *current) onNode368() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode368() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode368() +} + +func (c *current) onNode370() (any, error) { + return nil, nil + +} + +func (p *parser) callonNode370() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode370() +} + +func (c *current) onNode376(v any) (any, error) { + return nil, nil + +} + +func (p *parser) callonNode376() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode376(stack["v"]) +} + +func (c *current) onNode380() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode380() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode380() +} + +func (c *current) onNode365(v any) (any, error) { + return buildStringNode("", v, false, c.text, c.pos) + +} + +func (p *parser) callonNode365() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode365(stack["v"]) } func (c *current) onGroupNode8() (any, error) { diff --git a/pkg/kql/dictionary_test.go b/pkg/kql/dictionary_test.go index 81ace7b811..25f6e2f902 100644 --- a/pkg/kql/dictionary_test.go +++ b/pkg/kql/dictionary_test.go @@ -145,6 +145,22 @@ func TestParse_Spec(t *testing.T) { }, }, }, + { + name: `author="John Smith"`, + ast: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "author", Value: "John Smith", Exact: true}, + }, + }, + }, + { + name: `filename=budget.xlsx`, + ast: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "filename", Value: "budget.xlsx", Exact: true}, + }, + }, + }, // 3.2.3 Implicit Operator for Property Restriction { name: `author:"John Smith" filetype:docx`, @@ -423,6 +439,60 @@ func TestParse_Spec(t *testing.T) { } } +func TestParse_NumberRestrictionNode(t *testing.T) { + tests := []testCase{ + { + name: "format", + query: join([]string{ + `size>100`, + `size>"100"`, + `size>=15.5`, + `size<100`, + `size<=100`, + }), + ast: &ast.Ast{ + Nodes: []ast.Node{ + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: ">"}, + Value: 100, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: ">"}, + Value: 100, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: ">="}, + Value: 15.5, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: "<"}, + Value: 100, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: "<="}, + Value: 100, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + testKQL(t, tc) + }) + } +} + func TestParse_DateTimeRestrictionNode(t *testing.T) { tests := []testCase{ { diff --git a/pkg/kql/engine_suite_test.go b/pkg/kql/engine_suite_test.go deleted file mode 100644 index c4ad289b8b..0000000000 --- a/pkg/kql/engine_suite_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package kql - -import ( - "time" -) - -// PatchTimeNow is here to patch the package time now func, -// which is used in the test suite -func PatchTimeNow(t func() time.Time) { - timeNow = t -} diff --git a/pkg/kql/factory.go b/pkg/kql/factory.go index 2ac49113c4..5f55802d38 100644 --- a/pkg/kql/factory.go +++ b/pkg/kql/factory.go @@ -50,7 +50,7 @@ func buildAST(n any, text []byte, pos position) (*ast.Ast, error) { return a, nil } -func buildStringNode(k, v any, text []byte, pos position) (*ast.StringNode, error) { +func buildStringNode(k, v any, exact bool, text []byte, pos position) (*ast.StringNode, error) { b, err := base(text, pos) if err != nil { return nil, err @@ -70,6 +70,7 @@ func buildStringNode(k, v any, text []byte, pos position) (*ast.StringNode, erro Base: b, Key: key, Value: value, + Exact: exact, }, nil } @@ -101,6 +102,36 @@ func buildDateTimeNode(k, o, v any, text []byte, pos position) (*ast.DateTimeNod Value: value, }, nil } + +func buildNumberNode(k, o, v any, text []byte, pos position) (*ast.NumberNode, error) { + b, err := base(text, pos) + if err != nil { + return nil, err + } + + operator, err := toNode[*ast.OperatorNode](o) + if err != nil { + return nil, err + } + + key, err := toString(k) + if err != nil { + return nil, err + } + + value, err := toFloat(v) + if err != nil { + return nil, err + } + + return &ast.NumberNode{ + Base: b, + Key: key, + Operator: operator, + Value: value, + }, nil +} + func buildNaturalLanguageDateTimeNodes(k, v any, text []byte, pos position) ([]ast.Node, error) { b, err := base(text, pos) if err != nil { diff --git a/pkg/kql/kql.go b/pkg/kql/kql.go index 4cba6c7b91..a4132f4650 100644 --- a/pkg/kql/kql.go +++ b/pkg/kql/kql.go @@ -47,3 +47,9 @@ func (b Builder) Build(q string) (*ast.Ast, error) { // timeNow mirrors time.Now by default, the only reason why this exists // is to monkey patch it from the tests. See PatchTimeNow var timeNow = time.Now + +// PatchTimeNow pins the clock the natural language dates resolve against, +// so a test can hold "today" still while it runs +func PatchTimeNow(t func() time.Time) { + timeNow = t +} diff --git a/services/search/MIGRATION.md b/services/search/MIGRATION.md new file mode 100644 index 0000000000..a0cb646e61 --- /dev/null +++ b/services/search/MIGRATION.md @@ -0,0 +1,55 @@ +# Migrating the search index + +A version that changes how resources are indexed works on a new index and +leaves the old one untouched. The service starts normally, but the new index is +empty: search finds nothing until it is filled. The old index stays around +until you remove it. + +## v7.x.x to %%NEXT%% + +### OpenSearch + +The new index is `opencloud-resource-v3`. Fill it in one of two ways: + +- copy the old index, fast and keeps the extracted file contents, or +- index all spaces again, slower since every file is read once more, but drops + documents that no longer have a resource. + +The address below is the one from `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ADDRESSES`, +`opencloud-resource` the name from +`SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME`. + +```shell +# either copy the old index +curl -X POST "https://opensearch.example.com:9200/_reindex?wait_for_completion=false" \ + -H 'Content-Type: application/json' -d ' + {"source":{"index":"opencloud-resource"},"dest":{"index":"opencloud-resource-v3"}}' + +# the answer carries a task id, watch it while it runs +curl "https://opensearch.example.com:9200/_tasks/" + +# or index all spaces again, the service keeps running while it happens +opencloud search index --all-spaces +``` + +Once the new index is filled, remove the old one: + +```shell +curl -X DELETE "https://opensearch.example.com:9200/opencloud-resource" +``` + +### bleve + +The new index is the `bleve-v2` directory next to the old `bleve` one, both in +`$OC_BASE_DATA_PATH/search` by default (`SEARCH_ENGINE_BLEVE_DATA_PATH`). A +bleve index cannot be copied, index all spaces again: + +```shell +opencloud search index --all-spaces +``` + +Once the new index is filled, remove the old one: + +```shell +rm -r "$OC_BASE_DATA_PATH/search/bleve" +``` diff --git a/services/search/README.md b/services/search/README.md index 03e56c687a..c0e9036aac 100644 --- a/services/search/README.md +++ b/services/search/README.md @@ -55,6 +55,12 @@ Additionally, the following optional settings can be set: * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ENABLE_DEBUG_LOGGER=val`: Enable debug logging. * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_INSECURE=val`: Skip TLS certificate verification. +### Bringing an existing index over + +A version that changes how resources are indexed works on a new index and leaves the old one untouched. +The new index starts empty and has to be filled, see [MIGRATION.md](MIGRATION.md). New installations are +not affected. + ## Query language By default, [KQL](https://learn.microsoft.com/en-us/sharepoint/dev/general-development/keyword-query-language-kql-syntax-reference) is used as the query language. diff --git a/services/search/internal/opensearchtest/helper.go b/services/search/internal/opensearchtest/helper.go index 9ac497a2a3..b900b022c4 100644 --- a/services/search/internal/opensearchtest/helper.go +++ b/services/search/internal/opensearchtest/helper.go @@ -5,10 +5,7 @@ import ( "testing" "time" - opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "github.com/stretchr/testify/require" - - "github.com/opencloud-eu/opencloud/pkg/conversions" ) var TimeMustParse = func(t testing.TB, ts string) time.Time { @@ -23,14 +20,3 @@ func JSONMustMarshal(t testing.TB, data any) string { require.NoError(t, err, "failed to marshal data to JSON") return string(jsonData) } - -func SearchHitsMustBeConverted[T any](t testing.TB, hits []opensearchgoAPI.SearchHit) []T { - ts := make([]T, len(hits)) - for i, hit := range hits { - resource, err := conversions.To[T](hit.Source) - require.NoError(t, err) - ts[i] = resource - } - - return ts -} diff --git a/services/search/internal/opensearchtest/os.go b/services/search/internal/opensearchtest/os.go index 8d4ad649ad..16c3c10025 100644 --- a/services/search/internal/opensearchtest/os.go +++ b/services/search/internal/opensearchtest/os.go @@ -61,7 +61,6 @@ func (tc *TestClient) IndicesReset(ctx context.Context, indices []string) error } if len(indicesToDelete) == 0 { - // If no indices to delete, return nil return nil } @@ -138,76 +137,6 @@ func (tc *TestClient) IndicesCreate(ctx context.Context, index string, body io.R } } -func (tc *TestClient) IndicesCount(ctx context.Context, indices []string, body io.Reader) (int, error) { - if err := tc.IndicesRefresh(ctx, indices, []int{404}); err != nil { - return 0, err - } - - resp, err := tc.c.Indices.Count(ctx, &opensearchgoAPI.IndicesCountReq{ - Indices: indices, - Body: body, - }) - - switch { - case err != nil: - return 0, fmt.Errorf("failed to count documents in indices: %w", err) - default: - return resp.Count, nil - } -} - -func (tc *TestClient) DocumentCreate(ctx context.Context, index, id string, body io.Reader) error { - if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil { - return err - } - - _, err := tc.c.Document.Create(ctx, opensearchgoAPI.DocumentCreateReq{ - Index: index, - DocumentID: id, - Body: body, - }) - switch { - case err != nil: - return fmt.Errorf("failed to create document in index %s: %w", index, err) - default: - return nil - } -} - -func (tc *TestClient) Update(ctx context.Context, index, id string, body io.Reader) error { - if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil { - return err - } - - _, err := tc.c.Update(ctx, opensearchgoAPI.UpdateReq{ - Index: index, - DocumentID: id, - Body: body, - }) - switch { - case err != nil: - return fmt.Errorf("failed to update document in index %s: %w", index, err) - default: - return nil - } -} - -func (tc *TestClient) Search(ctx context.Context, index string, body io.Reader) (opensearchgoAPI.SearchHits, error) { - if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil { - return opensearchgoAPI.SearchHits{}, err - } - - resp, err := tc.c.Search(ctx, &opensearchgoAPI.SearchReq{ - Indices: []string{index}, - Body: body, - }) - if err != nil { - return opensearchgoAPI.SearchHits{}, fmt.Errorf("failed to search in index %s: %w", index, err) - } - - return resp.Hits, nil -} - type testRequireClient struct { tc *TestClient t testing.TB @@ -228,29 +157,3 @@ func (trc *testRequireClient) IndicesCreate(index string, body io.Reader) { func (trc *testRequireClient) IndicesDelete(indices []string) { require.NoError(trc.t, trc.tc.IndicesDelete(trc.t.Context(), indices)) } - -func (trc *testRequireClient) IndicesCount(indices []string, body io.Reader, expected int) { - count, err := trc.tc.IndicesCount(trc.t.Context(), indices, body) - - switch { - case expected <= 0: - require.True(trc.t, count <= 0, "expected indices to have no documents, but got a count of %d", count) - default: - require.Equal(trc.t, expected, count, "expected indices to have %d documents, but got %d", expected, count) - require.NoError(trc.t, err, "expected indices to have documents, but got an error") - } -} - -func (trc *testRequireClient) DocumentCreate(index, id string, body io.Reader) { - require.NoError(trc.t, trc.tc.DocumentCreate(trc.t.Context(), index, id, body)) -} - -func (trc *testRequireClient) Update(index, id string, body io.Reader) { - require.NoError(trc.t, trc.tc.Update(trc.t.Context(), index, id, body)) -} - -func (trc *testRequireClient) Search(index string, body io.Reader) opensearchgoAPI.SearchHits { - hits, err := trc.tc.Search(trc.t.Context(), index, body) - require.NoError(trc.t, err) - return hits -} diff --git a/services/search/internal/opensearchtest/testdata.go b/services/search/internal/opensearchtest/testdata.go index ef41c91190..84ceef3a93 100644 --- a/services/search/internal/opensearchtest/testdata.go +++ b/services/search/internal/opensearchtest/testdata.go @@ -16,16 +16,12 @@ var Testdata = struct { Resources resourceTestdata }{ Resources: resourceTestdata{ - Root: fromTestData[search.Resource]("resource_root.json"), - Folder: fromTestData[search.Resource]("resource_folder.json"), - File: fromTestData[search.Resource]("resource_file.json"), + File: fromTestData[search.Resource]("resource_file.json"), }, } type resourceTestdata struct { - Root search.Resource - File search.Resource - Folder search.Resource + File search.Resource } func fromTestData[D any](name string) D { diff --git a/services/search/pkg/bleve/index.go b/services/search/pkg/bleve/index.go index fea6986ae0..0aa61b4c48 100644 --- a/services/search/pkg/bleve/index.go +++ b/services/search/pkg/bleve/index.go @@ -8,8 +8,8 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + regexpCharFilter "github.com/blevesearch/bleve/v2/analysis/char/regexp" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" - "github.com/blevesearch/bleve/v2/analysis/token/porter" "github.com/blevesearch/bleve/v2/analysis/tokenizer/single" "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" "github.com/blevesearch/bleve/v2/mapping" @@ -18,10 +18,19 @@ import ( "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) +const ( + wildcardSuffix = ".wildcard" + indexVersion = "v2" +) + +func indexPath(root string) string { + return filepath.Join(root, "bleve-"+indexVersion) +} + func NewIndex(root string) (bleve.Index, error) { - destination := filepath.Join(root, "bleve") + destination := indexPath(root) index, err := bleve.Open(destination) - if errors.Is(bleve.ErrorIndexPathDoesNotExist, err) { + if errors.Is(err, bleve.ErrorIndexPathDoesNotExist) { indexMapping, err := NewMapping() if err != nil { return nil, err @@ -38,30 +47,61 @@ func NewIndex(root string) (bleve.Index, error) { } func NewMapping() (mapping.IndexMapping, error) { - nameMapping := bleve.NewTextFieldMapping() - nameMapping.Analyzer = "lowercaseKeyword" + words := func() *mapping.FieldMapping { + fm := bleve.NewTextFieldMapping() + fm.Analyzer = "lowercaseWords" + + return fm + } + + whole := func(field string) *mapping.FieldMapping { + fm := bleve.NewTextFieldMapping() + fm.Analyzer = "lowercaseKeyword" + fm.IncludeInAll = false + fm.Name = field + wildcardSuffix + + return fm + } lowercaseMapping := bleve.NewTextFieldMapping() lowercaseMapping.IncludeInAll = false lowercaseMapping.Analyzer = "lowercaseKeyword" - fulltextFieldMapping := bleve.NewTextFieldMapping() - fulltextFieldMapping.Analyzer = "fulltext" - fulltextFieldMapping.IncludeInAll = false + contentMapping := words() + contentMapping.IncludeInAll = false docMapping := bleve.NewDocumentMapping() - docMapping.AddFieldMappingsAt("Name", nameMapping) + docMapping.AddFieldMappingsAt("Name", + words(), + whole("Name"), + ) + docMapping.AddFieldMappingsAt("Title", + words(), + whole("Title"), + ) docMapping.AddFieldMappingsAt("Tags", lowercaseMapping) docMapping.AddFieldMappingsAt("Favorites", lowercaseMapping) - docMapping.AddFieldMappingsAt("Content", fulltextFieldMapping) + docMapping.AddFieldMappingsAt("Content", contentMapping) indexMapping := bleve.NewIndexMapping() indexMapping.DefaultAnalyzer = keyword.Name indexMapping.DefaultMapping = docMapping - err := indexMapping.AddCustomAnalyzer("lowercaseKeyword", + err := indexMapping.AddCustomCharFilter("dotToSpace", map[string]any{ - "type": custom.Name, - "tokenizer": single.Name, + "type": regexpCharFilter.Name, + "regexp": `\.`, + "replace": " ", + }, + ) + if err != nil { + return nil, err + } + + err = indexMapping.AddCustomAnalyzer("lowercaseWords", + map[string]any{ + "type": custom.Name, + "char_filters": []string{"dotToSpace"}, + "tokenizer": unicode.Name, "token_filters": []string{ lowercase.Name, }, @@ -71,13 +111,12 @@ func NewMapping() (mapping.IndexMapping, error) { return nil, err } - err = indexMapping.AddCustomAnalyzer("fulltext", + err = indexMapping.AddCustomAnalyzer("lowercaseKeyword", map[string]any{ "type": custom.Name, - "tokenizer": unicode.Name, + "tokenizer": single.Name, "token_filters": []string{ lowercase.Name, - porter.Name, }, }, ) diff --git a/services/search/pkg/bleve/index_test.go b/services/search/pkg/bleve/index_test.go new file mode 100644 index 0000000000..a62dfc2c28 --- /dev/null +++ b/services/search/pkg/bleve/index_test.go @@ -0,0 +1,39 @@ +package bleve_test + +import ( + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" +) + +var _ = Describe("Index", func() { + Describe("NewIndex", func() { + It("puts the index into a directory of its own generation", func() { + root := GinkgoT().TempDir() + + index, err := bleve.NewIndex(root) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(index.Close) + + Expect(index.Name()).To(Equal(filepath.Join(root, "bleve-v2"))) + Expect(filepath.Join(root, "bleve-v2")).To(BeADirectory()) + }) + + It("opens the index that is already there", func() { + root := GinkgoT().TempDir() + + index, err := bleve.NewIndex(root) + Expect(err).ToNot(HaveOccurred()) + Expect(index.Close()).To(Succeed()) + + reopened, err := bleve.NewIndex(root) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(reopened.Close) + + Expect(reopened.Name()).To(Equal(filepath.Join(root, "bleve-v2"))) + }) + }) +}) diff --git a/services/search/pkg/content/content.go b/services/search/pkg/content/content.go index e185351678..1c5d914231 100644 --- a/services/search/pkg/content/content.go +++ b/services/search/pkg/content/content.go @@ -18,7 +18,7 @@ type Document struct { Name string Content string Size uint64 - Mtime string + Mtime string `json:"Mtime,omitempty"` MimeType string Tags []string Favorites []string diff --git a/services/search/pkg/content/tika.go b/services/search/pkg/content/tika.go index 8b6ac5e60b..2b475efb17 100644 --- a/services/search/pkg/content/tika.go +++ b/services/search/pkg/content/tika.go @@ -11,8 +11,8 @@ import ( gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/google/go-tika/tika" - "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" libregraph "github.com/opencloud-eu/libre-graph-api-go" + "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" "github.com/opencloud-eu/opencloud/pkg/log" "github.com/opencloud-eu/opencloud/services/search/pkg/config" @@ -83,7 +83,11 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document, } for _, meta := range metas { - if title, err := getFirstValue(meta, "title"); err == nil { + title, err := getFirstValue(meta, "dc:title") + if err != nil { + title, err = getFirstValue(meta, "title") + } + if err == nil { doc.Title = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Title, title)) } diff --git a/services/search/pkg/content/tika_test.go b/services/search/pkg/content/tika_test.go index b6ee193528..286ffa8b9c 100644 --- a/services/search/pkg/content/tika_test.go +++ b/services/search/pkg/content/tika_test.go @@ -91,6 +91,28 @@ var _ = Describe("Tika", func() { Expect(doc.Content).To(Equal(body)) }) + It("adds the title", func() { + fullResponse = `[{"dc:title": "quarterly report", "X-TIKA:content": "some data"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Title).To(Equal("quarterly report")) + }) + + It("adds the title of an older tika", func() { + fullResponse = `[{"title": "quarterly report"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Title).To(Equal("quarterly report")) + }) + It("adds audio content", func() { fullResponse = `[ { diff --git a/services/search/pkg/opensearch/backend.go b/services/search/pkg/opensearch/backend.go index 000b24ebf1..6082643a93 100644 --- a/services/search/pkg/opensearch/backend.go +++ b/services/search/pkg/opensearch/backend.go @@ -33,7 +33,10 @@ type Backend struct { client *opensearchgoAPI.Client } -func NewBackend(index string, client *opensearchgoAPI.Client) (*Backend, error) { +// NewBackend creates a backend on the versioned generation of the named index. +func NewBackend(name string, client *opensearchgoAPI.Client) (*Backend, error) { + index := IndexName(name) + pingResp, err := client.Ping(context.TODO(), &opensearchgoAPI.PingReq{}) switch { case err != nil: @@ -172,7 +175,7 @@ func (b *Backend) DocCount() (uint64, error) { &opensearchgoAPI.IndicesCountReq{ Indices: []string{b.index}, }, - osu.NewTermQuery[bool]("Deleted").Value(false), + osu.NewMatchAllQuery(), ) if err != nil { return 0, fmt.Errorf("failed to build count request: %w", err) diff --git a/services/search/pkg/opensearch/backend_test.go b/services/search/pkg/opensearch/backend_test.go index 71ef3860fc..ecf728fe12 100644 --- a/services/search/pkg/opensearch/backend_test.go +++ b/services/search/pkg/opensearch/backend_test.go @@ -16,8 +16,6 @@ func TestOpenSearchBackend(t *testing.T) { RunSpecs(t, "OpenSearch Backend Suite") } -// what the engine does with its index is covered for both engines by -// services/search/pkg/parity; this is the one thing only OpenSearch can do var _ = Describe("Backend", func() { Describe("NewBackend", func() { It("fails to create if the cluster is not healthy", func() { diff --git a/services/search/pkg/opensearch/index.go b/services/search/pkg/opensearch/index.go index 73a1cf2eb9..277c9f5c5c 100644 --- a/services/search/pkg/opensearch/index.go +++ b/services/search/pkg/opensearch/index.go @@ -8,6 +8,7 @@ import ( "fmt" "path" "reflect" + "strings" "github.com/go-jose/go-jose/v3/json" opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" @@ -16,9 +17,8 @@ import ( var ( ErrManualActionRequired = errors.New("manual action required") - IndexManagerLatest = IndexIndexManagerResourceV2 - IndexIndexManagerResourceV1 IndexManager = "resource_v1.json" - IndexIndexManagerResourceV2 IndexManager = "resource_v2.json" + IndexManagerLatest = IndexIndexManagerResourceV3 + IndexIndexManagerResourceV3 IndexManager = "resource_v3.json" ) //go:embed internal/indexes/*.json @@ -26,6 +26,30 @@ var indexes embed.FS type IndexManager string +// Version is the part of the definition file name that says which generation it +// is, resource_v3.json carries v3. +func (m IndexManager) Version() string { + name := strings.TrimSuffix(string(m), path.Ext(string(m))) + _, version, found := strings.Cut(name, "_") + if !found { + return "" + } + + return version +} + +// IndexName puts the generation of the definition behind the configured name, +// so a new one starts on an index of its own instead of refusing to work with +// the one that is there. +func IndexName(name string) string { + version := IndexManagerLatest.Version() + if version == "" { + return name + } + + return name + "-" + version +} + func (m IndexManager) String() string { b, err := m.MarshalJSON() if err != nil { @@ -48,6 +72,43 @@ func (m IndexManager) MarshalJSON() ([]byte, error) { return body, nil } +func coveredAt(declared, index gjson.Result, declaredPath, indexPath string) (string, string, bool) { + declaredRaw := declared.Get(declaredPath).Raw + indexRaw := index.Get(indexPath).Raw + + var declaredValue, indexValue any + if err := json.Unmarshal([]byte(declaredRaw), &declaredValue); err != nil { + return declaredRaw, indexRaw, false + } + + if err := json.Unmarshal([]byte(indexRaw), &indexValue); err != nil { + return declaredRaw, indexRaw, false + } + + return declaredRaw, indexRaw, covered(declaredValue, indexValue) +} + +func covered(declared, index any) bool { + declaredMap, ok := declared.(map[string]any) + if !ok { + return reflect.DeepEqual(declared, index) + } + + indexMap, ok := index.(map[string]any) + if !ok { + return false + } + + for key, declaredValue := range declaredMap { + indexValue, ok := indexMap[key] + if !ok || !covered(declaredValue, indexValue) { + return false + } + } + + return true +} + func (m IndexManager) Apply(ctx context.Context, name string, client *opensearchgoAPI.Client) error { localIndexB, err := m.MarshalJSON() if err != nil { @@ -86,32 +147,16 @@ func (m IndexManager) Apply(ctx context.Context, name string, client *opensearch localIndexJson := gjson.ParseBytes(localIndexB) remoteIndexJson := gjson.ParseBytes(remoteIndexB) - compare := func(lvPath, rvPath string) (any, any, bool) { - lv := localIndexJson.Get(lvPath).Raw - rv := remoteIndexJson.Get(rvPath).Raw - - var lvv, rvv any - if err := json.Unmarshal([]byte(lv), &lvv); err != nil { - return nil, nil, false - } - - if err := json.Unmarshal([]byte(rv), &rvv); err != nil { - return nil, nil, false - } - - return lv, rv, reflect.DeepEqual(lvv, rvv) - } - var errs []error for k := range localIndexJson.Get("settings").Map() { - if lv, rv, ok := compare("settings."+k, "settings.index."+k); !ok { + if lv, rv, ok := coveredAt(localIndexJson, remoteIndexJson, "settings."+k, "settings.index."+k); !ok { errs = append(errs, fmt.Errorf("settings.%s local %s, remote %s", k, lv, rv)) } } for k := range localIndexJson.Get("mappings.properties").Map() { - if _, _, ok := compare("mappings.properties."+k, "mappings.properties."+k); !ok { + if _, _, ok := coveredAt(localIndexJson, remoteIndexJson, "mappings.properties."+k, "mappings.properties."+k); !ok { errs = append(errs, fmt.Errorf("mappings.properties.%s", k)) } } diff --git a/services/search/pkg/opensearch/index_test.go b/services/search/pkg/opensearch/index_test.go index 6a813572a2..faba72f674 100644 --- a/services/search/pkg/opensearch/index_test.go +++ b/services/search/pkg/opensearch/index_test.go @@ -47,6 +47,34 @@ func TestIndexManager(t *testing.T) { require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client())) }) + t.Run("accepts an index that carries more than the definition declares", func(t *testing.T) { + indexManager := opensearch.IndexManagerLatest + indexName := "opencloud-test-resource" + + tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + + body, err := sjson.Set(indexManager.String(), "mappings.properties.Path.fields.raw.type", "keyword") + require.NoError(t, err) + tc.Require.IndicesCreate(indexName, strings.NewReader(body)) + + require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client())) + }) + + t.Run("fails when the index misses something the definition declares", func(t *testing.T) { + indexManager := opensearch.IndexManagerLatest + indexName := "opencloud-test-resource" + + tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + + body, err := sjson.Delete(indexManager.String(), "mappings.properties.Path.analyzer") + require.NoError(t, err) + tc.Require.IndicesCreate(indexName, strings.NewReader(body)) + + require.ErrorIs(t, indexManager.Apply(t.Context(), indexName, tc.Client()), opensearch.ErrManualActionRequired) + }) + t.Run("fails to create index if it already exists but is not up to date", func(t *testing.T) { indexManager := opensearch.IndexManagerLatest indexName := "opencloud-test-resource" diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand.go b/services/search/pkg/opensearch/internal/convert/kql_expand.go index 5688f6dfe6..17258512c9 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand.go @@ -4,8 +4,11 @@ import ( "fmt" "reflect" "slices" + "strconv" "strings" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/opencloud-eu/opencloud/pkg/ast" ) @@ -53,6 +56,8 @@ func (e kqlExpander) expand(nodes []ast.Node, defaultKey string) ([]ast.Node, er cnode.Key = e.remapKey(cnode.Key, defaultKey) case *ast.BooleanNode: cnode.Key = e.remapKey(cnode.Key, defaultKey) + case *ast.NumberNode: + cnode.Key = e.remapKey(cnode.Key, defaultKey) } if unfoldedNodes != nil { @@ -73,6 +78,7 @@ func (_ kqlExpander) remapKey(current string, defaultKey string) string { key, ok := map[string]string{ "": defaultKey, // Default case if current is empty + "title": "Title", "rootid": "RootID", "path": "Path", "id": "ID", @@ -85,7 +91,8 @@ func (_ kqlExpander) remapKey(current string, defaultKey string) string { "tags": "Tags", "content": "Content", "hidden": "Hidden", - }[current] + "favorite": "Favorites", + }[strings.ToLower(current)] if !ok { return current // Return the original key if not found } @@ -94,15 +101,21 @@ func (_ kqlExpander) remapKey(current string, defaultKey string) string { } func (_ kqlExpander) lowerValue(key, value string) string { - if slices.Contains([]string{"Hidden"}, key) { - return value // ignore certain keys and return the original value + if slices.Contains([]string{"Name", "Title", "Tags", "Content", "MimeType", "Type", "Hidden"}, key) { + return strings.ToLower(value) } - return strings.ToLower(value) + return value } func (_ kqlExpander) unfoldValue(key, value string) []ast.Node { result, ok := map[string][]ast.Node{ + "Type:file": { + &ast.StringNode{Key: key, Value: strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10)}, + }, + "Type:folder": { + &ast.StringNode{Key: key, Value: strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10)}, + }, "MimeType:file": { &ast.OperatorNode{Value: "NOT"}, &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, @@ -139,8 +152,6 @@ func (_ kqlExpander) unfoldValue(key, value string) []ast.Node { &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"}, - &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"}, }}, }, diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go index 1b3cb0b914..44534d7bcd 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go @@ -142,6 +142,7 @@ func TestExpandKQLAST(t *testing.T) { "tags": "Tags", "content": "Content", "hidden": "Hidden", + "favorite": "Favorites", "any": "any", // Example of an unknown key that should remain unchanged } { tests = append(tests, opensearchtest.TableTest[[]ast.Node, []ast.Node]{ @@ -244,35 +245,72 @@ func TestExpandKQLAST(t *testing.T) { t.Run("lowercases some values", func(t *testing.T) { tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ { - Name: "!Hidden: StringNode -> stringnode", + Name: "Name: StringNode -> stringnode", Got: []ast.Node{ - ast.StringNode{Key: "aBc", Value: "StringNode"}, + ast.StringNode{Key: "Name", Value: "StringNode"}, ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "aBc", Value: "StringNode"}, + ast.StringNode{Key: "Name", Value: "StringNode"}, }}, }, Want: []ast.Node{ - &ast.StringNode{Key: "aBc", Value: "stringnode"}, + &ast.StringNode{Key: "Name", Value: "stringnode"}, &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "aBc", Value: "stringnode"}, + &ast.StringNode{Key: "Name", Value: "stringnode"}, }}, }, }, { - Name: "Hidden: StringNode -> StringNode", + Name: "aBc: StringNode -> StringNode", Got: []ast.Node{ - ast.StringNode{Key: "Hidden", Value: "StringNode"}, + ast.StringNode{Key: "aBc", Value: "StringNode"}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "aBc", Value: "StringNode"}, + }, + }, + { + Name: "Path: ./Documents -> ./Documents", + Got: []ast.Node{ + ast.StringNode{Key: "Path", Value: "./Documents"}, ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "Hidden", Value: "StringNode"}, + ast.StringNode{Key: "Path", Value: "./Documents"}, }}, }, Want: []ast.Node{ - &ast.StringNode{Key: "Hidden", Value: "StringNode"}, + &ast.StringNode{Key: "Path", Value: "./Documents"}, &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "Hidden", Value: "StringNode"}, + &ast.StringNode{Key: "Path", Value: "./Documents"}, }}, }, }, + { + Name: "Hidden: TRUE -> true", + Got: []ast.Node{ + ast.StringNode{Key: "Hidden", Value: "TRUE"}, + ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + ast.StringNode{Key: "Hidden", Value: "TRUE"}, + }}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "true"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "true"}, + }}, + }, + }, + { + Name: "ID: 1$1!AB23 -> 1$1!AB23", + Got: []ast.Node{ + ast.StringNode{Key: "ID", Value: "1$1!AB23"}, + ast.StringNode{Key: "RootID", Value: "1$1!AB23"}, + ast.StringNode{Key: "ParentID", Value: "1$1!AB23"}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "ID", Value: "1$1!AB23"}, + &ast.StringNode{Key: "RootID", Value: "1$1!AB23"}, + &ast.StringNode{Key: "ParentID", Value: "1$1!AB23"}, + }, + }, } for _, test := range tests { @@ -384,8 +422,6 @@ func TestExpandKQLAST(t *testing.T) { &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadshee"}, - &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.numbers"}, }}, &ast.OperatorNode{Value: "AND"}, diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index d94c1ac5fd..e3fb25a807 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "slices" + "strconv" "strings" "time" @@ -101,6 +102,8 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { return stringNodeQuery(node), nil case *ast.DateTimeNode: return dateTimeNodeQuery(node) + case *ast.NumberNode: + return numberNodeQuery(node) case *ast.GroupNode: group, err := t.transpile(node.Nodes) if err != nil { @@ -115,22 +118,35 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { // stringNodeQuery picks the query a string node turns into. func stringNodeQuery(node *ast.StringNode) osu.Builder { - isWildcard := strings.Contains(node.Value, "*") + isWildcard := strings.ContainsAny(node.Value, "*?") switch { // Name: "*oo-bar", "*oo ba*", "*OO*" // Title: "*rterly rep*" // Tags: "*spaced tag*" - case isWildcard && slices.Contains([]string{"Name", "Title", "Tags"}, node.Key): - return osu.NewWildcardQuery(node.Key + ".keyword"). - Value(node.Value). - Params(&osu.WildcardQueryParams{CaseInsensitive: true}) + case isWildcard && slices.Contains([]string{"Name", "Title"}, node.Key): + patterns := []osu.Builder{wildcardOn(node.Key+".wildcard", node.Value)} + if !strings.HasSuffix(node.Value, "*") { + patterns = append(patterns, wildcardOn(node.Key+".wildcard", node.Value+".*")) + } + + return osu.NewBoolQuery(). + Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). + Should(patterns...) + // Tags: "*foo*", "*paced ta*" + case isWildcard && node.Key == "Tags": + return wildcardOn(node.Key+".wildcard", node.Value) // Path: "./foo*", MimeType: "*plain" case isWildcard: return osu.NewWildcardQuery(node.Key).Value(node.Value) + // Name: =new, Title: ="quarterly report" + case node.Exact && slices.Contains([]string{"Name", "Title"}, node.Key): + return osu.NewTermQuery[string](node.Key + ".wildcard"). + Value(node.Value). + Params(&osu.TermQueryParams{CaseInsensitive: true}) // Tags: "foo-bar", "spaced tag", "FOO-BAR" case node.Key == "Tags": - return osu.NewTermQuery[string](node.Key + ".keyword"). + return osu.NewTermQuery[string](node.Key + ".wildcard"). Value(node.Value). Params(&osu.TermQueryParams{CaseInsensitive: true}) // Name: "foo-bar", "foo bar" @@ -138,12 +154,62 @@ func stringNodeQuery(node *ast.StringNode) osu.Builder { // Content: "foo bar" case slices.Contains([]string{"Name", "Title", "Content"}, node.Key): return osu.NewMatchPhraseQuery(node.Key).Query(node.Value) - // Path: "./foo bar", MimeType: "text/plain" + // Size: "42", Type: "1" + case slices.Contains([]string{"Size", "Type"}, node.Key): + number, err := strconv.ParseInt(node.Value, 10, 64) + if err != nil { + return osu.NewMatchNoneQuery() + } + + return osu.NewTermQuery[int64](node.Key).Value(number) + // Path: "./foo bar/", the hierarchy tokens carry no trailing slash + case node.Key == "Path": + return osu.NewTermQuery[string](node.Key).Value(strings.TrimSuffix(node.Value, "/")) + // Hidden: "TRUE" arrives lowered, anything that is no bool matches nothing + case node.Key == "Hidden": + value, err := strconv.ParseBool(node.Value) + if err != nil { + return osu.NewMatchNoneQuery() + } + + return osu.NewTermQuery[bool](node.Key).Value(value) + // MimeType: "text/plain" default: return osu.NewTermQuery[string](node.Key).Value(node.Value) } } +func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) { + if node.Operator == nil { + return nil, fmt.Errorf("number node without operator: %w", ErrUnsupportedNodeType) + } + + if !slices.Contains([]string{"Size", "Type"}, node.Key) { + return osu.NewMatchNoneQuery(), nil + } + + query := osu.NewRangeQuery[float64](node.Key) + + switch node.Operator.Value { + case ">": + return query.Gt(node.Value), nil + case ">=": + return query.Gte(node.Value), nil + case "<": + return query.Lt(node.Value), nil + case "<=": + return query.Lte(node.Value), nil + } + + return nil, fmt.Errorf("unsupported operator %s for number node: %w", node.Operator.Value, ErrUnsupportedNodeType) +} + +func wildcardOn(field, value string) osu.Builder { + return osu.NewWildcardQuery(field). + Value(value). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}) +} + // dateTimeNodeQuery turns a date time node into a range query. func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) { if node.Operator == nil { diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go index 1b627c8133..9e03fd23df 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go @@ -58,12 +58,16 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { &ast.StringNode{Key: "Name", Value: "open*"}, }, }, - Want: osu.NewWildcardQuery("Name.keyword"). - Value("open*"). - Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + Want: osu.NewBoolQuery(). + Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). + Should( + osu.NewWildcardQuery("Name.wildcard"). + Value("open*"). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + ), }, { - Name: "wildcard query - string node without a keyword sub field", + Name: "wildcard query - string node without an unanalyzed sub field", Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "Content", Value: "open*"}, @@ -71,6 +75,62 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewWildcardQuery("Content").Value("open*"), }, + { + Name: "wildcard query - a question mark counts as a wildcard", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "fo?o"}, + }, + }, + Want: osu.NewBoolQuery(). + Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). + Should( + osu.NewWildcardQuery("Name.wildcard"). + Value("fo?o"). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + osu.NewWildcardQuery("Name.wildcard"). + Value("fo?o.*"). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + ), + }, + { + Name: "term query - an equals restriction matches the whole name", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "foo bar.txt", Exact: true}, + }, + }, + Want: osu.NewTermQuery[string]("Name.wildcard"). + Value("foo bar.txt"). + Params(&osu.TermQueryParams{CaseInsensitive: true}), + }, + { + Name: "term query - a path loses its trailing slash", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Path", Value: "./Documents/"}, + }, + }, + Want: osu.NewTermQuery[string]("Path").Value("./Documents"), + }, + { + Name: "term query - a hidden string turns into a bool", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "true"}, + }, + }, + Want: osu.NewTermQuery[bool]("Hidden").Value(true), + }, + { + Name: "match-none query - a hidden string that is no bool", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "banana"}, + }, + }, + Want: osu.NewMatchNoneQuery(), + }, { Name: "bool query", Got: &ast.Ast{ diff --git a/services/search/pkg/opensearch/internal/convert/opensearch_test.go b/services/search/pkg/opensearch/internal/convert/opensearch_test.go index d2fba09c70..8f034e327c 100644 --- a/services/search/pkg/opensearch/internal/convert/opensearch_test.go +++ b/services/search/pkg/opensearch/internal/convert/opensearch_test.go @@ -10,7 +10,7 @@ import ( "github.com/opencloud-eu/opencloud/pkg/conversions" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) diff --git a/services/search/pkg/opensearch/internal/indexes/resource_v3.json b/services/search/pkg/opensearch/internal/indexes/resource_v3.json new file mode 100644 index 0000000000..5180f6ad56 --- /dev/null +++ b/services/search/pkg/opensearch/internal/indexes/resource_v3.json @@ -0,0 +1,122 @@ +{ + "settings": { + "number_of_shards": "1", + "number_of_replicas": "1", + "analysis": { + "analyzer": { + "path_hierarchy": { + "tokenizer": "path_hierarchy", + "type": "custom" + }, + "name_words": { + "type": "custom", + "char_filter": [ + "dot_to_space" + ], + "tokenizer": "standard", + "filter": [ + "lowercase" + ] + } + }, + "tokenizer": { + "path_hierarchy": { + "type": "path_hierarchy" + } + }, + "normalizer": { + "lowercase": { + "type": "custom", + "filter": [ + "lowercase" + ] + } + }, + "char_filter": { + "dot_to_space": { + "type": "pattern_replace", + "pattern": "\\.", + "replacement": " " + } + } + } + }, + "mappings": { + "properties": { + "Content": { + "type": "text", + "analyzer": "name_words", + "term_vector": "with_positions_offsets" + }, + "ID": { + "type": "keyword" + }, + "ParentID": { + "type": "keyword" + }, + "RootID": { + "type": "keyword" + }, + "MimeType": { + "type": "wildcard" + }, + "Path": { + "type": "text", + "analyzer": "path_hierarchy" + }, + "Deleted": { + "type": "boolean" + }, + "Hidden": { + "type": "boolean" + }, + "Favorites": { + "type": "keyword" + }, + "Tags": { + "type": "text", + "fields": { + "wildcard": { + "type": "wildcard", + "normalizer": "lowercase" + } + } + }, + "Name": { + "type": "text", + "analyzer": "name_words", + "fields": { + "wildcard": { + "type": "wildcard", + "normalizer": "lowercase" + } + } + }, + "Title": { + "type": "text", + "analyzer": "name_words", + "fields": { + "wildcard": { + "type": "wildcard", + "normalizer": "lowercase" + } + } + }, + "Mtime": { + "type": "date", + "ignore_malformed": true + } + }, + "dynamic_templates": [ + { + "audio_facets": { + "path_match": "audio.*", + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + } + ] + } +} diff --git a/services/search/pkg/opensearch/internal/osu/query_match_none.go b/services/search/pkg/opensearch/internal/osu/query_match_none.go new file mode 100644 index 0000000000..ef071ac57c --- /dev/null +++ b/services/search/pkg/opensearch/internal/osu/query_match_none.go @@ -0,0 +1,55 @@ +package osu + +import ( + "encoding/json" +) + +// MatchNoneQuery matches no document. +type MatchNoneQuery struct{} + +// NewMatchNoneQuery creates a query that matches no document. +func NewMatchNoneQuery() *MatchNoneQuery { + return &MatchNoneQuery{} +} + +// Map returns the query as a map. +func (q *MatchNoneQuery) Map() (map[string]any, error) { + return map[string]any{ + "match_none": map[string]any{}, + }, nil +} + +// MarshalJSON returns the query as JSON. +func (q *MatchNoneQuery) MarshalJSON() ([]byte, error) { + data, err := q.Map() + if err != nil { + return nil, err + } + + return json.Marshal(data) +} + +// MatchAllQuery matches every document. +type MatchAllQuery struct{} + +// NewMatchAllQuery creates a query that matches every document. +func NewMatchAllQuery() *MatchAllQuery { + return &MatchAllQuery{} +} + +// Map returns the query as a map. +func (q *MatchAllQuery) Map() (map[string]any, error) { + return map[string]any{ + "match_all": map[string]any{}, + }, nil +} + +// MarshalJSON returns the query as JSON. +func (q *MatchAllQuery) MarshalJSON() ([]byte, error) { + data, err := q.Map() + if err != nil { + return nil, err + } + + return json.Marshal(data) +} diff --git a/services/search/pkg/opensearch/internal/osu/query_term_level_range.go b/services/search/pkg/opensearch/internal/osu/query_term_level_range.go index 3e81bb00f1..8a38db2317 100644 --- a/services/search/pkg/opensearch/internal/osu/query_term_level_range.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_range.go @@ -6,7 +6,8 @@ import ( "time" ) -type RangeQuery[T time.Time | string] struct { +// RangeQuery matches documents whose field value lies inside the bounds. +type RangeQuery[T time.Time | string | float64] struct { field string gt T gte T @@ -22,7 +23,8 @@ type RangeQueryParams struct { TimeZone string `json:"time_zone,omitempty"` } -func NewRangeQuery[T time.Time | string](field string) *RangeQuery[T] { +// NewRangeQuery creates a range query for the given field. +func NewRangeQuery[T time.Time | string | float64](field string) *RangeQuery[T] { return &RangeQuery[T]{field: field} } diff --git a/services/search/pkg/opensearch/internal/osu/request_test.go b/services/search/pkg/opensearch/internal/osu/request_test.go index f8e0010258..833536ff2e 100644 --- a/services/search/pkg/opensearch/internal/osu/request_test.go +++ b/services/search/pkg/opensearch/internal/osu/request_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" ) diff --git a/services/search/pkg/opensearch/opensearch_test.go b/services/search/pkg/opensearch/opensearch_test.go index cd5fb6fdbc..690c2d70f3 100644 --- a/services/search/pkg/opensearch/opensearch_test.go +++ b/services/search/pkg/opensearch/opensearch_test.go @@ -6,7 +6,7 @@ import ( "os" "testing" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/config" ) diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index a8acafba94..5487239b60 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -3,7 +3,9 @@ Written by the parity suite (`go test ./services/search/pkg/parity/`), do not edit. Every case runs against bleve and OpenSearch. `same?` is ✅ when both answer as expected, `❌ known` when an engine's divergence is documented in the case -(`engineOverrides`), `❌` when it is not. +(`engineOverrides`), `❌` when it is not. `✅ stale` when every engine +answers the expected value although the case still documents a +divergence, that override can come out. ## Queries @@ -24,15 +26,15 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| NAME-01 | `new` | new-folder | no match | new-folder | ❌ known | -| NAME-02 | `quarterly` | quarterly notes.txt | no match | quarterly notes.txt | ❌ known | -| NAME-03 | `report` | Report.txt | no match | no match | ❌ known | +| NAME-01 | `new` | new-folder | new-folder | new-folder | ✅ | +| NAME-02 | `quarterly` | quarterly notes.txt | quarterly notes.txt | quarterly notes.txt | ✅ | +| NAME-03 | `report` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-04 | `name:"*new-folder*"` | new-folder | new-folder | new-folder | ✅ | | NAME-05 | `name:"*w-fol*"` | new-folder | new-folder | new-folder | ✅ | | NAME-06 | `name:"*oo ba*"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | | NAME-07 | `name:"*REPORT*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-08 | `name:"*übung*"` | Übung.txt | Übung.txt | no match | ❌ known | -| NAME-09 | `name:"*ÜBUNG*"` | Übung.txt | Übung.txt | no match | ❌ known | +| NAME-08 | `name:"*übung*"` | Übung.txt | Übung.txt | Übung.txt | ✅ | +| NAME-09 | `name:"*ÜBUNG*"` | Übung.txt | Übung.txt | Übung.txt | ✅ | | NAME-10 | `name:"*a+b*"` | a+b.txt | a+b.txt | a+b.txt | ✅ | | NAME-11 | `name:"*c(d)*"` | c(d).txt | c(d).txt | c(d).txt | ✅ | | NAME-12 | `name:"*e&f*"` | e&f.txt | e&f.txt | e&f.txt | ✅ | @@ -41,30 +43,35 @@ Fixtures: | NAME-15 | `*folder*` | new-folder | new-folder | new-folder | ✅ | | NAME-16 | `name:"*foo bar*"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | | NAME-17 | `name:"foo bar.txt"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | -| NAME-18 | `name:"*needle*"` | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | no match | ❌ known | +| NAME-18 | `name:"*needle*"` | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | ✅ | | NAME-19 | `name:"report*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-20 | `name:"*report"` | Report.txt | no match | no match | ❌ known | +| NAME-20 | `name:"*report"` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-21 | `name:"Rep*rt.txt"` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-22 | `Name:"*report*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-23 | `NAME:"*report*"` | Report.txt | Report.txt | no match | ❌ known | -| NAME-24 | `name:Rep?rt.txt` | Report.txt | Report.txt | no match | ❌ known | -| NAME-25 | `name:"*eport"` | Report.txt | no match | no match | ❌ known | +| NAME-23 | `NAME:"*report*"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-24 | `name:Rep?rt.txt` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-25 | `name:"*eport"` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-26 | `name:"repor*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-27 | `REPORT` | Report.txt | no match | no match | ❌ known | -| NAME-28 | `name:REPORT` | Report.txt | no match | no match | ❌ known | +| NAME-27 | `REPORT` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-28 | `name:REPORT` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-29 | `name:"REPORT.TXT"` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-30 | `name:"FOO BAR.TXT"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | | NAME-31 | `name:"ÜBUNG.TXT"` | Übung.txt | Übung.txt | Übung.txt | ✅ | | NAME-32 | `name:"folder*"` | no match | no match | no match | ✅ | | NAME-33 | `name:"*new"` | no match | no match | no match | ✅ | -| NAME-34 | `name:new` | new-folder | no match | new-folder | ❌ known | -| NAME-35 | `name:"new"` | no match | no match | new-folder | ❌ known | +| NAME-34 | `name:new` | new-folder | new-folder | new-folder | ✅ | +| NAME-35 | `name:"new"` | new-folder | new-folder | new-folder | ✅ | | NAME-36 | `name:"new-folder"` | new-folder | new-folder | new-folder | ✅ | | NAME-37 | `name:"new-*"` | new-folder | new-folder | new-folder | ✅ | | NAME-38 | `name:"new*"` | new-folder | new-folder | new-folder | ✅ | | NAME-39 | `name:new-*` | new-folder | new-folder | new-folder | ✅ | | NAME-40 | `name:"*-folder"` | new-folder | new-folder | new-folder | ✅ | -| NAME-41 | `name:"Rep?rt.txt"` | Report.txt | Report.txt | no match | ❌ known | +| NAME-41 | `name:"Rep?rt.txt"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-42 | `name="Report.txt"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-43 | `name="REPORT.TXT"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-44 | `name="new"` | no match | no match | no match | ✅ | +| NAME-45 | `name="new-folder"` | new-folder | new-folder | new-folder | ✅ | +| NAME-46 | `name="foo bar.txt"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | ### extension @@ -76,10 +83,10 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| EXTENSION-01 | `txt` | report.txt | no match | no match | ❌ known | -| EXTENSION-02 | `md` | notes.md | no match | no match | ❌ known | +| EXTENSION-01 | `txt` | report.txt | report.txt | report.txt | ✅ | +| EXTENSION-02 | `md` | notes.md | notes.md | notes.md | ✅ | | EXTENSION-03 | `name:"*.txt"` | report.txt | report.txt | report.txt | ✅ | -| EXTENSION-04 | `report` | report.txt | no match | no match | ❌ known | +| EXTENSION-04 | `report` | report.txt | report.txt | report.txt | ✅ | ### tags @@ -102,7 +109,7 @@ Fixtures: | TAGS-06 | `tag:("spaced tag")` | spaced.txt | spaced.txt | spaced.txt | ✅ | | TAGS-07 | `tag:("*paced ta*")` | spaced.txt | spaced.txt | spaced.txt | ✅ | | TAGS-08 | `tag:("work")` | project | project | project | ✅ | -| TAGS-09 | `tag:("zzzzzzzzzzzzzzzzzzzzzzzzzz...zzzzzzzzzzzzzzzzneedle")` | longtag.txt | longtag.txt | no match | ❌ known | +| TAGS-09 | `tag:("zzzzzzzzzzzzzzzzzzzzzzzzzz...zzzzzzzzzzzzzzzzneedle")` | longtag.txt | longtag.txt | longtag.txt | ✅ | ### title @@ -113,12 +120,14 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| | TITLE-01 | `Title:"quarterly report"` | q1.html | q1.html | q1.html | ✅ | -| TITLE-02 | `Title:quarterly` | q1.html | no match | q1.html | ❌ known | -| TITLE-03 | `Title:QUARTERLY` | q1.html | no match | q1.html | ❌ known | +| TITLE-02 | `Title:quarterly` | q1.html | q1.html | q1.html | ✅ | +| TITLE-03 | `Title:QUARTERLY` | q1.html | q1.html | q1.html | ✅ | | TITLE-04 | `Title:quarterl*` | q1.html | q1.html | q1.html | ✅ | | TITLE-05 | `Title:"*ly rep*"` | q1.html | q1.html | q1.html | ✅ | -| TITLE-06 | `title:quarterly` | q1.html | no match | no match | ❌ known | -| TITLE-07 | `Title:"QUARTERLY REPORT"` | q1.html | no match | q1.html | ❌ known | +| TITLE-06 | `title:quarterly` | q1.html | q1.html | q1.html | ✅ | +| TITLE-07 | `Title:"QUARTERLY REPORT"` | q1.html | q1.html | q1.html | ✅ | +| TITLE-08 | `Title="quarterly report"` | q1.html | q1.html | q1.html | ✅ | +| TITLE-09 | `Title="quarterly"` | no match | no match | no match | ✅ | ### content @@ -129,16 +138,16 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| CONTENT-01 | `Content:report` | no match | monthly.txt | no match | ❌ known | +| CONTENT-01 | `Content:report` | no match | no match | no match | ✅ | | CONTENT-02 | `Content:REPORTS` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-03 | `Content:"monthly reports"` | monthly.txt | monthly.txt | monthly.txt | ✅ | -| CONTENT-04 | `Content:"reports monthly"` | no match | monthly.txt | no match | ❌ known | +| CONTENT-04 | `Content:"reports monthly"` | no match | no match | no match | ✅ | | CONTENT-05 | `Content:report*` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-06 | `Content:*eport*` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-07 | `Content:month*` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-08 | `Content:"https://opencloud.example.com/help"` | links.txt | links.txt | links.txt | ✅ | | CONTENT-09 | `Content:"alan@example.org"` | links.txt | links.txt | links.txt | ✅ | -| CONTENT-10 | `Content:opencloud` | links.txt | no match | no match | ❌ known | +| CONTENT-10 | `Content:opencloud` | links.txt | links.txt | links.txt | ✅ | ### favorites @@ -151,8 +160,8 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| FAVORITES-01 | `Favorites:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | no match | ❌ known | -| FAVORITES-02 | `favorite:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | no match | ❌ known | +| FAVORITES-01 | `Favorites:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | keepsakes, starred.txt | ✅ | +| FAVORITES-02 | `favorite:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | keepsakes, starred.txt | ✅ | | FAVORITES-03 | `Favorites:"somebody-else"` | no match | no match | no match | ✅ | ### mediatype @@ -167,7 +176,7 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| | MEDIATYPE-01 | `mediatype:text/markdown` | notes.md | notes.md | notes.md | ✅ | -| MEDIATYPE-02 | `mediatype:TEXT/MARKDOWN` | notes.md | no match | notes.md | ❌ known | +| MEDIATYPE-02 | `mediatype:TEXT/MARKDOWN` | notes.md | notes.md | notes.md | ✅ | | MEDIATYPE-03 | `mediatype:image/jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ | | MEDIATYPE-04 | `mediatype:*jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ | | MEDIATYPE-05 | `mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ | @@ -185,14 +194,14 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| PATH-01 | `path:"./parent"` | child.jpg, parent | parent | child.jpg, parent | ❌ known | +| PATH-01 | `path:"./parent"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ | | PATH-02 | `path:"./parent/child.jpg"` | child.jpg | child.jpg | child.jpg | ✅ | -| PATH-03 | `path:"./Parent"` | no match | no match | child.jpg, parent | ❌ known | +| PATH-03 | `path:"./Parent"` | no match | no match | no match | ✅ | | PATH-04 | `path:"*child*"` | child.jpg | child.jpg | child.jpg | ✅ | -| PATH-05 | `path:"./documents"` | docs-lower | docs-lower | docs-lower, docs-mixed, docs-upper | ❌ known | -| PATH-06 | `path:"./DOCUMENTS"` | docs-upper | docs-upper | docs-lower, docs-mixed, docs-upper | ❌ known | -| PATH-07 | `path:"./Documents"` | docs-mixed | docs-mixed | docs-lower, docs-mixed, docs-upper | ❌ known | -| PATH-08 | `path:"./parent/"` | child.jpg, parent | no match | no match | ❌ known | +| PATH-05 | `path:"./documents"` | docs-lower | docs-lower | docs-lower | ✅ | +| PATH-06 | `path:"./DOCUMENTS"` | docs-upper | docs-upper | docs-upper | ✅ | +| PATH-07 | `path:"./Documents"` | docs-mixed | docs-mixed | docs-mixed | ✅ | +| PATH-08 | `path:"./parent/"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ | ### fields @@ -213,16 +222,16 @@ Fixtures: | FIELDS-01 | `size:42` | small.txt | small.txt | small.txt | ✅ | | FIELDS-02 | `mtime<"2021-01-01T00:00:00Z"` | old.txt | old.txt | old.txt | ✅ | | FIELDS-03 | `id:"1$1!23"` | known.txt | known.txt | known.txt | ✅ | -| FIELDS-04 | `hidden:true` | hidden.txt | no match | hidden.txt | ❌ known | -| FIELDS-05 | `type:file` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | no match | error | ❌ known | -| FIELDS-06 | `type:folder` | box | no match | error | ❌ known | +| FIELDS-04 | `hidden:true` | hidden.txt | hidden.txt | hidden.txt | ✅ | +| FIELDS-05 | `type:file` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | ✅ | +| FIELDS-06 | `type:folder` | box | box | box | ✅ | | FIELDS-07 | `unknown:field` | no match | no match | no match | ✅ | -| FIELDS-08 | `type:File` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | no match | error | ❌ known | -| FIELDS-09 | `type:FOLDER` | box | no match | error | ❌ known | -| FIELDS-10 | `hidden:TRUE` | hidden.txt | no match | error | ❌ known | -| FIELDS-11 | `id:"1$1!AB-23"` | cased.txt | cased.txt | no match | ❌ known | +| FIELDS-08 | `type:File` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | ✅ | +| FIELDS-09 | `type:FOLDER` | box | box | box | ✅ | +| FIELDS-10 | `hidden:TRUE` | hidden.txt | hidden.txt | hidden.txt | ✅ | +| FIELDS-11 | `id:"1$1!AB-23"` | cased.txt | cased.txt | cased.txt | ✅ | | FIELDS-12 | `id:"1$1!ab-23"` | no match | no match | no match | ✅ | -| FIELDS-13 | `audio.artist:"Some Artist"` | song.mp3 | song.mp3 | no match | ❌ known | +| FIELDS-13 | `audio.artist:"Some Artist"` | song.mp3 | song.mp3 | song.mp3 | ✅ | | FIELDS-14 | `audio.artist:"some artist"` | no match | no match | no match | ✅ | ### deleted @@ -242,7 +251,7 @@ Fixtures: | DELETED-02 | `name:"*.txt"` | book.txt, kept.txt | book.txt, kept.txt | book.txt, kept.txt | ✅ | | DELETED-03 | `name:"*receipt*"` | no match | no match | no match | ✅ | | DELETED-04 | `path:"./bin"` | no match | no match | no match | ✅ | -| DELETED-05 | `path:"./shelf"` | book.txt, shelf | shelf | book.txt, shelf | ❌ known | +| DELETED-05 | `path:"./shelf"` | book.txt, shelf | book.txt, shelf | book.txt, shelf | ✅ | ### visibility @@ -255,13 +264,13 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| VISIBILITY-01 | `hidden:true` | .private, dotfile.txt, secret.txt | no match | .private, dotfile.txt, secret.txt | ❌ known | -| VISIBILITY-02 | `hidden:TRUE` | .private, dotfile.txt, secret.txt | no match | error | ❌ known | -| VISIBILITY-03 | `hidden:false` | visible.txt | no match | visible.txt | ❌ known | +| VISIBILITY-01 | `hidden:true` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ | +| VISIBILITY-02 | `hidden:TRUE` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ | +| VISIBILITY-03 | `hidden:false` | visible.txt | visible.txt | visible.txt | ✅ | | VISIBILITY-04 | `name:"*secret*"` | secret.txt | secret.txt | secret.txt | ✅ | -| VISIBILITY-05 | `path:"./.private"` | .private, secret.txt | .private | .private, secret.txt | ❌ known | -| VISIBILITY-06 | `hidden:banana` | no match | no match | error | ❌ known | -| VISIBILITY-07 | `hidden:"true"` | .private, dotfile.txt, secret.txt | no match | .private, dotfile.txt, secret.txt | ❌ known | +| VISIBILITY-05 | `path:"./.private"` | .private, secret.txt | .private, secret.txt | .private, secret.txt | ✅ | +| VISIBILITY-06 | `hidden:banana` | no match | no match | no match | ✅ | +| VISIBILITY-07 | `hidden:"true"` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ | ### boolean @@ -310,21 +319,21 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| | STRESS-01 | `name:"*report*" AND mediatype:document` | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | ✅ | -| STRESS-02 | `name:"*report*" AND NOT tag:("draft")` | quarterly report.docx | draft report.txt, quarterly report.docx | quarterly report.docx | ❌ known | +| STRESS-02 | `name:"*report*" AND NOT tag:("draft")` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | | STRESS-03 | `(tag:("final") OR tag:("draft")) AND mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ | | STRESS-04 | `mediatype:document AND mtime>"2021-01-01T00:00:00Z"` | notes.md, quarterly report.docx | notes.md, quarterly report.docx | notes.md, quarterly report.docx | ✅ | -| STRESS-05 | `name:"*report*" AND size>100` | quarterly report.docx | no match | no match | ❌ known | +| STRESS-05 | `name:"*report*" AND size>100` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | | STRESS-06 | `tag:("final") AND NOT mediatype:folder` | photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | ✅ | -| STRESS-07 | `hidden:true AND name:"*notes*"` | notes.md | no match | notes.md | ❌ known | -| STRESS-08 | `name:quarterly report` | quarterly report.docx | no match | no match | ❌ known | -| STRESS-09 | `name:"quarterly report"` | no match | no match | no match | ✅ | +| STRESS-07 | `hidden:true AND name:"*notes*"` | notes.md | notes.md | notes.md | ✅ | +| STRESS-08 | `name:quarterly report` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | +| STRESS-09 | `name:"quarterly report"` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | | STRESS-10 | `name:"quarterly report.docx"` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | -| STRESS-11 | `NOT tag:("draft")` | archive, notes.md, photo.jpg, quarterly report.docx | archive, draft report.txt, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | ❌ known | -| STRESS-12 | `tag:("final") OR hidden:true` | notes.md, photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | ❌ known | -| STRESS-13 | `(name:"*report*" OR name:"*notes..."draft") OR hidden:true)` | quarterly report.docx | notes.md, quarterly report.docx | quarterly report.docx | ❌ known | +| STRESS-11 | `NOT tag:("draft")` | archive, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | ✅ | +| STRESS-12 | `tag:("final") OR hidden:true` | notes.md, photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | ✅ | +| STRESS-13 | `(name:"*report*" OR name:"*notes..."draft") OR hidden:true)` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | | STRESS-14 | `mediatype:image OR (mediatype:document AND tag:("draft"))` | draft report.txt, photo.jpg | draft report.txt, photo.jpg | draft report.txt, photo.jpg | ✅ | -| STRESS-15 | `NOT (mediatype:folder OR hidden:true)` | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, notes.md, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | ❌ known | -| STRESS-16 | `name:"*report*" AND (size>100 OR tag:("draft"))` | draft report.txt, quarterly report.docx | draft report.txt | draft report.txt | ❌ known | +| STRESS-15 | `NOT (mediatype:folder OR hidden:true)` | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | ✅ | +| STRESS-16 | `name:"*report*" AND (size>100 OR tag:("draft"))` | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | ✅ | ### everything @@ -351,8 +360,8 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| RANGE-01 | `size>100` | big.txt | no match | no match | ❌ known | -| RANGE-02 | `size<100` | ancient.txt, small.txt | no match | no match | ❌ known | +| RANGE-01 | `size>100` | big.txt | big.txt | big.txt | ✅ | +| RANGE-02 | `size<100` | ancient.txt, small.txt | ancient.txt, small.txt | ancient.txt, small.txt | ✅ | | RANGE-03 | `mtime>"2021-01-01T00:00:00Z"` | big.txt, small.txt | big.txt, small.txt | big.txt, small.txt | ✅ | | RANGE-04 | `mtime<"2021-01-01T00:00:00Z"` | ancient.txt | ancient.txt | ancient.txt | ✅ | | RANGE-05 | `Mtime:"today"` | big.txt, small.txt | big.txt, small.txt | big.txt, small.txt | ✅ | @@ -403,7 +412,7 @@ Fixtures: | DELETE-01 | takes the resource out of the results, then `name:"*child*"` | no match | no match | no match | ✅ | | DELETE-02 | takes the descendants along, then `name:"*parent*"` | no match | no match | no match | ✅ | | DELETE-02 | takes the descendants along, then `name:"*child*"` | no match | no match | no match | ✅ | -| DELETE-03 | leaves the resource in the index, then `DocCount()` | 2 | 2 | 1 | ❌ known | +| DELETE-03 | leaves the resource in the index, then `DocCount()` | 2 | 2 | 2 | ✅ | | DELETE-04 | takes a resource out that was just written, then `name:"*fresh*"` | no match | no match | no match | ✅ | ### restore @@ -418,7 +427,7 @@ Fixtures: |---|---|---|---|---|---| | RESTORE-01 | brings the descendants back, then `name:"*parent*"` | parent | parent | parent | ✅ | | RESTORE-01 | brings the descendants back, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ | -| RESTORE-02 | leaves the hidden flag alone, then `hidden:true` | file.txt | no match | file.txt | ❌ known | +| RESTORE-02 | leaves the hidden flag alone, then `hidden:true` | file.txt | file.txt | file.txt | ✅ | ### purge @@ -502,11 +511,11 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| CASEPATH-01 | takes the descendants along when deleting, then `path:"./Documents"` | no match | no match | Documents, Picture.jpg | ❌ known | -| CASEPATH-02 | takes the descendants along when moving, then `path:"./Other Documents"` | Other Documents, Picture.jpg | Other Documents | no match | ❌ known | -| CASEPATH-02 | takes the descendants along when moving, then `path:"./Documents"` | no match | no match | Documents, Picture.jpg | ❌ known | -| CASEPATH-03 | reaches the descendants when purging, then `path:"./Documents"` | no match | no match | Documents, Picture.jpg | ❌ known | -| CASEPATH-03 | reaches the descendants when purging, then `DocCount()` | 0 | 0 | 2 | ❌ known | +| CASEPATH-01 | takes the descendants along when deleting, then `path:"./Documents"` | no match | no match | no match | ✅ | +| CASEPATH-02 | takes the descendants along when moving, then `path:"./Other Documents"` | Other Documents, Picture.jpg | Other Documents, Picture.jpg | Other Documents, Picture.jpg | ✅ | +| CASEPATH-02 | takes the descendants along when moving, then `path:"./Documents"` | no match | no match | no match | ✅ | +| CASEPATH-03 | reaches the descendants when purging, then `path:"./Documents"` | no match | no match | no match | ✅ | +| CASEPATH-03 | reaches the descendants when purging, then `DocCount()` | 0 | 0 | 0 | ✅ | ### hidden @@ -521,12 +530,12 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| HIDDEN-01 | follows a move into a dot folder, then `hidden:true` | child.pdf, parent | no match | child.pdf, parent | ❌ known | +| HIDDEN-01 | follows a move into a dot folder, then `hidden:true` | child.pdf, parent | child.pdf, parent | child.pdf, parent | ✅ | | HIDDEN-02 | follows a move into a plain folder, then `hidden:true` | no match | no match | no match | ✅ | -| HIDDEN-03 | follows a move renamed with a leading dot, then `hidden:true` | .parent, child.pdf | no match | .parent, child.pdf | ❌ known | +| HIDDEN-03 | follows a move renamed with a leading dot, then `hidden:true` | .parent, child.pdf | .parent, child.pdf | .parent, child.pdf | ✅ | | HIDDEN-04 | follows a move out of a dot folder, then `hidden:true` | no match | no match | no match | ✅ | | HIDDEN-05 | follows a move renamed without the leading dot, then `hidden:true` | no match | no match | no match | ✅ | -| HIDDEN-06 | follows a move within the same dot folder, then `hidden:true` | child.pdf, moved | no match | child.pdf, moved | ❌ known | +| HIDDEN-06 | follows a move within the same dot folder, then `hidden:true` | child.pdf, moved | child.pdf, moved | child.pdf, moved | ✅ | ### upsert @@ -610,7 +619,7 @@ Fixtures: | ENTITY-08 | `name:"bar.pdf"` reads `MimeType` | application/pdf | application/pdf | application/pdf | ✅ | | ENTITY-09 | `name:"bar.pdf"` reads `Deleted` | false | false | false | ✅ | | ENTITY-10 | `name:"bar.pdf"` reads `Score` | above zero | above zero | above zero | ✅ | -| ENTITY-11 | `path:"./parent"` reads `TotalMatches` | 2 | 1 | 2 | ❌ known | +| ENTITY-11 | `path:"./parent"` reads `TotalMatches` | 2 | 2 | 2 | ✅ | | ENTITY-12 | `name:"*notes*"` reads `Highlights` | "" | "" | "" | ✅ | | ENTITY-13 | `content:bar` reads `Highlights` | foo bar baz | foo bar baz | foo bar baz | ✅ | | ENTITY-14 | moved to another parent, then `name:"newname"` reads `ParentId` | 1$1!9 | 1$1!9 | 1$1!9 | ✅ | diff --git a/services/search/pkg/parity/engines_test.go b/services/search/pkg/parity/engines_test.go index a77fb61008..71fd99033a 100644 --- a/services/search/pkg/parity/engines_test.go +++ b/services/search/pkg/parity/engines_test.go @@ -88,16 +88,17 @@ func newBleve(fixtures []search.Resource) testEngine { return testEngine{name: "bleve", backend: backend, settle: func() {}} } -func newOpenSearch(index string, fixtures []search.Resource) testEngine { +func newOpenSearch(name string, fixtures []search.Resource) testEngine { GinkgoHelper() tc := opensearchtest.NewDefaultTestClient(GinkgoTB(), openSearchClient) + index := opensearch.IndexName(name) if err := tc.IndicesReset(context.Background(), []string{index}); err != nil { return testEngine{name: "opensearch", unavailable: err.Error()} } - backend, err := opensearch.NewBackend(index, tc.Client()) + backend, err := opensearch.NewBackend(name, tc.Client()) if err != nil { return testEngine{name: "opensearch", unavailable: err.Error()} } diff --git a/services/search/pkg/parity/lifecycle_casepath_test.go b/services/search/pkg/parity/lifecycle_casepath_test.go index 8347fcf318..95b3b36208 100644 --- a/services/search/pkg/parity/lifecycle_casepath_test.go +++ b/services/search/pkg/parity/lifecycle_casepath_test.go @@ -28,9 +28,6 @@ func casePathLifecycle() lifecycleGroup { id: 1, title: "takes the descendants along when deleting", do: func(e search.Engine) error { return e.Delete(folder.ID) }, expect: left(), - engineOverrides: map[string]lifecycleOverride{ - "opensearch": {expect: map[string][]string{`path:"./Documents"`: {"Documents", "Picture.jpg"}}}, - }, }, { id: 2, title: "takes the descendants along when moving", @@ -39,22 +36,12 @@ func casePathLifecycle() lifecycleGroup { {`path:"./Other Documents"`, []string{"Other Documents", "Picture.jpg"}}, {`path:"./Documents"`, nil}, }, - engineOverrides: map[string]lifecycleOverride{ - "bleve": {expect: map[string][]string{`path:"./Other Documents"`: {"Other Documents"}}}, - "opensearch": {expect: map[string][]string{`path:"./Other Documents"`: {}, `path:"./Documents"`: {"Documents", "Picture.jpg"}}}, - }, }, { id: 3, title: "reaches the descendants when purging", do: func(e search.Engine) error { return e.Purge(folder.ID, false) }, expect: left(), wantDocCount: conversions.ToPointer(uint64(0)), - engineOverrides: map[string]lifecycleOverride{ - "opensearch": { - expect: map[string][]string{`path:"./Documents"`: {"Documents", "Picture.jpg"}}, - wantDocCount: conversions.ToPointer(uint64(2)), - }, - }, }, }, } diff --git a/services/search/pkg/parity/lifecycle_delete_test.go b/services/search/pkg/parity/lifecycle_delete_test.go index c483deb901..3a4c49b2c6 100644 --- a/services/search/pkg/parity/lifecycle_delete_test.go +++ b/services/search/pkg/parity/lifecycle_delete_test.go @@ -26,9 +26,6 @@ func deleteLifecycle() lifecycleGroup { id: 3, title: "leaves the resource in the index", do: func(e search.Engine) error { return e.Delete(child.ID) }, wantDocCount: conversions.ToPointer(uint64(2)), - engineOverrides: map[string]lifecycleOverride{ - "opensearch": {wantDocCount: conversions.ToPointer(uint64(1))}, - }, }, { id: 4, title: "takes a resource out that was just written", diff --git a/services/search/pkg/parity/lifecycle_hidden_test.go b/services/search/pkg/parity/lifecycle_hidden_test.go index 59ec242088..c703d6ce74 100644 --- a/services/search/pkg/parity/lifecycle_hidden_test.go +++ b/services/search/pkg/parity/lifecycle_hidden_test.go @@ -29,14 +29,9 @@ func hiddenLifecycle() lifecycleGroup { child.Path = move.from + "/child.pdf" child.Hidden = parent.Hidden - var ( - hidden []string - overrides map[string]lifecycleOverride - ) + var hidden []string if move.hidden { hidden = []string{path.Base(move.target), "child.pdf"} - // bleve does not answer hidden:true at all today - overrides = map[string]lifecycleOverride{"bleve": {expect: map[string][]string{`hidden:true`: {}}}} } group.cases = append(group.cases, lifecycleCase{ @@ -45,8 +40,7 @@ func hiddenLifecycle() lifecycleGroup { do: func(e search.Engine) error { return e.Move(parent.ID, parent.ParentID, move.target) }, - expect: []expectation{{`hidden:true`, hidden}}, - engineOverrides: overrides, + expect: []expectation{{`hidden:true`, hidden}}, }) } diff --git a/services/search/pkg/parity/lifecycle_restore_test.go b/services/search/pkg/parity/lifecycle_restore_test.go index 79dd3e3f02..c85c4c596a 100644 --- a/services/search/pkg/parity/lifecycle_restore_test.go +++ b/services/search/pkg/parity/lifecycle_restore_test.go @@ -33,8 +33,7 @@ func restoreLifecycle() lifecycleGroup { return e.Restore(secret.ID) }, - expect: []expectation{{`hidden:true`, []string{"file.txt"}}}, - engineOverrides: map[string]lifecycleOverride{"bleve": {expect: map[string][]string{`hidden:true`: {}}}}, + expect: []expectation{{`hidden:true`, []string{"file.txt"}}}, }, }, } diff --git a/services/search/pkg/parity/matrix_test.go b/services/search/pkg/parity/matrix_test.go index 4b4d65fca2..be0c538597 100644 --- a/services/search/pkg/parity/matrix_test.go +++ b/services/search/pkg/parity/matrix_test.go @@ -173,7 +173,9 @@ func writeMatrix(report types.Report) { out.WriteString("Written by the parity suite (`go test ./services/search/pkg/parity/`), do not edit.\n") out.WriteString("Every case runs against bleve and OpenSearch. `same?` is ✅ when both answer as\n") out.WriteString("expected, `❌ known` when an engine's divergence is documented in the case\n") - out.WriteString("(`engineOverrides`), `❌` when it is not.\n") + out.WriteString("(`engineOverrides`), `❌` when it is not. `✅ stale` when every engine\n") + out.WriteString("answers the expected value although the case still documents a\n") + out.WriteString("divergence, that override can come out.\n") group, section := "", "" for _, row := range rows { @@ -426,6 +428,8 @@ func matrixVerdict(row *matrixResult) string { } switch { + case len(off) == 0 && len(row.Overrides) > 0: + return "✅ stale" case len(off) == 0: return "✅" case known: diff --git a/services/search/pkg/parity/query_content_test.go b/services/search/pkg/parity/query_content_test.go index d51d36d53b..010b5545c3 100644 --- a/services/search/pkg/parity/query_content_test.go +++ b/services/search/pkg/parity/query_content_test.go @@ -12,16 +12,16 @@ func contentGroup() queryGroup { fixtureDoc("links.txt", withContent("see https://opencloud.example.com/help or write to alan@example.org")), }, cases: []queryCase{ - {id: 1, query: `Content:report`, engineOverrides: map[string]override{"bleve": override{want: []string{"monthly.txt"}}}}, + {id: 1, query: `Content:report`}, {id: 2, query: `Content:REPORTS`, want: []string{"monthly.txt"}}, {id: 3, query: `Content:"monthly reports"`, want: []string{"monthly.txt"}}, - {id: 4, query: `Content:"reports monthly"`, engineOverrides: map[string]override{"bleve": override{want: []string{"monthly.txt"}}}}, + {id: 4, query: `Content:"reports monthly"`}, {id: 5, query: `Content:report*`, want: []string{"monthly.txt"}}, {id: 6, query: `Content:*eport*`, want: []string{"monthly.txt"}}, {id: 7, query: `Content:month*`, want: []string{"monthly.txt"}}, {id: 8, query: `Content:"https://opencloud.example.com/help"`, want: []string{"links.txt"}}, {id: 9, query: `Content:"alan@example.org"`, want: []string{"links.txt"}}, - {id: 10, query: `Content:opencloud`, want: []string{"links.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 10, query: `Content:opencloud`, want: []string{"links.txt"}}, }, } } diff --git a/services/search/pkg/parity/query_deleted_test.go b/services/search/pkg/parity/query_deleted_test.go index 905725bd8b..1e048b1e9d 100644 --- a/services/search/pkg/parity/query_deleted_test.go +++ b/services/search/pkg/parity/query_deleted_test.go @@ -20,7 +20,7 @@ func deletedGroup() queryGroup { {id: 2, query: `name:"*.txt"`, want: []string{"kept.txt", "book.txt"}}, {id: 3, query: `name:"*receipt*"`}, {id: 4, query: `path:"./bin"`}, - {id: 5, query: `path:"./shelf"`, want: []string{"shelf", "book.txt"}, engineOverrides: map[string]override{"bleve": override{want: []string{"shelf"}}}}, + {id: 5, query: `path:"./shelf"`, want: []string{"shelf", "book.txt"}}, }, } } diff --git a/services/search/pkg/parity/query_extension_test.go b/services/search/pkg/parity/query_extension_test.go index e02b5e30fa..a813640bcb 100644 --- a/services/search/pkg/parity/query_extension_test.go +++ b/services/search/pkg/parity/query_extension_test.go @@ -13,10 +13,10 @@ func extensionGroup() queryGroup { fixtureFolder("archive"), }, cases: []queryCase{ - {id: 1, query: `txt`, want: []string{"report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, - {id: 2, query: `md`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 1, query: `txt`, want: []string{"report.txt"}}, + {id: 2, query: `md`, want: []string{"notes.md"}}, {id: 3, query: `name:"*.txt"`, want: []string{"report.txt"}}, - {id: 4, query: `report`, want: []string{"report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 4, query: `report`, want: []string{"report.txt"}}, }, } } diff --git a/services/search/pkg/parity/query_favorites_test.go b/services/search/pkg/parity/query_favorites_test.go index ba1f4d7d1c..5413f3eaf4 100644 --- a/services/search/pkg/parity/query_favorites_test.go +++ b/services/search/pkg/parity/query_favorites_test.go @@ -14,8 +14,8 @@ func favoritesGroup() queryGroup { fixtureDoc("photo.jpg", withParent("1$1!keepsakes"), withPath("./keepsakes/photo.jpg"), withMime("image/jpeg")), }, cases: []queryCase{ - {id: 1, query: `Favorites:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}, engineOverrides: map[string]override{"opensearch": override{}}}, - {id: 2, query: `favorite:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 1, query: `Favorites:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}}, + {id: 2, query: `favorite:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}}, {id: 3, query: `Favorites:"somebody-else"`}, }, } diff --git a/services/search/pkg/parity/query_fields_test.go b/services/search/pkg/parity/query_fields_test.go index a643f0cfae..e5600893fb 100644 --- a/services/search/pkg/parity/query_fields_test.go +++ b/services/search/pkg/parity/query_fields_test.go @@ -24,17 +24,17 @@ func fieldsGroup() queryGroup { {id: 1, query: `size:42`, want: []string{"small.txt"}}, {id: 2, query: `mtime<"2021-01-01T00:00:00Z"`, want: []string{"old.txt"}}, {id: 3, query: `id:"1$1!23"`, want: []string{"known.txt"}}, - {id: 4, query: `hidden:true`, want: []string{"hidden.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 5, query: `type:file`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, - {id: 6, query: `type:folder`, want: []string{"box"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, + {id: 4, query: `hidden:true`, want: []string{"hidden.txt"}}, + {id: 5, query: `type:file`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt", "song.mp3"}}, + {id: 6, query: `type:folder`, want: []string{"box"}}, {id: 7, query: `unknown:field`}, - {id: 8, query: `type:File`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, - {id: 9, query: `type:FOLDER`, want: []string{"box"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, - {id: 10, query: `hidden:TRUE`, want: []string{"hidden.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, - {id: 11, query: `id:"1$1!AB-23"`, want: []string{"cased.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 8, query: `type:File`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt", "song.mp3"}}, + {id: 9, query: `type:FOLDER`, want: []string{"box"}}, + {id: 10, query: `hidden:TRUE`, want: []string{"hidden.txt"}}, + {id: 11, query: `id:"1$1!AB-23"`, want: []string{"cased.txt"}}, {id: 12, query: `id:"1$1!ab-23"`}, // a facet value keeps its case, the field is not marked lowercase - {id: 13, query: `audio.artist:"Some Artist"`, want: []string{"song.mp3"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 13, query: `audio.artist:"Some Artist"`, want: []string{"song.mp3"}}, {id: 14, query: `audio.artist:"some artist"`}, }, } diff --git a/services/search/pkg/parity/query_mediatype_test.go b/services/search/pkg/parity/query_mediatype_test.go index 11a5065d99..cd43ff89d5 100644 --- a/services/search/pkg/parity/query_mediatype_test.go +++ b/services/search/pkg/parity/query_mediatype_test.go @@ -15,7 +15,7 @@ func mediatypeGroup() queryGroup { }, cases: []queryCase{ {id: 1, query: `mediatype:text/markdown`, want: []string{"notes.md"}}, - {id: 2, query: `mediatype:TEXT/MARKDOWN`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 2, query: `mediatype:TEXT/MARKDOWN`, want: []string{"notes.md"}}, {id: 3, query: `mediatype:image/jpeg`, want: []string{"photo.jpg"}}, {id: 4, query: `mediatype:*jpeg`, want: []string{"photo.jpg"}}, {id: 5, query: `mediatype:image`, want: []string{"photo.jpg"}}, diff --git a/services/search/pkg/parity/query_name_test.go b/services/search/pkg/parity/query_name_test.go index 807041166d..b5f3acfa24 100644 --- a/services/search/pkg/parity/query_name_test.go +++ b/services/search/pkg/parity/query_name_test.go @@ -20,15 +20,15 @@ func nameGroup() queryGroup { fixtureDoc(fixtureLongName), }, cases: []queryCase{ - {id: 1, query: `new`, want: []string{"new-folder"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 2, query: `quarterly`, want: []string{"quarterly notes.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 3, query: `report`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 1, query: `new`, want: []string{"new-folder"}}, + {id: 2, query: `quarterly`, want: []string{"quarterly notes.txt"}}, + {id: 3, query: `report`, want: []string{"Report.txt"}}, {id: 4, query: `name:"*new-folder*"`, want: []string{"new-folder"}}, {id: 5, query: `name:"*w-fol*"`, want: []string{"new-folder"}}, {id: 6, query: `name:"*oo ba*"`, want: []string{"foo bar.txt"}}, {id: 7, query: `name:"*REPORT*"`, want: []string{"Report.txt"}}, - {id: 8, query: `name:"*übung*"`, want: []string{"Übung.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, - {id: 9, query: `name:"*ÜBUNG*"`, want: []string{"Übung.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 8, query: `name:"*übung*"`, want: []string{"Übung.txt"}}, + {id: 9, query: `name:"*ÜBUNG*"`, want: []string{"Übung.txt"}}, {id: 10, query: `name:"*a+b*"`, want: []string{"a+b.txt"}}, {id: 11, query: `name:"*c(d)*"`, want: []string{"c(d).txt"}}, {id: 12, query: `name:"*e&f*"`, want: []string{"e&f.txt"}}, @@ -37,30 +37,35 @@ func nameGroup() queryGroup { {id: 15, query: `*folder*`, want: []string{"new-folder"}}, {id: 16, query: `name:"*foo bar*"`, want: []string{"foo bar.txt"}}, {id: 17, query: `name:"foo bar.txt"`, want: []string{"foo bar.txt"}}, - {id: 18, query: `name:"*needle*"`, want: []string{fixtureLongName}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 18, query: `name:"*needle*"`, want: []string{fixtureLongName}}, {id: 19, query: `name:"report*"`, want: []string{"Report.txt"}}, - {id: 20, query: `name:"*report"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 20, query: `name:"*report"`, want: []string{"Report.txt"}}, {id: 21, query: `name:"Rep*rt.txt"`, want: []string{"Report.txt"}}, {id: 22, query: `Name:"*report*"`, want: []string{"Report.txt"}}, - {id: 23, query: `NAME:"*report*"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, - {id: 24, query: `name:Rep?rt.txt`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, - {id: 25, query: `name:"*eport"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 23, query: `NAME:"*report*"`, want: []string{"Report.txt"}}, + {id: 24, query: `name:Rep?rt.txt`, want: []string{"Report.txt"}}, + {id: 25, query: `name:"*eport"`, want: []string{"Report.txt"}}, {id: 26, query: `name:"repor*"`, want: []string{"Report.txt"}}, - {id: 27, query: `REPORT`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, - {id: 28, query: `name:REPORT`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 27, query: `REPORT`, want: []string{"Report.txt"}}, + {id: 28, query: `name:REPORT`, want: []string{"Report.txt"}}, {id: 29, query: `name:"REPORT.TXT"`, want: []string{"Report.txt"}}, {id: 30, query: `name:"FOO BAR.TXT"`, want: []string{"foo bar.txt"}}, {id: 31, query: `name:"ÜBUNG.TXT"`, want: []string{"Übung.txt"}}, {id: 32, query: `name:"folder*"`}, {id: 33, query: `name:"*new"`}, - {id: 34, query: `name:new`, want: []string{"new-folder"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 35, query: `name:"new"`, engineOverrides: map[string]override{"opensearch": override{want: []string{"new-folder"}}}}, + {id: 34, query: `name:new`, want: []string{"new-folder"}}, + {id: 35, query: `name:"new"`, want: []string{"new-folder"}}, {id: 36, query: `name:"new-folder"`, want: []string{"new-folder"}}, {id: 37, query: `name:"new-*"`, want: []string{"new-folder"}}, {id: 38, query: `name:"new*"`, want: []string{"new-folder"}}, {id: 39, query: `name:new-*`, want: []string{"new-folder"}}, {id: 40, query: `name:"*-folder"`, want: []string{"new-folder"}}, - {id: 41, query: `name:"Rep?rt.txt"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 41, query: `name:"Rep?rt.txt"`, want: []string{"Report.txt"}}, + {id: 42, query: `name="Report.txt"`, want: []string{"Report.txt"}}, + {id: 43, query: `name="REPORT.TXT"`, want: []string{"Report.txt"}}, + {id: 44, query: `name="new"`}, + {id: 45, query: `name="new-folder"`, want: []string{"new-folder"}}, + {id: 46, query: `name="foo bar.txt"`, want: []string{"foo bar.txt"}}, }, } } diff --git a/services/search/pkg/parity/query_path_test.go b/services/search/pkg/parity/query_path_test.go index 828d298951..e97e8e1596 100644 --- a/services/search/pkg/parity/query_path_test.go +++ b/services/search/pkg/parity/query_path_test.go @@ -15,14 +15,14 @@ func pathGroup() queryGroup { fixtureFolder("docs-mixed", withPath("./Documents")), }, cases: []queryCase{ - {id: 1, query: `path:"./parent"`, want: []string{"parent", "child.jpg"}, engineOverrides: map[string]override{"bleve": override{want: []string{"parent"}}}}, + {id: 1, query: `path:"./parent"`, want: []string{"parent", "child.jpg"}}, {id: 2, query: `path:"./parent/child.jpg"`, want: []string{"child.jpg"}}, - {id: 3, query: `path:"./Parent"`, engineOverrides: map[string]override{"opensearch": override{want: []string{"child.jpg", "parent"}}}}, + {id: 3, query: `path:"./Parent"`}, {id: 4, query: `path:"*child*"`, want: []string{"child.jpg"}}, - {id: 5, query: `path:"./documents"`, want: []string{"docs-lower"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"docs-lower", "docs-mixed", "docs-upper"}}}}, - {id: 6, query: `path:"./DOCUMENTS"`, want: []string{"docs-upper"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"docs-lower", "docs-mixed", "docs-upper"}}}}, - {id: 7, query: `path:"./Documents"`, want: []string{"docs-mixed"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"docs-lower", "docs-mixed", "docs-upper"}}}}, - {id: 8, query: `path:"./parent/"`, want: []string{"parent", "child.jpg"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 5, query: `path:"./documents"`, want: []string{"docs-lower"}}, + {id: 6, query: `path:"./DOCUMENTS"`, want: []string{"docs-upper"}}, + {id: 7, query: `path:"./Documents"`, want: []string{"docs-mixed"}}, + {id: 8, query: `path:"./parent/"`, want: []string{"parent", "child.jpg"}}, }, } } diff --git a/services/search/pkg/parity/query_range_test.go b/services/search/pkg/parity/query_range_test.go index 59a1618272..f5c7033012 100644 --- a/services/search/pkg/parity/query_range_test.go +++ b/services/search/pkg/parity/query_range_test.go @@ -13,8 +13,8 @@ func rangeGroup() queryGroup { fixtureDoc("ancient.txt", withSize(10), withMtime("2020-01-01T00:00:00Z")), }, cases: []queryCase{ - {id: 1, query: `size>100`, want: []string{"big.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, - {id: 2, query: `size<100`, want: []string{"small.txt", "ancient.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 1, query: `size>100`, want: []string{"big.txt"}}, + {id: 2, query: `size<100`, want: []string{"small.txt", "ancient.txt"}}, {id: 3, query: `mtime>"2021-01-01T00:00:00Z"`, want: []string{"small.txt", "big.txt"}}, {id: 4, query: `mtime<"2021-01-01T00:00:00Z"`, want: []string{"ancient.txt"}}, {id: 5, query: `Mtime:"today"`, want: []string{"small.txt", "big.txt"}}, diff --git a/services/search/pkg/parity/query_stress_test.go b/services/search/pkg/parity/query_stress_test.go index a739800ad2..52682d59ed 100644 --- a/services/search/pkg/parity/query_stress_test.go +++ b/services/search/pkg/parity/query_stress_test.go @@ -18,21 +18,21 @@ func stressGroup() queryGroup { }, cases: []queryCase{ {id: 1, query: `name:"*report*" AND mediatype:document`, want: []string{"quarterly report.docx", "draft report.txt"}}, - {id: 2, query: `name:"*report*" AND NOT tag:("draft")`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{want: []string{"draft report.txt", "quarterly report.docx"}}}}, + {id: 2, query: `name:"*report*" AND NOT tag:("draft")`, want: []string{"quarterly report.docx"}}, {id: 3, query: `(tag:("final") OR tag:("draft")) AND mediatype:image`, want: []string{"photo.jpg"}}, {id: 4, query: `mediatype:document AND mtime>"2021-01-01T00:00:00Z"`, want: []string{"quarterly report.docx", "notes.md"}}, - {id: 5, query: `name:"*report*" AND size>100`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 5, query: `name:"*report*" AND size>100`, want: []string{"quarterly report.docx"}}, {id: 6, query: `tag:("final") AND NOT mediatype:folder`, want: []string{"quarterly report.docx", "photo.jpg"}}, - {id: 7, query: `hidden:true AND name:"*notes*"`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 8, query: `name:quarterly report`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, - {id: 9, query: `name:"quarterly report"`}, + {id: 7, query: `hidden:true AND name:"*notes*"`, want: []string{"notes.md"}}, + {id: 8, query: `name:quarterly report`, want: []string{"quarterly report.docx"}}, + {id: 9, query: `name:"quarterly report"`, want: []string{"quarterly report.docx"}}, {id: 10, query: `name:"quarterly report.docx"`, want: []string{"quarterly report.docx"}}, - {id: 11, query: `NOT tag:("draft")`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md", "archive"}, engineOverrides: map[string]override{"bleve": override{want: []string{"archive", "draft report.txt", "notes.md", "photo.jpg", "quarterly report.docx"}}}}, - {id: 12, query: `tag:("final") OR hidden:true`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md"}, engineOverrides: map[string]override{"bleve": override{want: []string{"photo.jpg", "quarterly report.docx"}}}}, - {id: 13, query: `(name:"*report*" OR name:"*notes*") AND NOT (tag:("draft") OR hidden:true)`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{want: []string{"notes.md", "quarterly report.docx"}}}}, + {id: 11, query: `NOT tag:("draft")`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md", "archive"}}, + {id: 12, query: `tag:("final") OR hidden:true`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md"}}, + {id: 13, query: `(name:"*report*" OR name:"*notes*") AND NOT (tag:("draft") OR hidden:true)`, want: []string{"quarterly report.docx"}}, {id: 14, query: `mediatype:image OR (mediatype:document AND tag:("draft"))`, want: []string{"photo.jpg", "draft report.txt"}}, - {id: 15, query: `NOT (mediatype:folder OR hidden:true)`, want: []string{"quarterly report.docx", "draft report.txt", "photo.jpg"}, engineOverrides: map[string]override{"bleve": override{want: []string{"draft report.txt", "notes.md", "photo.jpg", "quarterly report.docx"}}}}, - {id: 16, query: `name:"*report*" AND (size>100 OR tag:("draft"))`, want: []string{"quarterly report.docx", "draft report.txt"}, engineOverrides: map[string]override{"bleve": override{want: []string{"draft report.txt"}}, "opensearch": override{want: []string{"draft report.txt"}}}}, + {id: 15, query: `NOT (mediatype:folder OR hidden:true)`, want: []string{"quarterly report.docx", "draft report.txt", "photo.jpg"}}, + {id: 16, query: `name:"*report*" AND (size>100 OR tag:("draft"))`, want: []string{"quarterly report.docx", "draft report.txt"}}, }, } } diff --git a/services/search/pkg/parity/query_tags_test.go b/services/search/pkg/parity/query_tags_test.go index 74cc3bc716..c8505074d3 100644 --- a/services/search/pkg/parity/query_tags_test.go +++ b/services/search/pkg/parity/query_tags_test.go @@ -28,7 +28,7 @@ func tagsGroup() queryGroup { {id: 6, query: `tag:("spaced tag")`, want: []string{"spaced.txt"}}, {id: 7, query: `tag:("*paced ta*")`, want: []string{"spaced.txt"}}, {id: 8, query: `tag:("work")`, want: []string{"project"}}, - {id: 9, query: `tag:("` + longTag + `")`, want: []string{"longtag.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 9, query: `tag:("` + longTag + `")`, want: []string{"longtag.txt"}}, }, } } diff --git a/services/search/pkg/parity/query_title_test.go b/services/search/pkg/parity/query_title_test.go index 8251c668db..727f69ac55 100644 --- a/services/search/pkg/parity/query_title_test.go +++ b/services/search/pkg/parity/query_title_test.go @@ -12,12 +12,14 @@ func titleGroup() queryGroup { }, cases: []queryCase{ {id: 1, query: `Title:"quarterly report"`, want: []string{"q1.html"}}, - {id: 2, query: `Title:quarterly`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 3, query: `Title:QUARTERLY`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 2, query: `Title:quarterly`, want: []string{"q1.html"}}, + {id: 3, query: `Title:QUARTERLY`, want: []string{"q1.html"}}, {id: 4, query: `Title:quarterl*`, want: []string{"q1.html"}}, {id: 5, query: `Title:"*ly rep*"`, want: []string{"q1.html"}}, - {id: 6, query: `title:quarterly`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, - {id: 7, query: `Title:"QUARTERLY REPORT"`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 6, query: `title:quarterly`, want: []string{"q1.html"}}, + {id: 7, query: `Title:"QUARTERLY REPORT"`, want: []string{"q1.html"}}, + {id: 8, query: `Title="quarterly report"`, want: []string{"q1.html"}}, + {id: 9, query: `Title="quarterly"`}, }, } } diff --git a/services/search/pkg/parity/query_visibility_test.go b/services/search/pkg/parity/query_visibility_test.go index 12939126c2..5a3d879946 100644 --- a/services/search/pkg/parity/query_visibility_test.go +++ b/services/search/pkg/parity/query_visibility_test.go @@ -14,13 +14,13 @@ func visibilityGroup() queryGroup { fixtureDoc("secret.txt", withParent("1$1!.private"), withPath("./.private/secret.txt"), isHidden()), }, cases: []queryCase{ - {id: 1, query: `hidden:true`, want: []string{"dotfile.txt", ".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 2, query: `hidden:TRUE`, want: []string{"dotfile.txt", ".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, - {id: 3, query: `hidden:false`, want: []string{"visible.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 1, query: `hidden:true`, want: []string{"dotfile.txt", ".private", "secret.txt"}}, + {id: 2, query: `hidden:TRUE`, want: []string{"dotfile.txt", ".private", "secret.txt"}}, + {id: 3, query: `hidden:false`, want: []string{"visible.txt"}}, {id: 4, query: `name:"*secret*"`, want: []string{"secret.txt"}}, - {id: 5, query: `path:"./.private"`, want: []string{".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{want: []string{".private"}}}}, - {id: 6, query: `hidden:banana`, engineOverrides: map[string]override{"opensearch": override{want: []string{"error"}}}}, - {id: 7, query: `hidden:"true"`, want: []string{"dotfile.txt", ".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 5, query: `path:"./.private"`, want: []string{".private", "secret.txt"}}, + {id: 6, query: `hidden:banana`}, + {id: 7, query: `hidden:"true"`, want: []string{"dotfile.txt", ".private", "secret.txt"}}, }, } } diff --git a/services/search/pkg/parity/response_entity_test.go b/services/search/pkg/parity/response_entity_test.go index 29ecffa8df..c32ecbf5df 100644 --- a/services/search/pkg/parity/response_entity_test.go +++ b/services/search/pkg/parity/response_entity_test.go @@ -72,7 +72,7 @@ func entityGroup() responseGroup { }), }, { - id: 11, query: `path:"./parent"`, reads: "TotalMatches", want: []string{"2"}, engineOverrides: map[string]override{"bleve": override{want: []string{"1"}}}, + id: 11, query: `path:"./parent"`, reads: "TotalMatches", want: []string{"2"}, read: func(resp *searchService.SearchIndexResponse) []string { return []string{fmt.Sprint(resp.GetTotalMatches())} }, diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index 310397cae6..0a84fabfc4 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -2,23 +2,26 @@ package bleve import ( "fmt" + "slices" + "strconv" "strings" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/blevesearch/bleve/v2" bleveQuery "github.com/blevesearch/bleve/v2/search/query" "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/pkg/kql" ) -// lowercaseFields lists the bleve fields whose index mapping uses a -// lowercasing analyzer. Values bound to these fields are pre-lowercased -// so query-side matching stays consistent with the index. -// Keep in sync with services/search/pkg/bleve/index.go NewMapping. var lowercaseFields = map[string]struct{}{ "Name": {}, + "Title": {}, "Tags": {}, "Favorites": {}, "Content": {}, + "MimeType": {}, + "Hidden": {}, } var _fields = map[string]string{ @@ -33,6 +36,7 @@ var _fields = map[string]string{ "tag": "Tags", "tags": "Tags", "content": "Content", + "title": "Title", "hidden": "Hidden", "favorite": "Favorites", } @@ -53,7 +57,6 @@ var bleveEscaper = strings.NewReplacer( `)`, `\)`, `{`, `\{`, `}`, `\}`, - `{`, `\}`, `[`, `\[`, `]`, `\]`, `^`, `\^`, @@ -106,14 +109,43 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { v = strings.ToLower(v) } + if k == "Type" { + v = resourceType(v) + } + var q bleveQuery.Query var group bool - switch k { - case "MimeType": + switch { + case k == "Hidden": + value, err := strconv.ParseBool(v) + if err != nil { + q = bleveQuery.NewMatchNoneQuery() + break + } + + bq := bleveQuery.NewBoolFieldQuery(value) + bq.SetField(k) + q = bq + case k == "MimeType": q, group = mimeType(k, v) if prev == nil { isGroup = group } + case slices.Contains([]string{"Name", "Title"}, k) && strings.ContainsAny(n.Value, "*?"): + patterns := []bleveQuery.Query{bleveQuery.NewQueryStringQuery(k + ".wildcard:" + v)} + if !strings.HasSuffix(v, "*") { + patterns = append(patterns, bleveQuery.NewQueryStringQuery(k+".wildcard:"+v+".*")) + } + + q = closed(bleveQuery.NewDisjunctionQuery(patterns)) + case n.Exact && !strings.ContainsAny(n.Value, "*?") && slices.Contains([]string{"Name", "Title"}, k): + q = bleveQuery.NewQueryStringQuery(k + ".wildcard:" + v) + case k == "Path" && !strings.ContainsAny(n.Value, "*?"): + q = pathAndBelow(k, n.Value) + case slices.Contains([]string{"Name", "Title", "Content"}, k) && !strings.ContainsAny(n.Value, "*?"): + q = phrase(k, n.Value) + case strings.Contains(n.Value, " ") && !strings.ContainsAny(n.Value, "*?"): + q = phrase(k, n.Value) default: q = bleveQuery.NewQueryStringQuery(k + ":" + v) } @@ -153,13 +185,25 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { continue } + if prev == nil { + prev = q + } else { + next = q + } + case *ast.NumberNode: + q := numberRange(getField(n.Key), n.Operator, n.Value) + if q == nil { + continue + } + if prev == nil { prev = q } else { next = q } case *ast.BooleanNode: - q := bleveQuery.NewQueryStringQuery(getField(n.Key) + fmt.Sprintf(":%v", n.Value)) + q := bleveQuery.NewBoolFieldQuery(n.Value) + q.SetField(getField(n.Key)) if prev == nil { prev = q } else { @@ -216,6 +260,10 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { func nextNode(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { if n, ok := nodes[offset].(*ast.GroupNode); ok { + if n.Key != "" { + n = normalizeGroupingProperty(n) + } + gq, _, err := walk(0, n.Nodes) if err != nil { return nil, 0, err @@ -277,6 +325,57 @@ func mapBinary(operator *ast.OperatorNode, ln, rn bleveQuery.Query, leftIsGroup }) } +func numberRange(field string, operator *ast.OperatorNode, value float64) bleveQuery.Query { + if operator == nil { + return nil + } + + inclusive, exclusive := true, false + + var q *bleveQuery.NumericRangeQuery + switch operator.Value { + case ">": + q = bleveQuery.NewNumericRangeInclusiveQuery(&value, nil, &exclusive, nil) + case ">=": + q = bleveQuery.NewNumericRangeInclusiveQuery(&value, nil, &inclusive, nil) + case "<": + q = bleveQuery.NewNumericRangeInclusiveQuery(nil, &value, nil, &exclusive) + case "<=": + q = bleveQuery.NewNumericRangeInclusiveQuery(nil, &value, nil, &inclusive) + default: + return nil + } + + q.SetField(field) + + return q +} + +func pathAndBelow(field, path string) bleveQuery.Query { + path = strings.TrimSuffix(path, "/") + + self := bleveQuery.NewTermQuery(path) + self.SetField(field) + + below := bleveQuery.NewPrefixQuery(path + "/") + below.SetField(field) + + return closed(bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{self, below})) +} + +func closed(q bleveQuery.Query) bleveQuery.Query { + // a bare disjunction reads as an open OR chain to mapBinary, a later OR + // would merge into it and widen the group + return bleveQuery.NewConjunctionQuery([]bleveQuery.Query{q}) +} + +func phrase(field, value string) bleveQuery.Query { + q := bleveQuery.NewMatchPhraseQuery(value) + q.SetField(field) + + return q +} + func getField(name string) string { if name == "" { return "Name" @@ -296,6 +395,17 @@ func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode { return group } +func resourceType(value string) string { + switch strings.ToLower(value) { + case "file": + return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10) + case "folder": + return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10) + default: + return value + } +} + func mimeType(k, v string) (bleveQuery.Query, bool) { switch v { case "file": @@ -321,7 +431,6 @@ func mimeType(k, v string) (bleveQuery.Query, bool) { "application/vnd.oasis.opendocument.spreadsheet", "text/csv", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "application/vnd.oasis.opendocument.spreadsheet", "application/vnd.apple.numbers", )), true case "presentation": diff --git a/services/search/pkg/query/bleve/compiler_test.go b/services/search/pkg/query/bleve/compiler_test.go index ff94e88726..30c283f694 100644 --- a/services/search/pkg/query/bleve/compiler_test.go +++ b/services/search/pkg/query/bleve/compiler_test.go @@ -1,6 +1,7 @@ package bleve import ( + "strings" "testing" "time" @@ -18,6 +19,29 @@ var timeMustParse = func(t *testing.T, ts string) time.Time { return tp } +func wildcardQuery(field, value string) query.Query { + patterns := []query.Query{query.NewQueryStringQuery(field + ".wildcard:" + value)} + if !strings.HasSuffix(value, "*") { + patterns = append(patterns, query.NewQueryStringQuery(field+".wildcard:"+value+".*")) + } + + return query.NewConjunctionQuery([]query.Query{query.NewDisjunctionQuery(patterns)}) +} + +func phraseQuery(field, value string) query.Query { + q := query.NewMatchPhraseQuery(value) + q.SetField(field) + + return q +} + +func boolFieldQuery(field string, value bool) query.Query { + q := query.NewBoolFieldQuery(value) + q.SetField(field) + + return q +} + func Test_compile(t *testing.T) { tests := []struct { name string @@ -33,7 +57,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:federated`), + phraseQuery("Name", "federated"), }), wantErr: false, }, @@ -45,7 +69,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith`), + phraseQuery("Name", "John Smith"), }), wantErr: false, }, @@ -59,8 +83,8 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith`), - query.NewQueryStringQuery(`Name:jane`), + phraseQuery("Name", "John Smith"), + phraseQuery("Name", "Jane"), }), wantErr: false, }, @@ -91,7 +115,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:moby\ di*`), + wildcardQuery("Name", `moby\ di*`), query.NewConjunctionQuery([]query.Query{ query.NewQueryStringQuery(`Tags:bestseller`), query.NewQueryStringQuery(`Tags:book`), @@ -112,10 +136,10 @@ func Test_compile(t *testing.T) { }, want: query.NewDisjunctionQuery([]query.Query{ query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:a`), - query.NewQueryStringQuery(`Name:b`), + phraseQuery("Name", "a"), + phraseQuery("Name", "b"), }), - query.NewQueryStringQuery(`Name:c`), + phraseQuery("Name", "c"), }), wantErr: false, }, @@ -131,10 +155,10 @@ func Test_compile(t *testing.T) { }, }, want: query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:a`), + phraseQuery("Name", "a"), query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:b`), - query.NewQueryStringQuery(`Name:c`), + phraseQuery("Name", "b"), + phraseQuery("Name", "c"), }), }), wantErr: false, @@ -156,11 +180,11 @@ func Test_compile(t *testing.T) { }, want: query.NewConjunctionQuery([]query.Query{ query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:a`), - query.NewQueryStringQuery(`Name:b`), - query.NewQueryStringQuery(`Name:c`), + phraseQuery("Name", "a"), + phraseQuery("Name", "b"), + phraseQuery("Name", "c"), }), - query.NewQueryStringQuery(`Name:d`), + phraseQuery("Name", "d"), }), wantErr: false, }, @@ -179,7 +203,7 @@ func Test_compile(t *testing.T) { }, want: query.NewConjunctionQuery([]query.Query{ query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:moby\ di*`), + wildcardQuery("Name", `moby\ di*`), query.NewQueryStringQuery(`Tags:bestseller`), }), query.NewQueryStringQuery(`Tags:book`), @@ -204,7 +228,7 @@ func Test_compile(t *testing.T) { }, want: query.NewConjunctionQuery([]query.Query{ query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:moby\ di*`), + wildcardQuery("Name", `moby\ di*`), query.NewQueryStringQuery(`Tags:bestseller`), }), query.NewQueryStringQuery(`Tags:book`), @@ -227,7 +251,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`author:John\ Smith`), + phraseQuery("author", "John Smith"), query.NewQueryStringQuery(`author:Jane`), }), wantErr: false, @@ -249,7 +273,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`author:John\ Smith`), + phraseQuery("author", "John Smith"), query.NewQueryStringQuery(`author:Jane`), query.NewQueryStringQuery(`Tags:bestseller`), }), @@ -293,9 +317,44 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith`), - query.NewQueryStringQuery(`Hidden:T`), - query.NewQueryStringQuery(`Hidden:T`), + phraseQuery("Name", "John Smith"), + boolFieldQuery("Hidden", true), + boolFieldQuery("Hidden", true), + }), + wantErr: false, + }, + { + name: `hidden:banana`, + args: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "hidden", Value: "banana"}, + }, + }, + want: query.NewConjunctionQuery([]query.Query{query.NewMatchNoneQuery()}), + wantErr: false, + }, + { + name: `name="Report.txt"`, + args: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "name", Value: "Report.txt", Exact: true}, + }, + }, + want: query.NewConjunctionQuery([]query.Query{query.NewQueryStringQuery(`Name.wildcard:report.txt`)}), + wantErr: false, + }, + { + name: `type:File`, + args: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "type", Value: "File"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "type", Value: "FOLDER"}, + }, + }, + want: query.NewDisjunctionQuery([]query.Query{ + query.NewQueryStringQuery(`Type:1`), + query.NewQueryStringQuery(`Type:2`), }), wantErr: false, }, @@ -424,7 +483,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/rtf`), query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), }), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), wantErr: false, }, @@ -450,7 +509,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), query.NewConjunctionQuery([]query.Query{ query.NewQueryStringQuery(`MimeType:application/pdf`), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), }), wantErr: false, @@ -480,7 +539,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), query.NewQueryStringQuery(`MimeType:application/pdf`), }), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), wantErr: false, }, @@ -509,7 +568,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/rtf`), query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), }), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), wantErr: false, }, @@ -521,7 +580,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith\ \+\-\=\&\|\>\<\!\(\)\{\}\[\]\^\"\~\:\ `), + phraseQuery("Name", "John Smith +-=&|>quarterly reportsome data" to "q1.html" + And user "Alice" has uploaded file with content "notessome data" to "q2.html" + When user "Alice" searches for 'Title:"quarterly report"' using the WebDAV API + Then the HTTP status code should be "207" + And the search result of user "Alice" should contain only these entries: + | q1.html | diff --git a/vendor/github.com/blevesearch/bleve/v2/analysis/char/regexp/regexp.go b/vendor/github.com/blevesearch/bleve/v2/analysis/char/regexp/regexp.go new file mode 100644 index 0000000000..a94236af9a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/char/regexp/regexp.go @@ -0,0 +1,65 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package regexp + +import ( + "fmt" + "regexp" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "regexp" + +type CharFilter struct { + r *regexp.Regexp + replacement []byte +} + +func New(r *regexp.Regexp, replacement []byte) *CharFilter { + return &CharFilter{ + r: r, + replacement: replacement, + } +} + +func (s *CharFilter) Filter(input []byte) []byte { + return s.r.ReplaceAll(input, s.replacement) +} + +func CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) { + regexpStr, ok := config["regexp"].(string) + if !ok { + return nil, fmt.Errorf("must specify regexp") + } + r, err := regexp.Compile(regexpStr) + if err != nil { + return nil, fmt.Errorf("unable to build regexp char filter: %v", err) + } + replaceBytes := []byte(" ") + replaceStr, ok := config["replace"].(string) + if ok { + replaceBytes = []byte(replaceStr) + } + return New(r, replaceBytes), nil +} + +func init() { + err := registry.RegisterCharFilter(Name, CharFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 972a95dca7..7cef9150b7 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -126,6 +126,7 @@ github.com/blevesearch/bleve/v2/analysis/analyzer/custom github.com/blevesearch/bleve/v2/analysis/analyzer/keyword github.com/blevesearch/bleve/v2/analysis/analyzer/standard github.com/blevesearch/bleve/v2/analysis/datetime/flexible +github.com/blevesearch/bleve/v2/analysis/char/regexp github.com/blevesearch/bleve/v2/analysis/datetime/optional github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds