From bd4287762ec3b7715582e5c6378bc72a09d726e7 Mon Sep 17 00:00:00 2001 From: Florian Schade Date: Mon, 31 Aug 2026 10:44:03 +0200 Subject: [PATCH] fix(search): answer the same on both engines both engines now agree on names, titles, tags, paths, types, sizes, dates, hidden flags, facet values and wildcards. quotes only delimit phrases and the equals operator matches the whole field value, following the kql spec. the index name carries a generation so a changed mapping starts on a fresh index, MIGRATION.md says how to fill it. --- pkg/ast/ast.go | 9 + pkg/ast/test/test.go | 1 + pkg/kql/cast.go | 10 + pkg/kql/dictionary.peg | 27 +- pkg/kql/dictionary_gen.go | 1411 +++++++++++------ pkg/kql/dictionary_test.go | 70 + pkg/kql/engine_suite_test.go | 11 - pkg/kql/factory.go | 33 +- pkg/kql/kql.go | 6 + services/search/MIGRATION.md | 55 + services/search/README.md | 6 + .../search/internal/opensearchtest/helper.go | 14 - services/search/internal/opensearchtest/os.go | 97 -- .../search/internal/opensearchtest/suite.go | 14 +- .../internal/opensearchtest/testdata.go | 8 +- services/search/pkg/bleve/index.go | 71 +- services/search/pkg/bleve/index_test.go | 39 + services/search/pkg/content/content.go | 2 +- services/search/pkg/content/tika.go | 8 +- services/search/pkg/content/tika_test.go | 22 + services/search/pkg/opensearch/backend.go | 7 +- .../search/pkg/opensearch/backend_test.go | 2 - services/search/pkg/opensearch/index.go | 87 +- services/search/pkg/opensearch/index_test.go | 28 + .../opensearch/internal/convert/kql_expand.go | 23 +- .../internal/convert/kql_expand_test.go | 60 +- .../internal/convert/kql_transpile.go | 80 +- .../internal/convert/kql_transpile_test.go | 68 +- .../internal/convert/opensearch_test.go | 2 +- .../internal/indexes/resource_v3.json | 122 ++ .../internal/osu/query_match_none.go | 55 + .../internal/osu/query_term_level_range.go | 6 +- .../opensearch/internal/osu/request_test.go | 2 +- .../search/pkg/opensearch/opensearch_test.go | 2 +- services/search/pkg/query/bleve/compiler.go | 127 +- .../search/pkg/query/bleve/compiler_test.go | 113 +- .../apiSearchContent/titleSearch.feature | 19 + .../bleve/v2/analysis/char/regexp/regexp.go | 65 + vendor/modules.txt | 1 + 39 files changed, 2040 insertions(+), 743 deletions(-) delete mode 100644 pkg/kql/engine_suite_test.go create mode 100644 services/search/MIGRATION.md create mode 100644 services/search/pkg/bleve/index_test.go create mode 100644 services/search/pkg/opensearch/internal/indexes/resource_v3.json create mode 100644 services/search/pkg/opensearch/internal/osu/query_match_none.go create mode 100644 tests/acceptance/features/apiSearchContent/titleSearch.feature create mode 100644 vendor/github.com/blevesearch/bleve/v2/analysis/char/regexp/regexp.go diff --git a/pkg/ast/ast.go b/pkg/ast/ast.go index 9aafe853aa..f1a7e3263e 100644 --- a/pkg/ast/ast.go +++ b/pkg/ast/ast.go @@ -43,6 +43,7 @@ type StringNode struct { *Base Key string Value string + Exact bool } // BooleanNode represents a bool value @@ -60,6 +61,14 @@ type DateTimeNode struct { Value time.Time } +// NumberNode represents a numeric value +type NumberNode struct { + *Base + Key string + Operator *OperatorNode + Value float64 +} + // OperatorNode represents an operator value like // AND, OR, NOT, =, <= ... and so on type OperatorNode struct { diff --git a/pkg/ast/test/test.go b/pkg/ast/test/test.go index d7fb985226..8a39b75022 100644 --- a/pkg/ast/test/test.go +++ b/pkg/ast/test/test.go @@ -21,6 +21,7 @@ func DiffAst(x, y any, opts ...cmp.Option) string { cmpopts.IgnoreFields(ast.GroupNode{}, "Base"), cmpopts.IgnoreFields(ast.BooleanNode{}, "Base"), cmpopts.IgnoreFields(ast.DateTimeNode{}, "Base"), + cmpopts.IgnoreFields(ast.NumberNode{}, "Base"), )..., ) } diff --git a/pkg/kql/cast.go b/pkg/kql/cast.go index c1fb185623..ecd0cac27f 100644 --- a/pkg/kql/cast.go +++ b/pkg/kql/cast.go @@ -2,6 +2,7 @@ package kql import ( "fmt" + "strconv" "time" "github.com/jinzhu/now" @@ -136,3 +137,12 @@ func toTimeRange(in any) (*time.Time, *time.Time, error) { return &from, &to, nil } + +func toFloat(v any) (float64, error) { + value, err := toString(v) + if err != nil { + return 0, err + } + + return strconv.ParseFloat(value, 64) +} diff --git a/pkg/kql/dictionary.peg b/pkg/kql/dictionary.peg index 5f59e78afb..c21f46f4c4 100644 --- a/pkg/kql/dictionary.peg +++ b/pkg/kql/dictionary.peg @@ -40,6 +40,7 @@ GroupNode <- PropertyRestrictionNodes <- YesNoPropertyRestrictionNode / DateTimeRestrictionNode / + NumberRestrictionNode / TextPropertyRestrictionNode YesNoPropertyRestrictionNode <- @@ -69,9 +70,22 @@ DateTimeRestrictionNode <- return buildNaturalLanguageDateTimeNodes(k, v, c.text, c.pos) } +NumberRestrictionNode <- + k:Key o:( + OperatorGreaterOrEqualNode / + OperatorLessOrEqualNode / + OperatorGreaterNode / + OperatorLessNode + ) '"'? v:Number '"'? { + return buildNumberNode(k, o, v, c.text, c.pos) + } + TextPropertyRestrictionNode <- - k:Key (OperatorColonNode / OperatorEqualNode) v:(String / [^ ()]+){ - return buildStringNode(k, v, c.text, c.pos) + k:Key OperatorEqualNode v:(String / [^ ()]+) { + return buildStringNode(k, v, true, c.text, c.pos) + } / + k:Key OperatorColonNode v:(String / [^ ()]+) { + return buildStringNode(k, v, false, c.text, c.pos) } //////////////////////////////////////////////////////// @@ -84,12 +98,12 @@ FreeTextKeywordNodes <- PhraseNode <- OperatorColonNode? _ v:String _ OperatorColonNode? { - return buildStringNode("", v, c.text, c.pos) + return buildStringNode("", v, false, c.text, c.pos) } WordNode <- OperatorColonNode? _ v:[^ :()]+ _ OperatorColonNode? { - return buildStringNode("", v, c.text, c.pos) + return buildStringNode("", v, false, c.text, c.pos) } //////////////////////////////////////////////////////// @@ -229,6 +243,11 @@ String <- return v, nil } +Number <- + [0-9]+ ("." [0-9]+)? { + return string(c.text), nil + } + Digit <- [0-9] { return c.text, nil diff --git a/pkg/kql/dictionary_gen.go b/pkg/kql/dictionary_gen.go index aec6f49d1d..9777ece435 100644 --- a/pkg/kql/dictionary_gen.go +++ b/pkg/kql/dictionary_gen.go @@ -43,12 +43,12 @@ var g = &grammar{ pos: position{line: 19, col: 6, offset: 351}, exprs: []any{ &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, run: (*parser).callonNodes3, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -75,27 +75,27 @@ var g = &grammar{ name: "GroupNode", }, &actionExpr{ - pos: position{line: 46, col: 5, offset: 1038}, + pos: position{line: 47, col: 5, offset: 1066}, run: (*parser).callonNode3, expr: &seqExpr{ - pos: position{line: 46, col: 5, offset: 1038}, + pos: position{line: 47, col: 5, offset: 1066}, exprs: []any{ &labeledExpr{ - pos: position{line: 46, col: 5, offset: 1038}, + pos: position{line: 47, col: 5, offset: 1066}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode6, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode9, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -104,23 +104,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode15, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -136,23 +136,23 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 46, col: 12, offset: 1045}, + pos: position{line: 47, col: 12, offset: 1073}, alternatives: []any{ &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonNode18, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", }, }, &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonNode20, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", @@ -161,19 +161,19 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 46, col: 51, offset: 1084}, + pos: position{line: 47, col: 51, offset: 1112}, label: "v", expr: &choiceExpr{ - pos: position{line: 46, col: 54, offset: 1087}, + pos: position{line: 47, col: 54, offset: 1115}, alternatives: []any{ &litMatcher{ - pos: position{line: 46, col: 54, offset: 1087}, + pos: position{line: 47, col: 54, offset: 1115}, val: "true", ignoreCase: false, want: "\"true\"", }, &litMatcher{ - pos: position{line: 46, col: 63, offset: 1096}, + pos: position{line: 47, col: 63, offset: 1124}, val: "false", ignoreCase: false, want: "\"false\"", @@ -185,27 +185,27 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 51, col: 5, offset: 1197}, + pos: position{line: 52, col: 5, offset: 1225}, run: (*parser).callonNode26, expr: &seqExpr{ - pos: position{line: 51, col: 5, offset: 1197}, + pos: position{line: 52, col: 5, offset: 1225}, exprs: []any{ &labeledExpr{ - pos: position{line: 51, col: 5, offset: 1197}, + pos: position{line: 52, col: 5, offset: 1225}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode29, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode32, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -214,23 +214,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode38, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -246,66 +246,66 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 51, col: 11, offset: 1203}, + pos: position{line: 52, col: 11, offset: 1231}, label: "o", expr: &choiceExpr{ - pos: position{line: 52, col: 9, offset: 1215}, + pos: position{line: 53, col: 9, offset: 1243}, alternatives: []any{ &actionExpr{ - pos: position{line: 145, col: 5, offset: 3346}, + pos: position{line: 159, col: 5, offset: 3745}, run: (*parser).callonNode42, expr: &litMatcher{ - pos: position{line: 145, col: 5, offset: 3346}, + pos: position{line: 159, col: 5, offset: 3745}, val: ">=", ignoreCase: false, want: "\">=\"", }, }, &actionExpr{ - pos: position{line: 135, col: 5, offset: 3162}, + pos: position{line: 149, col: 5, offset: 3561}, run: (*parser).callonNode44, expr: &litMatcher{ - pos: position{line: 135, col: 5, offset: 3162}, + pos: position{line: 149, col: 5, offset: 3561}, val: "<=", ignoreCase: false, want: "\"<=\"", }, }, &actionExpr{ - pos: position{line: 140, col: 5, offset: 3251}, + pos: position{line: 154, col: 5, offset: 3650}, run: (*parser).callonNode46, expr: &litMatcher{ - pos: position{line: 140, col: 5, offset: 3251}, + pos: position{line: 154, col: 5, offset: 3650}, val: ">", ignoreCase: false, want: "\">\"", }, }, &actionExpr{ - pos: position{line: 130, col: 5, offset: 3070}, + pos: position{line: 144, col: 5, offset: 3469}, run: (*parser).callonNode48, expr: &litMatcher{ - pos: position{line: 130, col: 5, offset: 3070}, + pos: position{line: 144, col: 5, offset: 3469}, val: "<", ignoreCase: false, want: "\"<\"", }, }, &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonNode50, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", }, }, &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonNode52, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -315,43 +315,43 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 58, col: 7, offset: 1395}, + pos: position{line: 59, col: 7, offset: 1423}, expr: &litMatcher{ - pos: position{line: 58, col: 7, offset: 1395}, + pos: position{line: 59, col: 7, offset: 1423}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, }, &labeledExpr{ - pos: position{line: 58, col: 12, offset: 1400}, + pos: position{line: 59, col: 12, offset: 1428}, label: "v", expr: &choiceExpr{ - pos: position{line: 59, col: 9, offset: 1412}, + pos: position{line: 60, col: 9, offset: 1440}, alternatives: []any{ &actionExpr{ - pos: position{line: 195, col: 5, offset: 4185}, + pos: position{line: 209, col: 5, offset: 4584}, run: (*parser).callonNode58, expr: &seqExpr{ - pos: position{line: 195, col: 5, offset: 4185}, + pos: position{line: 209, col: 5, offset: 4584}, exprs: []any{ &actionExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, run: (*parser).callonNode60, expr: &seqExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, exprs: []any{ &actionExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, run: (*parser).callonNode62, expr: &seqExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode64, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -359,10 +359,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode66, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -370,10 +370,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode68, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -381,10 +381,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode70, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -395,22 +395,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 14, offset: 3957}, + pos: position{line: 199, col: 14, offset: 4356}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, run: (*parser).callonNode73, expr: &seqExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode75, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -418,10 +418,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode77, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -432,22 +432,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 28, offset: 3971}, + pos: position{line: 199, col: 28, offset: 4370}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, run: (*parser).callonNode80, expr: &seqExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode82, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -455,10 +455,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode84, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -472,28 +472,28 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 195, col: 14, offset: 4194}, + pos: position{line: 209, col: 14, offset: 4593}, val: "T", ignoreCase: false, want: "\"T\"", }, &actionExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, run: (*parser).callonNode87, expr: &seqExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, exprs: []any{ &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode89, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode91, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -501,10 +501,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode93, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -515,22 +515,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 14, offset: 4044}, + pos: position{line: 204, col: 14, offset: 4443}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode96, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode98, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -538,10 +538,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode100, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -552,22 +552,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 29, offset: 4059}, + pos: position{line: 204, col: 29, offset: 4458}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, run: (*parser).callonNode103, expr: &seqExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode105, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -575,10 +575,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode107, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -589,23 +589,23 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 190, col: 44, offset: 4074}, + pos: position{line: 204, col: 44, offset: 4473}, expr: &seqExpr{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, exprs: []any{ &litMatcher{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 190, col: 49, offset: 4079}, + pos: position{line: 204, col: 49, offset: 4478}, expr: &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode113, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -617,35 +617,35 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, alternatives: []any{ &litMatcher{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, val: "Z", ignoreCase: false, want: "\"Z\"", }, &seqExpr{ - pos: position{line: 190, col: 65, offset: 4095}, + pos: position{line: 204, col: 65, offset: 4494}, exprs: []any{ &charClassMatcher{ - pos: position{line: 190, col: 66, offset: 4096}, + pos: position{line: 204, col: 66, offset: 4495}, val: "[+-]", chars: []rune{'+', '-'}, ignoreCase: false, inverted: false, }, &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode119, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode121, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -653,10 +653,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode123, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -667,22 +667,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 86, offset: 4116}, + pos: position{line: 204, col: 86, offset: 4515}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode126, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode128, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -690,10 +690,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode130, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -714,22 +714,22 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, run: (*parser).callonNode132, expr: &seqExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, exprs: []any{ &actionExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, run: (*parser).callonNode134, expr: &seqExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode136, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -737,10 +737,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode138, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -748,10 +748,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode140, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -759,10 +759,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode142, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -773,22 +773,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 14, offset: 3957}, + pos: position{line: 199, col: 14, offset: 4356}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, run: (*parser).callonNode145, expr: &seqExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode147, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -796,10 +796,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode149, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -810,22 +810,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 28, offset: 3971}, + pos: position{line: 199, col: 28, offset: 4370}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, run: (*parser).callonNode152, expr: &seqExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode154, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -833,10 +833,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode156, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -850,22 +850,22 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, run: (*parser).callonNode158, expr: &seqExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, exprs: []any{ &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode160, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode162, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -873,10 +873,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode164, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -887,22 +887,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 14, offset: 4044}, + pos: position{line: 204, col: 14, offset: 4443}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode167, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode169, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -910,10 +910,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode171, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -924,22 +924,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 29, offset: 4059}, + pos: position{line: 204, col: 29, offset: 4458}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, run: (*parser).callonNode174, expr: &seqExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode176, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -947,10 +947,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode178, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -961,23 +961,23 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 190, col: 44, offset: 4074}, + pos: position{line: 204, col: 44, offset: 4473}, expr: &seqExpr{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, exprs: []any{ &litMatcher{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 190, col: 49, offset: 4079}, + pos: position{line: 204, col: 49, offset: 4478}, expr: &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode184, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -989,35 +989,35 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, alternatives: []any{ &litMatcher{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, val: "Z", ignoreCase: false, want: "\"Z\"", }, &seqExpr{ - pos: position{line: 190, col: 65, offset: 4095}, + pos: position{line: 204, col: 65, offset: 4494}, exprs: []any{ &charClassMatcher{ - pos: position{line: 190, col: 66, offset: 4096}, + pos: position{line: 204, col: 66, offset: 4495}, val: "[+-]", chars: []rune{'+', '-'}, ignoreCase: false, inverted: false, }, &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode190, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode192, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1025,10 +1025,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode194, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1039,22 +1039,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 86, offset: 4116}, + pos: position{line: 204, col: 86, offset: 4515}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode197, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode199, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1062,10 +1062,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode201, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1086,9 +1086,9 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 62, col: 7, offset: 1465}, + pos: position{line: 63, col: 7, offset: 1493}, expr: &litMatcher{ - pos: position{line: 62, col: 7, offset: 1465}, + pos: position{line: 63, col: 7, offset: 1493}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1098,27 +1098,27 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 65, col: 5, offset: 1541}, + pos: position{line: 66, col: 5, offset: 1569}, run: (*parser).callonNode205, expr: &seqExpr{ - pos: position{line: 65, col: 5, offset: 1541}, + pos: position{line: 66, col: 5, offset: 1569}, exprs: []any{ &labeledExpr{ - pos: position{line: 65, col: 5, offset: 1541}, + pos: position{line: 66, col: 5, offset: 1569}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode208, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode211, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1127,23 +1127,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode217, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1159,23 +1159,23 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 66, col: 9, offset: 1557}, + pos: position{line: 67, col: 9, offset: 1585}, alternatives: []any{ &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonNode220, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", }, }, &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonNode222, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1184,79 +1184,79 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 68, col: 7, offset: 1609}, + pos: position{line: 69, col: 7, offset: 1637}, expr: &litMatcher{ - pos: position{line: 68, col: 7, offset: 1609}, + pos: position{line: 69, col: 7, offset: 1637}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, }, &labeledExpr{ - pos: position{line: 68, col: 12, offset: 1614}, + pos: position{line: 69, col: 12, offset: 1642}, label: "v", expr: &choiceExpr{ - pos: position{line: 200, col: 5, offset: 4273}, + pos: position{line: 214, col: 5, offset: 4672}, alternatives: []any{ &litMatcher{ - pos: position{line: 200, col: 5, offset: 4273}, + pos: position{line: 214, col: 5, offset: 4672}, val: "today", ignoreCase: false, want: "\"today\"", }, &litMatcher{ - pos: position{line: 201, col: 5, offset: 4287}, + pos: position{line: 215, col: 5, offset: 4686}, val: "yesterday", ignoreCase: false, want: "\"yesterday\"", }, &litMatcher{ - pos: position{line: 202, col: 5, offset: 4305}, + pos: position{line: 216, col: 5, offset: 4704}, val: "this week", ignoreCase: false, want: "\"this week\"", }, &litMatcher{ - pos: position{line: 203, col: 5, offset: 4323}, + pos: position{line: 217, col: 5, offset: 4722}, val: "last week", ignoreCase: false, want: "\"last week\"", }, &litMatcher{ - pos: position{line: 204, col: 5, offset: 4341}, + pos: position{line: 218, col: 5, offset: 4740}, val: "last 7 days", ignoreCase: false, want: "\"last 7 days\"", }, &litMatcher{ - pos: position{line: 205, col: 5, offset: 4361}, + pos: position{line: 219, col: 5, offset: 4760}, val: "this month", ignoreCase: false, want: "\"this month\"", }, &litMatcher{ - pos: position{line: 206, col: 5, offset: 4380}, + pos: position{line: 220, col: 5, offset: 4779}, val: "last month", ignoreCase: false, want: "\"last month\"", }, &litMatcher{ - pos: position{line: 207, col: 5, offset: 4399}, + pos: position{line: 221, col: 5, offset: 4798}, val: "last 30 days", ignoreCase: false, want: "\"last 30 days\"", }, &litMatcher{ - pos: position{line: 208, col: 5, offset: 4420}, + pos: position{line: 222, col: 5, offset: 4819}, val: "this year", ignoreCase: false, want: "\"this year\"", }, &actionExpr{ - pos: position{line: 209, col: 5, offset: 4438}, + pos: position{line: 223, col: 5, offset: 4837}, run: (*parser).callonNode237, expr: &litMatcher{ - pos: position{line: 209, col: 5, offset: 4438}, + pos: position{line: 223, col: 5, offset: 4837}, val: "last year", ignoreCase: false, want: "\"last year\"", @@ -1266,9 +1266,9 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 68, col: 38, offset: 1640}, + pos: position{line: 69, col: 38, offset: 1668}, expr: &litMatcher{ - pos: position{line: 68, col: 38, offset: 1640}, + pos: position{line: 69, col: 38, offset: 1668}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1278,27 +1278,27 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 73, col: 5, offset: 1759}, + pos: position{line: 74, col: 5, offset: 1781}, run: (*parser).callonNode241, expr: &seqExpr{ - pos: position{line: 73, col: 5, offset: 1759}, + pos: position{line: 74, col: 5, offset: 1781}, exprs: []any{ &labeledExpr{ - pos: position{line: 73, col: 5, offset: 1759}, + pos: position{line: 74, col: 5, offset: 1781}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode244, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode247, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1307,23 +1307,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode253, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1338,56 +1338,219 @@ var g = &grammar{ }, }, }, - &choiceExpr{ - pos: position{line: 73, col: 12, offset: 1766}, - alternatives: []any{ - &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode256, - expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, - val: ":", - ignoreCase: false, - want: "\":\"", + &labeledExpr{ + pos: position{line: 74, col: 11, offset: 1787}, + label: "o", + expr: &choiceExpr{ + pos: position{line: 75, col: 9, offset: 1799}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 159, col: 5, offset: 3745}, + run: (*parser).callonNode257, + expr: &litMatcher{ + pos: position{line: 159, col: 5, offset: 3745}, + val: ">=", + ignoreCase: false, + want: "\">=\"", + }, }, - }, - &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, - run: (*parser).callonNode258, - expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, - val: "=", - ignoreCase: false, - want: "\"=\"", + &actionExpr{ + pos: position{line: 149, col: 5, offset: 3561}, + run: (*parser).callonNode259, + expr: &litMatcher{ + pos: position{line: 149, col: 5, offset: 3561}, + val: "<=", + ignoreCase: false, + want: "\"<=\"", + }, + }, + &actionExpr{ + pos: position{line: 154, col: 5, offset: 3650}, + run: (*parser).callonNode261, + expr: &litMatcher{ + pos: position{line: 154, col: 5, offset: 3650}, + val: ">", + ignoreCase: false, + want: "\">\"", + }, + }, + &actionExpr{ + pos: position{line: 144, col: 5, offset: 3469}, + run: (*parser).callonNode263, + expr: &litMatcher{ + pos: position{line: 144, col: 5, offset: 3469}, + val: "<", + ignoreCase: false, + want: "\"<\"", + }, }, }, }, }, + &zeroOrOneExpr{ + pos: position{line: 79, col: 7, offset: 1923}, + expr: &litMatcher{ + pos: position{line: 79, col: 7, offset: 1923}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + }, &labeledExpr{ - pos: position{line: 73, col: 51, offset: 1805}, + pos: position{line: 79, col: 12, offset: 1928}, + label: "v", + expr: &actionExpr{ + pos: position{line: 247, col: 5, offset: 5207}, + run: (*parser).callonNode268, + expr: &seqExpr{ + pos: position{line: 247, col: 5, offset: 5207}, + exprs: []any{ + &oneOrMoreExpr{ + pos: position{line: 247, col: 5, offset: 5207}, + expr: &charClassMatcher{ + pos: position{line: 247, col: 5, offset: 5207}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + &zeroOrOneExpr{ + pos: position{line: 247, col: 12, offset: 5214}, + expr: &seqExpr{ + pos: position{line: 247, col: 13, offset: 5215}, + exprs: []any{ + &litMatcher{ + pos: position{line: 247, col: 13, offset: 5215}, + val: ".", + ignoreCase: false, + want: "\".\"", + }, + &oneOrMoreExpr{ + pos: position{line: 247, col: 17, offset: 5219}, + expr: &charClassMatcher{ + pos: position{line: 247, col: 17, offset: 5219}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + }, + }, + }, + }, + }, + &zeroOrOneExpr{ + pos: position{line: 79, col: 21, offset: 1937}, + expr: &litMatcher{ + pos: position{line: 79, col: 21, offset: 1937}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 84, col: 5, offset: 2041}, + run: (*parser).callonNode279, + expr: &seqExpr{ + pos: position{line: 84, col: 5, offset: 2041}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 84, col: 5, offset: 2041}, + label: "k", + expr: &actionExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + run: (*parser).callonNode282, + expr: &seqExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + exprs: []any{ + &oneOrMoreExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode285, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + &zeroOrMoreExpr{ + pos: position{line: 237, col: 11, offset: 5083}, + expr: &seqExpr{ + pos: position{line: 237, col: 12, offset: 5084}, + exprs: []any{ + &litMatcher{ + pos: position{line: 237, col: 12, offset: 5084}, + val: ".", + ignoreCase: false, + want: "\".\"", + }, + &oneOrMoreExpr{ + pos: position{line: 237, col: 16, offset: 5088}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode291, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 139, col: 5, offset: 3384}, + run: (*parser).callonNode293, + expr: &litMatcher{ + pos: position{line: 139, col: 5, offset: 3384}, + val: "=", + ignoreCase: false, + want: "\"=\"", + }, + }, + &labeledExpr{ + pos: position{line: 84, col: 29, offset: 2065}, label: "v", expr: &choiceExpr{ - pos: position{line: 73, col: 54, offset: 1808}, + pos: position{line: 84, col: 32, offset: 2068}, alternatives: []any{ &actionExpr{ - pos: position{line: 228, col: 5, offset: 4747}, - run: (*parser).callonNode262, + pos: position{line: 242, col: 5, offset: 5146}, + run: (*parser).callonNode297, expr: &seqExpr{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, exprs: []any{ &litMatcher{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, &labeledExpr{ - pos: position{line: 228, col: 9, offset: 4751}, + pos: position{line: 242, col: 9, offset: 5150}, label: "v", expr: &zeroOrMoreExpr{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, expr: &charClassMatcher{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, val: "[^\"]", chars: []rune{'"'}, ignoreCase: false, @@ -1396,7 +1559,7 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 228, col: 17, offset: 4759}, + pos: position{line: 242, col: 17, offset: 5158}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1405,9 +1568,9 @@ var g = &grammar{ }, }, &oneOrMoreExpr{ - pos: position{line: 73, col: 63, offset: 1817}, + pos: position{line: 84, col: 41, offset: 2077}, expr: &charClassMatcher{ - pos: position{line: 73, col: 63, offset: 1817}, + pos: position{line: 84, col: 41, offset: 2077}, val: "[^ ()]", chars: []rune{' ', '(', ')'}, ignoreCase: false, @@ -1421,19 +1584,147 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 105, col: 5, offset: 2609}, - run: (*parser).callonNode271, + pos: position{line: 87, col: 5, offset: 2158}, + run: (*parser).callonNode306, + expr: &seqExpr{ + pos: position{line: 87, col: 5, offset: 2158}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 87, col: 5, offset: 2158}, + label: "k", + expr: &actionExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + run: (*parser).callonNode309, + expr: &seqExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + exprs: []any{ + &oneOrMoreExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode312, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + &zeroOrMoreExpr{ + pos: position{line: 237, col: 11, offset: 5083}, + expr: &seqExpr{ + pos: position{line: 237, col: 12, offset: 5084}, + exprs: []any{ + &litMatcher{ + pos: position{line: 237, col: 12, offset: 5084}, + val: ".", + ignoreCase: false, + want: "\".\"", + }, + &oneOrMoreExpr{ + pos: position{line: 237, col: 16, offset: 5088}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode318, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode320, + expr: &litMatcher{ + pos: position{line: 134, col: 5, offset: 3298}, + val: ":", + ignoreCase: false, + want: "\":\"", + }, + }, + &labeledExpr{ + pos: position{line: 87, col: 29, offset: 2182}, + label: "v", + expr: &choiceExpr{ + pos: position{line: 87, col: 32, offset: 2185}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 242, col: 5, offset: 5146}, + run: (*parser).callonNode324, + expr: &seqExpr{ + pos: position{line: 242, col: 5, offset: 5146}, + exprs: []any{ + &litMatcher{ + pos: position{line: 242, col: 5, offset: 5146}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + &labeledExpr{ + pos: position{line: 242, col: 9, offset: 5150}, + label: "v", + expr: &zeroOrMoreExpr{ + pos: position{line: 242, col: 11, offset: 5152}, + expr: &charClassMatcher{ + pos: position{line: 242, col: 11, offset: 5152}, + val: "[^\"]", + chars: []rune{'"'}, + ignoreCase: false, + inverted: true, + }, + }, + }, + &litMatcher{ + pos: position{line: 242, col: 17, offset: 5158}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + }, + }, + }, + &oneOrMoreExpr{ + pos: position{line: 87, col: 41, offset: 2194}, + expr: &charClassMatcher{ + pos: position{line: 87, col: 41, offset: 2194}, + val: "[^ ()]", + chars: []rune{' ', '(', ')'}, + ignoreCase: false, + inverted: true, + }, + }, + }, + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 119, col: 5, offset: 3008}, + run: (*parser).callonNode333, expr: &choiceExpr{ - pos: position{line: 105, col: 6, offset: 2610}, + pos: position{line: 119, col: 6, offset: 3009}, alternatives: []any{ &litMatcher{ - pos: position{line: 105, col: 6, offset: 2610}, + pos: position{line: 119, col: 6, offset: 3009}, val: "AND", ignoreCase: false, want: "\"AND\"", }, &litMatcher{ - pos: position{line: 105, col: 14, offset: 2618}, + pos: position{line: 119, col: 14, offset: 3017}, val: "+", ignoreCase: false, want: "\"+\"", @@ -1442,19 +1733,19 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 110, col: 5, offset: 2710}, - run: (*parser).callonNode275, + pos: position{line: 124, col: 5, offset: 3109}, + run: (*parser).callonNode337, expr: &choiceExpr{ - pos: position{line: 110, col: 6, offset: 2711}, + pos: position{line: 124, col: 6, offset: 3110}, alternatives: []any{ &litMatcher{ - pos: position{line: 110, col: 6, offset: 2711}, + pos: position{line: 124, col: 6, offset: 3110}, val: "NOT", ignoreCase: false, want: "\"NOT\"", }, &litMatcher{ - pos: position{line: 110, col: 14, offset: 2719}, + pos: position{line: 124, col: 14, offset: 3118}, val: "-", ignoreCase: false, want: "\"-\"", @@ -1463,28 +1754,28 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 115, col: 5, offset: 2810}, - run: (*parser).callonNode279, + pos: position{line: 129, col: 5, offset: 3209}, + run: (*parser).callonNode341, expr: &litMatcher{ - pos: position{line: 115, col: 6, offset: 2811}, + pos: position{line: 129, col: 6, offset: 3210}, val: "OR", ignoreCase: false, want: "\"OR\"", }, }, &actionExpr{ - pos: position{line: 86, col: 6, offset: 2097}, - run: (*parser).callonNode281, + pos: position{line: 100, col: 6, offset: 2482}, + run: (*parser).callonNode343, expr: &seqExpr{ - pos: position{line: 86, col: 6, offset: 2097}, + pos: position{line: 100, col: 6, offset: 2482}, exprs: []any{ &zeroOrOneExpr{ - pos: position{line: 86, col: 6, offset: 2097}, + pos: position{line: 100, col: 6, offset: 2482}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode284, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode346, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1492,12 +1783,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode286, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode348, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1506,27 +1797,27 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 86, col: 27, offset: 2118}, + pos: position{line: 100, col: 27, offset: 2503}, label: "v", expr: &actionExpr{ - pos: position{line: 228, col: 5, offset: 4747}, - run: (*parser).callonNode290, + pos: position{line: 242, col: 5, offset: 5146}, + run: (*parser).callonNode352, expr: &seqExpr{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, exprs: []any{ &litMatcher{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, &labeledExpr{ - pos: position{line: 228, col: 9, offset: 4751}, + pos: position{line: 242, col: 9, offset: 5150}, label: "v", expr: &zeroOrMoreExpr{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, expr: &charClassMatcher{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, val: "[^\"]", chars: []rune{'"'}, ignoreCase: false, @@ -1535,7 +1826,7 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 228, col: 17, offset: 4759}, + pos: position{line: 242, col: 17, offset: 5158}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1545,12 +1836,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode297, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode359, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1559,12 +1850,12 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 86, col: 38, offset: 2129}, + pos: position{line: 100, col: 38, offset: 2514}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode301, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode363, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1575,18 +1866,18 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 91, col: 6, offset: 2227}, - run: (*parser).callonNode303, + pos: position{line: 105, col: 6, offset: 2619}, + run: (*parser).callonNode365, expr: &seqExpr{ - pos: position{line: 91, col: 6, offset: 2227}, + pos: position{line: 105, col: 6, offset: 2619}, exprs: []any{ &zeroOrOneExpr{ - pos: position{line: 91, col: 6, offset: 2227}, + pos: position{line: 105, col: 6, offset: 2619}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode306, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode368, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1594,12 +1885,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode308, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode370, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1608,12 +1899,12 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 91, col: 27, offset: 2248}, + pos: position{line: 105, col: 27, offset: 2640}, label: "v", expr: &oneOrMoreExpr{ - pos: position{line: 91, col: 29, offset: 2250}, + pos: position{line: 105, col: 29, offset: 2642}, expr: &charClassMatcher{ - pos: position{line: 91, col: 29, offset: 2250}, + pos: position{line: 105, col: 29, offset: 2642}, val: "[^ :()]", chars: []rune{' ', ':', '(', ')'}, ignoreCase: false, @@ -1622,12 +1913,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode314, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode376, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1636,12 +1927,12 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 91, col: 40, offset: 2261}, + pos: position{line: 105, col: 40, offset: 2653}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode318, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode380, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1669,18 +1960,18 @@ var g = &grammar{ expr: &zeroOrOneExpr{ pos: position{line: 32, col: 7, offset: 614}, expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonGroupNode5, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonGroupNode8, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1689,23 +1980,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonGroupNode14, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1727,20 +2018,20 @@ var g = &grammar{ pos: position{line: 32, col: 13, offset: 620}, alternatives: []any{ &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonGroupNode18, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", }, }, &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonGroupNode20, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", @@ -2733,118 +3024,118 @@ func (p *parser) callonNode244() (any, error) { return p.cur.onNode244() } -func (c *current) onNode256() (any, error) { +func (c *current) onNode257() (any, error) { return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode256() (any, error) { +func (p *parser) callonNode257() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode256() + return p.cur.onNode257() } -func (c *current) onNode258() (any, error) { +func (c *current) onNode259() (any, error) { return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode258() (any, error) { +func (p *parser) callonNode259() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode258() + return p.cur.onNode259() } -func (c *current) onNode262(v any) (any, error) { - return v, nil +func (c *current) onNode261() (any, error) { + return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode262() (any, error) { +func (p *parser) callonNode261() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode262(stack["v"]) + return p.cur.onNode261() } -func (c *current) onNode241(k, v any) (any, error) { - return buildStringNode(k, v, c.text, c.pos) +func (c *current) onNode263() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode263() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode263() +} + +func (c *current) onNode268() (any, error) { + return string(c.text), nil + +} + +func (p *parser) callonNode268() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode268() +} + +func (c *current) onNode241(k, o, v any) (any, error) { + return buildNumberNode(k, o, v, c.text, c.pos) } func (p *parser) callonNode241() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode241(stack["k"], stack["v"]) + return p.cur.onNode241(stack["k"], stack["o"], stack["v"]) } -func (c *current) onNode271() (any, error) { +func (c *current) onNode285() (any, error) { + return c.text, nil + +} + +func (p *parser) callonNode285() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode285() +} + +func (c *current) onNode291() (any, error) { + return c.text, nil + +} + +func (p *parser) callonNode291() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode291() +} + +func (c *current) onNode282() (any, error) { + return c.text, nil + +} + +func (p *parser) callonNode282() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode282() +} + +func (c *current) onNode293(k any) (any, error) { return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode271() (any, error) { +func (p *parser) callonNode293() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode271() -} - -func (c *current) onNode275() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode275() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode275() -} - -func (c *current) onNode279() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode279() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode279() -} - -func (c *current) onNode284() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode284() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode284() -} - -func (c *current) onNode286() (any, error) { - return nil, nil - -} - -func (p *parser) callonNode286() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode286() -} - -func (c *current) onNode290(v any) (any, error) { - return v, nil - -} - -func (p *parser) callonNode290() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode290(stack["v"]) + return p.cur.onNode293(stack["k"]) } func (c *current) onNode297(v any) (any, error) { - return nil, nil + return v, nil } @@ -2854,63 +3145,30 @@ func (p *parser) callonNode297() (any, error) { return p.cur.onNode297(stack["v"]) } -func (c *current) onNode301() (any, error) { - return buildOperatorNode(c.text, c.pos) +func (c *current) onNode279(k, v any) (any, error) { + return buildStringNode(k, v, true, c.text, c.pos) } -func (p *parser) callonNode301() (any, error) { +func (p *parser) callonNode279() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode301() + return p.cur.onNode279(stack["k"], stack["v"]) } -func (c *current) onNode281(v any) (any, error) { - return buildStringNode("", v, c.text, c.pos) +func (c *current) onNode312() (any, error) { + return c.text, nil } -func (p *parser) callonNode281() (any, error) { +func (p *parser) callonNode312() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode281(stack["v"]) -} - -func (c *current) onNode306() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode306() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode306() -} - -func (c *current) onNode308() (any, error) { - return nil, nil - -} - -func (p *parser) callonNode308() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode308() -} - -func (c *current) onNode314(v any) (any, error) { - return nil, nil - -} - -func (p *parser) callonNode314() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode314(stack["v"]) + return p.cur.onNode312() } func (c *current) onNode318() (any, error) { - return buildOperatorNode(c.text, c.pos) + return c.text, nil } @@ -2920,15 +3178,202 @@ func (p *parser) callonNode318() (any, error) { return p.cur.onNode318() } -func (c *current) onNode303(v any) (any, error) { - return buildStringNode("", v, c.text, c.pos) +func (c *current) onNode309() (any, error) { + return c.text, nil } -func (p *parser) callonNode303() (any, error) { +func (p *parser) callonNode309() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode303(stack["v"]) + return p.cur.onNode309() +} + +func (c *current) onNode320(k any) (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode320() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode320(stack["k"]) +} + +func (c *current) onNode324(v any) (any, error) { + return v, nil + +} + +func (p *parser) callonNode324() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode324(stack["v"]) +} + +func (c *current) onNode306(k, v any) (any, error) { + return buildStringNode(k, v, false, c.text, c.pos) + +} + +func (p *parser) callonNode306() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode306(stack["k"], stack["v"]) +} + +func (c *current) onNode333() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode333() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode333() +} + +func (c *current) onNode337() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode337() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode337() +} + +func (c *current) onNode341() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode341() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode341() +} + +func (c *current) onNode346() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode346() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode346() +} + +func (c *current) onNode348() (any, error) { + return nil, nil + +} + +func (p *parser) callonNode348() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode348() +} + +func (c *current) onNode352(v any) (any, error) { + return v, nil + +} + +func (p *parser) callonNode352() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode352(stack["v"]) +} + +func (c *current) onNode359(v any) (any, error) { + return nil, nil + +} + +func (p *parser) callonNode359() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode359(stack["v"]) +} + +func (c *current) onNode363() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode363() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode363() +} + +func (c *current) onNode343(v any) (any, error) { + return buildStringNode("", v, false, c.text, c.pos) + +} + +func (p *parser) callonNode343() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode343(stack["v"]) +} + +func (c *current) onNode368() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode368() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode368() +} + +func (c *current) onNode370() (any, error) { + return nil, nil + +} + +func (p *parser) callonNode370() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode370() +} + +func (c *current) onNode376(v any) (any, error) { + return nil, nil + +} + +func (p *parser) callonNode376() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode376(stack["v"]) +} + +func (c *current) onNode380() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode380() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode380() +} + +func (c *current) onNode365(v any) (any, error) { + return buildStringNode("", v, false, c.text, c.pos) + +} + +func (p *parser) callonNode365() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode365(stack["v"]) } func (c *current) onGroupNode8() (any, error) { diff --git a/pkg/kql/dictionary_test.go b/pkg/kql/dictionary_test.go index 81ace7b811..25f6e2f902 100644 --- a/pkg/kql/dictionary_test.go +++ b/pkg/kql/dictionary_test.go @@ -145,6 +145,22 @@ func TestParse_Spec(t *testing.T) { }, }, }, + { + name: `author="John Smith"`, + ast: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "author", Value: "John Smith", Exact: true}, + }, + }, + }, + { + name: `filename=budget.xlsx`, + ast: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "filename", Value: "budget.xlsx", Exact: true}, + }, + }, + }, // 3.2.3 Implicit Operator for Property Restriction { name: `author:"John Smith" filetype:docx`, @@ -423,6 +439,60 @@ func TestParse_Spec(t *testing.T) { } } +func TestParse_NumberRestrictionNode(t *testing.T) { + tests := []testCase{ + { + name: "format", + query: join([]string{ + `size>100`, + `size>"100"`, + `size>=15.5`, + `size<100`, + `size<=100`, + }), + ast: &ast.Ast{ + Nodes: []ast.Node{ + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: ">"}, + Value: 100, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: ">"}, + Value: 100, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: ">="}, + Value: 15.5, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: "<"}, + Value: 100, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: "<="}, + Value: 100, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + testKQL(t, tc) + }) + } +} + func TestParse_DateTimeRestrictionNode(t *testing.T) { tests := []testCase{ { diff --git a/pkg/kql/engine_suite_test.go b/pkg/kql/engine_suite_test.go deleted file mode 100644 index c4ad289b8b..0000000000 --- a/pkg/kql/engine_suite_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package kql - -import ( - "time" -) - -// PatchTimeNow is here to patch the package time now func, -// which is used in the test suite -func PatchTimeNow(t func() time.Time) { - timeNow = t -} diff --git a/pkg/kql/factory.go b/pkg/kql/factory.go index 2ac49113c4..5f55802d38 100644 --- a/pkg/kql/factory.go +++ b/pkg/kql/factory.go @@ -50,7 +50,7 @@ func buildAST(n any, text []byte, pos position) (*ast.Ast, error) { return a, nil } -func buildStringNode(k, v any, text []byte, pos position) (*ast.StringNode, error) { +func buildStringNode(k, v any, exact bool, text []byte, pos position) (*ast.StringNode, error) { b, err := base(text, pos) if err != nil { return nil, err @@ -70,6 +70,7 @@ func buildStringNode(k, v any, text []byte, pos position) (*ast.StringNode, erro Base: b, Key: key, Value: value, + Exact: exact, }, nil } @@ -101,6 +102,36 @@ func buildDateTimeNode(k, o, v any, text []byte, pos position) (*ast.DateTimeNod Value: value, }, nil } + +func buildNumberNode(k, o, v any, text []byte, pos position) (*ast.NumberNode, error) { + b, err := base(text, pos) + if err != nil { + return nil, err + } + + operator, err := toNode[*ast.OperatorNode](o) + if err != nil { + return nil, err + } + + key, err := toString(k) + if err != nil { + return nil, err + } + + value, err := toFloat(v) + if err != nil { + return nil, err + } + + return &ast.NumberNode{ + Base: b, + Key: key, + Operator: operator, + Value: value, + }, nil +} + func buildNaturalLanguageDateTimeNodes(k, v any, text []byte, pos position) ([]ast.Node, error) { b, err := base(text, pos) if err != nil { diff --git a/pkg/kql/kql.go b/pkg/kql/kql.go index 4cba6c7b91..a4132f4650 100644 --- a/pkg/kql/kql.go +++ b/pkg/kql/kql.go @@ -47,3 +47,9 @@ func (b Builder) Build(q string) (*ast.Ast, error) { // timeNow mirrors time.Now by default, the only reason why this exists // is to monkey patch it from the tests. See PatchTimeNow var timeNow = time.Now + +// PatchTimeNow pins the clock the natural language dates resolve against, +// so a test can hold "today" still while it runs +func PatchTimeNow(t func() time.Time) { + timeNow = t +} diff --git a/services/search/MIGRATION.md b/services/search/MIGRATION.md new file mode 100644 index 0000000000..a0cb646e61 --- /dev/null +++ b/services/search/MIGRATION.md @@ -0,0 +1,55 @@ +# Migrating the search index + +A version that changes how resources are indexed works on a new index and +leaves the old one untouched. The service starts normally, but the new index is +empty: search finds nothing until it is filled. The old index stays around +until you remove it. + +## v7.x.x to %%NEXT%% + +### OpenSearch + +The new index is `opencloud-resource-v3`. Fill it in one of two ways: + +- copy the old index, fast and keeps the extracted file contents, or +- index all spaces again, slower since every file is read once more, but drops + documents that no longer have a resource. + +The address below is the one from `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ADDRESSES`, +`opencloud-resource` the name from +`SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME`. + +```shell +# either copy the old index +curl -X POST "https://opensearch.example.com:9200/_reindex?wait_for_completion=false" \ + -H 'Content-Type: application/json' -d ' + {"source":{"index":"opencloud-resource"},"dest":{"index":"opencloud-resource-v3"}}' + +# the answer carries a task id, watch it while it runs +curl "https://opensearch.example.com:9200/_tasks/" + +# or index all spaces again, the service keeps running while it happens +opencloud search index --all-spaces +``` + +Once the new index is filled, remove the old one: + +```shell +curl -X DELETE "https://opensearch.example.com:9200/opencloud-resource" +``` + +### bleve + +The new index is the `bleve-v2` directory next to the old `bleve` one, both in +`$OC_BASE_DATA_PATH/search` by default (`SEARCH_ENGINE_BLEVE_DATA_PATH`). A +bleve index cannot be copied, index all spaces again: + +```shell +opencloud search index --all-spaces +``` + +Once the new index is filled, remove the old one: + +```shell +rm -r "$OC_BASE_DATA_PATH/search/bleve" +``` diff --git a/services/search/README.md b/services/search/README.md index 03e56c687a..c0e9036aac 100644 --- a/services/search/README.md +++ b/services/search/README.md @@ -55,6 +55,12 @@ Additionally, the following optional settings can be set: * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ENABLE_DEBUG_LOGGER=val`: Enable debug logging. * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_INSECURE=val`: Skip TLS certificate verification. +### Bringing an existing index over + +A version that changes how resources are indexed works on a new index and leaves the old one untouched. +The new index starts empty and has to be filled, see [MIGRATION.md](MIGRATION.md). New installations are +not affected. + ## Query language By default, [KQL](https://learn.microsoft.com/en-us/sharepoint/dev/general-development/keyword-query-language-kql-syntax-reference) is used as the query language. diff --git a/services/search/internal/opensearchtest/helper.go b/services/search/internal/opensearchtest/helper.go index 9ac497a2a3..b900b022c4 100644 --- a/services/search/internal/opensearchtest/helper.go +++ b/services/search/internal/opensearchtest/helper.go @@ -5,10 +5,7 @@ import ( "testing" "time" - opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "github.com/stretchr/testify/require" - - "github.com/opencloud-eu/opencloud/pkg/conversions" ) var TimeMustParse = func(t testing.TB, ts string) time.Time { @@ -23,14 +20,3 @@ func JSONMustMarshal(t testing.TB, data any) string { require.NoError(t, err, "failed to marshal data to JSON") return string(jsonData) } - -func SearchHitsMustBeConverted[T any](t testing.TB, hits []opensearchgoAPI.SearchHit) []T { - ts := make([]T, len(hits)) - for i, hit := range hits { - resource, err := conversions.To[T](hit.Source) - require.NoError(t, err) - ts[i] = resource - } - - return ts -} diff --git a/services/search/internal/opensearchtest/os.go b/services/search/internal/opensearchtest/os.go index 8d4ad649ad..16c3c10025 100644 --- a/services/search/internal/opensearchtest/os.go +++ b/services/search/internal/opensearchtest/os.go @@ -61,7 +61,6 @@ func (tc *TestClient) IndicesReset(ctx context.Context, indices []string) error } if len(indicesToDelete) == 0 { - // If no indices to delete, return nil return nil } @@ -138,76 +137,6 @@ func (tc *TestClient) IndicesCreate(ctx context.Context, index string, body io.R } } -func (tc *TestClient) IndicesCount(ctx context.Context, indices []string, body io.Reader) (int, error) { - if err := tc.IndicesRefresh(ctx, indices, []int{404}); err != nil { - return 0, err - } - - resp, err := tc.c.Indices.Count(ctx, &opensearchgoAPI.IndicesCountReq{ - Indices: indices, - Body: body, - }) - - switch { - case err != nil: - return 0, fmt.Errorf("failed to count documents in indices: %w", err) - default: - return resp.Count, nil - } -} - -func (tc *TestClient) DocumentCreate(ctx context.Context, index, id string, body io.Reader) error { - if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil { - return err - } - - _, err := tc.c.Document.Create(ctx, opensearchgoAPI.DocumentCreateReq{ - Index: index, - DocumentID: id, - Body: body, - }) - switch { - case err != nil: - return fmt.Errorf("failed to create document in index %s: %w", index, err) - default: - return nil - } -} - -func (tc *TestClient) Update(ctx context.Context, index, id string, body io.Reader) error { - if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil { - return err - } - - _, err := tc.c.Update(ctx, opensearchgoAPI.UpdateReq{ - Index: index, - DocumentID: id, - Body: body, - }) - switch { - case err != nil: - return fmt.Errorf("failed to update document in index %s: %w", index, err) - default: - return nil - } -} - -func (tc *TestClient) Search(ctx context.Context, index string, body io.Reader) (opensearchgoAPI.SearchHits, error) { - if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil { - return opensearchgoAPI.SearchHits{}, err - } - - resp, err := tc.c.Search(ctx, &opensearchgoAPI.SearchReq{ - Indices: []string{index}, - Body: body, - }) - if err != nil { - return opensearchgoAPI.SearchHits{}, fmt.Errorf("failed to search in index %s: %w", index, err) - } - - return resp.Hits, nil -} - type testRequireClient struct { tc *TestClient t testing.TB @@ -228,29 +157,3 @@ func (trc *testRequireClient) IndicesCreate(index string, body io.Reader) { func (trc *testRequireClient) IndicesDelete(indices []string) { require.NoError(trc.t, trc.tc.IndicesDelete(trc.t.Context(), indices)) } - -func (trc *testRequireClient) IndicesCount(indices []string, body io.Reader, expected int) { - count, err := trc.tc.IndicesCount(trc.t.Context(), indices, body) - - switch { - case expected <= 0: - require.True(trc.t, count <= 0, "expected indices to have no documents, but got a count of %d", count) - default: - require.Equal(trc.t, expected, count, "expected indices to have %d documents, but got %d", expected, count) - require.NoError(trc.t, err, "expected indices to have documents, but got an error") - } -} - -func (trc *testRequireClient) DocumentCreate(index, id string, body io.Reader) { - require.NoError(trc.t, trc.tc.DocumentCreate(trc.t.Context(), index, id, body)) -} - -func (trc *testRequireClient) Update(index, id string, body io.Reader) { - require.NoError(trc.t, trc.tc.Update(trc.t.Context(), index, id, body)) -} - -func (trc *testRequireClient) Search(index string, body io.Reader) opensearchgoAPI.SearchHits { - hits, err := trc.tc.Search(trc.t.Context(), index, body) - require.NoError(trc.t, err) - return hits -} diff --git a/services/search/internal/opensearchtest/suite.go b/services/search/internal/opensearchtest/suite.go index 433907292a..9b6e41e412 100644 --- a/services/search/internal/opensearchtest/suite.go +++ b/services/search/internal/opensearchtest/suite.go @@ -67,7 +67,6 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func keepContainer := os.Getenv("KEEP_TEST_CONTAINER") == "true" if keepContainer { - // the reaper would take the kept container down with the session if err := os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true"); err != nil { return nil, fmt.Errorf("failed to disable the testcontainers reaper: %w", err) } @@ -83,14 +82,6 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func opensearch.WithPassword(cfg.Engine.OpenSearch.Client.Password), testcontainers.WithName(containerName), testcontainers.WithReuseByName(containerName), - // test indexes are tiny; don't let a full host disk trip the flood-stage - // create-index / read-only blocks mid-run - testcontainers.WithEnv(map[string]string{ - "cluster.routing.allocation.disk.threshold_enabled": "false", - }), - // a health probe answers at once on a reused container, a log wait - // would sit out the log timeout on it; a cold boot takes well over the - // previous 5s testcontainers.WithWaitStrategy( wait.ForHTTP("/_cluster/health?wait_for_status=yellow&timeout=1s"). WithPort(openSearchPort). @@ -106,16 +97,13 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func address, err := container.Address(ctx) if err != nil { - _ = container.Terminate(ctx) // attempt to clean up the container + _ = container.Terminate(ctx) return nil, fmt.Errorf("failed to get OpenSearch container address: %w", err) } - // Ensure the address is set in the default configuration. cfg.Engine.OpenSearch.Client.Addresses = []string{address} return func() { - // KEEP_TEST_CONTAINER=true leaves the container up for the next run, - // which picks it up again by name instead of booting a fresh one if keepContainer { _, _ = fmt.Fprintf(os.Stderr, "keeping OpenSearch container %s\n", containerName) return diff --git a/services/search/internal/opensearchtest/testdata.go b/services/search/internal/opensearchtest/testdata.go index ef41c91190..84ceef3a93 100644 --- a/services/search/internal/opensearchtest/testdata.go +++ b/services/search/internal/opensearchtest/testdata.go @@ -16,16 +16,12 @@ var Testdata = struct { Resources resourceTestdata }{ Resources: resourceTestdata{ - Root: fromTestData[search.Resource]("resource_root.json"), - Folder: fromTestData[search.Resource]("resource_folder.json"), - File: fromTestData[search.Resource]("resource_file.json"), + File: fromTestData[search.Resource]("resource_file.json"), }, } type resourceTestdata struct { - Root search.Resource - File search.Resource - Folder search.Resource + File search.Resource } func fromTestData[D any](name string) D { diff --git a/services/search/pkg/bleve/index.go b/services/search/pkg/bleve/index.go index fea6986ae0..0aa61b4c48 100644 --- a/services/search/pkg/bleve/index.go +++ b/services/search/pkg/bleve/index.go @@ -8,8 +8,8 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + regexpCharFilter "github.com/blevesearch/bleve/v2/analysis/char/regexp" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" - "github.com/blevesearch/bleve/v2/analysis/token/porter" "github.com/blevesearch/bleve/v2/analysis/tokenizer/single" "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" "github.com/blevesearch/bleve/v2/mapping" @@ -18,10 +18,19 @@ import ( "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) +const ( + wildcardSuffix = ".wildcard" + indexVersion = "v2" +) + +func indexPath(root string) string { + return filepath.Join(root, "bleve-"+indexVersion) +} + func NewIndex(root string) (bleve.Index, error) { - destination := filepath.Join(root, "bleve") + destination := indexPath(root) index, err := bleve.Open(destination) - if errors.Is(bleve.ErrorIndexPathDoesNotExist, err) { + if errors.Is(err, bleve.ErrorIndexPathDoesNotExist) { indexMapping, err := NewMapping() if err != nil { return nil, err @@ -38,30 +47,61 @@ func NewIndex(root string) (bleve.Index, error) { } func NewMapping() (mapping.IndexMapping, error) { - nameMapping := bleve.NewTextFieldMapping() - nameMapping.Analyzer = "lowercaseKeyword" + words := func() *mapping.FieldMapping { + fm := bleve.NewTextFieldMapping() + fm.Analyzer = "lowercaseWords" + + return fm + } + + whole := func(field string) *mapping.FieldMapping { + fm := bleve.NewTextFieldMapping() + fm.Analyzer = "lowercaseKeyword" + fm.IncludeInAll = false + fm.Name = field + wildcardSuffix + + return fm + } lowercaseMapping := bleve.NewTextFieldMapping() lowercaseMapping.IncludeInAll = false lowercaseMapping.Analyzer = "lowercaseKeyword" - fulltextFieldMapping := bleve.NewTextFieldMapping() - fulltextFieldMapping.Analyzer = "fulltext" - fulltextFieldMapping.IncludeInAll = false + contentMapping := words() + contentMapping.IncludeInAll = false docMapping := bleve.NewDocumentMapping() - docMapping.AddFieldMappingsAt("Name", nameMapping) + docMapping.AddFieldMappingsAt("Name", + words(), + whole("Name"), + ) + docMapping.AddFieldMappingsAt("Title", + words(), + whole("Title"), + ) docMapping.AddFieldMappingsAt("Tags", lowercaseMapping) docMapping.AddFieldMappingsAt("Favorites", lowercaseMapping) - docMapping.AddFieldMappingsAt("Content", fulltextFieldMapping) + docMapping.AddFieldMappingsAt("Content", contentMapping) indexMapping := bleve.NewIndexMapping() indexMapping.DefaultAnalyzer = keyword.Name indexMapping.DefaultMapping = docMapping - err := indexMapping.AddCustomAnalyzer("lowercaseKeyword", + err := indexMapping.AddCustomCharFilter("dotToSpace", map[string]any{ - "type": custom.Name, - "tokenizer": single.Name, + "type": regexpCharFilter.Name, + "regexp": `\.`, + "replace": " ", + }, + ) + if err != nil { + return nil, err + } + + err = indexMapping.AddCustomAnalyzer("lowercaseWords", + map[string]any{ + "type": custom.Name, + "char_filters": []string{"dotToSpace"}, + "tokenizer": unicode.Name, "token_filters": []string{ lowercase.Name, }, @@ -71,13 +111,12 @@ func NewMapping() (mapping.IndexMapping, error) { return nil, err } - err = indexMapping.AddCustomAnalyzer("fulltext", + err = indexMapping.AddCustomAnalyzer("lowercaseKeyword", map[string]any{ "type": custom.Name, - "tokenizer": unicode.Name, + "tokenizer": single.Name, "token_filters": []string{ lowercase.Name, - porter.Name, }, }, ) diff --git a/services/search/pkg/bleve/index_test.go b/services/search/pkg/bleve/index_test.go new file mode 100644 index 0000000000..a62dfc2c28 --- /dev/null +++ b/services/search/pkg/bleve/index_test.go @@ -0,0 +1,39 @@ +package bleve_test + +import ( + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" +) + +var _ = Describe("Index", func() { + Describe("NewIndex", func() { + It("puts the index into a directory of its own generation", func() { + root := GinkgoT().TempDir() + + index, err := bleve.NewIndex(root) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(index.Close) + + Expect(index.Name()).To(Equal(filepath.Join(root, "bleve-v2"))) + Expect(filepath.Join(root, "bleve-v2")).To(BeADirectory()) + }) + + It("opens the index that is already there", func() { + root := GinkgoT().TempDir() + + index, err := bleve.NewIndex(root) + Expect(err).ToNot(HaveOccurred()) + Expect(index.Close()).To(Succeed()) + + reopened, err := bleve.NewIndex(root) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(reopened.Close) + + Expect(reopened.Name()).To(Equal(filepath.Join(root, "bleve-v2"))) + }) + }) +}) diff --git a/services/search/pkg/content/content.go b/services/search/pkg/content/content.go index e185351678..1c5d914231 100644 --- a/services/search/pkg/content/content.go +++ b/services/search/pkg/content/content.go @@ -18,7 +18,7 @@ type Document struct { Name string Content string Size uint64 - Mtime string + Mtime string `json:"Mtime,omitempty"` MimeType string Tags []string Favorites []string diff --git a/services/search/pkg/content/tika.go b/services/search/pkg/content/tika.go index 8b6ac5e60b..2b475efb17 100644 --- a/services/search/pkg/content/tika.go +++ b/services/search/pkg/content/tika.go @@ -11,8 +11,8 @@ import ( gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/google/go-tika/tika" - "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" libregraph "github.com/opencloud-eu/libre-graph-api-go" + "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" "github.com/opencloud-eu/opencloud/pkg/log" "github.com/opencloud-eu/opencloud/services/search/pkg/config" @@ -83,7 +83,11 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document, } for _, meta := range metas { - if title, err := getFirstValue(meta, "title"); err == nil { + title, err := getFirstValue(meta, "dc:title") + if err != nil { + title, err = getFirstValue(meta, "title") + } + if err == nil { doc.Title = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Title, title)) } diff --git a/services/search/pkg/content/tika_test.go b/services/search/pkg/content/tika_test.go index b6ee193528..286ffa8b9c 100644 --- a/services/search/pkg/content/tika_test.go +++ b/services/search/pkg/content/tika_test.go @@ -91,6 +91,28 @@ var _ = Describe("Tika", func() { Expect(doc.Content).To(Equal(body)) }) + It("adds the title", func() { + fullResponse = `[{"dc:title": "quarterly report", "X-TIKA:content": "some data"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Title).To(Equal("quarterly report")) + }) + + It("adds the title of an older tika", func() { + fullResponse = `[{"title": "quarterly report"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Title).To(Equal("quarterly report")) + }) + It("adds audio content", func() { fullResponse = `[ { diff --git a/services/search/pkg/opensearch/backend.go b/services/search/pkg/opensearch/backend.go index 000b24ebf1..6082643a93 100644 --- a/services/search/pkg/opensearch/backend.go +++ b/services/search/pkg/opensearch/backend.go @@ -33,7 +33,10 @@ type Backend struct { client *opensearchgoAPI.Client } -func NewBackend(index string, client *opensearchgoAPI.Client) (*Backend, error) { +// NewBackend creates a backend on the versioned generation of the named index. +func NewBackend(name string, client *opensearchgoAPI.Client) (*Backend, error) { + index := IndexName(name) + pingResp, err := client.Ping(context.TODO(), &opensearchgoAPI.PingReq{}) switch { case err != nil: @@ -172,7 +175,7 @@ func (b *Backend) DocCount() (uint64, error) { &opensearchgoAPI.IndicesCountReq{ Indices: []string{b.index}, }, - osu.NewTermQuery[bool]("Deleted").Value(false), + osu.NewMatchAllQuery(), ) if err != nil { return 0, fmt.Errorf("failed to build count request: %w", err) diff --git a/services/search/pkg/opensearch/backend_test.go b/services/search/pkg/opensearch/backend_test.go index 71ef3860fc..ecf728fe12 100644 --- a/services/search/pkg/opensearch/backend_test.go +++ b/services/search/pkg/opensearch/backend_test.go @@ -16,8 +16,6 @@ func TestOpenSearchBackend(t *testing.T) { RunSpecs(t, "OpenSearch Backend Suite") } -// what the engine does with its index is covered for both engines by -// services/search/pkg/parity; this is the one thing only OpenSearch can do var _ = Describe("Backend", func() { Describe("NewBackend", func() { It("fails to create if the cluster is not healthy", func() { diff --git a/services/search/pkg/opensearch/index.go b/services/search/pkg/opensearch/index.go index 73a1cf2eb9..277c9f5c5c 100644 --- a/services/search/pkg/opensearch/index.go +++ b/services/search/pkg/opensearch/index.go @@ -8,6 +8,7 @@ import ( "fmt" "path" "reflect" + "strings" "github.com/go-jose/go-jose/v3/json" opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" @@ -16,9 +17,8 @@ import ( var ( ErrManualActionRequired = errors.New("manual action required") - IndexManagerLatest = IndexIndexManagerResourceV2 - IndexIndexManagerResourceV1 IndexManager = "resource_v1.json" - IndexIndexManagerResourceV2 IndexManager = "resource_v2.json" + IndexManagerLatest = IndexIndexManagerResourceV3 + IndexIndexManagerResourceV3 IndexManager = "resource_v3.json" ) //go:embed internal/indexes/*.json @@ -26,6 +26,30 @@ var indexes embed.FS type IndexManager string +// Version is the part of the definition file name that says which generation it +// is, resource_v3.json carries v3. +func (m IndexManager) Version() string { + name := strings.TrimSuffix(string(m), path.Ext(string(m))) + _, version, found := strings.Cut(name, "_") + if !found { + return "" + } + + return version +} + +// IndexName puts the generation of the definition behind the configured name, +// so a new one starts on an index of its own instead of refusing to work with +// the one that is there. +func IndexName(name string) string { + version := IndexManagerLatest.Version() + if version == "" { + return name + } + + return name + "-" + version +} + func (m IndexManager) String() string { b, err := m.MarshalJSON() if err != nil { @@ -48,6 +72,43 @@ func (m IndexManager) MarshalJSON() ([]byte, error) { return body, nil } +func coveredAt(declared, index gjson.Result, declaredPath, indexPath string) (string, string, bool) { + declaredRaw := declared.Get(declaredPath).Raw + indexRaw := index.Get(indexPath).Raw + + var declaredValue, indexValue any + if err := json.Unmarshal([]byte(declaredRaw), &declaredValue); err != nil { + return declaredRaw, indexRaw, false + } + + if err := json.Unmarshal([]byte(indexRaw), &indexValue); err != nil { + return declaredRaw, indexRaw, false + } + + return declaredRaw, indexRaw, covered(declaredValue, indexValue) +} + +func covered(declared, index any) bool { + declaredMap, ok := declared.(map[string]any) + if !ok { + return reflect.DeepEqual(declared, index) + } + + indexMap, ok := index.(map[string]any) + if !ok { + return false + } + + for key, declaredValue := range declaredMap { + indexValue, ok := indexMap[key] + if !ok || !covered(declaredValue, indexValue) { + return false + } + } + + return true +} + func (m IndexManager) Apply(ctx context.Context, name string, client *opensearchgoAPI.Client) error { localIndexB, err := m.MarshalJSON() if err != nil { @@ -86,32 +147,16 @@ func (m IndexManager) Apply(ctx context.Context, name string, client *opensearch localIndexJson := gjson.ParseBytes(localIndexB) remoteIndexJson := gjson.ParseBytes(remoteIndexB) - compare := func(lvPath, rvPath string) (any, any, bool) { - lv := localIndexJson.Get(lvPath).Raw - rv := remoteIndexJson.Get(rvPath).Raw - - var lvv, rvv any - if err := json.Unmarshal([]byte(lv), &lvv); err != nil { - return nil, nil, false - } - - if err := json.Unmarshal([]byte(rv), &rvv); err != nil { - return nil, nil, false - } - - return lv, rv, reflect.DeepEqual(lvv, rvv) - } - var errs []error for k := range localIndexJson.Get("settings").Map() { - if lv, rv, ok := compare("settings."+k, "settings.index."+k); !ok { + if lv, rv, ok := coveredAt(localIndexJson, remoteIndexJson, "settings."+k, "settings.index."+k); !ok { errs = append(errs, fmt.Errorf("settings.%s local %s, remote %s", k, lv, rv)) } } for k := range localIndexJson.Get("mappings.properties").Map() { - if _, _, ok := compare("mappings.properties."+k, "mappings.properties."+k); !ok { + if _, _, ok := coveredAt(localIndexJson, remoteIndexJson, "mappings.properties."+k, "mappings.properties."+k); !ok { errs = append(errs, fmt.Errorf("mappings.properties.%s", k)) } } diff --git a/services/search/pkg/opensearch/index_test.go b/services/search/pkg/opensearch/index_test.go index 6a813572a2..faba72f674 100644 --- a/services/search/pkg/opensearch/index_test.go +++ b/services/search/pkg/opensearch/index_test.go @@ -47,6 +47,34 @@ func TestIndexManager(t *testing.T) { require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client())) }) + t.Run("accepts an index that carries more than the definition declares", func(t *testing.T) { + indexManager := opensearch.IndexManagerLatest + indexName := "opencloud-test-resource" + + tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + + body, err := sjson.Set(indexManager.String(), "mappings.properties.Path.fields.raw.type", "keyword") + require.NoError(t, err) + tc.Require.IndicesCreate(indexName, strings.NewReader(body)) + + require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client())) + }) + + t.Run("fails when the index misses something the definition declares", func(t *testing.T) { + indexManager := opensearch.IndexManagerLatest + indexName := "opencloud-test-resource" + + tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + + body, err := sjson.Delete(indexManager.String(), "mappings.properties.Path.analyzer") + require.NoError(t, err) + tc.Require.IndicesCreate(indexName, strings.NewReader(body)) + + require.ErrorIs(t, indexManager.Apply(t.Context(), indexName, tc.Client()), opensearch.ErrManualActionRequired) + }) + t.Run("fails to create index if it already exists but is not up to date", func(t *testing.T) { indexManager := opensearch.IndexManagerLatest indexName := "opencloud-test-resource" diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand.go b/services/search/pkg/opensearch/internal/convert/kql_expand.go index 5688f6dfe6..17258512c9 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand.go @@ -4,8 +4,11 @@ import ( "fmt" "reflect" "slices" + "strconv" "strings" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/opencloud-eu/opencloud/pkg/ast" ) @@ -53,6 +56,8 @@ func (e kqlExpander) expand(nodes []ast.Node, defaultKey string) ([]ast.Node, er cnode.Key = e.remapKey(cnode.Key, defaultKey) case *ast.BooleanNode: cnode.Key = e.remapKey(cnode.Key, defaultKey) + case *ast.NumberNode: + cnode.Key = e.remapKey(cnode.Key, defaultKey) } if unfoldedNodes != nil { @@ -73,6 +78,7 @@ func (_ kqlExpander) remapKey(current string, defaultKey string) string { key, ok := map[string]string{ "": defaultKey, // Default case if current is empty + "title": "Title", "rootid": "RootID", "path": "Path", "id": "ID", @@ -85,7 +91,8 @@ func (_ kqlExpander) remapKey(current string, defaultKey string) string { "tags": "Tags", "content": "Content", "hidden": "Hidden", - }[current] + "favorite": "Favorites", + }[strings.ToLower(current)] if !ok { return current // Return the original key if not found } @@ -94,15 +101,21 @@ func (_ kqlExpander) remapKey(current string, defaultKey string) string { } func (_ kqlExpander) lowerValue(key, value string) string { - if slices.Contains([]string{"Hidden"}, key) { - return value // ignore certain keys and return the original value + if slices.Contains([]string{"Name", "Title", "Tags", "Content", "MimeType", "Type", "Hidden"}, key) { + return strings.ToLower(value) } - return strings.ToLower(value) + return value } func (_ kqlExpander) unfoldValue(key, value string) []ast.Node { result, ok := map[string][]ast.Node{ + "Type:file": { + &ast.StringNode{Key: key, Value: strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10)}, + }, + "Type:folder": { + &ast.StringNode{Key: key, Value: strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10)}, + }, "MimeType:file": { &ast.OperatorNode{Value: "NOT"}, &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, @@ -139,8 +152,6 @@ func (_ kqlExpander) unfoldValue(key, value string) []ast.Node { &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"}, - &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"}, }}, }, diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go index 1b3cb0b914..44534d7bcd 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go @@ -142,6 +142,7 @@ func TestExpandKQLAST(t *testing.T) { "tags": "Tags", "content": "Content", "hidden": "Hidden", + "favorite": "Favorites", "any": "any", // Example of an unknown key that should remain unchanged } { tests = append(tests, opensearchtest.TableTest[[]ast.Node, []ast.Node]{ @@ -244,35 +245,72 @@ func TestExpandKQLAST(t *testing.T) { t.Run("lowercases some values", func(t *testing.T) { tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ { - Name: "!Hidden: StringNode -> stringnode", + Name: "Name: StringNode -> stringnode", Got: []ast.Node{ - ast.StringNode{Key: "aBc", Value: "StringNode"}, + ast.StringNode{Key: "Name", Value: "StringNode"}, ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "aBc", Value: "StringNode"}, + ast.StringNode{Key: "Name", Value: "StringNode"}, }}, }, Want: []ast.Node{ - &ast.StringNode{Key: "aBc", Value: "stringnode"}, + &ast.StringNode{Key: "Name", Value: "stringnode"}, &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "aBc", Value: "stringnode"}, + &ast.StringNode{Key: "Name", Value: "stringnode"}, }}, }, }, { - Name: "Hidden: StringNode -> StringNode", + Name: "aBc: StringNode -> StringNode", Got: []ast.Node{ - ast.StringNode{Key: "Hidden", Value: "StringNode"}, + ast.StringNode{Key: "aBc", Value: "StringNode"}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "aBc", Value: "StringNode"}, + }, + }, + { + Name: "Path: ./Documents -> ./Documents", + Got: []ast.Node{ + ast.StringNode{Key: "Path", Value: "./Documents"}, ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "Hidden", Value: "StringNode"}, + ast.StringNode{Key: "Path", Value: "./Documents"}, }}, }, Want: []ast.Node{ - &ast.StringNode{Key: "Hidden", Value: "StringNode"}, + &ast.StringNode{Key: "Path", Value: "./Documents"}, &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "Hidden", Value: "StringNode"}, + &ast.StringNode{Key: "Path", Value: "./Documents"}, }}, }, }, + { + Name: "Hidden: TRUE -> true", + Got: []ast.Node{ + ast.StringNode{Key: "Hidden", Value: "TRUE"}, + ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + ast.StringNode{Key: "Hidden", Value: "TRUE"}, + }}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "true"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "true"}, + }}, + }, + }, + { + Name: "ID: 1$1!AB23 -> 1$1!AB23", + Got: []ast.Node{ + ast.StringNode{Key: "ID", Value: "1$1!AB23"}, + ast.StringNode{Key: "RootID", Value: "1$1!AB23"}, + ast.StringNode{Key: "ParentID", Value: "1$1!AB23"}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "ID", Value: "1$1!AB23"}, + &ast.StringNode{Key: "RootID", Value: "1$1!AB23"}, + &ast.StringNode{Key: "ParentID", Value: "1$1!AB23"}, + }, + }, } for _, test := range tests { @@ -384,8 +422,6 @@ func TestExpandKQLAST(t *testing.T) { &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadshee"}, - &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.numbers"}, }}, &ast.OperatorNode{Value: "AND"}, diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index d94c1ac5fd..e3fb25a807 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "slices" + "strconv" "strings" "time" @@ -101,6 +102,8 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { return stringNodeQuery(node), nil case *ast.DateTimeNode: return dateTimeNodeQuery(node) + case *ast.NumberNode: + return numberNodeQuery(node) case *ast.GroupNode: group, err := t.transpile(node.Nodes) if err != nil { @@ -115,22 +118,35 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { // stringNodeQuery picks the query a string node turns into. func stringNodeQuery(node *ast.StringNode) osu.Builder { - isWildcard := strings.Contains(node.Value, "*") + isWildcard := strings.ContainsAny(node.Value, "*?") switch { // Name: "*oo-bar", "*oo ba*", "*OO*" // Title: "*rterly rep*" // Tags: "*spaced tag*" - case isWildcard && slices.Contains([]string{"Name", "Title", "Tags"}, node.Key): - return osu.NewWildcardQuery(node.Key + ".keyword"). - Value(node.Value). - Params(&osu.WildcardQueryParams{CaseInsensitive: true}) + case isWildcard && slices.Contains([]string{"Name", "Title"}, node.Key): + patterns := []osu.Builder{wildcardOn(node.Key+".wildcard", node.Value)} + if !strings.HasSuffix(node.Value, "*") { + patterns = append(patterns, wildcardOn(node.Key+".wildcard", node.Value+".*")) + } + + return osu.NewBoolQuery(). + Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). + Should(patterns...) + // Tags: "*foo*", "*paced ta*" + case isWildcard && node.Key == "Tags": + return wildcardOn(node.Key+".wildcard", node.Value) // Path: "./foo*", MimeType: "*plain" case isWildcard: return osu.NewWildcardQuery(node.Key).Value(node.Value) + // Name: =new, Title: ="quarterly report" + case node.Exact && slices.Contains([]string{"Name", "Title"}, node.Key): + return osu.NewTermQuery[string](node.Key + ".wildcard"). + Value(node.Value). + Params(&osu.TermQueryParams{CaseInsensitive: true}) // Tags: "foo-bar", "spaced tag", "FOO-BAR" case node.Key == "Tags": - return osu.NewTermQuery[string](node.Key + ".keyword"). + return osu.NewTermQuery[string](node.Key + ".wildcard"). Value(node.Value). Params(&osu.TermQueryParams{CaseInsensitive: true}) // Name: "foo-bar", "foo bar" @@ -138,12 +154,62 @@ func stringNodeQuery(node *ast.StringNode) osu.Builder { // Content: "foo bar" case slices.Contains([]string{"Name", "Title", "Content"}, node.Key): return osu.NewMatchPhraseQuery(node.Key).Query(node.Value) - // Path: "./foo bar", MimeType: "text/plain" + // Size: "42", Type: "1" + case slices.Contains([]string{"Size", "Type"}, node.Key): + number, err := strconv.ParseInt(node.Value, 10, 64) + if err != nil { + return osu.NewMatchNoneQuery() + } + + return osu.NewTermQuery[int64](node.Key).Value(number) + // Path: "./foo bar/", the hierarchy tokens carry no trailing slash + case node.Key == "Path": + return osu.NewTermQuery[string](node.Key).Value(strings.TrimSuffix(node.Value, "/")) + // Hidden: "TRUE" arrives lowered, anything that is no bool matches nothing + case node.Key == "Hidden": + value, err := strconv.ParseBool(node.Value) + if err != nil { + return osu.NewMatchNoneQuery() + } + + return osu.NewTermQuery[bool](node.Key).Value(value) + // MimeType: "text/plain" default: return osu.NewTermQuery[string](node.Key).Value(node.Value) } } +func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) { + if node.Operator == nil { + return nil, fmt.Errorf("number node without operator: %w", ErrUnsupportedNodeType) + } + + if !slices.Contains([]string{"Size", "Type"}, node.Key) { + return osu.NewMatchNoneQuery(), nil + } + + query := osu.NewRangeQuery[float64](node.Key) + + switch node.Operator.Value { + case ">": + return query.Gt(node.Value), nil + case ">=": + return query.Gte(node.Value), nil + case "<": + return query.Lt(node.Value), nil + case "<=": + return query.Lte(node.Value), nil + } + + return nil, fmt.Errorf("unsupported operator %s for number node: %w", node.Operator.Value, ErrUnsupportedNodeType) +} + +func wildcardOn(field, value string) osu.Builder { + return osu.NewWildcardQuery(field). + Value(value). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}) +} + // dateTimeNodeQuery turns a date time node into a range query. func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) { if node.Operator == nil { diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go index 1b627c8133..9e03fd23df 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go @@ -58,12 +58,16 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { &ast.StringNode{Key: "Name", Value: "open*"}, }, }, - Want: osu.NewWildcardQuery("Name.keyword"). - Value("open*"). - Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + Want: osu.NewBoolQuery(). + Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). + Should( + osu.NewWildcardQuery("Name.wildcard"). + Value("open*"). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + ), }, { - Name: "wildcard query - string node without a keyword sub field", + Name: "wildcard query - string node without an unanalyzed sub field", Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "Content", Value: "open*"}, @@ -71,6 +75,62 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewWildcardQuery("Content").Value("open*"), }, + { + Name: "wildcard query - a question mark counts as a wildcard", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "fo?o"}, + }, + }, + Want: osu.NewBoolQuery(). + Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). + Should( + osu.NewWildcardQuery("Name.wildcard"). + Value("fo?o"). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + osu.NewWildcardQuery("Name.wildcard"). + Value("fo?o.*"). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + ), + }, + { + Name: "term query - an equals restriction matches the whole name", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "foo bar.txt", Exact: true}, + }, + }, + Want: osu.NewTermQuery[string]("Name.wildcard"). + Value("foo bar.txt"). + Params(&osu.TermQueryParams{CaseInsensitive: true}), + }, + { + Name: "term query - a path loses its trailing slash", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Path", Value: "./Documents/"}, + }, + }, + Want: osu.NewTermQuery[string]("Path").Value("./Documents"), + }, + { + Name: "term query - a hidden string turns into a bool", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "true"}, + }, + }, + Want: osu.NewTermQuery[bool]("Hidden").Value(true), + }, + { + Name: "match-none query - a hidden string that is no bool", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "banana"}, + }, + }, + Want: osu.NewMatchNoneQuery(), + }, { Name: "bool query", Got: &ast.Ast{ diff --git a/services/search/pkg/opensearch/internal/convert/opensearch_test.go b/services/search/pkg/opensearch/internal/convert/opensearch_test.go index d2fba09c70..8f034e327c 100644 --- a/services/search/pkg/opensearch/internal/convert/opensearch_test.go +++ b/services/search/pkg/opensearch/internal/convert/opensearch_test.go @@ -10,7 +10,7 @@ import ( "github.com/opencloud-eu/opencloud/pkg/conversions" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) diff --git a/services/search/pkg/opensearch/internal/indexes/resource_v3.json b/services/search/pkg/opensearch/internal/indexes/resource_v3.json new file mode 100644 index 0000000000..5180f6ad56 --- /dev/null +++ b/services/search/pkg/opensearch/internal/indexes/resource_v3.json @@ -0,0 +1,122 @@ +{ + "settings": { + "number_of_shards": "1", + "number_of_replicas": "1", + "analysis": { + "analyzer": { + "path_hierarchy": { + "tokenizer": "path_hierarchy", + "type": "custom" + }, + "name_words": { + "type": "custom", + "char_filter": [ + "dot_to_space" + ], + "tokenizer": "standard", + "filter": [ + "lowercase" + ] + } + }, + "tokenizer": { + "path_hierarchy": { + "type": "path_hierarchy" + } + }, + "normalizer": { + "lowercase": { + "type": "custom", + "filter": [ + "lowercase" + ] + } + }, + "char_filter": { + "dot_to_space": { + "type": "pattern_replace", + "pattern": "\\.", + "replacement": " " + } + } + } + }, + "mappings": { + "properties": { + "Content": { + "type": "text", + "analyzer": "name_words", + "term_vector": "with_positions_offsets" + }, + "ID": { + "type": "keyword" + }, + "ParentID": { + "type": "keyword" + }, + "RootID": { + "type": "keyword" + }, + "MimeType": { + "type": "wildcard" + }, + "Path": { + "type": "text", + "analyzer": "path_hierarchy" + }, + "Deleted": { + "type": "boolean" + }, + "Hidden": { + "type": "boolean" + }, + "Favorites": { + "type": "keyword" + }, + "Tags": { + "type": "text", + "fields": { + "wildcard": { + "type": "wildcard", + "normalizer": "lowercase" + } + } + }, + "Name": { + "type": "text", + "analyzer": "name_words", + "fields": { + "wildcard": { + "type": "wildcard", + "normalizer": "lowercase" + } + } + }, + "Title": { + "type": "text", + "analyzer": "name_words", + "fields": { + "wildcard": { + "type": "wildcard", + "normalizer": "lowercase" + } + } + }, + "Mtime": { + "type": "date", + "ignore_malformed": true + } + }, + "dynamic_templates": [ + { + "audio_facets": { + "path_match": "audio.*", + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + } + ] + } +} diff --git a/services/search/pkg/opensearch/internal/osu/query_match_none.go b/services/search/pkg/opensearch/internal/osu/query_match_none.go new file mode 100644 index 0000000000..ef071ac57c --- /dev/null +++ b/services/search/pkg/opensearch/internal/osu/query_match_none.go @@ -0,0 +1,55 @@ +package osu + +import ( + "encoding/json" +) + +// MatchNoneQuery matches no document. +type MatchNoneQuery struct{} + +// NewMatchNoneQuery creates a query that matches no document. +func NewMatchNoneQuery() *MatchNoneQuery { + return &MatchNoneQuery{} +} + +// Map returns the query as a map. +func (q *MatchNoneQuery) Map() (map[string]any, error) { + return map[string]any{ + "match_none": map[string]any{}, + }, nil +} + +// MarshalJSON returns the query as JSON. +func (q *MatchNoneQuery) MarshalJSON() ([]byte, error) { + data, err := q.Map() + if err != nil { + return nil, err + } + + return json.Marshal(data) +} + +// MatchAllQuery matches every document. +type MatchAllQuery struct{} + +// NewMatchAllQuery creates a query that matches every document. +func NewMatchAllQuery() *MatchAllQuery { + return &MatchAllQuery{} +} + +// Map returns the query as a map. +func (q *MatchAllQuery) Map() (map[string]any, error) { + return map[string]any{ + "match_all": map[string]any{}, + }, nil +} + +// MarshalJSON returns the query as JSON. +func (q *MatchAllQuery) MarshalJSON() ([]byte, error) { + data, err := q.Map() + if err != nil { + return nil, err + } + + return json.Marshal(data) +} diff --git a/services/search/pkg/opensearch/internal/osu/query_term_level_range.go b/services/search/pkg/opensearch/internal/osu/query_term_level_range.go index 3e81bb00f1..8a38db2317 100644 --- a/services/search/pkg/opensearch/internal/osu/query_term_level_range.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_range.go @@ -6,7 +6,8 @@ import ( "time" ) -type RangeQuery[T time.Time | string] struct { +// RangeQuery matches documents whose field value lies inside the bounds. +type RangeQuery[T time.Time | string | float64] struct { field string gt T gte T @@ -22,7 +23,8 @@ type RangeQueryParams struct { TimeZone string `json:"time_zone,omitempty"` } -func NewRangeQuery[T time.Time | string](field string) *RangeQuery[T] { +// NewRangeQuery creates a range query for the given field. +func NewRangeQuery[T time.Time | string | float64](field string) *RangeQuery[T] { return &RangeQuery[T]{field: field} } diff --git a/services/search/pkg/opensearch/internal/osu/request_test.go b/services/search/pkg/opensearch/internal/osu/request_test.go index f8e0010258..833536ff2e 100644 --- a/services/search/pkg/opensearch/internal/osu/request_test.go +++ b/services/search/pkg/opensearch/internal/osu/request_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" ) diff --git a/services/search/pkg/opensearch/opensearch_test.go b/services/search/pkg/opensearch/opensearch_test.go index cd5fb6fdbc..690c2d70f3 100644 --- a/services/search/pkg/opensearch/opensearch_test.go +++ b/services/search/pkg/opensearch/opensearch_test.go @@ -6,7 +6,7 @@ import ( "os" "testing" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/config" ) diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index 310397cae6..0a84fabfc4 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -2,23 +2,26 @@ package bleve import ( "fmt" + "slices" + "strconv" "strings" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/blevesearch/bleve/v2" bleveQuery "github.com/blevesearch/bleve/v2/search/query" "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/pkg/kql" ) -// lowercaseFields lists the bleve fields whose index mapping uses a -// lowercasing analyzer. Values bound to these fields are pre-lowercased -// so query-side matching stays consistent with the index. -// Keep in sync with services/search/pkg/bleve/index.go NewMapping. var lowercaseFields = map[string]struct{}{ "Name": {}, + "Title": {}, "Tags": {}, "Favorites": {}, "Content": {}, + "MimeType": {}, + "Hidden": {}, } var _fields = map[string]string{ @@ -33,6 +36,7 @@ var _fields = map[string]string{ "tag": "Tags", "tags": "Tags", "content": "Content", + "title": "Title", "hidden": "Hidden", "favorite": "Favorites", } @@ -53,7 +57,6 @@ var bleveEscaper = strings.NewReplacer( `)`, `\)`, `{`, `\{`, `}`, `\}`, - `{`, `\}`, `[`, `\[`, `]`, `\]`, `^`, `\^`, @@ -106,14 +109,43 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { v = strings.ToLower(v) } + if k == "Type" { + v = resourceType(v) + } + var q bleveQuery.Query var group bool - switch k { - case "MimeType": + switch { + case k == "Hidden": + value, err := strconv.ParseBool(v) + if err != nil { + q = bleveQuery.NewMatchNoneQuery() + break + } + + bq := bleveQuery.NewBoolFieldQuery(value) + bq.SetField(k) + q = bq + case k == "MimeType": q, group = mimeType(k, v) if prev == nil { isGroup = group } + case slices.Contains([]string{"Name", "Title"}, k) && strings.ContainsAny(n.Value, "*?"): + patterns := []bleveQuery.Query{bleveQuery.NewQueryStringQuery(k + ".wildcard:" + v)} + if !strings.HasSuffix(v, "*") { + patterns = append(patterns, bleveQuery.NewQueryStringQuery(k+".wildcard:"+v+".*")) + } + + q = closed(bleveQuery.NewDisjunctionQuery(patterns)) + case n.Exact && !strings.ContainsAny(n.Value, "*?") && slices.Contains([]string{"Name", "Title"}, k): + q = bleveQuery.NewQueryStringQuery(k + ".wildcard:" + v) + case k == "Path" && !strings.ContainsAny(n.Value, "*?"): + q = pathAndBelow(k, n.Value) + case slices.Contains([]string{"Name", "Title", "Content"}, k) && !strings.ContainsAny(n.Value, "*?"): + q = phrase(k, n.Value) + case strings.Contains(n.Value, " ") && !strings.ContainsAny(n.Value, "*?"): + q = phrase(k, n.Value) default: q = bleveQuery.NewQueryStringQuery(k + ":" + v) } @@ -153,13 +185,25 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { continue } + if prev == nil { + prev = q + } else { + next = q + } + case *ast.NumberNode: + q := numberRange(getField(n.Key), n.Operator, n.Value) + if q == nil { + continue + } + if prev == nil { prev = q } else { next = q } case *ast.BooleanNode: - q := bleveQuery.NewQueryStringQuery(getField(n.Key) + fmt.Sprintf(":%v", n.Value)) + q := bleveQuery.NewBoolFieldQuery(n.Value) + q.SetField(getField(n.Key)) if prev == nil { prev = q } else { @@ -216,6 +260,10 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { func nextNode(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { if n, ok := nodes[offset].(*ast.GroupNode); ok { + if n.Key != "" { + n = normalizeGroupingProperty(n) + } + gq, _, err := walk(0, n.Nodes) if err != nil { return nil, 0, err @@ -277,6 +325,57 @@ func mapBinary(operator *ast.OperatorNode, ln, rn bleveQuery.Query, leftIsGroup }) } +func numberRange(field string, operator *ast.OperatorNode, value float64) bleveQuery.Query { + if operator == nil { + return nil + } + + inclusive, exclusive := true, false + + var q *bleveQuery.NumericRangeQuery + switch operator.Value { + case ">": + q = bleveQuery.NewNumericRangeInclusiveQuery(&value, nil, &exclusive, nil) + case ">=": + q = bleveQuery.NewNumericRangeInclusiveQuery(&value, nil, &inclusive, nil) + case "<": + q = bleveQuery.NewNumericRangeInclusiveQuery(nil, &value, nil, &exclusive) + case "<=": + q = bleveQuery.NewNumericRangeInclusiveQuery(nil, &value, nil, &inclusive) + default: + return nil + } + + q.SetField(field) + + return q +} + +func pathAndBelow(field, path string) bleveQuery.Query { + path = strings.TrimSuffix(path, "/") + + self := bleveQuery.NewTermQuery(path) + self.SetField(field) + + below := bleveQuery.NewPrefixQuery(path + "/") + below.SetField(field) + + return closed(bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{self, below})) +} + +func closed(q bleveQuery.Query) bleveQuery.Query { + // a bare disjunction reads as an open OR chain to mapBinary, a later OR + // would merge into it and widen the group + return bleveQuery.NewConjunctionQuery([]bleveQuery.Query{q}) +} + +func phrase(field, value string) bleveQuery.Query { + q := bleveQuery.NewMatchPhraseQuery(value) + q.SetField(field) + + return q +} + func getField(name string) string { if name == "" { return "Name" @@ -296,6 +395,17 @@ func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode { return group } +func resourceType(value string) string { + switch strings.ToLower(value) { + case "file": + return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10) + case "folder": + return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10) + default: + return value + } +} + func mimeType(k, v string) (bleveQuery.Query, bool) { switch v { case "file": @@ -321,7 +431,6 @@ func mimeType(k, v string) (bleveQuery.Query, bool) { "application/vnd.oasis.opendocument.spreadsheet", "text/csv", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "application/vnd.oasis.opendocument.spreadsheet", "application/vnd.apple.numbers", )), true case "presentation": diff --git a/services/search/pkg/query/bleve/compiler_test.go b/services/search/pkg/query/bleve/compiler_test.go index ff94e88726..30c283f694 100644 --- a/services/search/pkg/query/bleve/compiler_test.go +++ b/services/search/pkg/query/bleve/compiler_test.go @@ -1,6 +1,7 @@ package bleve import ( + "strings" "testing" "time" @@ -18,6 +19,29 @@ var timeMustParse = func(t *testing.T, ts string) time.Time { return tp } +func wildcardQuery(field, value string) query.Query { + patterns := []query.Query{query.NewQueryStringQuery(field + ".wildcard:" + value)} + if !strings.HasSuffix(value, "*") { + patterns = append(patterns, query.NewQueryStringQuery(field+".wildcard:"+value+".*")) + } + + return query.NewConjunctionQuery([]query.Query{query.NewDisjunctionQuery(patterns)}) +} + +func phraseQuery(field, value string) query.Query { + q := query.NewMatchPhraseQuery(value) + q.SetField(field) + + return q +} + +func boolFieldQuery(field string, value bool) query.Query { + q := query.NewBoolFieldQuery(value) + q.SetField(field) + + return q +} + func Test_compile(t *testing.T) { tests := []struct { name string @@ -33,7 +57,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:federated`), + phraseQuery("Name", "federated"), }), wantErr: false, }, @@ -45,7 +69,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith`), + phraseQuery("Name", "John Smith"), }), wantErr: false, }, @@ -59,8 +83,8 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith`), - query.NewQueryStringQuery(`Name:jane`), + phraseQuery("Name", "John Smith"), + phraseQuery("Name", "Jane"), }), wantErr: false, }, @@ -91,7 +115,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:moby\ di*`), + wildcardQuery("Name", `moby\ di*`), query.NewConjunctionQuery([]query.Query{ query.NewQueryStringQuery(`Tags:bestseller`), query.NewQueryStringQuery(`Tags:book`), @@ -112,10 +136,10 @@ func Test_compile(t *testing.T) { }, want: query.NewDisjunctionQuery([]query.Query{ query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:a`), - query.NewQueryStringQuery(`Name:b`), + phraseQuery("Name", "a"), + phraseQuery("Name", "b"), }), - query.NewQueryStringQuery(`Name:c`), + phraseQuery("Name", "c"), }), wantErr: false, }, @@ -131,10 +155,10 @@ func Test_compile(t *testing.T) { }, }, want: query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:a`), + phraseQuery("Name", "a"), query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:b`), - query.NewQueryStringQuery(`Name:c`), + phraseQuery("Name", "b"), + phraseQuery("Name", "c"), }), }), wantErr: false, @@ -156,11 +180,11 @@ func Test_compile(t *testing.T) { }, want: query.NewConjunctionQuery([]query.Query{ query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:a`), - query.NewQueryStringQuery(`Name:b`), - query.NewQueryStringQuery(`Name:c`), + phraseQuery("Name", "a"), + phraseQuery("Name", "b"), + phraseQuery("Name", "c"), }), - query.NewQueryStringQuery(`Name:d`), + phraseQuery("Name", "d"), }), wantErr: false, }, @@ -179,7 +203,7 @@ func Test_compile(t *testing.T) { }, want: query.NewConjunctionQuery([]query.Query{ query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:moby\ di*`), + wildcardQuery("Name", `moby\ di*`), query.NewQueryStringQuery(`Tags:bestseller`), }), query.NewQueryStringQuery(`Tags:book`), @@ -204,7 +228,7 @@ func Test_compile(t *testing.T) { }, want: query.NewConjunctionQuery([]query.Query{ query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:moby\ di*`), + wildcardQuery("Name", `moby\ di*`), query.NewQueryStringQuery(`Tags:bestseller`), }), query.NewQueryStringQuery(`Tags:book`), @@ -227,7 +251,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`author:John\ Smith`), + phraseQuery("author", "John Smith"), query.NewQueryStringQuery(`author:Jane`), }), wantErr: false, @@ -249,7 +273,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`author:John\ Smith`), + phraseQuery("author", "John Smith"), query.NewQueryStringQuery(`author:Jane`), query.NewQueryStringQuery(`Tags:bestseller`), }), @@ -293,9 +317,44 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith`), - query.NewQueryStringQuery(`Hidden:T`), - query.NewQueryStringQuery(`Hidden:T`), + phraseQuery("Name", "John Smith"), + boolFieldQuery("Hidden", true), + boolFieldQuery("Hidden", true), + }), + wantErr: false, + }, + { + name: `hidden:banana`, + args: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "hidden", Value: "banana"}, + }, + }, + want: query.NewConjunctionQuery([]query.Query{query.NewMatchNoneQuery()}), + wantErr: false, + }, + { + name: `name="Report.txt"`, + args: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "name", Value: "Report.txt", Exact: true}, + }, + }, + want: query.NewConjunctionQuery([]query.Query{query.NewQueryStringQuery(`Name.wildcard:report.txt`)}), + wantErr: false, + }, + { + name: `type:File`, + args: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "type", Value: "File"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "type", Value: "FOLDER"}, + }, + }, + want: query.NewDisjunctionQuery([]query.Query{ + query.NewQueryStringQuery(`Type:1`), + query.NewQueryStringQuery(`Type:2`), }), wantErr: false, }, @@ -424,7 +483,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/rtf`), query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), }), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), wantErr: false, }, @@ -450,7 +509,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), query.NewConjunctionQuery([]query.Query{ query.NewQueryStringQuery(`MimeType:application/pdf`), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), }), wantErr: false, @@ -480,7 +539,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), query.NewQueryStringQuery(`MimeType:application/pdf`), }), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), wantErr: false, }, @@ -509,7 +568,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/rtf`), query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), }), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), wantErr: false, }, @@ -521,7 +580,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith\ \+\-\=\&\|\>\<\!\(\)\{\}\[\]\^\"\~\:\ `), + phraseQuery("Name", "John Smith +-=&|>quarterly reportsome data" to "q1.html" + And user "Alice" has uploaded file with content "notessome data" to "q2.html" + When user "Alice" searches for 'Title:"quarterly report"' using the WebDAV API + Then the HTTP status code should be "207" + And the search result of user "Alice" should contain only these entries: + | q1.html | diff --git a/vendor/github.com/blevesearch/bleve/v2/analysis/char/regexp/regexp.go b/vendor/github.com/blevesearch/bleve/v2/analysis/char/regexp/regexp.go new file mode 100644 index 0000000000..a94236af9a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/char/regexp/regexp.go @@ -0,0 +1,65 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package regexp + +import ( + "fmt" + "regexp" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "regexp" + +type CharFilter struct { + r *regexp.Regexp + replacement []byte +} + +func New(r *regexp.Regexp, replacement []byte) *CharFilter { + return &CharFilter{ + r: r, + replacement: replacement, + } +} + +func (s *CharFilter) Filter(input []byte) []byte { + return s.r.ReplaceAll(input, s.replacement) +} + +func CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) { + regexpStr, ok := config["regexp"].(string) + if !ok { + return nil, fmt.Errorf("must specify regexp") + } + r, err := regexp.Compile(regexpStr) + if err != nil { + return nil, fmt.Errorf("unable to build regexp char filter: %v", err) + } + replaceBytes := []byte(" ") + replaceStr, ok := config["replace"].(string) + if ok { + replaceBytes = []byte(replaceStr) + } + return New(r, replaceBytes), nil +} + +func init() { + err := registry.RegisterCharFilter(Name, CharFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index f13e2c9497..beaceb675a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -126,6 +126,7 @@ github.com/blevesearch/bleve/v2/analysis/analyzer/custom github.com/blevesearch/bleve/v2/analysis/analyzer/keyword github.com/blevesearch/bleve/v2/analysis/analyzer/standard github.com/blevesearch/bleve/v2/analysis/datetime/flexible +github.com/blevesearch/bleve/v2/analysis/char/regexp github.com/blevesearch/bleve/v2/analysis/datetime/optional github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds