From bd4287762ec3b7715582e5c6378bc72a09d726e7 Mon Sep 17 00:00:00 2001 From: Florian Schade Date: Mon, 31 Aug 2026 10:44:03 +0200 Subject: [PATCH 1/4] fix(search): answer the same on both engines both engines now agree on names, titles, tags, paths, types, sizes, dates, hidden flags, facet values and wildcards. quotes only delimit phrases and the equals operator matches the whole field value, following the kql spec. the index name carries a generation so a changed mapping starts on a fresh index, MIGRATION.md says how to fill it. --- pkg/ast/ast.go | 9 + pkg/ast/test/test.go | 1 + pkg/kql/cast.go | 10 + pkg/kql/dictionary.peg | 27 +- pkg/kql/dictionary_gen.go | 1411 +++++++++++------ pkg/kql/dictionary_test.go | 70 + pkg/kql/engine_suite_test.go | 11 - pkg/kql/factory.go | 33 +- pkg/kql/kql.go | 6 + services/search/MIGRATION.md | 55 + services/search/README.md | 6 + .../search/internal/opensearchtest/helper.go | 14 - services/search/internal/opensearchtest/os.go | 97 -- .../search/internal/opensearchtest/suite.go | 14 +- .../internal/opensearchtest/testdata.go | 8 +- services/search/pkg/bleve/index.go | 71 +- services/search/pkg/bleve/index_test.go | 39 + services/search/pkg/content/content.go | 2 +- services/search/pkg/content/tika.go | 8 +- services/search/pkg/content/tika_test.go | 22 + services/search/pkg/opensearch/backend.go | 7 +- .../search/pkg/opensearch/backend_test.go | 2 - services/search/pkg/opensearch/index.go | 87 +- services/search/pkg/opensearch/index_test.go | 28 + .../opensearch/internal/convert/kql_expand.go | 23 +- .../internal/convert/kql_expand_test.go | 60 +- .../internal/convert/kql_transpile.go | 80 +- .../internal/convert/kql_transpile_test.go | 68 +- .../internal/convert/opensearch_test.go | 2 +- .../internal/indexes/resource_v3.json | 122 ++ .../internal/osu/query_match_none.go | 55 + .../internal/osu/query_term_level_range.go | 6 +- .../opensearch/internal/osu/request_test.go | 2 +- .../search/pkg/opensearch/opensearch_test.go | 2 +- services/search/pkg/query/bleve/compiler.go | 127 +- .../search/pkg/query/bleve/compiler_test.go | 113 +- .../apiSearchContent/titleSearch.feature | 19 + .../bleve/v2/analysis/char/regexp/regexp.go | 65 + vendor/modules.txt | 1 + 39 files changed, 2040 insertions(+), 743 deletions(-) delete mode 100644 pkg/kql/engine_suite_test.go create mode 100644 services/search/MIGRATION.md create mode 100644 services/search/pkg/bleve/index_test.go create mode 100644 services/search/pkg/opensearch/internal/indexes/resource_v3.json create mode 100644 services/search/pkg/opensearch/internal/osu/query_match_none.go create mode 100644 tests/acceptance/features/apiSearchContent/titleSearch.feature create mode 100644 vendor/github.com/blevesearch/bleve/v2/analysis/char/regexp/regexp.go diff --git a/pkg/ast/ast.go b/pkg/ast/ast.go index 9aafe853aa..f1a7e3263e 100644 --- a/pkg/ast/ast.go +++ b/pkg/ast/ast.go @@ -43,6 +43,7 @@ type StringNode struct { *Base Key string Value string + Exact bool } // BooleanNode represents a bool value @@ -60,6 +61,14 @@ type DateTimeNode struct { Value time.Time } +// NumberNode represents a numeric value +type NumberNode struct { + *Base + Key string + Operator *OperatorNode + Value float64 +} + // OperatorNode represents an operator value like // AND, OR, NOT, =, <= ... and so on type OperatorNode struct { diff --git a/pkg/ast/test/test.go b/pkg/ast/test/test.go index d7fb985226..8a39b75022 100644 --- a/pkg/ast/test/test.go +++ b/pkg/ast/test/test.go @@ -21,6 +21,7 @@ func DiffAst(x, y any, opts ...cmp.Option) string { cmpopts.IgnoreFields(ast.GroupNode{}, "Base"), cmpopts.IgnoreFields(ast.BooleanNode{}, "Base"), cmpopts.IgnoreFields(ast.DateTimeNode{}, "Base"), + cmpopts.IgnoreFields(ast.NumberNode{}, "Base"), )..., ) } diff --git a/pkg/kql/cast.go b/pkg/kql/cast.go index c1fb185623..ecd0cac27f 100644 --- a/pkg/kql/cast.go +++ b/pkg/kql/cast.go @@ -2,6 +2,7 @@ package kql import ( "fmt" + "strconv" "time" "github.com/jinzhu/now" @@ -136,3 +137,12 @@ func toTimeRange(in any) (*time.Time, *time.Time, error) { return &from, &to, nil } + +func toFloat(v any) (float64, error) { + value, err := toString(v) + if err != nil { + return 0, err + } + + return strconv.ParseFloat(value, 64) +} diff --git a/pkg/kql/dictionary.peg b/pkg/kql/dictionary.peg index 5f59e78afb..c21f46f4c4 100644 --- a/pkg/kql/dictionary.peg +++ b/pkg/kql/dictionary.peg @@ -40,6 +40,7 @@ GroupNode <- PropertyRestrictionNodes <- YesNoPropertyRestrictionNode / DateTimeRestrictionNode / + NumberRestrictionNode / TextPropertyRestrictionNode YesNoPropertyRestrictionNode <- @@ -69,9 +70,22 @@ DateTimeRestrictionNode <- return buildNaturalLanguageDateTimeNodes(k, v, c.text, c.pos) } +NumberRestrictionNode <- + k:Key o:( + OperatorGreaterOrEqualNode / + OperatorLessOrEqualNode / + OperatorGreaterNode / + OperatorLessNode + ) '"'? v:Number '"'? { + return buildNumberNode(k, o, v, c.text, c.pos) + } + TextPropertyRestrictionNode <- - k:Key (OperatorColonNode / OperatorEqualNode) v:(String / [^ ()]+){ - return buildStringNode(k, v, c.text, c.pos) + k:Key OperatorEqualNode v:(String / [^ ()]+) { + return buildStringNode(k, v, true, c.text, c.pos) + } / + k:Key OperatorColonNode v:(String / [^ ()]+) { + return buildStringNode(k, v, false, c.text, c.pos) } //////////////////////////////////////////////////////// @@ -84,12 +98,12 @@ FreeTextKeywordNodes <- PhraseNode <- OperatorColonNode? _ v:String _ OperatorColonNode? { - return buildStringNode("", v, c.text, c.pos) + return buildStringNode("", v, false, c.text, c.pos) } WordNode <- OperatorColonNode? _ v:[^ :()]+ _ OperatorColonNode? { - return buildStringNode("", v, c.text, c.pos) + return buildStringNode("", v, false, c.text, c.pos) } //////////////////////////////////////////////////////// @@ -229,6 +243,11 @@ String <- return v, nil } +Number <- + [0-9]+ ("." [0-9]+)? { + return string(c.text), nil + } + Digit <- [0-9] { return c.text, nil diff --git a/pkg/kql/dictionary_gen.go b/pkg/kql/dictionary_gen.go index aec6f49d1d..9777ece435 100644 --- a/pkg/kql/dictionary_gen.go +++ b/pkg/kql/dictionary_gen.go @@ -43,12 +43,12 @@ var g = &grammar{ pos: position{line: 19, col: 6, offset: 351}, exprs: []any{ &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, run: (*parser).callonNodes3, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -75,27 +75,27 @@ var g = &grammar{ name: "GroupNode", }, &actionExpr{ - pos: position{line: 46, col: 5, offset: 1038}, + pos: position{line: 47, col: 5, offset: 1066}, run: (*parser).callonNode3, expr: &seqExpr{ - pos: position{line: 46, col: 5, offset: 1038}, + pos: position{line: 47, col: 5, offset: 1066}, exprs: []any{ &labeledExpr{ - pos: position{line: 46, col: 5, offset: 1038}, + pos: position{line: 47, col: 5, offset: 1066}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode6, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode9, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -104,23 +104,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode15, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -136,23 +136,23 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 46, col: 12, offset: 1045}, + pos: position{line: 47, col: 12, offset: 1073}, alternatives: []any{ &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonNode18, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", }, }, &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonNode20, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", @@ -161,19 +161,19 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 46, col: 51, offset: 1084}, + pos: position{line: 47, col: 51, offset: 1112}, label: "v", expr: &choiceExpr{ - pos: position{line: 46, col: 54, offset: 1087}, + pos: position{line: 47, col: 54, offset: 1115}, alternatives: []any{ &litMatcher{ - pos: position{line: 46, col: 54, offset: 1087}, + pos: position{line: 47, col: 54, offset: 1115}, val: "true", ignoreCase: false, want: "\"true\"", }, &litMatcher{ - pos: position{line: 46, col: 63, offset: 1096}, + pos: position{line: 47, col: 63, offset: 1124}, val: "false", ignoreCase: false, want: "\"false\"", @@ -185,27 +185,27 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 51, col: 5, offset: 1197}, + pos: position{line: 52, col: 5, offset: 1225}, run: (*parser).callonNode26, expr: &seqExpr{ - pos: position{line: 51, col: 5, offset: 1197}, + pos: position{line: 52, col: 5, offset: 1225}, exprs: []any{ &labeledExpr{ - pos: position{line: 51, col: 5, offset: 1197}, + pos: position{line: 52, col: 5, offset: 1225}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode29, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode32, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -214,23 +214,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode38, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -246,66 +246,66 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 51, col: 11, offset: 1203}, + pos: position{line: 52, col: 11, offset: 1231}, label: "o", expr: &choiceExpr{ - pos: position{line: 52, col: 9, offset: 1215}, + pos: position{line: 53, col: 9, offset: 1243}, alternatives: []any{ &actionExpr{ - pos: position{line: 145, col: 5, offset: 3346}, + pos: position{line: 159, col: 5, offset: 3745}, run: (*parser).callonNode42, expr: &litMatcher{ - pos: position{line: 145, col: 5, offset: 3346}, + pos: position{line: 159, col: 5, offset: 3745}, val: ">=", ignoreCase: false, want: "\">=\"", }, }, &actionExpr{ - pos: position{line: 135, col: 5, offset: 3162}, + pos: position{line: 149, col: 5, offset: 3561}, run: (*parser).callonNode44, expr: &litMatcher{ - pos: position{line: 135, col: 5, offset: 3162}, + pos: position{line: 149, col: 5, offset: 3561}, val: "<=", ignoreCase: false, want: "\"<=\"", }, }, &actionExpr{ - pos: position{line: 140, col: 5, offset: 3251}, + pos: position{line: 154, col: 5, offset: 3650}, run: (*parser).callonNode46, expr: &litMatcher{ - pos: position{line: 140, col: 5, offset: 3251}, + pos: position{line: 154, col: 5, offset: 3650}, val: ">", ignoreCase: false, want: "\">\"", }, }, &actionExpr{ - pos: position{line: 130, col: 5, offset: 3070}, + pos: position{line: 144, col: 5, offset: 3469}, run: (*parser).callonNode48, expr: &litMatcher{ - pos: position{line: 130, col: 5, offset: 3070}, + pos: position{line: 144, col: 5, offset: 3469}, val: "<", ignoreCase: false, want: "\"<\"", }, }, &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonNode50, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", }, }, &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonNode52, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -315,43 +315,43 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 58, col: 7, offset: 1395}, + pos: position{line: 59, col: 7, offset: 1423}, expr: &litMatcher{ - pos: position{line: 58, col: 7, offset: 1395}, + pos: position{line: 59, col: 7, offset: 1423}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, }, &labeledExpr{ - pos: position{line: 58, col: 12, offset: 1400}, + pos: position{line: 59, col: 12, offset: 1428}, label: "v", expr: &choiceExpr{ - pos: position{line: 59, col: 9, offset: 1412}, + pos: position{line: 60, col: 9, offset: 1440}, alternatives: []any{ &actionExpr{ - pos: position{line: 195, col: 5, offset: 4185}, + pos: position{line: 209, col: 5, offset: 4584}, run: (*parser).callonNode58, expr: &seqExpr{ - pos: position{line: 195, col: 5, offset: 4185}, + pos: position{line: 209, col: 5, offset: 4584}, exprs: []any{ &actionExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, run: (*parser).callonNode60, expr: &seqExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, exprs: []any{ &actionExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, run: (*parser).callonNode62, expr: &seqExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode64, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -359,10 +359,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode66, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -370,10 +370,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode68, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -381,10 +381,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode70, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -395,22 +395,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 14, offset: 3957}, + pos: position{line: 199, col: 14, offset: 4356}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, run: (*parser).callonNode73, expr: &seqExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode75, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -418,10 +418,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode77, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -432,22 +432,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 28, offset: 3971}, + pos: position{line: 199, col: 28, offset: 4370}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, run: (*parser).callonNode80, expr: &seqExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode82, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -455,10 +455,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode84, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -472,28 +472,28 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 195, col: 14, offset: 4194}, + pos: position{line: 209, col: 14, offset: 4593}, val: "T", ignoreCase: false, want: "\"T\"", }, &actionExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, run: (*parser).callonNode87, expr: &seqExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, exprs: []any{ &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode89, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode91, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -501,10 +501,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode93, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -515,22 +515,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 14, offset: 4044}, + pos: position{line: 204, col: 14, offset: 4443}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode96, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode98, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -538,10 +538,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode100, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -552,22 +552,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 29, offset: 4059}, + pos: position{line: 204, col: 29, offset: 4458}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, run: (*parser).callonNode103, expr: &seqExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode105, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -575,10 +575,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode107, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -589,23 +589,23 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 190, col: 44, offset: 4074}, + pos: position{line: 204, col: 44, offset: 4473}, expr: &seqExpr{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, exprs: []any{ &litMatcher{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 190, col: 49, offset: 4079}, + pos: position{line: 204, col: 49, offset: 4478}, expr: &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode113, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -617,35 +617,35 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, alternatives: []any{ &litMatcher{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, val: "Z", ignoreCase: false, want: "\"Z\"", }, &seqExpr{ - pos: position{line: 190, col: 65, offset: 4095}, + pos: position{line: 204, col: 65, offset: 4494}, exprs: []any{ &charClassMatcher{ - pos: position{line: 190, col: 66, offset: 4096}, + pos: position{line: 204, col: 66, offset: 4495}, val: "[+-]", chars: []rune{'+', '-'}, ignoreCase: false, inverted: false, }, &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode119, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode121, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -653,10 +653,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode123, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -667,22 +667,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 86, offset: 4116}, + pos: position{line: 204, col: 86, offset: 4515}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode126, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode128, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -690,10 +690,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode130, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -714,22 +714,22 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, run: (*parser).callonNode132, expr: &seqExpr{ - pos: position{line: 185, col: 5, offset: 3948}, + pos: position{line: 199, col: 5, offset: 4347}, exprs: []any{ &actionExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, run: (*parser).callonNode134, expr: &seqExpr{ - pos: position{line: 155, col: 5, offset: 3548}, + pos: position{line: 169, col: 5, offset: 3947}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode136, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -737,10 +737,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode138, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -748,10 +748,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode140, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -759,10 +759,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode142, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -773,22 +773,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 14, offset: 3957}, + pos: position{line: 199, col: 14, offset: 4356}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, run: (*parser).callonNode145, expr: &seqExpr{ - pos: position{line: 160, col: 5, offset: 3625}, + pos: position{line: 174, col: 5, offset: 4024}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode147, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -796,10 +796,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode149, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -810,22 +810,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 185, col: 28, offset: 3971}, + pos: position{line: 199, col: 28, offset: 4370}, val: "-", ignoreCase: false, want: "\"-\"", }, &actionExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, run: (*parser).callonNode152, expr: &seqExpr{ - pos: position{line: 165, col: 5, offset: 3688}, + pos: position{line: 179, col: 5, offset: 4087}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode154, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -833,10 +833,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode156, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -850,22 +850,22 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, run: (*parser).callonNode158, expr: &seqExpr{ - pos: position{line: 190, col: 5, offset: 4035}, + pos: position{line: 204, col: 5, offset: 4434}, exprs: []any{ &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode160, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode162, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -873,10 +873,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode164, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -887,22 +887,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 14, offset: 4044}, + pos: position{line: 204, col: 14, offset: 4443}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode167, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode169, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -910,10 +910,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode171, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -924,22 +924,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 29, offset: 4059}, + pos: position{line: 204, col: 29, offset: 4458}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, run: (*parser).callonNode174, expr: &seqExpr{ - pos: position{line: 180, col: 5, offset: 3884}, + pos: position{line: 194, col: 5, offset: 4283}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode176, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -947,10 +947,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode178, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -961,23 +961,23 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 190, col: 44, offset: 4074}, + pos: position{line: 204, col: 44, offset: 4473}, expr: &seqExpr{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, exprs: []any{ &litMatcher{ - pos: position{line: 190, col: 45, offset: 4075}, + pos: position{line: 204, col: 45, offset: 4474}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 190, col: 49, offset: 4079}, + pos: position{line: 204, col: 49, offset: 4478}, expr: &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode184, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -989,35 +989,35 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, alternatives: []any{ &litMatcher{ - pos: position{line: 190, col: 59, offset: 4089}, + pos: position{line: 204, col: 59, offset: 4488}, val: "Z", ignoreCase: false, want: "\"Z\"", }, &seqExpr{ - pos: position{line: 190, col: 65, offset: 4095}, + pos: position{line: 204, col: 65, offset: 4494}, exprs: []any{ &charClassMatcher{ - pos: position{line: 190, col: 66, offset: 4096}, + pos: position{line: 204, col: 66, offset: 4495}, val: "[+-]", chars: []rune{'+', '-'}, ignoreCase: false, inverted: false, }, &actionExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, run: (*parser).callonNode190, expr: &seqExpr{ - pos: position{line: 170, col: 5, offset: 3752}, + pos: position{line: 184, col: 5, offset: 4151}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode192, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1025,10 +1025,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode194, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1039,22 +1039,22 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 190, col: 86, offset: 4116}, + pos: position{line: 204, col: 86, offset: 4515}, val: ":", ignoreCase: false, want: "\":\"", }, &actionExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, run: (*parser).callonNode197, expr: &seqExpr{ - pos: position{line: 175, col: 5, offset: 3818}, + pos: position{line: 189, col: 5, offset: 4217}, exprs: []any{ &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode199, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1062,10 +1062,10 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, run: (*parser).callonNode201, expr: &charClassMatcher{ - pos: position{line: 233, col: 5, offset: 4807}, + pos: position{line: 252, col: 5, offset: 5285}, val: "[0-9]", ranges: []rune{'0', '9'}, ignoreCase: false, @@ -1086,9 +1086,9 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 62, col: 7, offset: 1465}, + pos: position{line: 63, col: 7, offset: 1493}, expr: &litMatcher{ - pos: position{line: 62, col: 7, offset: 1465}, + pos: position{line: 63, col: 7, offset: 1493}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1098,27 +1098,27 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 65, col: 5, offset: 1541}, + pos: position{line: 66, col: 5, offset: 1569}, run: (*parser).callonNode205, expr: &seqExpr{ - pos: position{line: 65, col: 5, offset: 1541}, + pos: position{line: 66, col: 5, offset: 1569}, exprs: []any{ &labeledExpr{ - pos: position{line: 65, col: 5, offset: 1541}, + pos: position{line: 66, col: 5, offset: 1569}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode208, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode211, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1127,23 +1127,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode217, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1159,23 +1159,23 @@ var g = &grammar{ }, }, &choiceExpr{ - pos: position{line: 66, col: 9, offset: 1557}, + pos: position{line: 67, col: 9, offset: 1585}, alternatives: []any{ &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonNode220, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", }, }, &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonNode222, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1184,79 +1184,79 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 68, col: 7, offset: 1609}, + pos: position{line: 69, col: 7, offset: 1637}, expr: &litMatcher{ - pos: position{line: 68, col: 7, offset: 1609}, + pos: position{line: 69, col: 7, offset: 1637}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, }, &labeledExpr{ - pos: position{line: 68, col: 12, offset: 1614}, + pos: position{line: 69, col: 12, offset: 1642}, label: "v", expr: &choiceExpr{ - pos: position{line: 200, col: 5, offset: 4273}, + pos: position{line: 214, col: 5, offset: 4672}, alternatives: []any{ &litMatcher{ - pos: position{line: 200, col: 5, offset: 4273}, + pos: position{line: 214, col: 5, offset: 4672}, val: "today", ignoreCase: false, want: "\"today\"", }, &litMatcher{ - pos: position{line: 201, col: 5, offset: 4287}, + pos: position{line: 215, col: 5, offset: 4686}, val: "yesterday", ignoreCase: false, want: "\"yesterday\"", }, &litMatcher{ - pos: position{line: 202, col: 5, offset: 4305}, + pos: position{line: 216, col: 5, offset: 4704}, val: "this week", ignoreCase: false, want: "\"this week\"", }, &litMatcher{ - pos: position{line: 203, col: 5, offset: 4323}, + pos: position{line: 217, col: 5, offset: 4722}, val: "last week", ignoreCase: false, want: "\"last week\"", }, &litMatcher{ - pos: position{line: 204, col: 5, offset: 4341}, + pos: position{line: 218, col: 5, offset: 4740}, val: "last 7 days", ignoreCase: false, want: "\"last 7 days\"", }, &litMatcher{ - pos: position{line: 205, col: 5, offset: 4361}, + pos: position{line: 219, col: 5, offset: 4760}, val: "this month", ignoreCase: false, want: "\"this month\"", }, &litMatcher{ - pos: position{line: 206, col: 5, offset: 4380}, + pos: position{line: 220, col: 5, offset: 4779}, val: "last month", ignoreCase: false, want: "\"last month\"", }, &litMatcher{ - pos: position{line: 207, col: 5, offset: 4399}, + pos: position{line: 221, col: 5, offset: 4798}, val: "last 30 days", ignoreCase: false, want: "\"last 30 days\"", }, &litMatcher{ - pos: position{line: 208, col: 5, offset: 4420}, + pos: position{line: 222, col: 5, offset: 4819}, val: "this year", ignoreCase: false, want: "\"this year\"", }, &actionExpr{ - pos: position{line: 209, col: 5, offset: 4438}, + pos: position{line: 223, col: 5, offset: 4837}, run: (*parser).callonNode237, expr: &litMatcher{ - pos: position{line: 209, col: 5, offset: 4438}, + pos: position{line: 223, col: 5, offset: 4837}, val: "last year", ignoreCase: false, want: "\"last year\"", @@ -1266,9 +1266,9 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 68, col: 38, offset: 1640}, + pos: position{line: 69, col: 38, offset: 1668}, expr: &litMatcher{ - pos: position{line: 68, col: 38, offset: 1640}, + pos: position{line: 69, col: 38, offset: 1668}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1278,27 +1278,27 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 73, col: 5, offset: 1759}, + pos: position{line: 74, col: 5, offset: 1781}, run: (*parser).callonNode241, expr: &seqExpr{ - pos: position{line: 73, col: 5, offset: 1759}, + pos: position{line: 74, col: 5, offset: 1781}, exprs: []any{ &labeledExpr{ - pos: position{line: 73, col: 5, offset: 1759}, + pos: position{line: 74, col: 5, offset: 1781}, label: "k", expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonNode244, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode247, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1307,23 +1307,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonNode253, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1338,56 +1338,219 @@ var g = &grammar{ }, }, }, - &choiceExpr{ - pos: position{line: 73, col: 12, offset: 1766}, - alternatives: []any{ - &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode256, - expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, - val: ":", - ignoreCase: false, - want: "\":\"", + &labeledExpr{ + pos: position{line: 74, col: 11, offset: 1787}, + label: "o", + expr: &choiceExpr{ + pos: position{line: 75, col: 9, offset: 1799}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 159, col: 5, offset: 3745}, + run: (*parser).callonNode257, + expr: &litMatcher{ + pos: position{line: 159, col: 5, offset: 3745}, + val: ">=", + ignoreCase: false, + want: "\">=\"", + }, }, - }, - &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, - run: (*parser).callonNode258, - expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, - val: "=", - ignoreCase: false, - want: "\"=\"", + &actionExpr{ + pos: position{line: 149, col: 5, offset: 3561}, + run: (*parser).callonNode259, + expr: &litMatcher{ + pos: position{line: 149, col: 5, offset: 3561}, + val: "<=", + ignoreCase: false, + want: "\"<=\"", + }, + }, + &actionExpr{ + pos: position{line: 154, col: 5, offset: 3650}, + run: (*parser).callonNode261, + expr: &litMatcher{ + pos: position{line: 154, col: 5, offset: 3650}, + val: ">", + ignoreCase: false, + want: "\">\"", + }, + }, + &actionExpr{ + pos: position{line: 144, col: 5, offset: 3469}, + run: (*parser).callonNode263, + expr: &litMatcher{ + pos: position{line: 144, col: 5, offset: 3469}, + val: "<", + ignoreCase: false, + want: "\"<\"", + }, }, }, }, }, + &zeroOrOneExpr{ + pos: position{line: 79, col: 7, offset: 1923}, + expr: &litMatcher{ + pos: position{line: 79, col: 7, offset: 1923}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + }, &labeledExpr{ - pos: position{line: 73, col: 51, offset: 1805}, + pos: position{line: 79, col: 12, offset: 1928}, + label: "v", + expr: &actionExpr{ + pos: position{line: 247, col: 5, offset: 5207}, + run: (*parser).callonNode268, + expr: &seqExpr{ + pos: position{line: 247, col: 5, offset: 5207}, + exprs: []any{ + &oneOrMoreExpr{ + pos: position{line: 247, col: 5, offset: 5207}, + expr: &charClassMatcher{ + pos: position{line: 247, col: 5, offset: 5207}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + &zeroOrOneExpr{ + pos: position{line: 247, col: 12, offset: 5214}, + expr: &seqExpr{ + pos: position{line: 247, col: 13, offset: 5215}, + exprs: []any{ + &litMatcher{ + pos: position{line: 247, col: 13, offset: 5215}, + val: ".", + ignoreCase: false, + want: "\".\"", + }, + &oneOrMoreExpr{ + pos: position{line: 247, col: 17, offset: 5219}, + expr: &charClassMatcher{ + pos: position{line: 247, col: 17, offset: 5219}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + }, + }, + }, + }, + }, + &zeroOrOneExpr{ + pos: position{line: 79, col: 21, offset: 1937}, + expr: &litMatcher{ + pos: position{line: 79, col: 21, offset: 1937}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 84, col: 5, offset: 2041}, + run: (*parser).callonNode279, + expr: &seqExpr{ + pos: position{line: 84, col: 5, offset: 2041}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 84, col: 5, offset: 2041}, + label: "k", + expr: &actionExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + run: (*parser).callonNode282, + expr: &seqExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + exprs: []any{ + &oneOrMoreExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode285, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + &zeroOrMoreExpr{ + pos: position{line: 237, col: 11, offset: 5083}, + expr: &seqExpr{ + pos: position{line: 237, col: 12, offset: 5084}, + exprs: []any{ + &litMatcher{ + pos: position{line: 237, col: 12, offset: 5084}, + val: ".", + ignoreCase: false, + want: "\".\"", + }, + &oneOrMoreExpr{ + pos: position{line: 237, col: 16, offset: 5088}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode291, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 139, col: 5, offset: 3384}, + run: (*parser).callonNode293, + expr: &litMatcher{ + pos: position{line: 139, col: 5, offset: 3384}, + val: "=", + ignoreCase: false, + want: "\"=\"", + }, + }, + &labeledExpr{ + pos: position{line: 84, col: 29, offset: 2065}, label: "v", expr: &choiceExpr{ - pos: position{line: 73, col: 54, offset: 1808}, + pos: position{line: 84, col: 32, offset: 2068}, alternatives: []any{ &actionExpr{ - pos: position{line: 228, col: 5, offset: 4747}, - run: (*parser).callonNode262, + pos: position{line: 242, col: 5, offset: 5146}, + run: (*parser).callonNode297, expr: &seqExpr{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, exprs: []any{ &litMatcher{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, &labeledExpr{ - pos: position{line: 228, col: 9, offset: 4751}, + pos: position{line: 242, col: 9, offset: 5150}, label: "v", expr: &zeroOrMoreExpr{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, expr: &charClassMatcher{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, val: "[^\"]", chars: []rune{'"'}, ignoreCase: false, @@ -1396,7 +1559,7 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 228, col: 17, offset: 4759}, + pos: position{line: 242, col: 17, offset: 5158}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1405,9 +1568,9 @@ var g = &grammar{ }, }, &oneOrMoreExpr{ - pos: position{line: 73, col: 63, offset: 1817}, + pos: position{line: 84, col: 41, offset: 2077}, expr: &charClassMatcher{ - pos: position{line: 73, col: 63, offset: 1817}, + pos: position{line: 84, col: 41, offset: 2077}, val: "[^ ()]", chars: []rune{' ', '(', ')'}, ignoreCase: false, @@ -1421,19 +1584,147 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 105, col: 5, offset: 2609}, - run: (*parser).callonNode271, + pos: position{line: 87, col: 5, offset: 2158}, + run: (*parser).callonNode306, + expr: &seqExpr{ + pos: position{line: 87, col: 5, offset: 2158}, + exprs: []any{ + &labeledExpr{ + pos: position{line: 87, col: 5, offset: 2158}, + label: "k", + expr: &actionExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + run: (*parser).callonNode309, + expr: &seqExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + exprs: []any{ + &oneOrMoreExpr{ + pos: position{line: 237, col: 5, offset: 5077}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode312, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + &zeroOrMoreExpr{ + pos: position{line: 237, col: 11, offset: 5083}, + expr: &seqExpr{ + pos: position{line: 237, col: 12, offset: 5084}, + exprs: []any{ + &litMatcher{ + pos: position{line: 237, col: 12, offset: 5084}, + val: ".", + ignoreCase: false, + want: "\".\"", + }, + &oneOrMoreExpr{ + pos: position{line: 237, col: 16, offset: 5088}, + expr: &actionExpr{ + pos: position{line: 232, col: 5, offset: 5021}, + run: (*parser).callonNode318, + expr: &charClassMatcher{ + pos: position{line: 232, col: 5, offset: 5021}, + val: "[A-Za-z]", + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode320, + expr: &litMatcher{ + pos: position{line: 134, col: 5, offset: 3298}, + val: ":", + ignoreCase: false, + want: "\":\"", + }, + }, + &labeledExpr{ + pos: position{line: 87, col: 29, offset: 2182}, + label: "v", + expr: &choiceExpr{ + pos: position{line: 87, col: 32, offset: 2185}, + alternatives: []any{ + &actionExpr{ + pos: position{line: 242, col: 5, offset: 5146}, + run: (*parser).callonNode324, + expr: &seqExpr{ + pos: position{line: 242, col: 5, offset: 5146}, + exprs: []any{ + &litMatcher{ + pos: position{line: 242, col: 5, offset: 5146}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + &labeledExpr{ + pos: position{line: 242, col: 9, offset: 5150}, + label: "v", + expr: &zeroOrMoreExpr{ + pos: position{line: 242, col: 11, offset: 5152}, + expr: &charClassMatcher{ + pos: position{line: 242, col: 11, offset: 5152}, + val: "[^\"]", + chars: []rune{'"'}, + ignoreCase: false, + inverted: true, + }, + }, + }, + &litMatcher{ + pos: position{line: 242, col: 17, offset: 5158}, + val: "\"", + ignoreCase: false, + want: "\"\\\"\"", + }, + }, + }, + }, + &oneOrMoreExpr{ + pos: position{line: 87, col: 41, offset: 2194}, + expr: &charClassMatcher{ + pos: position{line: 87, col: 41, offset: 2194}, + val: "[^ ()]", + chars: []rune{' ', '(', ')'}, + ignoreCase: false, + inverted: true, + }, + }, + }, + }, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 119, col: 5, offset: 3008}, + run: (*parser).callonNode333, expr: &choiceExpr{ - pos: position{line: 105, col: 6, offset: 2610}, + pos: position{line: 119, col: 6, offset: 3009}, alternatives: []any{ &litMatcher{ - pos: position{line: 105, col: 6, offset: 2610}, + pos: position{line: 119, col: 6, offset: 3009}, val: "AND", ignoreCase: false, want: "\"AND\"", }, &litMatcher{ - pos: position{line: 105, col: 14, offset: 2618}, + pos: position{line: 119, col: 14, offset: 3017}, val: "+", ignoreCase: false, want: "\"+\"", @@ -1442,19 +1733,19 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 110, col: 5, offset: 2710}, - run: (*parser).callonNode275, + pos: position{line: 124, col: 5, offset: 3109}, + run: (*parser).callonNode337, expr: &choiceExpr{ - pos: position{line: 110, col: 6, offset: 2711}, + pos: position{line: 124, col: 6, offset: 3110}, alternatives: []any{ &litMatcher{ - pos: position{line: 110, col: 6, offset: 2711}, + pos: position{line: 124, col: 6, offset: 3110}, val: "NOT", ignoreCase: false, want: "\"NOT\"", }, &litMatcher{ - pos: position{line: 110, col: 14, offset: 2719}, + pos: position{line: 124, col: 14, offset: 3118}, val: "-", ignoreCase: false, want: "\"-\"", @@ -1463,28 +1754,28 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 115, col: 5, offset: 2810}, - run: (*parser).callonNode279, + pos: position{line: 129, col: 5, offset: 3209}, + run: (*parser).callonNode341, expr: &litMatcher{ - pos: position{line: 115, col: 6, offset: 2811}, + pos: position{line: 129, col: 6, offset: 3210}, val: "OR", ignoreCase: false, want: "\"OR\"", }, }, &actionExpr{ - pos: position{line: 86, col: 6, offset: 2097}, - run: (*parser).callonNode281, + pos: position{line: 100, col: 6, offset: 2482}, + run: (*parser).callonNode343, expr: &seqExpr{ - pos: position{line: 86, col: 6, offset: 2097}, + pos: position{line: 100, col: 6, offset: 2482}, exprs: []any{ &zeroOrOneExpr{ - pos: position{line: 86, col: 6, offset: 2097}, + pos: position{line: 100, col: 6, offset: 2482}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode284, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode346, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1492,12 +1783,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode286, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode348, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1506,27 +1797,27 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 86, col: 27, offset: 2118}, + pos: position{line: 100, col: 27, offset: 2503}, label: "v", expr: &actionExpr{ - pos: position{line: 228, col: 5, offset: 4747}, - run: (*parser).callonNode290, + pos: position{line: 242, col: 5, offset: 5146}, + run: (*parser).callonNode352, expr: &seqExpr{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, exprs: []any{ &litMatcher{ - pos: position{line: 228, col: 5, offset: 4747}, + pos: position{line: 242, col: 5, offset: 5146}, val: "\"", ignoreCase: false, want: "\"\\\"\"", }, &labeledExpr{ - pos: position{line: 228, col: 9, offset: 4751}, + pos: position{line: 242, col: 9, offset: 5150}, label: "v", expr: &zeroOrMoreExpr{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, expr: &charClassMatcher{ - pos: position{line: 228, col: 11, offset: 4753}, + pos: position{line: 242, col: 11, offset: 5152}, val: "[^\"]", chars: []rune{'"'}, ignoreCase: false, @@ -1535,7 +1826,7 @@ var g = &grammar{ }, }, &litMatcher{ - pos: position{line: 228, col: 17, offset: 4759}, + pos: position{line: 242, col: 17, offset: 5158}, val: "\"", ignoreCase: false, want: "\"\\\"\"", @@ -1545,12 +1836,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode297, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode359, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1559,12 +1850,12 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 86, col: 38, offset: 2129}, + pos: position{line: 100, col: 38, offset: 2514}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode301, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode363, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1575,18 +1866,18 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 91, col: 6, offset: 2227}, - run: (*parser).callonNode303, + pos: position{line: 105, col: 6, offset: 2619}, + run: (*parser).callonNode365, expr: &seqExpr{ - pos: position{line: 91, col: 6, offset: 2227}, + pos: position{line: 105, col: 6, offset: 2619}, exprs: []any{ &zeroOrOneExpr{ - pos: position{line: 91, col: 6, offset: 2227}, + pos: position{line: 105, col: 6, offset: 2619}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode306, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode368, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1594,12 +1885,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode308, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode370, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1608,12 +1899,12 @@ var g = &grammar{ }, }, &labeledExpr{ - pos: position{line: 91, col: 27, offset: 2248}, + pos: position{line: 105, col: 27, offset: 2640}, label: "v", expr: &oneOrMoreExpr{ - pos: position{line: 91, col: 29, offset: 2250}, + pos: position{line: 105, col: 29, offset: 2642}, expr: &charClassMatcher{ - pos: position{line: 91, col: 29, offset: 2250}, + pos: position{line: 105, col: 29, offset: 2642}, val: "[^ :()]", chars: []rune{' ', ':', '(', ')'}, ignoreCase: false, @@ -1622,12 +1913,12 @@ var g = &grammar{ }, }, &actionExpr{ - pos: position{line: 238, col: 5, offset: 4858}, - run: (*parser).callonNode314, + pos: position{line: 257, col: 5, offset: 5336}, + run: (*parser).callonNode376, expr: &zeroOrMoreExpr{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, expr: &charClassMatcher{ - pos: position{line: 238, col: 5, offset: 4858}, + pos: position{line: 257, col: 5, offset: 5336}, val: "[ \\t]", chars: []rune{' ', '\t'}, ignoreCase: false, @@ -1636,12 +1927,12 @@ var g = &grammar{ }, }, &zeroOrOneExpr{ - pos: position{line: 91, col: 40, offset: 2261}, + pos: position{line: 105, col: 40, offset: 2653}, expr: &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, - run: (*parser).callonNode318, + pos: position{line: 134, col: 5, offset: 3298}, + run: (*parser).callonNode380, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", @@ -1669,18 +1960,18 @@ var g = &grammar{ expr: &zeroOrOneExpr{ pos: position{line: 32, col: 7, offset: 614}, expr: &actionExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, run: (*parser).callonGroupNode5, expr: &seqExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, exprs: []any{ &oneOrMoreExpr{ - pos: position{line: 223, col: 5, offset: 4678}, + pos: position{line: 237, col: 5, offset: 5077}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonGroupNode8, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1689,23 +1980,23 @@ var g = &grammar{ }, }, &zeroOrMoreExpr{ - pos: position{line: 223, col: 11, offset: 4684}, + pos: position{line: 237, col: 11, offset: 5083}, expr: &seqExpr{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, exprs: []any{ &litMatcher{ - pos: position{line: 223, col: 12, offset: 4685}, + pos: position{line: 237, col: 12, offset: 5084}, val: ".", ignoreCase: false, want: "\".\"", }, &oneOrMoreExpr{ - pos: position{line: 223, col: 16, offset: 4689}, + pos: position{line: 237, col: 16, offset: 5088}, expr: &actionExpr{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, run: (*parser).callonGroupNode14, expr: &charClassMatcher{ - pos: position{line: 218, col: 5, offset: 4622}, + pos: position{line: 232, col: 5, offset: 5021}, val: "[A-Za-z]", ranges: []rune{'A', 'Z', 'a', 'z'}, ignoreCase: false, @@ -1727,20 +2018,20 @@ var g = &grammar{ pos: position{line: 32, col: 13, offset: 620}, alternatives: []any{ &actionExpr{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, run: (*parser).callonGroupNode18, expr: &litMatcher{ - pos: position{line: 120, col: 5, offset: 2899}, + pos: position{line: 134, col: 5, offset: 3298}, val: ":", ignoreCase: false, want: "\":\"", }, }, &actionExpr{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, run: (*parser).callonGroupNode20, expr: &litMatcher{ - pos: position{line: 125, col: 5, offset: 2985}, + pos: position{line: 139, col: 5, offset: 3384}, val: "=", ignoreCase: false, want: "\"=\"", @@ -2733,118 +3024,118 @@ func (p *parser) callonNode244() (any, error) { return p.cur.onNode244() } -func (c *current) onNode256() (any, error) { +func (c *current) onNode257() (any, error) { return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode256() (any, error) { +func (p *parser) callonNode257() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode256() + return p.cur.onNode257() } -func (c *current) onNode258() (any, error) { +func (c *current) onNode259() (any, error) { return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode258() (any, error) { +func (p *parser) callonNode259() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode258() + return p.cur.onNode259() } -func (c *current) onNode262(v any) (any, error) { - return v, nil +func (c *current) onNode261() (any, error) { + return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode262() (any, error) { +func (p *parser) callonNode261() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode262(stack["v"]) + return p.cur.onNode261() } -func (c *current) onNode241(k, v any) (any, error) { - return buildStringNode(k, v, c.text, c.pos) +func (c *current) onNode263() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode263() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode263() +} + +func (c *current) onNode268() (any, error) { + return string(c.text), nil + +} + +func (p *parser) callonNode268() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode268() +} + +func (c *current) onNode241(k, o, v any) (any, error) { + return buildNumberNode(k, o, v, c.text, c.pos) } func (p *parser) callonNode241() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode241(stack["k"], stack["v"]) + return p.cur.onNode241(stack["k"], stack["o"], stack["v"]) } -func (c *current) onNode271() (any, error) { +func (c *current) onNode285() (any, error) { + return c.text, nil + +} + +func (p *parser) callonNode285() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode285() +} + +func (c *current) onNode291() (any, error) { + return c.text, nil + +} + +func (p *parser) callonNode291() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode291() +} + +func (c *current) onNode282() (any, error) { + return c.text, nil + +} + +func (p *parser) callonNode282() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode282() +} + +func (c *current) onNode293(k any) (any, error) { return buildOperatorNode(c.text, c.pos) } -func (p *parser) callonNode271() (any, error) { +func (p *parser) callonNode293() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode271() -} - -func (c *current) onNode275() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode275() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode275() -} - -func (c *current) onNode279() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode279() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode279() -} - -func (c *current) onNode284() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode284() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode284() -} - -func (c *current) onNode286() (any, error) { - return nil, nil - -} - -func (p *parser) callonNode286() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode286() -} - -func (c *current) onNode290(v any) (any, error) { - return v, nil - -} - -func (p *parser) callonNode290() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode290(stack["v"]) + return p.cur.onNode293(stack["k"]) } func (c *current) onNode297(v any) (any, error) { - return nil, nil + return v, nil } @@ -2854,63 +3145,30 @@ func (p *parser) callonNode297() (any, error) { return p.cur.onNode297(stack["v"]) } -func (c *current) onNode301() (any, error) { - return buildOperatorNode(c.text, c.pos) +func (c *current) onNode279(k, v any) (any, error) { + return buildStringNode(k, v, true, c.text, c.pos) } -func (p *parser) callonNode301() (any, error) { +func (p *parser) callonNode279() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode301() + return p.cur.onNode279(stack["k"], stack["v"]) } -func (c *current) onNode281(v any) (any, error) { - return buildStringNode("", v, c.text, c.pos) +func (c *current) onNode312() (any, error) { + return c.text, nil } -func (p *parser) callonNode281() (any, error) { +func (p *parser) callonNode312() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode281(stack["v"]) -} - -func (c *current) onNode306() (any, error) { - return buildOperatorNode(c.text, c.pos) - -} - -func (p *parser) callonNode306() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode306() -} - -func (c *current) onNode308() (any, error) { - return nil, nil - -} - -func (p *parser) callonNode308() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode308() -} - -func (c *current) onNode314(v any) (any, error) { - return nil, nil - -} - -func (p *parser) callonNode314() (any, error) { - stack := p.vstack[len(p.vstack)-1] - _ = stack - return p.cur.onNode314(stack["v"]) + return p.cur.onNode312() } func (c *current) onNode318() (any, error) { - return buildOperatorNode(c.text, c.pos) + return c.text, nil } @@ -2920,15 +3178,202 @@ func (p *parser) callonNode318() (any, error) { return p.cur.onNode318() } -func (c *current) onNode303(v any) (any, error) { - return buildStringNode("", v, c.text, c.pos) +func (c *current) onNode309() (any, error) { + return c.text, nil } -func (p *parser) callonNode303() (any, error) { +func (p *parser) callonNode309() (any, error) { stack := p.vstack[len(p.vstack)-1] _ = stack - return p.cur.onNode303(stack["v"]) + return p.cur.onNode309() +} + +func (c *current) onNode320(k any) (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode320() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode320(stack["k"]) +} + +func (c *current) onNode324(v any) (any, error) { + return v, nil + +} + +func (p *parser) callonNode324() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode324(stack["v"]) +} + +func (c *current) onNode306(k, v any) (any, error) { + return buildStringNode(k, v, false, c.text, c.pos) + +} + +func (p *parser) callonNode306() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode306(stack["k"], stack["v"]) +} + +func (c *current) onNode333() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode333() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode333() +} + +func (c *current) onNode337() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode337() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode337() +} + +func (c *current) onNode341() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode341() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode341() +} + +func (c *current) onNode346() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode346() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode346() +} + +func (c *current) onNode348() (any, error) { + return nil, nil + +} + +func (p *parser) callonNode348() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode348() +} + +func (c *current) onNode352(v any) (any, error) { + return v, nil + +} + +func (p *parser) callonNode352() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode352(stack["v"]) +} + +func (c *current) onNode359(v any) (any, error) { + return nil, nil + +} + +func (p *parser) callonNode359() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode359(stack["v"]) +} + +func (c *current) onNode363() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode363() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode363() +} + +func (c *current) onNode343(v any) (any, error) { + return buildStringNode("", v, false, c.text, c.pos) + +} + +func (p *parser) callonNode343() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode343(stack["v"]) +} + +func (c *current) onNode368() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode368() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode368() +} + +func (c *current) onNode370() (any, error) { + return nil, nil + +} + +func (p *parser) callonNode370() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode370() +} + +func (c *current) onNode376(v any) (any, error) { + return nil, nil + +} + +func (p *parser) callonNode376() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode376(stack["v"]) +} + +func (c *current) onNode380() (any, error) { + return buildOperatorNode(c.text, c.pos) + +} + +func (p *parser) callonNode380() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode380() +} + +func (c *current) onNode365(v any) (any, error) { + return buildStringNode("", v, false, c.text, c.pos) + +} + +func (p *parser) callonNode365() (any, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNode365(stack["v"]) } func (c *current) onGroupNode8() (any, error) { diff --git a/pkg/kql/dictionary_test.go b/pkg/kql/dictionary_test.go index 81ace7b811..25f6e2f902 100644 --- a/pkg/kql/dictionary_test.go +++ b/pkg/kql/dictionary_test.go @@ -145,6 +145,22 @@ func TestParse_Spec(t *testing.T) { }, }, }, + { + name: `author="John Smith"`, + ast: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "author", Value: "John Smith", Exact: true}, + }, + }, + }, + { + name: `filename=budget.xlsx`, + ast: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "filename", Value: "budget.xlsx", Exact: true}, + }, + }, + }, // 3.2.3 Implicit Operator for Property Restriction { name: `author:"John Smith" filetype:docx`, @@ -423,6 +439,60 @@ func TestParse_Spec(t *testing.T) { } } +func TestParse_NumberRestrictionNode(t *testing.T) { + tests := []testCase{ + { + name: "format", + query: join([]string{ + `size>100`, + `size>"100"`, + `size>=15.5`, + `size<100`, + `size<=100`, + }), + ast: &ast.Ast{ + Nodes: []ast.Node{ + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: ">"}, + Value: 100, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: ">"}, + Value: 100, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: ">="}, + Value: 15.5, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: "<"}, + Value: 100, + }, + &ast.OperatorNode{Value: kql.BoolAND}, + &ast.NumberNode{ + Key: "size", + Operator: &ast.OperatorNode{Value: "<="}, + Value: 100, + }, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + testKQL(t, tc) + }) + } +} + func TestParse_DateTimeRestrictionNode(t *testing.T) { tests := []testCase{ { diff --git a/pkg/kql/engine_suite_test.go b/pkg/kql/engine_suite_test.go deleted file mode 100644 index c4ad289b8b..0000000000 --- a/pkg/kql/engine_suite_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package kql - -import ( - "time" -) - -// PatchTimeNow is here to patch the package time now func, -// which is used in the test suite -func PatchTimeNow(t func() time.Time) { - timeNow = t -} diff --git a/pkg/kql/factory.go b/pkg/kql/factory.go index 2ac49113c4..5f55802d38 100644 --- a/pkg/kql/factory.go +++ b/pkg/kql/factory.go @@ -50,7 +50,7 @@ func buildAST(n any, text []byte, pos position) (*ast.Ast, error) { return a, nil } -func buildStringNode(k, v any, text []byte, pos position) (*ast.StringNode, error) { +func buildStringNode(k, v any, exact bool, text []byte, pos position) (*ast.StringNode, error) { b, err := base(text, pos) if err != nil { return nil, err @@ -70,6 +70,7 @@ func buildStringNode(k, v any, text []byte, pos position) (*ast.StringNode, erro Base: b, Key: key, Value: value, + Exact: exact, }, nil } @@ -101,6 +102,36 @@ func buildDateTimeNode(k, o, v any, text []byte, pos position) (*ast.DateTimeNod Value: value, }, nil } + +func buildNumberNode(k, o, v any, text []byte, pos position) (*ast.NumberNode, error) { + b, err := base(text, pos) + if err != nil { + return nil, err + } + + operator, err := toNode[*ast.OperatorNode](o) + if err != nil { + return nil, err + } + + key, err := toString(k) + if err != nil { + return nil, err + } + + value, err := toFloat(v) + if err != nil { + return nil, err + } + + return &ast.NumberNode{ + Base: b, + Key: key, + Operator: operator, + Value: value, + }, nil +} + func buildNaturalLanguageDateTimeNodes(k, v any, text []byte, pos position) ([]ast.Node, error) { b, err := base(text, pos) if err != nil { diff --git a/pkg/kql/kql.go b/pkg/kql/kql.go index 4cba6c7b91..a4132f4650 100644 --- a/pkg/kql/kql.go +++ b/pkg/kql/kql.go @@ -47,3 +47,9 @@ func (b Builder) Build(q string) (*ast.Ast, error) { // timeNow mirrors time.Now by default, the only reason why this exists // is to monkey patch it from the tests. See PatchTimeNow var timeNow = time.Now + +// PatchTimeNow pins the clock the natural language dates resolve against, +// so a test can hold "today" still while it runs +func PatchTimeNow(t func() time.Time) { + timeNow = t +} diff --git a/services/search/MIGRATION.md b/services/search/MIGRATION.md new file mode 100644 index 0000000000..a0cb646e61 --- /dev/null +++ b/services/search/MIGRATION.md @@ -0,0 +1,55 @@ +# Migrating the search index + +A version that changes how resources are indexed works on a new index and +leaves the old one untouched. The service starts normally, but the new index is +empty: search finds nothing until it is filled. The old index stays around +until you remove it. + +## v7.x.x to %%NEXT%% + +### OpenSearch + +The new index is `opencloud-resource-v3`. Fill it in one of two ways: + +- copy the old index, fast and keeps the extracted file contents, or +- index all spaces again, slower since every file is read once more, but drops + documents that no longer have a resource. + +The address below is the one from `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ADDRESSES`, +`opencloud-resource` the name from +`SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME`. + +```shell +# either copy the old index +curl -X POST "https://opensearch.example.com:9200/_reindex?wait_for_completion=false" \ + -H 'Content-Type: application/json' -d ' + {"source":{"index":"opencloud-resource"},"dest":{"index":"opencloud-resource-v3"}}' + +# the answer carries a task id, watch it while it runs +curl "https://opensearch.example.com:9200/_tasks/" + +# or index all spaces again, the service keeps running while it happens +opencloud search index --all-spaces +``` + +Once the new index is filled, remove the old one: + +```shell +curl -X DELETE "https://opensearch.example.com:9200/opencloud-resource" +``` + +### bleve + +The new index is the `bleve-v2` directory next to the old `bleve` one, both in +`$OC_BASE_DATA_PATH/search` by default (`SEARCH_ENGINE_BLEVE_DATA_PATH`). A +bleve index cannot be copied, index all spaces again: + +```shell +opencloud search index --all-spaces +``` + +Once the new index is filled, remove the old one: + +```shell +rm -r "$OC_BASE_DATA_PATH/search/bleve" +``` diff --git a/services/search/README.md b/services/search/README.md index 03e56c687a..c0e9036aac 100644 --- a/services/search/README.md +++ b/services/search/README.md @@ -55,6 +55,12 @@ Additionally, the following optional settings can be set: * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ENABLE_DEBUG_LOGGER=val`: Enable debug logging. * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_INSECURE=val`: Skip TLS certificate verification. +### Bringing an existing index over + +A version that changes how resources are indexed works on a new index and leaves the old one untouched. +The new index starts empty and has to be filled, see [MIGRATION.md](MIGRATION.md). New installations are +not affected. + ## Query language By default, [KQL](https://learn.microsoft.com/en-us/sharepoint/dev/general-development/keyword-query-language-kql-syntax-reference) is used as the query language. diff --git a/services/search/internal/opensearchtest/helper.go b/services/search/internal/opensearchtest/helper.go index 9ac497a2a3..b900b022c4 100644 --- a/services/search/internal/opensearchtest/helper.go +++ b/services/search/internal/opensearchtest/helper.go @@ -5,10 +5,7 @@ import ( "testing" "time" - opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "github.com/stretchr/testify/require" - - "github.com/opencloud-eu/opencloud/pkg/conversions" ) var TimeMustParse = func(t testing.TB, ts string) time.Time { @@ -23,14 +20,3 @@ func JSONMustMarshal(t testing.TB, data any) string { require.NoError(t, err, "failed to marshal data to JSON") return string(jsonData) } - -func SearchHitsMustBeConverted[T any](t testing.TB, hits []opensearchgoAPI.SearchHit) []T { - ts := make([]T, len(hits)) - for i, hit := range hits { - resource, err := conversions.To[T](hit.Source) - require.NoError(t, err) - ts[i] = resource - } - - return ts -} diff --git a/services/search/internal/opensearchtest/os.go b/services/search/internal/opensearchtest/os.go index 8d4ad649ad..16c3c10025 100644 --- a/services/search/internal/opensearchtest/os.go +++ b/services/search/internal/opensearchtest/os.go @@ -61,7 +61,6 @@ func (tc *TestClient) IndicesReset(ctx context.Context, indices []string) error } if len(indicesToDelete) == 0 { - // If no indices to delete, return nil return nil } @@ -138,76 +137,6 @@ func (tc *TestClient) IndicesCreate(ctx context.Context, index string, body io.R } } -func (tc *TestClient) IndicesCount(ctx context.Context, indices []string, body io.Reader) (int, error) { - if err := tc.IndicesRefresh(ctx, indices, []int{404}); err != nil { - return 0, err - } - - resp, err := tc.c.Indices.Count(ctx, &opensearchgoAPI.IndicesCountReq{ - Indices: indices, - Body: body, - }) - - switch { - case err != nil: - return 0, fmt.Errorf("failed to count documents in indices: %w", err) - default: - return resp.Count, nil - } -} - -func (tc *TestClient) DocumentCreate(ctx context.Context, index, id string, body io.Reader) error { - if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil { - return err - } - - _, err := tc.c.Document.Create(ctx, opensearchgoAPI.DocumentCreateReq{ - Index: index, - DocumentID: id, - Body: body, - }) - switch { - case err != nil: - return fmt.Errorf("failed to create document in index %s: %w", index, err) - default: - return nil - } -} - -func (tc *TestClient) Update(ctx context.Context, index, id string, body io.Reader) error { - if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil { - return err - } - - _, err := tc.c.Update(ctx, opensearchgoAPI.UpdateReq{ - Index: index, - DocumentID: id, - Body: body, - }) - switch { - case err != nil: - return fmt.Errorf("failed to update document in index %s: %w", index, err) - default: - return nil - } -} - -func (tc *TestClient) Search(ctx context.Context, index string, body io.Reader) (opensearchgoAPI.SearchHits, error) { - if err := tc.IndicesRefresh(ctx, []string{index}, []int{404}); err != nil { - return opensearchgoAPI.SearchHits{}, err - } - - resp, err := tc.c.Search(ctx, &opensearchgoAPI.SearchReq{ - Indices: []string{index}, - Body: body, - }) - if err != nil { - return opensearchgoAPI.SearchHits{}, fmt.Errorf("failed to search in index %s: %w", index, err) - } - - return resp.Hits, nil -} - type testRequireClient struct { tc *TestClient t testing.TB @@ -228,29 +157,3 @@ func (trc *testRequireClient) IndicesCreate(index string, body io.Reader) { func (trc *testRequireClient) IndicesDelete(indices []string) { require.NoError(trc.t, trc.tc.IndicesDelete(trc.t.Context(), indices)) } - -func (trc *testRequireClient) IndicesCount(indices []string, body io.Reader, expected int) { - count, err := trc.tc.IndicesCount(trc.t.Context(), indices, body) - - switch { - case expected <= 0: - require.True(trc.t, count <= 0, "expected indices to have no documents, but got a count of %d", count) - default: - require.Equal(trc.t, expected, count, "expected indices to have %d documents, but got %d", expected, count) - require.NoError(trc.t, err, "expected indices to have documents, but got an error") - } -} - -func (trc *testRequireClient) DocumentCreate(index, id string, body io.Reader) { - require.NoError(trc.t, trc.tc.DocumentCreate(trc.t.Context(), index, id, body)) -} - -func (trc *testRequireClient) Update(index, id string, body io.Reader) { - require.NoError(trc.t, trc.tc.Update(trc.t.Context(), index, id, body)) -} - -func (trc *testRequireClient) Search(index string, body io.Reader) opensearchgoAPI.SearchHits { - hits, err := trc.tc.Search(trc.t.Context(), index, body) - require.NoError(trc.t, err) - return hits -} diff --git a/services/search/internal/opensearchtest/suite.go b/services/search/internal/opensearchtest/suite.go index 433907292a..9b6e41e412 100644 --- a/services/search/internal/opensearchtest/suite.go +++ b/services/search/internal/opensearchtest/suite.go @@ -67,7 +67,6 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func keepContainer := os.Getenv("KEEP_TEST_CONTAINER") == "true" if keepContainer { - // the reaper would take the kept container down with the session if err := os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true"); err != nil { return nil, fmt.Errorf("failed to disable the testcontainers reaper: %w", err) } @@ -83,14 +82,6 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func opensearch.WithPassword(cfg.Engine.OpenSearch.Client.Password), testcontainers.WithName(containerName), testcontainers.WithReuseByName(containerName), - // test indexes are tiny; don't let a full host disk trip the flood-stage - // create-index / read-only blocks mid-run - testcontainers.WithEnv(map[string]string{ - "cluster.routing.allocation.disk.threshold_enabled": "false", - }), - // a health probe answers at once on a reused container, a log wait - // would sit out the log timeout on it; a cold boot takes well over the - // previous 5s testcontainers.WithWaitStrategy( wait.ForHTTP("/_cluster/health?wait_for_status=yellow&timeout=1s"). WithPort(openSearchPort). @@ -106,16 +97,13 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func address, err := container.Address(ctx) if err != nil { - _ = container.Terminate(ctx) // attempt to clean up the container + _ = container.Terminate(ctx) return nil, fmt.Errorf("failed to get OpenSearch container address: %w", err) } - // Ensure the address is set in the default configuration. cfg.Engine.OpenSearch.Client.Addresses = []string{address} return func() { - // KEEP_TEST_CONTAINER=true leaves the container up for the next run, - // which picks it up again by name instead of booting a fresh one if keepContainer { _, _ = fmt.Fprintf(os.Stderr, "keeping OpenSearch container %s\n", containerName) return diff --git a/services/search/internal/opensearchtest/testdata.go b/services/search/internal/opensearchtest/testdata.go index ef41c91190..84ceef3a93 100644 --- a/services/search/internal/opensearchtest/testdata.go +++ b/services/search/internal/opensearchtest/testdata.go @@ -16,16 +16,12 @@ var Testdata = struct { Resources resourceTestdata }{ Resources: resourceTestdata{ - Root: fromTestData[search.Resource]("resource_root.json"), - Folder: fromTestData[search.Resource]("resource_folder.json"), - File: fromTestData[search.Resource]("resource_file.json"), + File: fromTestData[search.Resource]("resource_file.json"), }, } type resourceTestdata struct { - Root search.Resource - File search.Resource - Folder search.Resource + File search.Resource } func fromTestData[D any](name string) D { diff --git a/services/search/pkg/bleve/index.go b/services/search/pkg/bleve/index.go index fea6986ae0..0aa61b4c48 100644 --- a/services/search/pkg/bleve/index.go +++ b/services/search/pkg/bleve/index.go @@ -8,8 +8,8 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + regexpCharFilter "github.com/blevesearch/bleve/v2/analysis/char/regexp" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" - "github.com/blevesearch/bleve/v2/analysis/token/porter" "github.com/blevesearch/bleve/v2/analysis/tokenizer/single" "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" "github.com/blevesearch/bleve/v2/mapping" @@ -18,10 +18,19 @@ import ( "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) +const ( + wildcardSuffix = ".wildcard" + indexVersion = "v2" +) + +func indexPath(root string) string { + return filepath.Join(root, "bleve-"+indexVersion) +} + func NewIndex(root string) (bleve.Index, error) { - destination := filepath.Join(root, "bleve") + destination := indexPath(root) index, err := bleve.Open(destination) - if errors.Is(bleve.ErrorIndexPathDoesNotExist, err) { + if errors.Is(err, bleve.ErrorIndexPathDoesNotExist) { indexMapping, err := NewMapping() if err != nil { return nil, err @@ -38,30 +47,61 @@ func NewIndex(root string) (bleve.Index, error) { } func NewMapping() (mapping.IndexMapping, error) { - nameMapping := bleve.NewTextFieldMapping() - nameMapping.Analyzer = "lowercaseKeyword" + words := func() *mapping.FieldMapping { + fm := bleve.NewTextFieldMapping() + fm.Analyzer = "lowercaseWords" + + return fm + } + + whole := func(field string) *mapping.FieldMapping { + fm := bleve.NewTextFieldMapping() + fm.Analyzer = "lowercaseKeyword" + fm.IncludeInAll = false + fm.Name = field + wildcardSuffix + + return fm + } lowercaseMapping := bleve.NewTextFieldMapping() lowercaseMapping.IncludeInAll = false lowercaseMapping.Analyzer = "lowercaseKeyword" - fulltextFieldMapping := bleve.NewTextFieldMapping() - fulltextFieldMapping.Analyzer = "fulltext" - fulltextFieldMapping.IncludeInAll = false + contentMapping := words() + contentMapping.IncludeInAll = false docMapping := bleve.NewDocumentMapping() - docMapping.AddFieldMappingsAt("Name", nameMapping) + docMapping.AddFieldMappingsAt("Name", + words(), + whole("Name"), + ) + docMapping.AddFieldMappingsAt("Title", + words(), + whole("Title"), + ) docMapping.AddFieldMappingsAt("Tags", lowercaseMapping) docMapping.AddFieldMappingsAt("Favorites", lowercaseMapping) - docMapping.AddFieldMappingsAt("Content", fulltextFieldMapping) + docMapping.AddFieldMappingsAt("Content", contentMapping) indexMapping := bleve.NewIndexMapping() indexMapping.DefaultAnalyzer = keyword.Name indexMapping.DefaultMapping = docMapping - err := indexMapping.AddCustomAnalyzer("lowercaseKeyword", + err := indexMapping.AddCustomCharFilter("dotToSpace", map[string]any{ - "type": custom.Name, - "tokenizer": single.Name, + "type": regexpCharFilter.Name, + "regexp": `\.`, + "replace": " ", + }, + ) + if err != nil { + return nil, err + } + + err = indexMapping.AddCustomAnalyzer("lowercaseWords", + map[string]any{ + "type": custom.Name, + "char_filters": []string{"dotToSpace"}, + "tokenizer": unicode.Name, "token_filters": []string{ lowercase.Name, }, @@ -71,13 +111,12 @@ func NewMapping() (mapping.IndexMapping, error) { return nil, err } - err = indexMapping.AddCustomAnalyzer("fulltext", + err = indexMapping.AddCustomAnalyzer("lowercaseKeyword", map[string]any{ "type": custom.Name, - "tokenizer": unicode.Name, + "tokenizer": single.Name, "token_filters": []string{ lowercase.Name, - porter.Name, }, }, ) diff --git a/services/search/pkg/bleve/index_test.go b/services/search/pkg/bleve/index_test.go new file mode 100644 index 0000000000..a62dfc2c28 --- /dev/null +++ b/services/search/pkg/bleve/index_test.go @@ -0,0 +1,39 @@ +package bleve_test + +import ( + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" +) + +var _ = Describe("Index", func() { + Describe("NewIndex", func() { + It("puts the index into a directory of its own generation", func() { + root := GinkgoT().TempDir() + + index, err := bleve.NewIndex(root) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(index.Close) + + Expect(index.Name()).To(Equal(filepath.Join(root, "bleve-v2"))) + Expect(filepath.Join(root, "bleve-v2")).To(BeADirectory()) + }) + + It("opens the index that is already there", func() { + root := GinkgoT().TempDir() + + index, err := bleve.NewIndex(root) + Expect(err).ToNot(HaveOccurred()) + Expect(index.Close()).To(Succeed()) + + reopened, err := bleve.NewIndex(root) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(reopened.Close) + + Expect(reopened.Name()).To(Equal(filepath.Join(root, "bleve-v2"))) + }) + }) +}) diff --git a/services/search/pkg/content/content.go b/services/search/pkg/content/content.go index e185351678..1c5d914231 100644 --- a/services/search/pkg/content/content.go +++ b/services/search/pkg/content/content.go @@ -18,7 +18,7 @@ type Document struct { Name string Content string Size uint64 - Mtime string + Mtime string `json:"Mtime,omitempty"` MimeType string Tags []string Favorites []string diff --git a/services/search/pkg/content/tika.go b/services/search/pkg/content/tika.go index 8b6ac5e60b..2b475efb17 100644 --- a/services/search/pkg/content/tika.go +++ b/services/search/pkg/content/tika.go @@ -11,8 +11,8 @@ import ( gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/google/go-tika/tika" - "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" libregraph "github.com/opencloud-eu/libre-graph-api-go" + "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" "github.com/opencloud-eu/opencloud/pkg/log" "github.com/opencloud-eu/opencloud/services/search/pkg/config" @@ -83,7 +83,11 @@ func (t Tika) Extract(ctx context.Context, ri *provider.ResourceInfo) (Document, } for _, meta := range metas { - if title, err := getFirstValue(meta, "title"); err == nil { + title, err := getFirstValue(meta, "dc:title") + if err != nil { + title, err = getFirstValue(meta, "title") + } + if err == nil { doc.Title = strings.TrimSpace(fmt.Sprintf("%s %s", doc.Title, title)) } diff --git a/services/search/pkg/content/tika_test.go b/services/search/pkg/content/tika_test.go index b6ee193528..286ffa8b9c 100644 --- a/services/search/pkg/content/tika_test.go +++ b/services/search/pkg/content/tika_test.go @@ -91,6 +91,28 @@ var _ = Describe("Tika", func() { Expect(doc.Content).To(Equal(body)) }) + It("adds the title", func() { + fullResponse = `[{"dc:title": "quarterly report", "X-TIKA:content": "some data"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Title).To(Equal("quarterly report")) + }) + + It("adds the title of an older tika", func() { + fullResponse = `[{"title": "quarterly report"}]` + + doc, err := tika.Extract(context.TODO(), &provider.ResourceInfo{ + Type: provider.ResourceType_RESOURCE_TYPE_FILE, + Size: 1, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Title).To(Equal("quarterly report")) + }) + It("adds audio content", func() { fullResponse = `[ { diff --git a/services/search/pkg/opensearch/backend.go b/services/search/pkg/opensearch/backend.go index 000b24ebf1..6082643a93 100644 --- a/services/search/pkg/opensearch/backend.go +++ b/services/search/pkg/opensearch/backend.go @@ -33,7 +33,10 @@ type Backend struct { client *opensearchgoAPI.Client } -func NewBackend(index string, client *opensearchgoAPI.Client) (*Backend, error) { +// NewBackend creates a backend on the versioned generation of the named index. +func NewBackend(name string, client *opensearchgoAPI.Client) (*Backend, error) { + index := IndexName(name) + pingResp, err := client.Ping(context.TODO(), &opensearchgoAPI.PingReq{}) switch { case err != nil: @@ -172,7 +175,7 @@ func (b *Backend) DocCount() (uint64, error) { &opensearchgoAPI.IndicesCountReq{ Indices: []string{b.index}, }, - osu.NewTermQuery[bool]("Deleted").Value(false), + osu.NewMatchAllQuery(), ) if err != nil { return 0, fmt.Errorf("failed to build count request: %w", err) diff --git a/services/search/pkg/opensearch/backend_test.go b/services/search/pkg/opensearch/backend_test.go index 71ef3860fc..ecf728fe12 100644 --- a/services/search/pkg/opensearch/backend_test.go +++ b/services/search/pkg/opensearch/backend_test.go @@ -16,8 +16,6 @@ func TestOpenSearchBackend(t *testing.T) { RunSpecs(t, "OpenSearch Backend Suite") } -// what the engine does with its index is covered for both engines by -// services/search/pkg/parity; this is the one thing only OpenSearch can do var _ = Describe("Backend", func() { Describe("NewBackend", func() { It("fails to create if the cluster is not healthy", func() { diff --git a/services/search/pkg/opensearch/index.go b/services/search/pkg/opensearch/index.go index 73a1cf2eb9..277c9f5c5c 100644 --- a/services/search/pkg/opensearch/index.go +++ b/services/search/pkg/opensearch/index.go @@ -8,6 +8,7 @@ import ( "fmt" "path" "reflect" + "strings" "github.com/go-jose/go-jose/v3/json" opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" @@ -16,9 +17,8 @@ import ( var ( ErrManualActionRequired = errors.New("manual action required") - IndexManagerLatest = IndexIndexManagerResourceV2 - IndexIndexManagerResourceV1 IndexManager = "resource_v1.json" - IndexIndexManagerResourceV2 IndexManager = "resource_v2.json" + IndexManagerLatest = IndexIndexManagerResourceV3 + IndexIndexManagerResourceV3 IndexManager = "resource_v3.json" ) //go:embed internal/indexes/*.json @@ -26,6 +26,30 @@ var indexes embed.FS type IndexManager string +// Version is the part of the definition file name that says which generation it +// is, resource_v3.json carries v3. +func (m IndexManager) Version() string { + name := strings.TrimSuffix(string(m), path.Ext(string(m))) + _, version, found := strings.Cut(name, "_") + if !found { + return "" + } + + return version +} + +// IndexName puts the generation of the definition behind the configured name, +// so a new one starts on an index of its own instead of refusing to work with +// the one that is there. +func IndexName(name string) string { + version := IndexManagerLatest.Version() + if version == "" { + return name + } + + return name + "-" + version +} + func (m IndexManager) String() string { b, err := m.MarshalJSON() if err != nil { @@ -48,6 +72,43 @@ func (m IndexManager) MarshalJSON() ([]byte, error) { return body, nil } +func coveredAt(declared, index gjson.Result, declaredPath, indexPath string) (string, string, bool) { + declaredRaw := declared.Get(declaredPath).Raw + indexRaw := index.Get(indexPath).Raw + + var declaredValue, indexValue any + if err := json.Unmarshal([]byte(declaredRaw), &declaredValue); err != nil { + return declaredRaw, indexRaw, false + } + + if err := json.Unmarshal([]byte(indexRaw), &indexValue); err != nil { + return declaredRaw, indexRaw, false + } + + return declaredRaw, indexRaw, covered(declaredValue, indexValue) +} + +func covered(declared, index any) bool { + declaredMap, ok := declared.(map[string]any) + if !ok { + return reflect.DeepEqual(declared, index) + } + + indexMap, ok := index.(map[string]any) + if !ok { + return false + } + + for key, declaredValue := range declaredMap { + indexValue, ok := indexMap[key] + if !ok || !covered(declaredValue, indexValue) { + return false + } + } + + return true +} + func (m IndexManager) Apply(ctx context.Context, name string, client *opensearchgoAPI.Client) error { localIndexB, err := m.MarshalJSON() if err != nil { @@ -86,32 +147,16 @@ func (m IndexManager) Apply(ctx context.Context, name string, client *opensearch localIndexJson := gjson.ParseBytes(localIndexB) remoteIndexJson := gjson.ParseBytes(remoteIndexB) - compare := func(lvPath, rvPath string) (any, any, bool) { - lv := localIndexJson.Get(lvPath).Raw - rv := remoteIndexJson.Get(rvPath).Raw - - var lvv, rvv any - if err := json.Unmarshal([]byte(lv), &lvv); err != nil { - return nil, nil, false - } - - if err := json.Unmarshal([]byte(rv), &rvv); err != nil { - return nil, nil, false - } - - return lv, rv, reflect.DeepEqual(lvv, rvv) - } - var errs []error for k := range localIndexJson.Get("settings").Map() { - if lv, rv, ok := compare("settings."+k, "settings.index."+k); !ok { + if lv, rv, ok := coveredAt(localIndexJson, remoteIndexJson, "settings."+k, "settings.index."+k); !ok { errs = append(errs, fmt.Errorf("settings.%s local %s, remote %s", k, lv, rv)) } } for k := range localIndexJson.Get("mappings.properties").Map() { - if _, _, ok := compare("mappings.properties."+k, "mappings.properties."+k); !ok { + if _, _, ok := coveredAt(localIndexJson, remoteIndexJson, "mappings.properties."+k, "mappings.properties."+k); !ok { errs = append(errs, fmt.Errorf("mappings.properties.%s", k)) } } diff --git a/services/search/pkg/opensearch/index_test.go b/services/search/pkg/opensearch/index_test.go index 6a813572a2..faba72f674 100644 --- a/services/search/pkg/opensearch/index_test.go +++ b/services/search/pkg/opensearch/index_test.go @@ -47,6 +47,34 @@ func TestIndexManager(t *testing.T) { require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client())) }) + t.Run("accepts an index that carries more than the definition declares", func(t *testing.T) { + indexManager := opensearch.IndexManagerLatest + indexName := "opencloud-test-resource" + + tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + + body, err := sjson.Set(indexManager.String(), "mappings.properties.Path.fields.raw.type", "keyword") + require.NoError(t, err) + tc.Require.IndicesCreate(indexName, strings.NewReader(body)) + + require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client())) + }) + + t.Run("fails when the index misses something the definition declares", func(t *testing.T) { + indexManager := opensearch.IndexManagerLatest + indexName := "opencloud-test-resource" + + tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + + body, err := sjson.Delete(indexManager.String(), "mappings.properties.Path.analyzer") + require.NoError(t, err) + tc.Require.IndicesCreate(indexName, strings.NewReader(body)) + + require.ErrorIs(t, indexManager.Apply(t.Context(), indexName, tc.Client()), opensearch.ErrManualActionRequired) + }) + t.Run("fails to create index if it already exists but is not up to date", func(t *testing.T) { indexManager := opensearch.IndexManagerLatest indexName := "opencloud-test-resource" diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand.go b/services/search/pkg/opensearch/internal/convert/kql_expand.go index 5688f6dfe6..17258512c9 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand.go @@ -4,8 +4,11 @@ import ( "fmt" "reflect" "slices" + "strconv" "strings" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/opencloud-eu/opencloud/pkg/ast" ) @@ -53,6 +56,8 @@ func (e kqlExpander) expand(nodes []ast.Node, defaultKey string) ([]ast.Node, er cnode.Key = e.remapKey(cnode.Key, defaultKey) case *ast.BooleanNode: cnode.Key = e.remapKey(cnode.Key, defaultKey) + case *ast.NumberNode: + cnode.Key = e.remapKey(cnode.Key, defaultKey) } if unfoldedNodes != nil { @@ -73,6 +78,7 @@ func (_ kqlExpander) remapKey(current string, defaultKey string) string { key, ok := map[string]string{ "": defaultKey, // Default case if current is empty + "title": "Title", "rootid": "RootID", "path": "Path", "id": "ID", @@ -85,7 +91,8 @@ func (_ kqlExpander) remapKey(current string, defaultKey string) string { "tags": "Tags", "content": "Content", "hidden": "Hidden", - }[current] + "favorite": "Favorites", + }[strings.ToLower(current)] if !ok { return current // Return the original key if not found } @@ -94,15 +101,21 @@ func (_ kqlExpander) remapKey(current string, defaultKey string) string { } func (_ kqlExpander) lowerValue(key, value string) string { - if slices.Contains([]string{"Hidden"}, key) { - return value // ignore certain keys and return the original value + if slices.Contains([]string{"Name", "Title", "Tags", "Content", "MimeType", "Type", "Hidden"}, key) { + return strings.ToLower(value) } - return strings.ToLower(value) + return value } func (_ kqlExpander) unfoldValue(key, value string) []ast.Node { result, ok := map[string][]ast.Node{ + "Type:file": { + &ast.StringNode{Key: key, Value: strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10)}, + }, + "Type:folder": { + &ast.StringNode{Key: key, Value: strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10)}, + }, "MimeType:file": { &ast.OperatorNode{Value: "NOT"}, &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, @@ -139,8 +152,6 @@ func (_ kqlExpander) unfoldValue(key, value string) []ast.Node { &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"}, - &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"}, }}, }, diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go index 1b3cb0b914..44534d7bcd 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go @@ -142,6 +142,7 @@ func TestExpandKQLAST(t *testing.T) { "tags": "Tags", "content": "Content", "hidden": "Hidden", + "favorite": "Favorites", "any": "any", // Example of an unknown key that should remain unchanged } { tests = append(tests, opensearchtest.TableTest[[]ast.Node, []ast.Node]{ @@ -244,35 +245,72 @@ func TestExpandKQLAST(t *testing.T) { t.Run("lowercases some values", func(t *testing.T) { tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ { - Name: "!Hidden: StringNode -> stringnode", + Name: "Name: StringNode -> stringnode", Got: []ast.Node{ - ast.StringNode{Key: "aBc", Value: "StringNode"}, + ast.StringNode{Key: "Name", Value: "StringNode"}, ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "aBc", Value: "StringNode"}, + ast.StringNode{Key: "Name", Value: "StringNode"}, }}, }, Want: []ast.Node{ - &ast.StringNode{Key: "aBc", Value: "stringnode"}, + &ast.StringNode{Key: "Name", Value: "stringnode"}, &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "aBc", Value: "stringnode"}, + &ast.StringNode{Key: "Name", Value: "stringnode"}, }}, }, }, { - Name: "Hidden: StringNode -> StringNode", + Name: "aBc: StringNode -> StringNode", Got: []ast.Node{ - ast.StringNode{Key: "Hidden", Value: "StringNode"}, + ast.StringNode{Key: "aBc", Value: "StringNode"}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "aBc", Value: "StringNode"}, + }, + }, + { + Name: "Path: ./Documents -> ./Documents", + Got: []ast.Node{ + ast.StringNode{Key: "Path", Value: "./Documents"}, ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "Hidden", Value: "StringNode"}, + ast.StringNode{Key: "Path", Value: "./Documents"}, }}, }, Want: []ast.Node{ - &ast.StringNode{Key: "Hidden", Value: "StringNode"}, + &ast.StringNode{Key: "Path", Value: "./Documents"}, &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "Hidden", Value: "StringNode"}, + &ast.StringNode{Key: "Path", Value: "./Documents"}, }}, }, }, + { + Name: "Hidden: TRUE -> true", + Got: []ast.Node{ + ast.StringNode{Key: "Hidden", Value: "TRUE"}, + ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + ast.StringNode{Key: "Hidden", Value: "TRUE"}, + }}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "true"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "true"}, + }}, + }, + }, + { + Name: "ID: 1$1!AB23 -> 1$1!AB23", + Got: []ast.Node{ + ast.StringNode{Key: "ID", Value: "1$1!AB23"}, + ast.StringNode{Key: "RootID", Value: "1$1!AB23"}, + ast.StringNode{Key: "ParentID", Value: "1$1!AB23"}, + }, + Want: []ast.Node{ + &ast.StringNode{Key: "ID", Value: "1$1!AB23"}, + &ast.StringNode{Key: "RootID", Value: "1$1!AB23"}, + &ast.StringNode{Key: "ParentID", Value: "1$1!AB23"}, + }, + }, } for _, test := range tests { @@ -384,8 +422,6 @@ func TestExpandKQLAST(t *testing.T) { &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadshee"}, - &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.numbers"}, }}, &ast.OperatorNode{Value: "AND"}, diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index d94c1ac5fd..e3fb25a807 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "slices" + "strconv" "strings" "time" @@ -101,6 +102,8 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { return stringNodeQuery(node), nil case *ast.DateTimeNode: return dateTimeNodeQuery(node) + case *ast.NumberNode: + return numberNodeQuery(node) case *ast.GroupNode: group, err := t.transpile(node.Nodes) if err != nil { @@ -115,22 +118,35 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { // stringNodeQuery picks the query a string node turns into. func stringNodeQuery(node *ast.StringNode) osu.Builder { - isWildcard := strings.Contains(node.Value, "*") + isWildcard := strings.ContainsAny(node.Value, "*?") switch { // Name: "*oo-bar", "*oo ba*", "*OO*" // Title: "*rterly rep*" // Tags: "*spaced tag*" - case isWildcard && slices.Contains([]string{"Name", "Title", "Tags"}, node.Key): - return osu.NewWildcardQuery(node.Key + ".keyword"). - Value(node.Value). - Params(&osu.WildcardQueryParams{CaseInsensitive: true}) + case isWildcard && slices.Contains([]string{"Name", "Title"}, node.Key): + patterns := []osu.Builder{wildcardOn(node.Key+".wildcard", node.Value)} + if !strings.HasSuffix(node.Value, "*") { + patterns = append(patterns, wildcardOn(node.Key+".wildcard", node.Value+".*")) + } + + return osu.NewBoolQuery(). + Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). + Should(patterns...) + // Tags: "*foo*", "*paced ta*" + case isWildcard && node.Key == "Tags": + return wildcardOn(node.Key+".wildcard", node.Value) // Path: "./foo*", MimeType: "*plain" case isWildcard: return osu.NewWildcardQuery(node.Key).Value(node.Value) + // Name: =new, Title: ="quarterly report" + case node.Exact && slices.Contains([]string{"Name", "Title"}, node.Key): + return osu.NewTermQuery[string](node.Key + ".wildcard"). + Value(node.Value). + Params(&osu.TermQueryParams{CaseInsensitive: true}) // Tags: "foo-bar", "spaced tag", "FOO-BAR" case node.Key == "Tags": - return osu.NewTermQuery[string](node.Key + ".keyword"). + return osu.NewTermQuery[string](node.Key + ".wildcard"). Value(node.Value). Params(&osu.TermQueryParams{CaseInsensitive: true}) // Name: "foo-bar", "foo bar" @@ -138,12 +154,62 @@ func stringNodeQuery(node *ast.StringNode) osu.Builder { // Content: "foo bar" case slices.Contains([]string{"Name", "Title", "Content"}, node.Key): return osu.NewMatchPhraseQuery(node.Key).Query(node.Value) - // Path: "./foo bar", MimeType: "text/plain" + // Size: "42", Type: "1" + case slices.Contains([]string{"Size", "Type"}, node.Key): + number, err := strconv.ParseInt(node.Value, 10, 64) + if err != nil { + return osu.NewMatchNoneQuery() + } + + return osu.NewTermQuery[int64](node.Key).Value(number) + // Path: "./foo bar/", the hierarchy tokens carry no trailing slash + case node.Key == "Path": + return osu.NewTermQuery[string](node.Key).Value(strings.TrimSuffix(node.Value, "/")) + // Hidden: "TRUE" arrives lowered, anything that is no bool matches nothing + case node.Key == "Hidden": + value, err := strconv.ParseBool(node.Value) + if err != nil { + return osu.NewMatchNoneQuery() + } + + return osu.NewTermQuery[bool](node.Key).Value(value) + // MimeType: "text/plain" default: return osu.NewTermQuery[string](node.Key).Value(node.Value) } } +func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) { + if node.Operator == nil { + return nil, fmt.Errorf("number node without operator: %w", ErrUnsupportedNodeType) + } + + if !slices.Contains([]string{"Size", "Type"}, node.Key) { + return osu.NewMatchNoneQuery(), nil + } + + query := osu.NewRangeQuery[float64](node.Key) + + switch node.Operator.Value { + case ">": + return query.Gt(node.Value), nil + case ">=": + return query.Gte(node.Value), nil + case "<": + return query.Lt(node.Value), nil + case "<=": + return query.Lte(node.Value), nil + } + + return nil, fmt.Errorf("unsupported operator %s for number node: %w", node.Operator.Value, ErrUnsupportedNodeType) +} + +func wildcardOn(field, value string) osu.Builder { + return osu.NewWildcardQuery(field). + Value(value). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}) +} + // dateTimeNodeQuery turns a date time node into a range query. func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) { if node.Operator == nil { diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go index 1b627c8133..9e03fd23df 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go @@ -58,12 +58,16 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { &ast.StringNode{Key: "Name", Value: "open*"}, }, }, - Want: osu.NewWildcardQuery("Name.keyword"). - Value("open*"). - Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + Want: osu.NewBoolQuery(). + Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). + Should( + osu.NewWildcardQuery("Name.wildcard"). + Value("open*"). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + ), }, { - Name: "wildcard query - string node without a keyword sub field", + Name: "wildcard query - string node without an unanalyzed sub field", Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "Content", Value: "open*"}, @@ -71,6 +75,62 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewWildcardQuery("Content").Value("open*"), }, + { + Name: "wildcard query - a question mark counts as a wildcard", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "fo?o"}, + }, + }, + Want: osu.NewBoolQuery(). + Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). + Should( + osu.NewWildcardQuery("Name.wildcard"). + Value("fo?o"). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + osu.NewWildcardQuery("Name.wildcard"). + Value("fo?o.*"). + Params(&osu.WildcardQueryParams{CaseInsensitive: true}), + ), + }, + { + Name: "term query - an equals restriction matches the whole name", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "foo bar.txt", Exact: true}, + }, + }, + Want: osu.NewTermQuery[string]("Name.wildcard"). + Value("foo bar.txt"). + Params(&osu.TermQueryParams{CaseInsensitive: true}), + }, + { + Name: "term query - a path loses its trailing slash", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Path", Value: "./Documents/"}, + }, + }, + Want: osu.NewTermQuery[string]("Path").Value("./Documents"), + }, + { + Name: "term query - a hidden string turns into a bool", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "true"}, + }, + }, + Want: osu.NewTermQuery[bool]("Hidden").Value(true), + }, + { + Name: "match-none query - a hidden string that is no bool", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Hidden", Value: "banana"}, + }, + }, + Want: osu.NewMatchNoneQuery(), + }, { Name: "bool query", Got: &ast.Ast{ diff --git a/services/search/pkg/opensearch/internal/convert/opensearch_test.go b/services/search/pkg/opensearch/internal/convert/opensearch_test.go index d2fba09c70..8f034e327c 100644 --- a/services/search/pkg/opensearch/internal/convert/opensearch_test.go +++ b/services/search/pkg/opensearch/internal/convert/opensearch_test.go @@ -10,7 +10,7 @@ import ( "github.com/opencloud-eu/opencloud/pkg/conversions" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) diff --git a/services/search/pkg/opensearch/internal/indexes/resource_v3.json b/services/search/pkg/opensearch/internal/indexes/resource_v3.json new file mode 100644 index 0000000000..5180f6ad56 --- /dev/null +++ b/services/search/pkg/opensearch/internal/indexes/resource_v3.json @@ -0,0 +1,122 @@ +{ + "settings": { + "number_of_shards": "1", + "number_of_replicas": "1", + "analysis": { + "analyzer": { + "path_hierarchy": { + "tokenizer": "path_hierarchy", + "type": "custom" + }, + "name_words": { + "type": "custom", + "char_filter": [ + "dot_to_space" + ], + "tokenizer": "standard", + "filter": [ + "lowercase" + ] + } + }, + "tokenizer": { + "path_hierarchy": { + "type": "path_hierarchy" + } + }, + "normalizer": { + "lowercase": { + "type": "custom", + "filter": [ + "lowercase" + ] + } + }, + "char_filter": { + "dot_to_space": { + "type": "pattern_replace", + "pattern": "\\.", + "replacement": " " + } + } + } + }, + "mappings": { + "properties": { + "Content": { + "type": "text", + "analyzer": "name_words", + "term_vector": "with_positions_offsets" + }, + "ID": { + "type": "keyword" + }, + "ParentID": { + "type": "keyword" + }, + "RootID": { + "type": "keyword" + }, + "MimeType": { + "type": "wildcard" + }, + "Path": { + "type": "text", + "analyzer": "path_hierarchy" + }, + "Deleted": { + "type": "boolean" + }, + "Hidden": { + "type": "boolean" + }, + "Favorites": { + "type": "keyword" + }, + "Tags": { + "type": "text", + "fields": { + "wildcard": { + "type": "wildcard", + "normalizer": "lowercase" + } + } + }, + "Name": { + "type": "text", + "analyzer": "name_words", + "fields": { + "wildcard": { + "type": "wildcard", + "normalizer": "lowercase" + } + } + }, + "Title": { + "type": "text", + "analyzer": "name_words", + "fields": { + "wildcard": { + "type": "wildcard", + "normalizer": "lowercase" + } + } + }, + "Mtime": { + "type": "date", + "ignore_malformed": true + } + }, + "dynamic_templates": [ + { + "audio_facets": { + "path_match": "audio.*", + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + } + ] + } +} diff --git a/services/search/pkg/opensearch/internal/osu/query_match_none.go b/services/search/pkg/opensearch/internal/osu/query_match_none.go new file mode 100644 index 0000000000..ef071ac57c --- /dev/null +++ b/services/search/pkg/opensearch/internal/osu/query_match_none.go @@ -0,0 +1,55 @@ +package osu + +import ( + "encoding/json" +) + +// MatchNoneQuery matches no document. +type MatchNoneQuery struct{} + +// NewMatchNoneQuery creates a query that matches no document. +func NewMatchNoneQuery() *MatchNoneQuery { + return &MatchNoneQuery{} +} + +// Map returns the query as a map. +func (q *MatchNoneQuery) Map() (map[string]any, error) { + return map[string]any{ + "match_none": map[string]any{}, + }, nil +} + +// MarshalJSON returns the query as JSON. +func (q *MatchNoneQuery) MarshalJSON() ([]byte, error) { + data, err := q.Map() + if err != nil { + return nil, err + } + + return json.Marshal(data) +} + +// MatchAllQuery matches every document. +type MatchAllQuery struct{} + +// NewMatchAllQuery creates a query that matches every document. +func NewMatchAllQuery() *MatchAllQuery { + return &MatchAllQuery{} +} + +// Map returns the query as a map. +func (q *MatchAllQuery) Map() (map[string]any, error) { + return map[string]any{ + "match_all": map[string]any{}, + }, nil +} + +// MarshalJSON returns the query as JSON. +func (q *MatchAllQuery) MarshalJSON() ([]byte, error) { + data, err := q.Map() + if err != nil { + return nil, err + } + + return json.Marshal(data) +} diff --git a/services/search/pkg/opensearch/internal/osu/query_term_level_range.go b/services/search/pkg/opensearch/internal/osu/query_term_level_range.go index 3e81bb00f1..8a38db2317 100644 --- a/services/search/pkg/opensearch/internal/osu/query_term_level_range.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_range.go @@ -6,7 +6,8 @@ import ( "time" ) -type RangeQuery[T time.Time | string] struct { +// RangeQuery matches documents whose field value lies inside the bounds. +type RangeQuery[T time.Time | string | float64] struct { field string gt T gte T @@ -22,7 +23,8 @@ type RangeQueryParams struct { TimeZone string `json:"time_zone,omitempty"` } -func NewRangeQuery[T time.Time | string](field string) *RangeQuery[T] { +// NewRangeQuery creates a range query for the given field. +func NewRangeQuery[T time.Time | string | float64](field string) *RangeQuery[T] { return &RangeQuery[T]{field: field} } diff --git a/services/search/pkg/opensearch/internal/osu/request_test.go b/services/search/pkg/opensearch/internal/osu/request_test.go index f8e0010258..833536ff2e 100644 --- a/services/search/pkg/opensearch/internal/osu/request_test.go +++ b/services/search/pkg/opensearch/internal/osu/request_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" ) diff --git a/services/search/pkg/opensearch/opensearch_test.go b/services/search/pkg/opensearch/opensearch_test.go index cd5fb6fdbc..690c2d70f3 100644 --- a/services/search/pkg/opensearch/opensearch_test.go +++ b/services/search/pkg/opensearch/opensearch_test.go @@ -6,7 +6,7 @@ import ( "os" "testing" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/config" ) diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index 310397cae6..0a84fabfc4 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -2,23 +2,26 @@ package bleve import ( "fmt" + "slices" + "strconv" "strings" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/blevesearch/bleve/v2" bleveQuery "github.com/blevesearch/bleve/v2/search/query" "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/pkg/kql" ) -// lowercaseFields lists the bleve fields whose index mapping uses a -// lowercasing analyzer. Values bound to these fields are pre-lowercased -// so query-side matching stays consistent with the index. -// Keep in sync with services/search/pkg/bleve/index.go NewMapping. var lowercaseFields = map[string]struct{}{ "Name": {}, + "Title": {}, "Tags": {}, "Favorites": {}, "Content": {}, + "MimeType": {}, + "Hidden": {}, } var _fields = map[string]string{ @@ -33,6 +36,7 @@ var _fields = map[string]string{ "tag": "Tags", "tags": "Tags", "content": "Content", + "title": "Title", "hidden": "Hidden", "favorite": "Favorites", } @@ -53,7 +57,6 @@ var bleveEscaper = strings.NewReplacer( `)`, `\)`, `{`, `\{`, `}`, `\}`, - `{`, `\}`, `[`, `\[`, `]`, `\]`, `^`, `\^`, @@ -106,14 +109,43 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { v = strings.ToLower(v) } + if k == "Type" { + v = resourceType(v) + } + var q bleveQuery.Query var group bool - switch k { - case "MimeType": + switch { + case k == "Hidden": + value, err := strconv.ParseBool(v) + if err != nil { + q = bleveQuery.NewMatchNoneQuery() + break + } + + bq := bleveQuery.NewBoolFieldQuery(value) + bq.SetField(k) + q = bq + case k == "MimeType": q, group = mimeType(k, v) if prev == nil { isGroup = group } + case slices.Contains([]string{"Name", "Title"}, k) && strings.ContainsAny(n.Value, "*?"): + patterns := []bleveQuery.Query{bleveQuery.NewQueryStringQuery(k + ".wildcard:" + v)} + if !strings.HasSuffix(v, "*") { + patterns = append(patterns, bleveQuery.NewQueryStringQuery(k+".wildcard:"+v+".*")) + } + + q = closed(bleveQuery.NewDisjunctionQuery(patterns)) + case n.Exact && !strings.ContainsAny(n.Value, "*?") && slices.Contains([]string{"Name", "Title"}, k): + q = bleveQuery.NewQueryStringQuery(k + ".wildcard:" + v) + case k == "Path" && !strings.ContainsAny(n.Value, "*?"): + q = pathAndBelow(k, n.Value) + case slices.Contains([]string{"Name", "Title", "Content"}, k) && !strings.ContainsAny(n.Value, "*?"): + q = phrase(k, n.Value) + case strings.Contains(n.Value, " ") && !strings.ContainsAny(n.Value, "*?"): + q = phrase(k, n.Value) default: q = bleveQuery.NewQueryStringQuery(k + ":" + v) } @@ -153,13 +185,25 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { continue } + if prev == nil { + prev = q + } else { + next = q + } + case *ast.NumberNode: + q := numberRange(getField(n.Key), n.Operator, n.Value) + if q == nil { + continue + } + if prev == nil { prev = q } else { next = q } case *ast.BooleanNode: - q := bleveQuery.NewQueryStringQuery(getField(n.Key) + fmt.Sprintf(":%v", n.Value)) + q := bleveQuery.NewBoolFieldQuery(n.Value) + q.SetField(getField(n.Key)) if prev == nil { prev = q } else { @@ -216,6 +260,10 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { func nextNode(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { if n, ok := nodes[offset].(*ast.GroupNode); ok { + if n.Key != "" { + n = normalizeGroupingProperty(n) + } + gq, _, err := walk(0, n.Nodes) if err != nil { return nil, 0, err @@ -277,6 +325,57 @@ func mapBinary(operator *ast.OperatorNode, ln, rn bleveQuery.Query, leftIsGroup }) } +func numberRange(field string, operator *ast.OperatorNode, value float64) bleveQuery.Query { + if operator == nil { + return nil + } + + inclusive, exclusive := true, false + + var q *bleveQuery.NumericRangeQuery + switch operator.Value { + case ">": + q = bleveQuery.NewNumericRangeInclusiveQuery(&value, nil, &exclusive, nil) + case ">=": + q = bleveQuery.NewNumericRangeInclusiveQuery(&value, nil, &inclusive, nil) + case "<": + q = bleveQuery.NewNumericRangeInclusiveQuery(nil, &value, nil, &exclusive) + case "<=": + q = bleveQuery.NewNumericRangeInclusiveQuery(nil, &value, nil, &inclusive) + default: + return nil + } + + q.SetField(field) + + return q +} + +func pathAndBelow(field, path string) bleveQuery.Query { + path = strings.TrimSuffix(path, "/") + + self := bleveQuery.NewTermQuery(path) + self.SetField(field) + + below := bleveQuery.NewPrefixQuery(path + "/") + below.SetField(field) + + return closed(bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{self, below})) +} + +func closed(q bleveQuery.Query) bleveQuery.Query { + // a bare disjunction reads as an open OR chain to mapBinary, a later OR + // would merge into it and widen the group + return bleveQuery.NewConjunctionQuery([]bleveQuery.Query{q}) +} + +func phrase(field, value string) bleveQuery.Query { + q := bleveQuery.NewMatchPhraseQuery(value) + q.SetField(field) + + return q +} + func getField(name string) string { if name == "" { return "Name" @@ -296,6 +395,17 @@ func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode { return group } +func resourceType(value string) string { + switch strings.ToLower(value) { + case "file": + return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10) + case "folder": + return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10) + default: + return value + } +} + func mimeType(k, v string) (bleveQuery.Query, bool) { switch v { case "file": @@ -321,7 +431,6 @@ func mimeType(k, v string) (bleveQuery.Query, bool) { "application/vnd.oasis.opendocument.spreadsheet", "text/csv", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "application/vnd.oasis.opendocument.spreadsheet", "application/vnd.apple.numbers", )), true case "presentation": diff --git a/services/search/pkg/query/bleve/compiler_test.go b/services/search/pkg/query/bleve/compiler_test.go index ff94e88726..30c283f694 100644 --- a/services/search/pkg/query/bleve/compiler_test.go +++ b/services/search/pkg/query/bleve/compiler_test.go @@ -1,6 +1,7 @@ package bleve import ( + "strings" "testing" "time" @@ -18,6 +19,29 @@ var timeMustParse = func(t *testing.T, ts string) time.Time { return tp } +func wildcardQuery(field, value string) query.Query { + patterns := []query.Query{query.NewQueryStringQuery(field + ".wildcard:" + value)} + if !strings.HasSuffix(value, "*") { + patterns = append(patterns, query.NewQueryStringQuery(field+".wildcard:"+value+".*")) + } + + return query.NewConjunctionQuery([]query.Query{query.NewDisjunctionQuery(patterns)}) +} + +func phraseQuery(field, value string) query.Query { + q := query.NewMatchPhraseQuery(value) + q.SetField(field) + + return q +} + +func boolFieldQuery(field string, value bool) query.Query { + q := query.NewBoolFieldQuery(value) + q.SetField(field) + + return q +} + func Test_compile(t *testing.T) { tests := []struct { name string @@ -33,7 +57,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:federated`), + phraseQuery("Name", "federated"), }), wantErr: false, }, @@ -45,7 +69,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith`), + phraseQuery("Name", "John Smith"), }), wantErr: false, }, @@ -59,8 +83,8 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith`), - query.NewQueryStringQuery(`Name:jane`), + phraseQuery("Name", "John Smith"), + phraseQuery("Name", "Jane"), }), wantErr: false, }, @@ -91,7 +115,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:moby\ di*`), + wildcardQuery("Name", `moby\ di*`), query.NewConjunctionQuery([]query.Query{ query.NewQueryStringQuery(`Tags:bestseller`), query.NewQueryStringQuery(`Tags:book`), @@ -112,10 +136,10 @@ func Test_compile(t *testing.T) { }, want: query.NewDisjunctionQuery([]query.Query{ query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:a`), - query.NewQueryStringQuery(`Name:b`), + phraseQuery("Name", "a"), + phraseQuery("Name", "b"), }), - query.NewQueryStringQuery(`Name:c`), + phraseQuery("Name", "c"), }), wantErr: false, }, @@ -131,10 +155,10 @@ func Test_compile(t *testing.T) { }, }, want: query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:a`), + phraseQuery("Name", "a"), query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:b`), - query.NewQueryStringQuery(`Name:c`), + phraseQuery("Name", "b"), + phraseQuery("Name", "c"), }), }), wantErr: false, @@ -156,11 +180,11 @@ func Test_compile(t *testing.T) { }, want: query.NewConjunctionQuery([]query.Query{ query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:a`), - query.NewQueryStringQuery(`Name:b`), - query.NewQueryStringQuery(`Name:c`), + phraseQuery("Name", "a"), + phraseQuery("Name", "b"), + phraseQuery("Name", "c"), }), - query.NewQueryStringQuery(`Name:d`), + phraseQuery("Name", "d"), }), wantErr: false, }, @@ -179,7 +203,7 @@ func Test_compile(t *testing.T) { }, want: query.NewConjunctionQuery([]query.Query{ query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:moby\ di*`), + wildcardQuery("Name", `moby\ di*`), query.NewQueryStringQuery(`Tags:bestseller`), }), query.NewQueryStringQuery(`Tags:book`), @@ -204,7 +228,7 @@ func Test_compile(t *testing.T) { }, want: query.NewConjunctionQuery([]query.Query{ query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:moby\ di*`), + wildcardQuery("Name", `moby\ di*`), query.NewQueryStringQuery(`Tags:bestseller`), }), query.NewQueryStringQuery(`Tags:book`), @@ -227,7 +251,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`author:John\ Smith`), + phraseQuery("author", "John Smith"), query.NewQueryStringQuery(`author:Jane`), }), wantErr: false, @@ -249,7 +273,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`author:John\ Smith`), + phraseQuery("author", "John Smith"), query.NewQueryStringQuery(`author:Jane`), query.NewQueryStringQuery(`Tags:bestseller`), }), @@ -293,9 +317,44 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith`), - query.NewQueryStringQuery(`Hidden:T`), - query.NewQueryStringQuery(`Hidden:T`), + phraseQuery("Name", "John Smith"), + boolFieldQuery("Hidden", true), + boolFieldQuery("Hidden", true), + }), + wantErr: false, + }, + { + name: `hidden:banana`, + args: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "hidden", Value: "banana"}, + }, + }, + want: query.NewConjunctionQuery([]query.Query{query.NewMatchNoneQuery()}), + wantErr: false, + }, + { + name: `name="Report.txt"`, + args: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "name", Value: "Report.txt", Exact: true}, + }, + }, + want: query.NewConjunctionQuery([]query.Query{query.NewQueryStringQuery(`Name.wildcard:report.txt`)}), + wantErr: false, + }, + { + name: `type:File`, + args: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "type", Value: "File"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "type", Value: "FOLDER"}, + }, + }, + want: query.NewDisjunctionQuery([]query.Query{ + query.NewQueryStringQuery(`Type:1`), + query.NewQueryStringQuery(`Type:2`), }), wantErr: false, }, @@ -424,7 +483,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/rtf`), query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), }), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), wantErr: false, }, @@ -450,7 +509,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), query.NewConjunctionQuery([]query.Query{ query.NewQueryStringQuery(`MimeType:application/pdf`), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), }), wantErr: false, @@ -480,7 +539,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), query.NewQueryStringQuery(`MimeType:application/pdf`), }), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), wantErr: false, }, @@ -509,7 +568,7 @@ func Test_compile(t *testing.T) { query.NewQueryStringQuery(`MimeType:application/rtf`), query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`), }), - query.NewQueryStringQuery(`Name:*tdd*`), + wildcardQuery("Name", `*tdd*`), }), wantErr: false, }, @@ -521,7 +580,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name:john\ smith\ \+\-\=\&\|\>\<\!\(\)\{\}\[\]\^\"\~\:\ `), + phraseQuery("Name", "John Smith +-=&|>quarterly reportsome data" to "q1.html" + And user "Alice" has uploaded file with content "notessome data" to "q2.html" + When user "Alice" searches for 'Title:"quarterly report"' using the WebDAV API + Then the HTTP status code should be "207" + And the search result of user "Alice" should contain only these entries: + | q1.html | diff --git a/vendor/github.com/blevesearch/bleve/v2/analysis/char/regexp/regexp.go b/vendor/github.com/blevesearch/bleve/v2/analysis/char/regexp/regexp.go new file mode 100644 index 0000000000..a94236af9a --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/v2/analysis/char/regexp/regexp.go @@ -0,0 +1,65 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package regexp + +import ( + "fmt" + "regexp" + + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" +) + +const Name = "regexp" + +type CharFilter struct { + r *regexp.Regexp + replacement []byte +} + +func New(r *regexp.Regexp, replacement []byte) *CharFilter { + return &CharFilter{ + r: r, + replacement: replacement, + } +} + +func (s *CharFilter) Filter(input []byte) []byte { + return s.r.ReplaceAll(input, s.replacement) +} + +func CharFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.CharFilter, error) { + regexpStr, ok := config["regexp"].(string) + if !ok { + return nil, fmt.Errorf("must specify regexp") + } + r, err := regexp.Compile(regexpStr) + if err != nil { + return nil, fmt.Errorf("unable to build regexp char filter: %v", err) + } + replaceBytes := []byte(" ") + replaceStr, ok := config["replace"].(string) + if ok { + replaceBytes = []byte(replaceStr) + } + return New(r, replaceBytes), nil +} + +func init() { + err := registry.RegisterCharFilter(Name, CharFilterConstructor) + if err != nil { + panic(err) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index f13e2c9497..beaceb675a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -126,6 +126,7 @@ github.com/blevesearch/bleve/v2/analysis/analyzer/custom github.com/blevesearch/bleve/v2/analysis/analyzer/keyword github.com/blevesearch/bleve/v2/analysis/analyzer/standard github.com/blevesearch/bleve/v2/analysis/datetime/flexible +github.com/blevesearch/bleve/v2/analysis/char/regexp github.com/blevesearch/bleve/v2/analysis/datetime/optional github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds From 0106a700c2ac8e879d3bea6eb503c01344d15935 Mon Sep 17 00:00:00 2001 From: Florian Schade Date: Mon, 31 Aug 2026 10:44:20 +0200 Subject: [PATCH 2/4] test(search): follow the versioned index and pin the equals operator --- services/search/pkg/parity/README.md | 157 +++++++++--------- services/search/pkg/parity/engines_test.go | 5 +- services/search/pkg/parity/matrix_test.go | 6 +- .../search/pkg/parity/query_fields_test.go | 4 +- services/search/pkg/parity/query_name_test.go | 7 +- .../search/pkg/parity/query_stress_test.go | 2 +- .../search/pkg/parity/query_title_test.go | 2 + 7 files changed, 102 insertions(+), 81 deletions(-) diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index a8acafba94..df08b45860 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -3,7 +3,9 @@ Written by the parity suite (`go test ./services/search/pkg/parity/`), do not edit. Every case runs against bleve and OpenSearch. `same?` is ✅ when both answer as expected, `❌ known` when an engine's divergence is documented in the case -(`engineOverrides`), `❌` when it is not. +(`engineOverrides`), `❌` when it is not. `✅ stale` when every engine +answers the expected value although the case still documents a +divergence, that override can come out. ## Queries @@ -24,15 +26,15 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| NAME-01 | `new` | new-folder | no match | new-folder | ❌ known | -| NAME-02 | `quarterly` | quarterly notes.txt | no match | quarterly notes.txt | ❌ known | -| NAME-03 | `report` | Report.txt | no match | no match | ❌ known | +| NAME-01 | `new` | new-folder | new-folder | new-folder | ✅ stale | +| NAME-02 | `quarterly` | quarterly notes.txt | quarterly notes.txt | quarterly notes.txt | ✅ stale | +| NAME-03 | `report` | Report.txt | Report.txt | Report.txt | ✅ stale | | NAME-04 | `name:"*new-folder*"` | new-folder | new-folder | new-folder | ✅ | | NAME-05 | `name:"*w-fol*"` | new-folder | new-folder | new-folder | ✅ | | NAME-06 | `name:"*oo ba*"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | | NAME-07 | `name:"*REPORT*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-08 | `name:"*übung*"` | Übung.txt | Übung.txt | no match | ❌ known | -| NAME-09 | `name:"*ÜBUNG*"` | Übung.txt | Übung.txt | no match | ❌ known | +| NAME-08 | `name:"*übung*"` | Übung.txt | Übung.txt | Übung.txt | ✅ stale | +| NAME-09 | `name:"*ÜBUNG*"` | Übung.txt | Übung.txt | Übung.txt | ✅ stale | | NAME-10 | `name:"*a+b*"` | a+b.txt | a+b.txt | a+b.txt | ✅ | | NAME-11 | `name:"*c(d)*"` | c(d).txt | c(d).txt | c(d).txt | ✅ | | NAME-12 | `name:"*e&f*"` | e&f.txt | e&f.txt | e&f.txt | ✅ | @@ -41,30 +43,35 @@ Fixtures: | NAME-15 | `*folder*` | new-folder | new-folder | new-folder | ✅ | | NAME-16 | `name:"*foo bar*"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | | NAME-17 | `name:"foo bar.txt"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | -| NAME-18 | `name:"*needle*"` | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | no match | ❌ known | +| NAME-18 | `name:"*needle*"` | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | ✅ stale | | NAME-19 | `name:"report*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-20 | `name:"*report"` | Report.txt | no match | no match | ❌ known | +| NAME-20 | `name:"*report"` | Report.txt | Report.txt | Report.txt | ✅ stale | | NAME-21 | `name:"Rep*rt.txt"` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-22 | `Name:"*report*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-23 | `NAME:"*report*"` | Report.txt | Report.txt | no match | ❌ known | -| NAME-24 | `name:Rep?rt.txt` | Report.txt | Report.txt | no match | ❌ known | -| NAME-25 | `name:"*eport"` | Report.txt | no match | no match | ❌ known | +| NAME-23 | `NAME:"*report*"` | Report.txt | Report.txt | Report.txt | ✅ stale | +| NAME-24 | `name:Rep?rt.txt` | Report.txt | Report.txt | Report.txt | ✅ stale | +| NAME-25 | `name:"*eport"` | Report.txt | Report.txt | Report.txt | ✅ stale | | NAME-26 | `name:"repor*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-27 | `REPORT` | Report.txt | no match | no match | ❌ known | -| NAME-28 | `name:REPORT` | Report.txt | no match | no match | ❌ known | +| NAME-27 | `REPORT` | Report.txt | Report.txt | Report.txt | ✅ stale | +| NAME-28 | `name:REPORT` | Report.txt | Report.txt | Report.txt | ✅ stale | | NAME-29 | `name:"REPORT.TXT"` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-30 | `name:"FOO BAR.TXT"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | | NAME-31 | `name:"ÜBUNG.TXT"` | Übung.txt | Übung.txt | Übung.txt | ✅ | | NAME-32 | `name:"folder*"` | no match | no match | no match | ✅ | | NAME-33 | `name:"*new"` | no match | no match | no match | ✅ | -| NAME-34 | `name:new` | new-folder | no match | new-folder | ❌ known | -| NAME-35 | `name:"new"` | no match | no match | new-folder | ❌ known | +| NAME-34 | `name:new` | new-folder | new-folder | new-folder | ✅ stale | +| NAME-35 | `name:"new"` | new-folder | new-folder | new-folder | ✅ stale | | NAME-36 | `name:"new-folder"` | new-folder | new-folder | new-folder | ✅ | | NAME-37 | `name:"new-*"` | new-folder | new-folder | new-folder | ✅ | | NAME-38 | `name:"new*"` | new-folder | new-folder | new-folder | ✅ | | NAME-39 | `name:new-*` | new-folder | new-folder | new-folder | ✅ | | NAME-40 | `name:"*-folder"` | new-folder | new-folder | new-folder | ✅ | -| NAME-41 | `name:"Rep?rt.txt"` | Report.txt | Report.txt | no match | ❌ known | +| NAME-41 | `name:"Rep?rt.txt"` | Report.txt | Report.txt | Report.txt | ✅ stale | +| NAME-42 | `name="Report.txt"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-43 | `name="REPORT.TXT"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-44 | `name="new"` | no match | no match | no match | ✅ | +| NAME-45 | `name="new-folder"` | new-folder | new-folder | new-folder | ✅ | +| NAME-46 | `name="foo bar.txt"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | ### extension @@ -76,10 +83,10 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| EXTENSION-01 | `txt` | report.txt | no match | no match | ❌ known | -| EXTENSION-02 | `md` | notes.md | no match | no match | ❌ known | +| EXTENSION-01 | `txt` | report.txt | report.txt | report.txt | ✅ stale | +| EXTENSION-02 | `md` | notes.md | notes.md | notes.md | ✅ stale | | EXTENSION-03 | `name:"*.txt"` | report.txt | report.txt | report.txt | ✅ | -| EXTENSION-04 | `report` | report.txt | no match | no match | ❌ known | +| EXTENSION-04 | `report` | report.txt | report.txt | report.txt | ✅ stale | ### tags @@ -102,7 +109,7 @@ Fixtures: | TAGS-06 | `tag:("spaced tag")` | spaced.txt | spaced.txt | spaced.txt | ✅ | | TAGS-07 | `tag:("*paced ta*")` | spaced.txt | spaced.txt | spaced.txt | ✅ | | TAGS-08 | `tag:("work")` | project | project | project | ✅ | -| TAGS-09 | `tag:("zzzzzzzzzzzzzzzzzzzzzzzzzz...zzzzzzzzzzzzzzzzneedle")` | longtag.txt | longtag.txt | no match | ❌ known | +| TAGS-09 | `tag:("zzzzzzzzzzzzzzzzzzzzzzzzzz...zzzzzzzzzzzzzzzzneedle")` | longtag.txt | longtag.txt | longtag.txt | ✅ stale | ### title @@ -113,12 +120,14 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| | TITLE-01 | `Title:"quarterly report"` | q1.html | q1.html | q1.html | ✅ | -| TITLE-02 | `Title:quarterly` | q1.html | no match | q1.html | ❌ known | -| TITLE-03 | `Title:QUARTERLY` | q1.html | no match | q1.html | ❌ known | +| TITLE-02 | `Title:quarterly` | q1.html | q1.html | q1.html | ✅ stale | +| TITLE-03 | `Title:QUARTERLY` | q1.html | q1.html | q1.html | ✅ stale | | TITLE-04 | `Title:quarterl*` | q1.html | q1.html | q1.html | ✅ | | TITLE-05 | `Title:"*ly rep*"` | q1.html | q1.html | q1.html | ✅ | -| TITLE-06 | `title:quarterly` | q1.html | no match | no match | ❌ known | -| TITLE-07 | `Title:"QUARTERLY REPORT"` | q1.html | no match | q1.html | ❌ known | +| TITLE-06 | `title:quarterly` | q1.html | q1.html | q1.html | ✅ stale | +| TITLE-07 | `Title:"QUARTERLY REPORT"` | q1.html | q1.html | q1.html | ✅ stale | +| TITLE-08 | `Title="quarterly report"` | q1.html | q1.html | q1.html | ✅ | +| TITLE-09 | `Title="quarterly"` | no match | no match | no match | ✅ | ### content @@ -129,16 +138,16 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| CONTENT-01 | `Content:report` | no match | monthly.txt | no match | ❌ known | +| CONTENT-01 | `Content:report` | no match | no match | no match | ✅ stale | | CONTENT-02 | `Content:REPORTS` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-03 | `Content:"monthly reports"` | monthly.txt | monthly.txt | monthly.txt | ✅ | -| CONTENT-04 | `Content:"reports monthly"` | no match | monthly.txt | no match | ❌ known | +| CONTENT-04 | `Content:"reports monthly"` | no match | no match | no match | ✅ stale | | CONTENT-05 | `Content:report*` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-06 | `Content:*eport*` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-07 | `Content:month*` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-08 | `Content:"https://opencloud.example.com/help"` | links.txt | links.txt | links.txt | ✅ | | CONTENT-09 | `Content:"alan@example.org"` | links.txt | links.txt | links.txt | ✅ | -| CONTENT-10 | `Content:opencloud` | links.txt | no match | no match | ❌ known | +| CONTENT-10 | `Content:opencloud` | links.txt | links.txt | links.txt | ✅ stale | ### favorites @@ -151,8 +160,8 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| FAVORITES-01 | `Favorites:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | no match | ❌ known | -| FAVORITES-02 | `favorite:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | no match | ❌ known | +| FAVORITES-01 | `Favorites:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | keepsakes, starred.txt | ✅ stale | +| FAVORITES-02 | `favorite:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | keepsakes, starred.txt | ✅ stale | | FAVORITES-03 | `Favorites:"somebody-else"` | no match | no match | no match | ✅ | ### mediatype @@ -167,7 +176,7 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| | MEDIATYPE-01 | `mediatype:text/markdown` | notes.md | notes.md | notes.md | ✅ | -| MEDIATYPE-02 | `mediatype:TEXT/MARKDOWN` | notes.md | no match | notes.md | ❌ known | +| MEDIATYPE-02 | `mediatype:TEXT/MARKDOWN` | notes.md | notes.md | notes.md | ✅ stale | | MEDIATYPE-03 | `mediatype:image/jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ | | MEDIATYPE-04 | `mediatype:*jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ | | MEDIATYPE-05 | `mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ | @@ -185,14 +194,14 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| PATH-01 | `path:"./parent"` | child.jpg, parent | parent | child.jpg, parent | ❌ known | +| PATH-01 | `path:"./parent"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ stale | | PATH-02 | `path:"./parent/child.jpg"` | child.jpg | child.jpg | child.jpg | ✅ | -| PATH-03 | `path:"./Parent"` | no match | no match | child.jpg, parent | ❌ known | +| PATH-03 | `path:"./Parent"` | no match | no match | no match | ✅ stale | | PATH-04 | `path:"*child*"` | child.jpg | child.jpg | child.jpg | ✅ | -| PATH-05 | `path:"./documents"` | docs-lower | docs-lower | docs-lower, docs-mixed, docs-upper | ❌ known | -| PATH-06 | `path:"./DOCUMENTS"` | docs-upper | docs-upper | docs-lower, docs-mixed, docs-upper | ❌ known | -| PATH-07 | `path:"./Documents"` | docs-mixed | docs-mixed | docs-lower, docs-mixed, docs-upper | ❌ known | -| PATH-08 | `path:"./parent/"` | child.jpg, parent | no match | no match | ❌ known | +| PATH-05 | `path:"./documents"` | docs-lower | docs-lower | docs-lower | ✅ stale | +| PATH-06 | `path:"./DOCUMENTS"` | docs-upper | docs-upper | docs-upper | ✅ stale | +| PATH-07 | `path:"./Documents"` | docs-mixed | docs-mixed | docs-mixed | ✅ stale | +| PATH-08 | `path:"./parent/"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ stale | ### fields @@ -213,16 +222,16 @@ Fixtures: | FIELDS-01 | `size:42` | small.txt | small.txt | small.txt | ✅ | | FIELDS-02 | `mtime<"2021-01-01T00:00:00Z"` | old.txt | old.txt | old.txt | ✅ | | FIELDS-03 | `id:"1$1!23"` | known.txt | known.txt | known.txt | ✅ | -| FIELDS-04 | `hidden:true` | hidden.txt | no match | hidden.txt | ❌ known | -| FIELDS-05 | `type:file` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | no match | error | ❌ known | -| FIELDS-06 | `type:folder` | box | no match | error | ❌ known | +| FIELDS-04 | `hidden:true` | hidden.txt | hidden.txt | hidden.txt | ✅ stale | +| FIELDS-05 | `type:file` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | ✅ stale | +| FIELDS-06 | `type:folder` | box | box | box | ✅ stale | | FIELDS-07 | `unknown:field` | no match | no match | no match | ✅ | -| FIELDS-08 | `type:File` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | no match | error | ❌ known | -| FIELDS-09 | `type:FOLDER` | box | no match | error | ❌ known | -| FIELDS-10 | `hidden:TRUE` | hidden.txt | no match | error | ❌ known | -| FIELDS-11 | `id:"1$1!AB-23"` | cased.txt | cased.txt | no match | ❌ known | +| FIELDS-08 | `type:File` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | ✅ stale | +| FIELDS-09 | `type:FOLDER` | box | box | box | ✅ stale | +| FIELDS-10 | `hidden:TRUE` | hidden.txt | hidden.txt | hidden.txt | ✅ stale | +| FIELDS-11 | `id:"1$1!AB-23"` | cased.txt | cased.txt | cased.txt | ✅ stale | | FIELDS-12 | `id:"1$1!ab-23"` | no match | no match | no match | ✅ | -| FIELDS-13 | `audio.artist:"Some Artist"` | song.mp3 | song.mp3 | no match | ❌ known | +| FIELDS-13 | `audio.artist:"Some Artist"` | song.mp3 | song.mp3 | song.mp3 | ✅ stale | | FIELDS-14 | `audio.artist:"some artist"` | no match | no match | no match | ✅ | ### deleted @@ -242,7 +251,7 @@ Fixtures: | DELETED-02 | `name:"*.txt"` | book.txt, kept.txt | book.txt, kept.txt | book.txt, kept.txt | ✅ | | DELETED-03 | `name:"*receipt*"` | no match | no match | no match | ✅ | | DELETED-04 | `path:"./bin"` | no match | no match | no match | ✅ | -| DELETED-05 | `path:"./shelf"` | book.txt, shelf | shelf | book.txt, shelf | ❌ known | +| DELETED-05 | `path:"./shelf"` | book.txt, shelf | book.txt, shelf | book.txt, shelf | ✅ stale | ### visibility @@ -255,13 +264,13 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| VISIBILITY-01 | `hidden:true` | .private, dotfile.txt, secret.txt | no match | .private, dotfile.txt, secret.txt | ❌ known | -| VISIBILITY-02 | `hidden:TRUE` | .private, dotfile.txt, secret.txt | no match | error | ❌ known | -| VISIBILITY-03 | `hidden:false` | visible.txt | no match | visible.txt | ❌ known | +| VISIBILITY-01 | `hidden:true` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ stale | +| VISIBILITY-02 | `hidden:TRUE` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ stale | +| VISIBILITY-03 | `hidden:false` | visible.txt | visible.txt | visible.txt | ✅ stale | | VISIBILITY-04 | `name:"*secret*"` | secret.txt | secret.txt | secret.txt | ✅ | -| VISIBILITY-05 | `path:"./.private"` | .private, secret.txt | .private | .private, secret.txt | ❌ known | -| VISIBILITY-06 | `hidden:banana` | no match | no match | error | ❌ known | -| VISIBILITY-07 | `hidden:"true"` | .private, dotfile.txt, secret.txt | no match | .private, dotfile.txt, secret.txt | ❌ known | +| VISIBILITY-05 | `path:"./.private"` | .private, secret.txt | .private, secret.txt | .private, secret.txt | ✅ stale | +| VISIBILITY-06 | `hidden:banana` | no match | no match | no match | ✅ stale | +| VISIBILITY-07 | `hidden:"true"` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ stale | ### boolean @@ -310,21 +319,21 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| | STRESS-01 | `name:"*report*" AND mediatype:document` | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | ✅ | -| STRESS-02 | `name:"*report*" AND NOT tag:("draft")` | quarterly report.docx | draft report.txt, quarterly report.docx | quarterly report.docx | ❌ known | +| STRESS-02 | `name:"*report*" AND NOT tag:("draft")` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ stale | | STRESS-03 | `(tag:("final") OR tag:("draft")) AND mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ | | STRESS-04 | `mediatype:document AND mtime>"2021-01-01T00:00:00Z"` | notes.md, quarterly report.docx | notes.md, quarterly report.docx | notes.md, quarterly report.docx | ✅ | -| STRESS-05 | `name:"*report*" AND size>100` | quarterly report.docx | no match | no match | ❌ known | +| STRESS-05 | `name:"*report*" AND size>100` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ stale | | STRESS-06 | `tag:("final") AND NOT mediatype:folder` | photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | ✅ | -| STRESS-07 | `hidden:true AND name:"*notes*"` | notes.md | no match | notes.md | ❌ known | -| STRESS-08 | `name:quarterly report` | quarterly report.docx | no match | no match | ❌ known | -| STRESS-09 | `name:"quarterly report"` | no match | no match | no match | ✅ | +| STRESS-07 | `hidden:true AND name:"*notes*"` | notes.md | notes.md | notes.md | ✅ stale | +| STRESS-08 | `name:quarterly report` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ stale | +| STRESS-09 | `name:"quarterly report"` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | | STRESS-10 | `name:"quarterly report.docx"` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | -| STRESS-11 | `NOT tag:("draft")` | archive, notes.md, photo.jpg, quarterly report.docx | archive, draft report.txt, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | ❌ known | -| STRESS-12 | `tag:("final") OR hidden:true` | notes.md, photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | ❌ known | -| STRESS-13 | `(name:"*report*" OR name:"*notes..."draft") OR hidden:true)` | quarterly report.docx | notes.md, quarterly report.docx | quarterly report.docx | ❌ known | +| STRESS-11 | `NOT tag:("draft")` | archive, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | ✅ stale | +| STRESS-12 | `tag:("final") OR hidden:true` | notes.md, photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | ✅ stale | +| STRESS-13 | `(name:"*report*" OR name:"*notes..."draft") OR hidden:true)` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ stale | | STRESS-14 | `mediatype:image OR (mediatype:document AND tag:("draft"))` | draft report.txt, photo.jpg | draft report.txt, photo.jpg | draft report.txt, photo.jpg | ✅ | -| STRESS-15 | `NOT (mediatype:folder OR hidden:true)` | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, notes.md, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | ❌ known | -| STRESS-16 | `name:"*report*" AND (size>100 OR tag:("draft"))` | draft report.txt, quarterly report.docx | draft report.txt | draft report.txt | ❌ known | +| STRESS-15 | `NOT (mediatype:folder OR hidden:true)` | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | ✅ stale | +| STRESS-16 | `name:"*report*" AND (size>100 OR tag:("draft"))` | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | ✅ stale | ### everything @@ -351,8 +360,8 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| RANGE-01 | `size>100` | big.txt | no match | no match | ❌ known | -| RANGE-02 | `size<100` | ancient.txt, small.txt | no match | no match | ❌ known | +| RANGE-01 | `size>100` | big.txt | big.txt | big.txt | ✅ stale | +| RANGE-02 | `size<100` | ancient.txt, small.txt | ancient.txt, small.txt | ancient.txt, small.txt | ✅ stale | | RANGE-03 | `mtime>"2021-01-01T00:00:00Z"` | big.txt, small.txt | big.txt, small.txt | big.txt, small.txt | ✅ | | RANGE-04 | `mtime<"2021-01-01T00:00:00Z"` | ancient.txt | ancient.txt | ancient.txt | ✅ | | RANGE-05 | `Mtime:"today"` | big.txt, small.txt | big.txt, small.txt | big.txt, small.txt | ✅ | @@ -403,7 +412,7 @@ Fixtures: | DELETE-01 | takes the resource out of the results, then `name:"*child*"` | no match | no match | no match | ✅ | | DELETE-02 | takes the descendants along, then `name:"*parent*"` | no match | no match | no match | ✅ | | DELETE-02 | takes the descendants along, then `name:"*child*"` | no match | no match | no match | ✅ | -| DELETE-03 | leaves the resource in the index, then `DocCount()` | 2 | 2 | 1 | ❌ known | +| DELETE-03 | leaves the resource in the index, then `DocCount()` | 2 | 2 | 2 | ✅ stale | | DELETE-04 | takes a resource out that was just written, then `name:"*fresh*"` | no match | no match | no match | ✅ | ### restore @@ -418,7 +427,7 @@ Fixtures: |---|---|---|---|---|---| | RESTORE-01 | brings the descendants back, then `name:"*parent*"` | parent | parent | parent | ✅ | | RESTORE-01 | brings the descendants back, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ | -| RESTORE-02 | leaves the hidden flag alone, then `hidden:true` | file.txt | no match | file.txt | ❌ known | +| RESTORE-02 | leaves the hidden flag alone, then `hidden:true` | file.txt | file.txt | file.txt | ✅ stale | ### purge @@ -502,11 +511,11 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| CASEPATH-01 | takes the descendants along when deleting, then `path:"./Documents"` | no match | no match | Documents, Picture.jpg | ❌ known | -| CASEPATH-02 | takes the descendants along when moving, then `path:"./Other Documents"` | Other Documents, Picture.jpg | Other Documents | no match | ❌ known | -| CASEPATH-02 | takes the descendants along when moving, then `path:"./Documents"` | no match | no match | Documents, Picture.jpg | ❌ known | -| CASEPATH-03 | reaches the descendants when purging, then `path:"./Documents"` | no match | no match | Documents, Picture.jpg | ❌ known | -| CASEPATH-03 | reaches the descendants when purging, then `DocCount()` | 0 | 0 | 2 | ❌ known | +| CASEPATH-01 | takes the descendants along when deleting, then `path:"./Documents"` | no match | no match | no match | ✅ stale | +| CASEPATH-02 | takes the descendants along when moving, then `path:"./Other Documents"` | Other Documents, Picture.jpg | Other Documents, Picture.jpg | Other Documents, Picture.jpg | ✅ stale | +| CASEPATH-02 | takes the descendants along when moving, then `path:"./Documents"` | no match | no match | no match | ✅ stale | +| CASEPATH-03 | reaches the descendants when purging, then `path:"./Documents"` | no match | no match | no match | ✅ stale | +| CASEPATH-03 | reaches the descendants when purging, then `DocCount()` | 0 | 0 | 0 | ✅ stale | ### hidden @@ -521,12 +530,12 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| HIDDEN-01 | follows a move into a dot folder, then `hidden:true` | child.pdf, parent | no match | child.pdf, parent | ❌ known | +| HIDDEN-01 | follows a move into a dot folder, then `hidden:true` | child.pdf, parent | child.pdf, parent | child.pdf, parent | ✅ stale | | HIDDEN-02 | follows a move into a plain folder, then `hidden:true` | no match | no match | no match | ✅ | -| HIDDEN-03 | follows a move renamed with a leading dot, then `hidden:true` | .parent, child.pdf | no match | .parent, child.pdf | ❌ known | +| HIDDEN-03 | follows a move renamed with a leading dot, then `hidden:true` | .parent, child.pdf | .parent, child.pdf | .parent, child.pdf | ✅ stale | | HIDDEN-04 | follows a move out of a dot folder, then `hidden:true` | no match | no match | no match | ✅ | | HIDDEN-05 | follows a move renamed without the leading dot, then `hidden:true` | no match | no match | no match | ✅ | -| HIDDEN-06 | follows a move within the same dot folder, then `hidden:true` | child.pdf, moved | no match | child.pdf, moved | ❌ known | +| HIDDEN-06 | follows a move within the same dot folder, then `hidden:true` | child.pdf, moved | child.pdf, moved | child.pdf, moved | ✅ stale | ### upsert @@ -610,7 +619,7 @@ Fixtures: | ENTITY-08 | `name:"bar.pdf"` reads `MimeType` | application/pdf | application/pdf | application/pdf | ✅ | | ENTITY-09 | `name:"bar.pdf"` reads `Deleted` | false | false | false | ✅ | | ENTITY-10 | `name:"bar.pdf"` reads `Score` | above zero | above zero | above zero | ✅ | -| ENTITY-11 | `path:"./parent"` reads `TotalMatches` | 2 | 1 | 2 | ❌ known | +| ENTITY-11 | `path:"./parent"` reads `TotalMatches` | 2 | 2 | 2 | ✅ stale | | ENTITY-12 | `name:"*notes*"` reads `Highlights` | "" | "" | "" | ✅ | | ENTITY-13 | `content:bar` reads `Highlights` | foo bar baz | foo bar baz | foo bar baz | ✅ | | ENTITY-14 | moved to another parent, then `name:"newname"` reads `ParentId` | 1$1!9 | 1$1!9 | 1$1!9 | ✅ | diff --git a/services/search/pkg/parity/engines_test.go b/services/search/pkg/parity/engines_test.go index a77fb61008..71fd99033a 100644 --- a/services/search/pkg/parity/engines_test.go +++ b/services/search/pkg/parity/engines_test.go @@ -88,16 +88,17 @@ func newBleve(fixtures []search.Resource) testEngine { return testEngine{name: "bleve", backend: backend, settle: func() {}} } -func newOpenSearch(index string, fixtures []search.Resource) testEngine { +func newOpenSearch(name string, fixtures []search.Resource) testEngine { GinkgoHelper() tc := opensearchtest.NewDefaultTestClient(GinkgoTB(), openSearchClient) + index := opensearch.IndexName(name) if err := tc.IndicesReset(context.Background(), []string{index}); err != nil { return testEngine{name: "opensearch", unavailable: err.Error()} } - backend, err := opensearch.NewBackend(index, tc.Client()) + backend, err := opensearch.NewBackend(name, tc.Client()) if err != nil { return testEngine{name: "opensearch", unavailable: err.Error()} } diff --git a/services/search/pkg/parity/matrix_test.go b/services/search/pkg/parity/matrix_test.go index 4b4d65fca2..be0c538597 100644 --- a/services/search/pkg/parity/matrix_test.go +++ b/services/search/pkg/parity/matrix_test.go @@ -173,7 +173,9 @@ func writeMatrix(report types.Report) { out.WriteString("Written by the parity suite (`go test ./services/search/pkg/parity/`), do not edit.\n") out.WriteString("Every case runs against bleve and OpenSearch. `same?` is ✅ when both answer as\n") out.WriteString("expected, `❌ known` when an engine's divergence is documented in the case\n") - out.WriteString("(`engineOverrides`), `❌` when it is not.\n") + out.WriteString("(`engineOverrides`), `❌` when it is not. `✅ stale` when every engine\n") + out.WriteString("answers the expected value although the case still documents a\n") + out.WriteString("divergence, that override can come out.\n") group, section := "", "" for _, row := range rows { @@ -426,6 +428,8 @@ func matrixVerdict(row *matrixResult) string { } switch { + case len(off) == 0 && len(row.Overrides) > 0: + return "✅ stale" case len(off) == 0: return "✅" case known: diff --git a/services/search/pkg/parity/query_fields_test.go b/services/search/pkg/parity/query_fields_test.go index a643f0cfae..187376a66a 100644 --- a/services/search/pkg/parity/query_fields_test.go +++ b/services/search/pkg/parity/query_fields_test.go @@ -25,10 +25,10 @@ func fieldsGroup() queryGroup { {id: 2, query: `mtime<"2021-01-01T00:00:00Z"`, want: []string{"old.txt"}}, {id: 3, query: `id:"1$1!23"`, want: []string{"known.txt"}}, {id: 4, query: `hidden:true`, want: []string{"hidden.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 5, query: `type:file`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, + {id: 5, query: `type:file`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt", "song.mp3"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, {id: 6, query: `type:folder`, want: []string{"box"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, {id: 7, query: `unknown:field`}, - {id: 8, query: `type:File`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, + {id: 8, query: `type:File`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt", "song.mp3"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, {id: 9, query: `type:FOLDER`, want: []string{"box"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, {id: 10, query: `hidden:TRUE`, want: []string{"hidden.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, {id: 11, query: `id:"1$1!AB-23"`, want: []string{"cased.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, diff --git a/services/search/pkg/parity/query_name_test.go b/services/search/pkg/parity/query_name_test.go index 807041166d..934fb4b48e 100644 --- a/services/search/pkg/parity/query_name_test.go +++ b/services/search/pkg/parity/query_name_test.go @@ -54,13 +54,18 @@ func nameGroup() queryGroup { {id: 32, query: `name:"folder*"`}, {id: 33, query: `name:"*new"`}, {id: 34, query: `name:new`, want: []string{"new-folder"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 35, query: `name:"new"`, engineOverrides: map[string]override{"opensearch": override{want: []string{"new-folder"}}}}, + {id: 35, query: `name:"new"`, want: []string{"new-folder"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"new-folder"}}}}, {id: 36, query: `name:"new-folder"`, want: []string{"new-folder"}}, {id: 37, query: `name:"new-*"`, want: []string{"new-folder"}}, {id: 38, query: `name:"new*"`, want: []string{"new-folder"}}, {id: 39, query: `name:new-*`, want: []string{"new-folder"}}, {id: 40, query: `name:"*-folder"`, want: []string{"new-folder"}}, {id: 41, query: `name:"Rep?rt.txt"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 42, query: `name="Report.txt"`, want: []string{"Report.txt"}}, + {id: 43, query: `name="REPORT.TXT"`, want: []string{"Report.txt"}}, + {id: 44, query: `name="new"`}, + {id: 45, query: `name="new-folder"`, want: []string{"new-folder"}}, + {id: 46, query: `name="foo bar.txt"`, want: []string{"foo bar.txt"}}, }, } } diff --git a/services/search/pkg/parity/query_stress_test.go b/services/search/pkg/parity/query_stress_test.go index a739800ad2..978391f345 100644 --- a/services/search/pkg/parity/query_stress_test.go +++ b/services/search/pkg/parity/query_stress_test.go @@ -25,7 +25,7 @@ func stressGroup() queryGroup { {id: 6, query: `tag:("final") AND NOT mediatype:folder`, want: []string{"quarterly report.docx", "photo.jpg"}}, {id: 7, query: `hidden:true AND name:"*notes*"`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}}}, {id: 8, query: `name:quarterly report`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, - {id: 9, query: `name:"quarterly report"`}, + {id: 9, query: `name:"quarterly report"`, want: []string{"quarterly report.docx"}}, {id: 10, query: `name:"quarterly report.docx"`, want: []string{"quarterly report.docx"}}, {id: 11, query: `NOT tag:("draft")`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md", "archive"}, engineOverrides: map[string]override{"bleve": override{want: []string{"archive", "draft report.txt", "notes.md", "photo.jpg", "quarterly report.docx"}}}}, {id: 12, query: `tag:("final") OR hidden:true`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md"}, engineOverrides: map[string]override{"bleve": override{want: []string{"photo.jpg", "quarterly report.docx"}}}}, diff --git a/services/search/pkg/parity/query_title_test.go b/services/search/pkg/parity/query_title_test.go index 8251c668db..640d0b82df 100644 --- a/services/search/pkg/parity/query_title_test.go +++ b/services/search/pkg/parity/query_title_test.go @@ -18,6 +18,8 @@ func titleGroup() queryGroup { {id: 5, query: `Title:"*ly rep*"`, want: []string{"q1.html"}}, {id: 6, query: `title:quarterly`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, {id: 7, query: `Title:"QUARTERLY REPORT"`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 8, query: `Title="quarterly report"`, want: []string{"q1.html"}}, + {id: 9, query: `Title="quarterly"`}, }, } } From 5c153ff74a85438484e6792d437f816a1e2e829e Mon Sep 17 00:00:00 2001 From: Florian Schade Date: Mon, 31 Aug 2026 11:15:03 +0200 Subject: [PATCH 3/4] test(search): drop the engine overrides, both engines answer alike --- services/search/pkg/parity/README.md | 144 +++++++++--------- .../pkg/parity/lifecycle_casepath_test.go | 13 -- .../pkg/parity/lifecycle_delete_test.go | 3 - .../pkg/parity/lifecycle_hidden_test.go | 10 +- .../pkg/parity/lifecycle_restore_test.go | 3 +- .../search/pkg/parity/query_content_test.go | 6 +- .../search/pkg/parity/query_deleted_test.go | 2 +- .../search/pkg/parity/query_extension_test.go | 6 +- .../search/pkg/parity/query_favorites_test.go | 4 +- .../search/pkg/parity/query_fields_test.go | 16 +- .../search/pkg/parity/query_mediatype_test.go | 2 +- services/search/pkg/parity/query_name_test.go | 30 ++-- services/search/pkg/parity/query_path_test.go | 12 +- .../search/pkg/parity/query_range_test.go | 4 +- .../search/pkg/parity/query_stress_test.go | 18 +-- services/search/pkg/parity/query_tags_test.go | 2 +- .../search/pkg/parity/query_title_test.go | 8 +- .../pkg/parity/query_visibility_test.go | 12 +- .../search/pkg/parity/response_entity_test.go | 2 +- 19 files changed, 137 insertions(+), 160 deletions(-) diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index df08b45860..5487239b60 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -26,15 +26,15 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| NAME-01 | `new` | new-folder | new-folder | new-folder | ✅ stale | -| NAME-02 | `quarterly` | quarterly notes.txt | quarterly notes.txt | quarterly notes.txt | ✅ stale | -| NAME-03 | `report` | Report.txt | Report.txt | Report.txt | ✅ stale | +| NAME-01 | `new` | new-folder | new-folder | new-folder | ✅ | +| NAME-02 | `quarterly` | quarterly notes.txt | quarterly notes.txt | quarterly notes.txt | ✅ | +| NAME-03 | `report` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-04 | `name:"*new-folder*"` | new-folder | new-folder | new-folder | ✅ | | NAME-05 | `name:"*w-fol*"` | new-folder | new-folder | new-folder | ✅ | | NAME-06 | `name:"*oo ba*"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | | NAME-07 | `name:"*REPORT*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-08 | `name:"*übung*"` | Übung.txt | Übung.txt | Übung.txt | ✅ stale | -| NAME-09 | `name:"*ÜBUNG*"` | Übung.txt | Übung.txt | Übung.txt | ✅ stale | +| NAME-08 | `name:"*übung*"` | Übung.txt | Übung.txt | Übung.txt | ✅ | +| NAME-09 | `name:"*ÜBUNG*"` | Übung.txt | Übung.txt | Übung.txt | ✅ | | NAME-10 | `name:"*a+b*"` | a+b.txt | a+b.txt | a+b.txt | ✅ | | NAME-11 | `name:"*c(d)*"` | c(d).txt | c(d).txt | c(d).txt | ✅ | | NAME-12 | `name:"*e&f*"` | e&f.txt | e&f.txt | e&f.txt | ✅ | @@ -43,30 +43,30 @@ Fixtures: | NAME-15 | `*folder*` | new-folder | new-folder | new-folder | ✅ | | NAME-16 | `name:"*foo bar*"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | | NAME-17 | `name:"foo bar.txt"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | -| NAME-18 | `name:"*needle*"` | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | ✅ stale | +| NAME-18 | `name:"*needle*"` | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | ✅ | | NAME-19 | `name:"report*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-20 | `name:"*report"` | Report.txt | Report.txt | Report.txt | ✅ stale | +| NAME-20 | `name:"*report"` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-21 | `name:"Rep*rt.txt"` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-22 | `Name:"*report*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-23 | `NAME:"*report*"` | Report.txt | Report.txt | Report.txt | ✅ stale | -| NAME-24 | `name:Rep?rt.txt` | Report.txt | Report.txt | Report.txt | ✅ stale | -| NAME-25 | `name:"*eport"` | Report.txt | Report.txt | Report.txt | ✅ stale | +| NAME-23 | `NAME:"*report*"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-24 | `name:Rep?rt.txt` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-25 | `name:"*eport"` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-26 | `name:"repor*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-27 | `REPORT` | Report.txt | Report.txt | Report.txt | ✅ stale | -| NAME-28 | `name:REPORT` | Report.txt | Report.txt | Report.txt | ✅ stale | +| NAME-27 | `REPORT` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-28 | `name:REPORT` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-29 | `name:"REPORT.TXT"` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-30 | `name:"FOO BAR.TXT"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | | NAME-31 | `name:"ÜBUNG.TXT"` | Übung.txt | Übung.txt | Übung.txt | ✅ | | NAME-32 | `name:"folder*"` | no match | no match | no match | ✅ | | NAME-33 | `name:"*new"` | no match | no match | no match | ✅ | -| NAME-34 | `name:new` | new-folder | new-folder | new-folder | ✅ stale | -| NAME-35 | `name:"new"` | new-folder | new-folder | new-folder | ✅ stale | +| NAME-34 | `name:new` | new-folder | new-folder | new-folder | ✅ | +| NAME-35 | `name:"new"` | new-folder | new-folder | new-folder | ✅ | | NAME-36 | `name:"new-folder"` | new-folder | new-folder | new-folder | ✅ | | NAME-37 | `name:"new-*"` | new-folder | new-folder | new-folder | ✅ | | NAME-38 | `name:"new*"` | new-folder | new-folder | new-folder | ✅ | | NAME-39 | `name:new-*` | new-folder | new-folder | new-folder | ✅ | | NAME-40 | `name:"*-folder"` | new-folder | new-folder | new-folder | ✅ | -| NAME-41 | `name:"Rep?rt.txt"` | Report.txt | Report.txt | Report.txt | ✅ stale | +| NAME-41 | `name:"Rep?rt.txt"` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-42 | `name="Report.txt"` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-43 | `name="REPORT.TXT"` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-44 | `name="new"` | no match | no match | no match | ✅ | @@ -83,10 +83,10 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| EXTENSION-01 | `txt` | report.txt | report.txt | report.txt | ✅ stale | -| EXTENSION-02 | `md` | notes.md | notes.md | notes.md | ✅ stale | +| EXTENSION-01 | `txt` | report.txt | report.txt | report.txt | ✅ | +| EXTENSION-02 | `md` | notes.md | notes.md | notes.md | ✅ | | EXTENSION-03 | `name:"*.txt"` | report.txt | report.txt | report.txt | ✅ | -| EXTENSION-04 | `report` | report.txt | report.txt | report.txt | ✅ stale | +| EXTENSION-04 | `report` | report.txt | report.txt | report.txt | ✅ | ### tags @@ -109,7 +109,7 @@ Fixtures: | TAGS-06 | `tag:("spaced tag")` | spaced.txt | spaced.txt | spaced.txt | ✅ | | TAGS-07 | `tag:("*paced ta*")` | spaced.txt | spaced.txt | spaced.txt | ✅ | | TAGS-08 | `tag:("work")` | project | project | project | ✅ | -| TAGS-09 | `tag:("zzzzzzzzzzzzzzzzzzzzzzzzzz...zzzzzzzzzzzzzzzzneedle")` | longtag.txt | longtag.txt | longtag.txt | ✅ stale | +| TAGS-09 | `tag:("zzzzzzzzzzzzzzzzzzzzzzzzzz...zzzzzzzzzzzzzzzzneedle")` | longtag.txt | longtag.txt | longtag.txt | ✅ | ### title @@ -120,12 +120,12 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| | TITLE-01 | `Title:"quarterly report"` | q1.html | q1.html | q1.html | ✅ | -| TITLE-02 | `Title:quarterly` | q1.html | q1.html | q1.html | ✅ stale | -| TITLE-03 | `Title:QUARTERLY` | q1.html | q1.html | q1.html | ✅ stale | +| TITLE-02 | `Title:quarterly` | q1.html | q1.html | q1.html | ✅ | +| TITLE-03 | `Title:QUARTERLY` | q1.html | q1.html | q1.html | ✅ | | TITLE-04 | `Title:quarterl*` | q1.html | q1.html | q1.html | ✅ | | TITLE-05 | `Title:"*ly rep*"` | q1.html | q1.html | q1.html | ✅ | -| TITLE-06 | `title:quarterly` | q1.html | q1.html | q1.html | ✅ stale | -| TITLE-07 | `Title:"QUARTERLY REPORT"` | q1.html | q1.html | q1.html | ✅ stale | +| TITLE-06 | `title:quarterly` | q1.html | q1.html | q1.html | ✅ | +| TITLE-07 | `Title:"QUARTERLY REPORT"` | q1.html | q1.html | q1.html | ✅ | | TITLE-08 | `Title="quarterly report"` | q1.html | q1.html | q1.html | ✅ | | TITLE-09 | `Title="quarterly"` | no match | no match | no match | ✅ | @@ -138,16 +138,16 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| CONTENT-01 | `Content:report` | no match | no match | no match | ✅ stale | +| CONTENT-01 | `Content:report` | no match | no match | no match | ✅ | | CONTENT-02 | `Content:REPORTS` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-03 | `Content:"monthly reports"` | monthly.txt | monthly.txt | monthly.txt | ✅ | -| CONTENT-04 | `Content:"reports monthly"` | no match | no match | no match | ✅ stale | +| CONTENT-04 | `Content:"reports monthly"` | no match | no match | no match | ✅ | | CONTENT-05 | `Content:report*` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-06 | `Content:*eport*` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-07 | `Content:month*` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-08 | `Content:"https://opencloud.example.com/help"` | links.txt | links.txt | links.txt | ✅ | | CONTENT-09 | `Content:"alan@example.org"` | links.txt | links.txt | links.txt | ✅ | -| CONTENT-10 | `Content:opencloud` | links.txt | links.txt | links.txt | ✅ stale | +| CONTENT-10 | `Content:opencloud` | links.txt | links.txt | links.txt | ✅ | ### favorites @@ -160,8 +160,8 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| FAVORITES-01 | `Favorites:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | keepsakes, starred.txt | ✅ stale | -| FAVORITES-02 | `favorite:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | keepsakes, starred.txt | ✅ stale | +| FAVORITES-01 | `Favorites:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | keepsakes, starred.txt | ✅ | +| FAVORITES-02 | `favorite:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | keepsakes, starred.txt | ✅ | | FAVORITES-03 | `Favorites:"somebody-else"` | no match | no match | no match | ✅ | ### mediatype @@ -176,7 +176,7 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| | MEDIATYPE-01 | `mediatype:text/markdown` | notes.md | notes.md | notes.md | ✅ | -| MEDIATYPE-02 | `mediatype:TEXT/MARKDOWN` | notes.md | notes.md | notes.md | ✅ stale | +| MEDIATYPE-02 | `mediatype:TEXT/MARKDOWN` | notes.md | notes.md | notes.md | ✅ | | MEDIATYPE-03 | `mediatype:image/jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ | | MEDIATYPE-04 | `mediatype:*jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ | | MEDIATYPE-05 | `mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ | @@ -194,14 +194,14 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| PATH-01 | `path:"./parent"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ stale | +| PATH-01 | `path:"./parent"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ | | PATH-02 | `path:"./parent/child.jpg"` | child.jpg | child.jpg | child.jpg | ✅ | -| PATH-03 | `path:"./Parent"` | no match | no match | no match | ✅ stale | +| PATH-03 | `path:"./Parent"` | no match | no match | no match | ✅ | | PATH-04 | `path:"*child*"` | child.jpg | child.jpg | child.jpg | ✅ | -| PATH-05 | `path:"./documents"` | docs-lower | docs-lower | docs-lower | ✅ stale | -| PATH-06 | `path:"./DOCUMENTS"` | docs-upper | docs-upper | docs-upper | ✅ stale | -| PATH-07 | `path:"./Documents"` | docs-mixed | docs-mixed | docs-mixed | ✅ stale | -| PATH-08 | `path:"./parent/"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ stale | +| PATH-05 | `path:"./documents"` | docs-lower | docs-lower | docs-lower | ✅ | +| PATH-06 | `path:"./DOCUMENTS"` | docs-upper | docs-upper | docs-upper | ✅ | +| PATH-07 | `path:"./Documents"` | docs-mixed | docs-mixed | docs-mixed | ✅ | +| PATH-08 | `path:"./parent/"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ | ### fields @@ -222,16 +222,16 @@ Fixtures: | FIELDS-01 | `size:42` | small.txt | small.txt | small.txt | ✅ | | FIELDS-02 | `mtime<"2021-01-01T00:00:00Z"` | old.txt | old.txt | old.txt | ✅ | | FIELDS-03 | `id:"1$1!23"` | known.txt | known.txt | known.txt | ✅ | -| FIELDS-04 | `hidden:true` | hidden.txt | hidden.txt | hidden.txt | ✅ stale | -| FIELDS-05 | `type:file` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | ✅ stale | -| FIELDS-06 | `type:folder` | box | box | box | ✅ stale | +| FIELDS-04 | `hidden:true` | hidden.txt | hidden.txt | hidden.txt | ✅ | +| FIELDS-05 | `type:file` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | ✅ | +| FIELDS-06 | `type:folder` | box | box | box | ✅ | | FIELDS-07 | `unknown:field` | no match | no match | no match | ✅ | -| FIELDS-08 | `type:File` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | ✅ stale | -| FIELDS-09 | `type:FOLDER` | box | box | box | ✅ stale | -| FIELDS-10 | `hidden:TRUE` | hidden.txt | hidden.txt | hidden.txt | ✅ stale | -| FIELDS-11 | `id:"1$1!AB-23"` | cased.txt | cased.txt | cased.txt | ✅ stale | +| FIELDS-08 | `type:File` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt, song.mp3 | ✅ | +| FIELDS-09 | `type:FOLDER` | box | box | box | ✅ | +| FIELDS-10 | `hidden:TRUE` | hidden.txt | hidden.txt | hidden.txt | ✅ | +| FIELDS-11 | `id:"1$1!AB-23"` | cased.txt | cased.txt | cased.txt | ✅ | | FIELDS-12 | `id:"1$1!ab-23"` | no match | no match | no match | ✅ | -| FIELDS-13 | `audio.artist:"Some Artist"` | song.mp3 | song.mp3 | song.mp3 | ✅ stale | +| FIELDS-13 | `audio.artist:"Some Artist"` | song.mp3 | song.mp3 | song.mp3 | ✅ | | FIELDS-14 | `audio.artist:"some artist"` | no match | no match | no match | ✅ | ### deleted @@ -251,7 +251,7 @@ Fixtures: | DELETED-02 | `name:"*.txt"` | book.txt, kept.txt | book.txt, kept.txt | book.txt, kept.txt | ✅ | | DELETED-03 | `name:"*receipt*"` | no match | no match | no match | ✅ | | DELETED-04 | `path:"./bin"` | no match | no match | no match | ✅ | -| DELETED-05 | `path:"./shelf"` | book.txt, shelf | book.txt, shelf | book.txt, shelf | ✅ stale | +| DELETED-05 | `path:"./shelf"` | book.txt, shelf | book.txt, shelf | book.txt, shelf | ✅ | ### visibility @@ -264,13 +264,13 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| VISIBILITY-01 | `hidden:true` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ stale | -| VISIBILITY-02 | `hidden:TRUE` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ stale | -| VISIBILITY-03 | `hidden:false` | visible.txt | visible.txt | visible.txt | ✅ stale | +| VISIBILITY-01 | `hidden:true` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ | +| VISIBILITY-02 | `hidden:TRUE` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ | +| VISIBILITY-03 | `hidden:false` | visible.txt | visible.txt | visible.txt | ✅ | | VISIBILITY-04 | `name:"*secret*"` | secret.txt | secret.txt | secret.txt | ✅ | -| VISIBILITY-05 | `path:"./.private"` | .private, secret.txt | .private, secret.txt | .private, secret.txt | ✅ stale | -| VISIBILITY-06 | `hidden:banana` | no match | no match | no match | ✅ stale | -| VISIBILITY-07 | `hidden:"true"` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ stale | +| VISIBILITY-05 | `path:"./.private"` | .private, secret.txt | .private, secret.txt | .private, secret.txt | ✅ | +| VISIBILITY-06 | `hidden:banana` | no match | no match | no match | ✅ | +| VISIBILITY-07 | `hidden:"true"` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ | ### boolean @@ -319,21 +319,21 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| | STRESS-01 | `name:"*report*" AND mediatype:document` | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | ✅ | -| STRESS-02 | `name:"*report*" AND NOT tag:("draft")` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ stale | +| STRESS-02 | `name:"*report*" AND NOT tag:("draft")` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | | STRESS-03 | `(tag:("final") OR tag:("draft")) AND mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ | | STRESS-04 | `mediatype:document AND mtime>"2021-01-01T00:00:00Z"` | notes.md, quarterly report.docx | notes.md, quarterly report.docx | notes.md, quarterly report.docx | ✅ | -| STRESS-05 | `name:"*report*" AND size>100` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ stale | +| STRESS-05 | `name:"*report*" AND size>100` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | | STRESS-06 | `tag:("final") AND NOT mediatype:folder` | photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | ✅ | -| STRESS-07 | `hidden:true AND name:"*notes*"` | notes.md | notes.md | notes.md | ✅ stale | -| STRESS-08 | `name:quarterly report` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ stale | +| STRESS-07 | `hidden:true AND name:"*notes*"` | notes.md | notes.md | notes.md | ✅ | +| STRESS-08 | `name:quarterly report` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | | STRESS-09 | `name:"quarterly report"` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | | STRESS-10 | `name:"quarterly report.docx"` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | -| STRESS-11 | `NOT tag:("draft")` | archive, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | ✅ stale | -| STRESS-12 | `tag:("final") OR hidden:true` | notes.md, photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | ✅ stale | -| STRESS-13 | `(name:"*report*" OR name:"*notes..."draft") OR hidden:true)` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ stale | +| STRESS-11 | `NOT tag:("draft")` | archive, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | ✅ | +| STRESS-12 | `tag:("final") OR hidden:true` | notes.md, photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | ✅ | +| STRESS-13 | `(name:"*report*" OR name:"*notes..."draft") OR hidden:true)` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | | STRESS-14 | `mediatype:image OR (mediatype:document AND tag:("draft"))` | draft report.txt, photo.jpg | draft report.txt, photo.jpg | draft report.txt, photo.jpg | ✅ | -| STRESS-15 | `NOT (mediatype:folder OR hidden:true)` | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | ✅ stale | -| STRESS-16 | `name:"*report*" AND (size>100 OR tag:("draft"))` | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | ✅ stale | +| STRESS-15 | `NOT (mediatype:folder OR hidden:true)` | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | ✅ | +| STRESS-16 | `name:"*report*" AND (size>100 OR tag:("draft"))` | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | ✅ | ### everything @@ -360,8 +360,8 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| RANGE-01 | `size>100` | big.txt | big.txt | big.txt | ✅ stale | -| RANGE-02 | `size<100` | ancient.txt, small.txt | ancient.txt, small.txt | ancient.txt, small.txt | ✅ stale | +| RANGE-01 | `size>100` | big.txt | big.txt | big.txt | ✅ | +| RANGE-02 | `size<100` | ancient.txt, small.txt | ancient.txt, small.txt | ancient.txt, small.txt | ✅ | | RANGE-03 | `mtime>"2021-01-01T00:00:00Z"` | big.txt, small.txt | big.txt, small.txt | big.txt, small.txt | ✅ | | RANGE-04 | `mtime<"2021-01-01T00:00:00Z"` | ancient.txt | ancient.txt | ancient.txt | ✅ | | RANGE-05 | `Mtime:"today"` | big.txt, small.txt | big.txt, small.txt | big.txt, small.txt | ✅ | @@ -412,7 +412,7 @@ Fixtures: | DELETE-01 | takes the resource out of the results, then `name:"*child*"` | no match | no match | no match | ✅ | | DELETE-02 | takes the descendants along, then `name:"*parent*"` | no match | no match | no match | ✅ | | DELETE-02 | takes the descendants along, then `name:"*child*"` | no match | no match | no match | ✅ | -| DELETE-03 | leaves the resource in the index, then `DocCount()` | 2 | 2 | 2 | ✅ stale | +| DELETE-03 | leaves the resource in the index, then `DocCount()` | 2 | 2 | 2 | ✅ | | DELETE-04 | takes a resource out that was just written, then `name:"*fresh*"` | no match | no match | no match | ✅ | ### restore @@ -427,7 +427,7 @@ Fixtures: |---|---|---|---|---|---| | RESTORE-01 | brings the descendants back, then `name:"*parent*"` | parent | parent | parent | ✅ | | RESTORE-01 | brings the descendants back, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ | -| RESTORE-02 | leaves the hidden flag alone, then `hidden:true` | file.txt | file.txt | file.txt | ✅ stale | +| RESTORE-02 | leaves the hidden flag alone, then `hidden:true` | file.txt | file.txt | file.txt | ✅ | ### purge @@ -511,11 +511,11 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| CASEPATH-01 | takes the descendants along when deleting, then `path:"./Documents"` | no match | no match | no match | ✅ stale | -| CASEPATH-02 | takes the descendants along when moving, then `path:"./Other Documents"` | Other Documents, Picture.jpg | Other Documents, Picture.jpg | Other Documents, Picture.jpg | ✅ stale | -| CASEPATH-02 | takes the descendants along when moving, then `path:"./Documents"` | no match | no match | no match | ✅ stale | -| CASEPATH-03 | reaches the descendants when purging, then `path:"./Documents"` | no match | no match | no match | ✅ stale | -| CASEPATH-03 | reaches the descendants when purging, then `DocCount()` | 0 | 0 | 0 | ✅ stale | +| CASEPATH-01 | takes the descendants along when deleting, then `path:"./Documents"` | no match | no match | no match | ✅ | +| CASEPATH-02 | takes the descendants along when moving, then `path:"./Other Documents"` | Other Documents, Picture.jpg | Other Documents, Picture.jpg | Other Documents, Picture.jpg | ✅ | +| CASEPATH-02 | takes the descendants along when moving, then `path:"./Documents"` | no match | no match | no match | ✅ | +| CASEPATH-03 | reaches the descendants when purging, then `path:"./Documents"` | no match | no match | no match | ✅ | +| CASEPATH-03 | reaches the descendants when purging, then `DocCount()` | 0 | 0 | 0 | ✅ | ### hidden @@ -530,12 +530,12 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| HIDDEN-01 | follows a move into a dot folder, then `hidden:true` | child.pdf, parent | child.pdf, parent | child.pdf, parent | ✅ stale | +| HIDDEN-01 | follows a move into a dot folder, then `hidden:true` | child.pdf, parent | child.pdf, parent | child.pdf, parent | ✅ | | HIDDEN-02 | follows a move into a plain folder, then `hidden:true` | no match | no match | no match | ✅ | -| HIDDEN-03 | follows a move renamed with a leading dot, then `hidden:true` | .parent, child.pdf | .parent, child.pdf | .parent, child.pdf | ✅ stale | +| HIDDEN-03 | follows a move renamed with a leading dot, then `hidden:true` | .parent, child.pdf | .parent, child.pdf | .parent, child.pdf | ✅ | | HIDDEN-04 | follows a move out of a dot folder, then `hidden:true` | no match | no match | no match | ✅ | | HIDDEN-05 | follows a move renamed without the leading dot, then `hidden:true` | no match | no match | no match | ✅ | -| HIDDEN-06 | follows a move within the same dot folder, then `hidden:true` | child.pdf, moved | child.pdf, moved | child.pdf, moved | ✅ stale | +| HIDDEN-06 | follows a move within the same dot folder, then `hidden:true` | child.pdf, moved | child.pdf, moved | child.pdf, moved | ✅ | ### upsert @@ -619,7 +619,7 @@ Fixtures: | ENTITY-08 | `name:"bar.pdf"` reads `MimeType` | application/pdf | application/pdf | application/pdf | ✅ | | ENTITY-09 | `name:"bar.pdf"` reads `Deleted` | false | false | false | ✅ | | ENTITY-10 | `name:"bar.pdf"` reads `Score` | above zero | above zero | above zero | ✅ | -| ENTITY-11 | `path:"./parent"` reads `TotalMatches` | 2 | 2 | 2 | ✅ stale | +| ENTITY-11 | `path:"./parent"` reads `TotalMatches` | 2 | 2 | 2 | ✅ | | ENTITY-12 | `name:"*notes*"` reads `Highlights` | "" | "" | "" | ✅ | | ENTITY-13 | `content:bar` reads `Highlights` | foo bar baz | foo bar baz | foo bar baz | ✅ | | ENTITY-14 | moved to another parent, then `name:"newname"` reads `ParentId` | 1$1!9 | 1$1!9 | 1$1!9 | ✅ | diff --git a/services/search/pkg/parity/lifecycle_casepath_test.go b/services/search/pkg/parity/lifecycle_casepath_test.go index 8347fcf318..95b3b36208 100644 --- a/services/search/pkg/parity/lifecycle_casepath_test.go +++ b/services/search/pkg/parity/lifecycle_casepath_test.go @@ -28,9 +28,6 @@ func casePathLifecycle() lifecycleGroup { id: 1, title: "takes the descendants along when deleting", do: func(e search.Engine) error { return e.Delete(folder.ID) }, expect: left(), - engineOverrides: map[string]lifecycleOverride{ - "opensearch": {expect: map[string][]string{`path:"./Documents"`: {"Documents", "Picture.jpg"}}}, - }, }, { id: 2, title: "takes the descendants along when moving", @@ -39,22 +36,12 @@ func casePathLifecycle() lifecycleGroup { {`path:"./Other Documents"`, []string{"Other Documents", "Picture.jpg"}}, {`path:"./Documents"`, nil}, }, - engineOverrides: map[string]lifecycleOverride{ - "bleve": {expect: map[string][]string{`path:"./Other Documents"`: {"Other Documents"}}}, - "opensearch": {expect: map[string][]string{`path:"./Other Documents"`: {}, `path:"./Documents"`: {"Documents", "Picture.jpg"}}}, - }, }, { id: 3, title: "reaches the descendants when purging", do: func(e search.Engine) error { return e.Purge(folder.ID, false) }, expect: left(), wantDocCount: conversions.ToPointer(uint64(0)), - engineOverrides: map[string]lifecycleOverride{ - "opensearch": { - expect: map[string][]string{`path:"./Documents"`: {"Documents", "Picture.jpg"}}, - wantDocCount: conversions.ToPointer(uint64(2)), - }, - }, }, }, } diff --git a/services/search/pkg/parity/lifecycle_delete_test.go b/services/search/pkg/parity/lifecycle_delete_test.go index c483deb901..3a4c49b2c6 100644 --- a/services/search/pkg/parity/lifecycle_delete_test.go +++ b/services/search/pkg/parity/lifecycle_delete_test.go @@ -26,9 +26,6 @@ func deleteLifecycle() lifecycleGroup { id: 3, title: "leaves the resource in the index", do: func(e search.Engine) error { return e.Delete(child.ID) }, wantDocCount: conversions.ToPointer(uint64(2)), - engineOverrides: map[string]lifecycleOverride{ - "opensearch": {wantDocCount: conversions.ToPointer(uint64(1))}, - }, }, { id: 4, title: "takes a resource out that was just written", diff --git a/services/search/pkg/parity/lifecycle_hidden_test.go b/services/search/pkg/parity/lifecycle_hidden_test.go index 59ec242088..c703d6ce74 100644 --- a/services/search/pkg/parity/lifecycle_hidden_test.go +++ b/services/search/pkg/parity/lifecycle_hidden_test.go @@ -29,14 +29,9 @@ func hiddenLifecycle() lifecycleGroup { child.Path = move.from + "/child.pdf" child.Hidden = parent.Hidden - var ( - hidden []string - overrides map[string]lifecycleOverride - ) + var hidden []string if move.hidden { hidden = []string{path.Base(move.target), "child.pdf"} - // bleve does not answer hidden:true at all today - overrides = map[string]lifecycleOverride{"bleve": {expect: map[string][]string{`hidden:true`: {}}}} } group.cases = append(group.cases, lifecycleCase{ @@ -45,8 +40,7 @@ func hiddenLifecycle() lifecycleGroup { do: func(e search.Engine) error { return e.Move(parent.ID, parent.ParentID, move.target) }, - expect: []expectation{{`hidden:true`, hidden}}, - engineOverrides: overrides, + expect: []expectation{{`hidden:true`, hidden}}, }) } diff --git a/services/search/pkg/parity/lifecycle_restore_test.go b/services/search/pkg/parity/lifecycle_restore_test.go index 79dd3e3f02..c85c4c596a 100644 --- a/services/search/pkg/parity/lifecycle_restore_test.go +++ b/services/search/pkg/parity/lifecycle_restore_test.go @@ -33,8 +33,7 @@ func restoreLifecycle() lifecycleGroup { return e.Restore(secret.ID) }, - expect: []expectation{{`hidden:true`, []string{"file.txt"}}}, - engineOverrides: map[string]lifecycleOverride{"bleve": {expect: map[string][]string{`hidden:true`: {}}}}, + expect: []expectation{{`hidden:true`, []string{"file.txt"}}}, }, }, } diff --git a/services/search/pkg/parity/query_content_test.go b/services/search/pkg/parity/query_content_test.go index d51d36d53b..010b5545c3 100644 --- a/services/search/pkg/parity/query_content_test.go +++ b/services/search/pkg/parity/query_content_test.go @@ -12,16 +12,16 @@ func contentGroup() queryGroup { fixtureDoc("links.txt", withContent("see https://opencloud.example.com/help or write to alan@example.org")), }, cases: []queryCase{ - {id: 1, query: `Content:report`, engineOverrides: map[string]override{"bleve": override{want: []string{"monthly.txt"}}}}, + {id: 1, query: `Content:report`}, {id: 2, query: `Content:REPORTS`, want: []string{"monthly.txt"}}, {id: 3, query: `Content:"monthly reports"`, want: []string{"monthly.txt"}}, - {id: 4, query: `Content:"reports monthly"`, engineOverrides: map[string]override{"bleve": override{want: []string{"monthly.txt"}}}}, + {id: 4, query: `Content:"reports monthly"`}, {id: 5, query: `Content:report*`, want: []string{"monthly.txt"}}, {id: 6, query: `Content:*eport*`, want: []string{"monthly.txt"}}, {id: 7, query: `Content:month*`, want: []string{"monthly.txt"}}, {id: 8, query: `Content:"https://opencloud.example.com/help"`, want: []string{"links.txt"}}, {id: 9, query: `Content:"alan@example.org"`, want: []string{"links.txt"}}, - {id: 10, query: `Content:opencloud`, want: []string{"links.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 10, query: `Content:opencloud`, want: []string{"links.txt"}}, }, } } diff --git a/services/search/pkg/parity/query_deleted_test.go b/services/search/pkg/parity/query_deleted_test.go index 905725bd8b..1e048b1e9d 100644 --- a/services/search/pkg/parity/query_deleted_test.go +++ b/services/search/pkg/parity/query_deleted_test.go @@ -20,7 +20,7 @@ func deletedGroup() queryGroup { {id: 2, query: `name:"*.txt"`, want: []string{"kept.txt", "book.txt"}}, {id: 3, query: `name:"*receipt*"`}, {id: 4, query: `path:"./bin"`}, - {id: 5, query: `path:"./shelf"`, want: []string{"shelf", "book.txt"}, engineOverrides: map[string]override{"bleve": override{want: []string{"shelf"}}}}, + {id: 5, query: `path:"./shelf"`, want: []string{"shelf", "book.txt"}}, }, } } diff --git a/services/search/pkg/parity/query_extension_test.go b/services/search/pkg/parity/query_extension_test.go index e02b5e30fa..a813640bcb 100644 --- a/services/search/pkg/parity/query_extension_test.go +++ b/services/search/pkg/parity/query_extension_test.go @@ -13,10 +13,10 @@ func extensionGroup() queryGroup { fixtureFolder("archive"), }, cases: []queryCase{ - {id: 1, query: `txt`, want: []string{"report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, - {id: 2, query: `md`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 1, query: `txt`, want: []string{"report.txt"}}, + {id: 2, query: `md`, want: []string{"notes.md"}}, {id: 3, query: `name:"*.txt"`, want: []string{"report.txt"}}, - {id: 4, query: `report`, want: []string{"report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 4, query: `report`, want: []string{"report.txt"}}, }, } } diff --git a/services/search/pkg/parity/query_favorites_test.go b/services/search/pkg/parity/query_favorites_test.go index ba1f4d7d1c..5413f3eaf4 100644 --- a/services/search/pkg/parity/query_favorites_test.go +++ b/services/search/pkg/parity/query_favorites_test.go @@ -14,8 +14,8 @@ func favoritesGroup() queryGroup { fixtureDoc("photo.jpg", withParent("1$1!keepsakes"), withPath("./keepsakes/photo.jpg"), withMime("image/jpeg")), }, cases: []queryCase{ - {id: 1, query: `Favorites:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}, engineOverrides: map[string]override{"opensearch": override{}}}, - {id: 2, query: `favorite:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 1, query: `Favorites:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}}, + {id: 2, query: `favorite:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}}, {id: 3, query: `Favorites:"somebody-else"`}, }, } diff --git a/services/search/pkg/parity/query_fields_test.go b/services/search/pkg/parity/query_fields_test.go index 187376a66a..e5600893fb 100644 --- a/services/search/pkg/parity/query_fields_test.go +++ b/services/search/pkg/parity/query_fields_test.go @@ -24,17 +24,17 @@ func fieldsGroup() queryGroup { {id: 1, query: `size:42`, want: []string{"small.txt"}}, {id: 2, query: `mtime<"2021-01-01T00:00:00Z"`, want: []string{"old.txt"}}, {id: 3, query: `id:"1$1!23"`, want: []string{"known.txt"}}, - {id: 4, query: `hidden:true`, want: []string{"hidden.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 5, query: `type:file`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt", "song.mp3"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, - {id: 6, query: `type:folder`, want: []string{"box"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, + {id: 4, query: `hidden:true`, want: []string{"hidden.txt"}}, + {id: 5, query: `type:file`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt", "song.mp3"}}, + {id: 6, query: `type:folder`, want: []string{"box"}}, {id: 7, query: `unknown:field`}, - {id: 8, query: `type:File`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt", "song.mp3"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, - {id: 9, query: `type:FOLDER`, want: []string{"box"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, - {id: 10, query: `hidden:TRUE`, want: []string{"hidden.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, - {id: 11, query: `id:"1$1!AB-23"`, want: []string{"cased.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 8, query: `type:File`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt", "song.mp3"}}, + {id: 9, query: `type:FOLDER`, want: []string{"box"}}, + {id: 10, query: `hidden:TRUE`, want: []string{"hidden.txt"}}, + {id: 11, query: `id:"1$1!AB-23"`, want: []string{"cased.txt"}}, {id: 12, query: `id:"1$1!ab-23"`}, // a facet value keeps its case, the field is not marked lowercase - {id: 13, query: `audio.artist:"Some Artist"`, want: []string{"song.mp3"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 13, query: `audio.artist:"Some Artist"`, want: []string{"song.mp3"}}, {id: 14, query: `audio.artist:"some artist"`}, }, } diff --git a/services/search/pkg/parity/query_mediatype_test.go b/services/search/pkg/parity/query_mediatype_test.go index 11a5065d99..cd43ff89d5 100644 --- a/services/search/pkg/parity/query_mediatype_test.go +++ b/services/search/pkg/parity/query_mediatype_test.go @@ -15,7 +15,7 @@ func mediatypeGroup() queryGroup { }, cases: []queryCase{ {id: 1, query: `mediatype:text/markdown`, want: []string{"notes.md"}}, - {id: 2, query: `mediatype:TEXT/MARKDOWN`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 2, query: `mediatype:TEXT/MARKDOWN`, want: []string{"notes.md"}}, {id: 3, query: `mediatype:image/jpeg`, want: []string{"photo.jpg"}}, {id: 4, query: `mediatype:*jpeg`, want: []string{"photo.jpg"}}, {id: 5, query: `mediatype:image`, want: []string{"photo.jpg"}}, diff --git a/services/search/pkg/parity/query_name_test.go b/services/search/pkg/parity/query_name_test.go index 934fb4b48e..b5f3acfa24 100644 --- a/services/search/pkg/parity/query_name_test.go +++ b/services/search/pkg/parity/query_name_test.go @@ -20,15 +20,15 @@ func nameGroup() queryGroup { fixtureDoc(fixtureLongName), }, cases: []queryCase{ - {id: 1, query: `new`, want: []string{"new-folder"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 2, query: `quarterly`, want: []string{"quarterly notes.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 3, query: `report`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 1, query: `new`, want: []string{"new-folder"}}, + {id: 2, query: `quarterly`, want: []string{"quarterly notes.txt"}}, + {id: 3, query: `report`, want: []string{"Report.txt"}}, {id: 4, query: `name:"*new-folder*"`, want: []string{"new-folder"}}, {id: 5, query: `name:"*w-fol*"`, want: []string{"new-folder"}}, {id: 6, query: `name:"*oo ba*"`, want: []string{"foo bar.txt"}}, {id: 7, query: `name:"*REPORT*"`, want: []string{"Report.txt"}}, - {id: 8, query: `name:"*übung*"`, want: []string{"Übung.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, - {id: 9, query: `name:"*ÜBUNG*"`, want: []string{"Übung.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 8, query: `name:"*übung*"`, want: []string{"Übung.txt"}}, + {id: 9, query: `name:"*ÜBUNG*"`, want: []string{"Übung.txt"}}, {id: 10, query: `name:"*a+b*"`, want: []string{"a+b.txt"}}, {id: 11, query: `name:"*c(d)*"`, want: []string{"c(d).txt"}}, {id: 12, query: `name:"*e&f*"`, want: []string{"e&f.txt"}}, @@ -37,30 +37,30 @@ func nameGroup() queryGroup { {id: 15, query: `*folder*`, want: []string{"new-folder"}}, {id: 16, query: `name:"*foo bar*"`, want: []string{"foo bar.txt"}}, {id: 17, query: `name:"foo bar.txt"`, want: []string{"foo bar.txt"}}, - {id: 18, query: `name:"*needle*"`, want: []string{fixtureLongName}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 18, query: `name:"*needle*"`, want: []string{fixtureLongName}}, {id: 19, query: `name:"report*"`, want: []string{"Report.txt"}}, - {id: 20, query: `name:"*report"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 20, query: `name:"*report"`, want: []string{"Report.txt"}}, {id: 21, query: `name:"Rep*rt.txt"`, want: []string{"Report.txt"}}, {id: 22, query: `Name:"*report*"`, want: []string{"Report.txt"}}, - {id: 23, query: `NAME:"*report*"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, - {id: 24, query: `name:Rep?rt.txt`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, - {id: 25, query: `name:"*eport"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 23, query: `NAME:"*report*"`, want: []string{"Report.txt"}}, + {id: 24, query: `name:Rep?rt.txt`, want: []string{"Report.txt"}}, + {id: 25, query: `name:"*eport"`, want: []string{"Report.txt"}}, {id: 26, query: `name:"repor*"`, want: []string{"Report.txt"}}, - {id: 27, query: `REPORT`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, - {id: 28, query: `name:REPORT`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 27, query: `REPORT`, want: []string{"Report.txt"}}, + {id: 28, query: `name:REPORT`, want: []string{"Report.txt"}}, {id: 29, query: `name:"REPORT.TXT"`, want: []string{"Report.txt"}}, {id: 30, query: `name:"FOO BAR.TXT"`, want: []string{"foo bar.txt"}}, {id: 31, query: `name:"ÜBUNG.TXT"`, want: []string{"Übung.txt"}}, {id: 32, query: `name:"folder*"`}, {id: 33, query: `name:"*new"`}, - {id: 34, query: `name:new`, want: []string{"new-folder"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 35, query: `name:"new"`, want: []string{"new-folder"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"new-folder"}}}}, + {id: 34, query: `name:new`, want: []string{"new-folder"}}, + {id: 35, query: `name:"new"`, want: []string{"new-folder"}}, {id: 36, query: `name:"new-folder"`, want: []string{"new-folder"}}, {id: 37, query: `name:"new-*"`, want: []string{"new-folder"}}, {id: 38, query: `name:"new*"`, want: []string{"new-folder"}}, {id: 39, query: `name:new-*`, want: []string{"new-folder"}}, {id: 40, query: `name:"*-folder"`, want: []string{"new-folder"}}, - {id: 41, query: `name:"Rep?rt.txt"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 41, query: `name:"Rep?rt.txt"`, want: []string{"Report.txt"}}, {id: 42, query: `name="Report.txt"`, want: []string{"Report.txt"}}, {id: 43, query: `name="REPORT.TXT"`, want: []string{"Report.txt"}}, {id: 44, query: `name="new"`}, diff --git a/services/search/pkg/parity/query_path_test.go b/services/search/pkg/parity/query_path_test.go index 828d298951..e97e8e1596 100644 --- a/services/search/pkg/parity/query_path_test.go +++ b/services/search/pkg/parity/query_path_test.go @@ -15,14 +15,14 @@ func pathGroup() queryGroup { fixtureFolder("docs-mixed", withPath("./Documents")), }, cases: []queryCase{ - {id: 1, query: `path:"./parent"`, want: []string{"parent", "child.jpg"}, engineOverrides: map[string]override{"bleve": override{want: []string{"parent"}}}}, + {id: 1, query: `path:"./parent"`, want: []string{"parent", "child.jpg"}}, {id: 2, query: `path:"./parent/child.jpg"`, want: []string{"child.jpg"}}, - {id: 3, query: `path:"./Parent"`, engineOverrides: map[string]override{"opensearch": override{want: []string{"child.jpg", "parent"}}}}, + {id: 3, query: `path:"./Parent"`}, {id: 4, query: `path:"*child*"`, want: []string{"child.jpg"}}, - {id: 5, query: `path:"./documents"`, want: []string{"docs-lower"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"docs-lower", "docs-mixed", "docs-upper"}}}}, - {id: 6, query: `path:"./DOCUMENTS"`, want: []string{"docs-upper"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"docs-lower", "docs-mixed", "docs-upper"}}}}, - {id: 7, query: `path:"./Documents"`, want: []string{"docs-mixed"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"docs-lower", "docs-mixed", "docs-upper"}}}}, - {id: 8, query: `path:"./parent/"`, want: []string{"parent", "child.jpg"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 5, query: `path:"./documents"`, want: []string{"docs-lower"}}, + {id: 6, query: `path:"./DOCUMENTS"`, want: []string{"docs-upper"}}, + {id: 7, query: `path:"./Documents"`, want: []string{"docs-mixed"}}, + {id: 8, query: `path:"./parent/"`, want: []string{"parent", "child.jpg"}}, }, } } diff --git a/services/search/pkg/parity/query_range_test.go b/services/search/pkg/parity/query_range_test.go index 59a1618272..f5c7033012 100644 --- a/services/search/pkg/parity/query_range_test.go +++ b/services/search/pkg/parity/query_range_test.go @@ -13,8 +13,8 @@ func rangeGroup() queryGroup { fixtureDoc("ancient.txt", withSize(10), withMtime("2020-01-01T00:00:00Z")), }, cases: []queryCase{ - {id: 1, query: `size>100`, want: []string{"big.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, - {id: 2, query: `size<100`, want: []string{"small.txt", "ancient.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 1, query: `size>100`, want: []string{"big.txt"}}, + {id: 2, query: `size<100`, want: []string{"small.txt", "ancient.txt"}}, {id: 3, query: `mtime>"2021-01-01T00:00:00Z"`, want: []string{"small.txt", "big.txt"}}, {id: 4, query: `mtime<"2021-01-01T00:00:00Z"`, want: []string{"ancient.txt"}}, {id: 5, query: `Mtime:"today"`, want: []string{"small.txt", "big.txt"}}, diff --git a/services/search/pkg/parity/query_stress_test.go b/services/search/pkg/parity/query_stress_test.go index 978391f345..52682d59ed 100644 --- a/services/search/pkg/parity/query_stress_test.go +++ b/services/search/pkg/parity/query_stress_test.go @@ -18,21 +18,21 @@ func stressGroup() queryGroup { }, cases: []queryCase{ {id: 1, query: `name:"*report*" AND mediatype:document`, want: []string{"quarterly report.docx", "draft report.txt"}}, - {id: 2, query: `name:"*report*" AND NOT tag:("draft")`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{want: []string{"draft report.txt", "quarterly report.docx"}}}}, + {id: 2, query: `name:"*report*" AND NOT tag:("draft")`, want: []string{"quarterly report.docx"}}, {id: 3, query: `(tag:("final") OR tag:("draft")) AND mediatype:image`, want: []string{"photo.jpg"}}, {id: 4, query: `mediatype:document AND mtime>"2021-01-01T00:00:00Z"`, want: []string{"quarterly report.docx", "notes.md"}}, - {id: 5, query: `name:"*report*" AND size>100`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 5, query: `name:"*report*" AND size>100`, want: []string{"quarterly report.docx"}}, {id: 6, query: `tag:("final") AND NOT mediatype:folder`, want: []string{"quarterly report.docx", "photo.jpg"}}, - {id: 7, query: `hidden:true AND name:"*notes*"`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 8, query: `name:quarterly report`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 7, query: `hidden:true AND name:"*notes*"`, want: []string{"notes.md"}}, + {id: 8, query: `name:quarterly report`, want: []string{"quarterly report.docx"}}, {id: 9, query: `name:"quarterly report"`, want: []string{"quarterly report.docx"}}, {id: 10, query: `name:"quarterly report.docx"`, want: []string{"quarterly report.docx"}}, - {id: 11, query: `NOT tag:("draft")`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md", "archive"}, engineOverrides: map[string]override{"bleve": override{want: []string{"archive", "draft report.txt", "notes.md", "photo.jpg", "quarterly report.docx"}}}}, - {id: 12, query: `tag:("final") OR hidden:true`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md"}, engineOverrides: map[string]override{"bleve": override{want: []string{"photo.jpg", "quarterly report.docx"}}}}, - {id: 13, query: `(name:"*report*" OR name:"*notes*") AND NOT (tag:("draft") OR hidden:true)`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{want: []string{"notes.md", "quarterly report.docx"}}}}, + {id: 11, query: `NOT tag:("draft")`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md", "archive"}}, + {id: 12, query: `tag:("final") OR hidden:true`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md"}}, + {id: 13, query: `(name:"*report*" OR name:"*notes*") AND NOT (tag:("draft") OR hidden:true)`, want: []string{"quarterly report.docx"}}, {id: 14, query: `mediatype:image OR (mediatype:document AND tag:("draft"))`, want: []string{"photo.jpg", "draft report.txt"}}, - {id: 15, query: `NOT (mediatype:folder OR hidden:true)`, want: []string{"quarterly report.docx", "draft report.txt", "photo.jpg"}, engineOverrides: map[string]override{"bleve": override{want: []string{"draft report.txt", "notes.md", "photo.jpg", "quarterly report.docx"}}}}, - {id: 16, query: `name:"*report*" AND (size>100 OR tag:("draft"))`, want: []string{"quarterly report.docx", "draft report.txt"}, engineOverrides: map[string]override{"bleve": override{want: []string{"draft report.txt"}}, "opensearch": override{want: []string{"draft report.txt"}}}}, + {id: 15, query: `NOT (mediatype:folder OR hidden:true)`, want: []string{"quarterly report.docx", "draft report.txt", "photo.jpg"}}, + {id: 16, query: `name:"*report*" AND (size>100 OR tag:("draft"))`, want: []string{"quarterly report.docx", "draft report.txt"}}, }, } } diff --git a/services/search/pkg/parity/query_tags_test.go b/services/search/pkg/parity/query_tags_test.go index 74cc3bc716..c8505074d3 100644 --- a/services/search/pkg/parity/query_tags_test.go +++ b/services/search/pkg/parity/query_tags_test.go @@ -28,7 +28,7 @@ func tagsGroup() queryGroup { {id: 6, query: `tag:("spaced tag")`, want: []string{"spaced.txt"}}, {id: 7, query: `tag:("*paced ta*")`, want: []string{"spaced.txt"}}, {id: 8, query: `tag:("work")`, want: []string{"project"}}, - {id: 9, query: `tag:("` + longTag + `")`, want: []string{"longtag.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 9, query: `tag:("` + longTag + `")`, want: []string{"longtag.txt"}}, }, } } diff --git a/services/search/pkg/parity/query_title_test.go b/services/search/pkg/parity/query_title_test.go index 640d0b82df..727f69ac55 100644 --- a/services/search/pkg/parity/query_title_test.go +++ b/services/search/pkg/parity/query_title_test.go @@ -12,12 +12,12 @@ func titleGroup() queryGroup { }, cases: []queryCase{ {id: 1, query: `Title:"quarterly report"`, want: []string{"q1.html"}}, - {id: 2, query: `Title:quarterly`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 3, query: `Title:QUARTERLY`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 2, query: `Title:quarterly`, want: []string{"q1.html"}}, + {id: 3, query: `Title:QUARTERLY`, want: []string{"q1.html"}}, {id: 4, query: `Title:quarterl*`, want: []string{"q1.html"}}, {id: 5, query: `Title:"*ly rep*"`, want: []string{"q1.html"}}, - {id: 6, query: `title:quarterly`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, - {id: 7, query: `Title:"QUARTERLY REPORT"`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 6, query: `title:quarterly`, want: []string{"q1.html"}}, + {id: 7, query: `Title:"QUARTERLY REPORT"`, want: []string{"q1.html"}}, {id: 8, query: `Title="quarterly report"`, want: []string{"q1.html"}}, {id: 9, query: `Title="quarterly"`}, }, diff --git a/services/search/pkg/parity/query_visibility_test.go b/services/search/pkg/parity/query_visibility_test.go index 12939126c2..5a3d879946 100644 --- a/services/search/pkg/parity/query_visibility_test.go +++ b/services/search/pkg/parity/query_visibility_test.go @@ -14,13 +14,13 @@ func visibilityGroup() queryGroup { fixtureDoc("secret.txt", withParent("1$1!.private"), withPath("./.private/secret.txt"), isHidden()), }, cases: []queryCase{ - {id: 1, query: `hidden:true`, want: []string{"dotfile.txt", ".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, - {id: 2, query: `hidden:TRUE`, want: []string{"dotfile.txt", ".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, - {id: 3, query: `hidden:false`, want: []string{"visible.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 1, query: `hidden:true`, want: []string{"dotfile.txt", ".private", "secret.txt"}}, + {id: 2, query: `hidden:TRUE`, want: []string{"dotfile.txt", ".private", "secret.txt"}}, + {id: 3, query: `hidden:false`, want: []string{"visible.txt"}}, {id: 4, query: `name:"*secret*"`, want: []string{"secret.txt"}}, - {id: 5, query: `path:"./.private"`, want: []string{".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{want: []string{".private"}}}}, - {id: 6, query: `hidden:banana`, engineOverrides: map[string]override{"opensearch": override{want: []string{"error"}}}}, - {id: 7, query: `hidden:"true"`, want: []string{"dotfile.txt", ".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 5, query: `path:"./.private"`, want: []string{".private", "secret.txt"}}, + {id: 6, query: `hidden:banana`}, + {id: 7, query: `hidden:"true"`, want: []string{"dotfile.txt", ".private", "secret.txt"}}, }, } } diff --git a/services/search/pkg/parity/response_entity_test.go b/services/search/pkg/parity/response_entity_test.go index 29ecffa8df..c32ecbf5df 100644 --- a/services/search/pkg/parity/response_entity_test.go +++ b/services/search/pkg/parity/response_entity_test.go @@ -72,7 +72,7 @@ func entityGroup() responseGroup { }), }, { - id: 11, query: `path:"./parent"`, reads: "TotalMatches", want: []string{"2"}, engineOverrides: map[string]override{"bleve": override{want: []string{"1"}}}, + id: 11, query: `path:"./parent"`, reads: "TotalMatches", want: []string{"2"}, read: func(resp *searchService.SearchIndexResponse) []string { return []string{fmt.Sprint(resp.GetTotalMatches())} }, From 1bde3d14bb58cd2aab2b3fdb6b1d36aa2a9005ff Mon Sep 17 00:00:00 2001 From: Florian Schade Date: Mon, 31 Aug 2026 11:57:19 +0200 Subject: [PATCH 4/4] fix(search): bring back rebase removals --- services/search/internal/opensearchtest/suite.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/services/search/internal/opensearchtest/suite.go b/services/search/internal/opensearchtest/suite.go index 9b6e41e412..433907292a 100644 --- a/services/search/internal/opensearchtest/suite.go +++ b/services/search/internal/opensearchtest/suite.go @@ -67,6 +67,7 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func keepContainer := os.Getenv("KEEP_TEST_CONTAINER") == "true" if keepContainer { + // the reaper would take the kept container down with the session if err := os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true"); err != nil { return nil, fmt.Errorf("failed to disable the testcontainers reaper: %w", err) } @@ -82,6 +83,14 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func opensearch.WithPassword(cfg.Engine.OpenSearch.Client.Password), testcontainers.WithName(containerName), testcontainers.WithReuseByName(containerName), + // test indexes are tiny; don't let a full host disk trip the flood-stage + // create-index / read-only blocks mid-run + testcontainers.WithEnv(map[string]string{ + "cluster.routing.allocation.disk.threshold_enabled": "false", + }), + // a health probe answers at once on a reused container, a log wait + // would sit out the log timeout on it; a cold boot takes well over the + // previous 5s testcontainers.WithWaitStrategy( wait.ForHTTP("/_cluster/health?wait_for_status=yellow&timeout=1s"). WithPort(openSearchPort). @@ -97,13 +106,16 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func address, err := container.Address(ctx) if err != nil { - _ = container.Terminate(ctx) + _ = container.Terminate(ctx) // attempt to clean up the container return nil, fmt.Errorf("failed to get OpenSearch container address: %w", err) } + // Ensure the address is set in the default configuration. cfg.Engine.OpenSearch.Client.Addresses = []string{address} return func() { + // KEEP_TEST_CONTAINER=true leaves the container up for the next run, + // which picks it up again by name instead of booting a fresh one if keepContainer { _, _ = fmt.Fprintf(os.Stderr, "keeping OpenSearch container %s\n", containerName) return