From 0ca3a10dafe4bc9384b4079a2948b818339e0f71 Mon Sep 17 00:00:00 2001 From: Ilja Neumann Date: Fri, 9 Oct 2020 15:44:28 +0200 Subject: [PATCH] Remove bleave provider and tokenizer --- accounts/pkg/provider/bleve.go | 128 ----------------------------- accounts/pkg/provider/tokenizer.go | 38 --------- 2 files changed, 166 deletions(-) delete mode 100644 accounts/pkg/provider/bleve.go delete mode 100644 accounts/pkg/provider/tokenizer.go diff --git a/accounts/pkg/provider/bleve.go b/accounts/pkg/provider/bleve.go deleted file mode 100644 index 5657e94096..0000000000 --- a/accounts/pkg/provider/bleve.go +++ /dev/null @@ -1,128 +0,0 @@ -package provider - -import ( - "errors" - "fmt" - "strconv" - "strings" - - "github.com/CiscoM31/godata" - "github.com/blevesearch/bleve" - "github.com/blevesearch/bleve/search/query" -) - -func init() { - // add (ap)prox filter - godata.GlobalFilterTokenizer = FilterTokenizer() - godata.GlobalFilterParser.DefineOperator("ap", 2, godata.OpAssociationLeft, 4, false) -} - -// BuildBleveQuery converts a GoDataFilterQuery into a bleve query -func BuildBleveQuery(r *godata.GoDataFilterQuery) (query.Query, error) { - return recursiveBuildQuery(r.Tree) -} - -// Builds the filter recursively using DFS -func recursiveBuildQuery(n *godata.ParseNode) (query.Query, error) { - if n.Token.Type == godata.FilterTokenFunc { - switch n.Token.Value { - case "startswith": - if len(n.Children) != 2 { - return nil, errors.New("startswith match must have two children") - } - if n.Children[0].Token.Type != godata.FilterTokenLiteral { - return nil, errors.New("startswith expected a literal as the first param") - } - if n.Children[1].Token.Type != godata.FilterTokenString { - return nil, errors.New("startswith expected a string as the second param") - } // remove enclosing ' of string tokens (looks like 'some ol'' string') - value := n.Children[1].Token.Value[1 : len(n.Children[1].Token.Value)-1] - // unescape '' as ' - unescaped := strings.ReplaceAll(value, "''", "'") - q := bleve.NewPrefixQuery(unescaped) - q.SetField(n.Children[0].Token.Value) - return q, nil - // TODO contains as regex? - // TODO endswith as regex? - default: - return nil, godata.NotImplementedError(n.Token.Value + " is not implemented.") - } - } - if n.Token.Type == godata.FilterTokenLogical { - switch n.Token.Value { - case "eq": - if len(n.Children) != 2 { - return nil, errors.New("equality match must have two children") - } - if n.Children[0].Token.Type != godata.FilterTokenLiteral { - return nil, errors.New("equality expected a literal on the lhs") - } - if n.Children[1].Token.Type == godata.FilterTokenString { - // for escape rules see http://docs.oasis-open.org/odata/odata/v4.01/cs01/part2-url-conventions/odata-v4.01-cs01-part2-url-conventions.html#sec_URLComponents - // remove enclosing ' of string tokens (looks like 'some ol'' string') - value := n.Children[1].Token.Value[1 : len(n.Children[1].Token.Value)-1] - // unescape '' as ' - unescaped := strings.ReplaceAll(value, "''", "'") - // use a match query, so the field mapping, e.g. lowercase is applied to the value - // remember we defined the field mapping for `preferred_name` to be lowercase - // a term query like `preferred_name eq 'Artur'` would use `Artur` to search in the index and come up empty - // a match query will apply the field mapping (lowercasing `Artur` to `artur`) before doing the search - // TODO there is a mismatch between the LDAP and odata filters: - // - LDAP matching rules depend on the attribute: see https://ldapwiki.com/wiki/MatchingRule - // - odata has functions like `startswith`, `contains`, `tolower`, `toupper`, `matchesPattern` andy more: see http://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#sec_BuiltinQueryFunctions - // - ocis-glauth should do the mapping between LDAP and odata filter - q := bleve.NewMatchQuery(unescaped) - q.SetField(n.Children[0].Token.Value) - return q, nil - } else if n.Children[1].Token.Type == godata.FilterTokenInteger { - v, err := strconv.ParseFloat(n.Children[1].Token.Value, 64) - if err != nil { - return nil, err - } - incl := true - q := bleve.NewNumericRangeInclusiveQuery(&v, &v, &incl, &incl) - q.SetField(n.Children[0].Token.Value) - return q, nil - } - return nil, fmt.Errorf("equality expected a string or int on the rhs, got %d", n.Children[1].Token.Type) - case "and": - q := query.NewConjunctionQuery([]query.Query{}) - for _, child := range n.Children { - subQuery, err := recursiveBuildQuery(child) - if err != nil { - return nil, err - } - if subQuery != nil { - q.AddQuery(subQuery) - } - } - return q, nil - case "or": - q := query.NewDisjunctionQuery([]query.Query{}) - for _, child := range n.Children { - subQuery, err := recursiveBuildQuery(child) - if err != nil { - return nil, err - } - if subQuery != nil { - q.AddQuery(subQuery) - } - } - return q, nil - case "Not": - if len(n.Children) != 1 { - return nil, errors.New("not filter must have only one child") - } - subQuery, err := recursiveBuildQuery(n.Children[0]) - if err != nil { - return nil, err - } - q := query.NewBooleanQuery(nil, nil, []query.Query{subQuery}) - return q, nil - default: - return nil, godata.NotImplementedError(n.Token.Value + " is not implemented.") - } - } - - return nil, godata.NotImplementedError(n.Token.Value + " is not implemented.") -} diff --git a/accounts/pkg/provider/tokenizer.go b/accounts/pkg/provider/tokenizer.go deleted file mode 100644 index f4d7c59eff..0000000000 --- a/accounts/pkg/provider/tokenizer.go +++ /dev/null @@ -1,38 +0,0 @@ -package provider - -import "github.com/CiscoM31/godata" - -// FilterTokenizer creates a tokenizer capable of tokenizing filter statements -// TODO disable tokens we don't handle anyway -func FilterTokenizer() *godata.Tokenizer { - t := godata.Tokenizer{} - t.Add("^[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2}T[0-9]{2,2}:[0-9]{2,2}(:[0-9]{2,2}(.[0-9]+)?)?(Z|[+-][0-9]{2,2}:[0-9]{2,2})", godata.FilterTokenDateTime) - t.Add("^-?[0-9]{4,4}-[0-9]{2,2}-[0-9]{2,2}", godata.FilterTokenDate) - t.Add("^[0-9]{2,2}:[0-9]{2,2}(:[0-9]{2,2}(.[0-9]+)?)?", godata.FilterTokenTime) - t.Add("^\\(", godata.FilterTokenOpenParen) - t.Add("^\\)", godata.FilterTokenCloseParen) - t.Add("^/", godata.FilterTokenNav) - t.Add("^:", godata.FilterTokenColon) - t.Add("^,", godata.FilterTokenComma) - t.Add("^(geo.distance|geo.intersects|geo.length)", godata.FilterTokenFunc) - t.Add("^(substringof|substring|length|indexof)", godata.FilterTokenFunc) - // only change from the global tokenizer is the added ap - t.Add("^(eq|ne|gt|ge|lt|le|and|or|not|has|in|ap)", godata.FilterTokenLogical) - t.Add("^(add|sub|mul|divby|div|mod)", godata.FilterTokenOp) - t.Add("^(contains|endswith|startswith|tolower|toupper|"+ - "trim|concat|year|month|day|hour|minute|second|fractionalseconds|date|"+ - "time|totaloffsetminutes|now|maxdatetime|mindatetime|totalseconds|round|"+ - "floor|ceiling|isof|cast)", godata.FilterTokenFunc) - t.Add("^(any|all)", godata.FilterTokenLambda) - t.Add("^null", godata.FilterTokenNull) - t.Add("^\\$it", godata.FilterTokenIt) - t.Add("^\\$root", godata.FilterTokenRoot) - t.Add("^-?[0-9]+\\.[0-9]+", godata.FilterTokenFloat) - t.Add("^-?[0-9]+", godata.FilterTokenInteger) - t.Add("^'(''|[^'])*'", godata.FilterTokenString) - t.Add("^(true|false)", godata.FilterTokenBoolean) - t.Add("^@*[a-zA-Z][a-zA-Z0-9_.]*", godata.FilterTokenLiteral) // The optional '@' character is used to identify parameter aliases - t.Ignore("^ ", godata.FilterTokenWhitespace) - - return &t -}