[full-ci] enhancement: add support for natural language kql date ranges (#7263)

* enhancement: add more kql spec tests and simplify ast normalization

* enhancement: kql parser error if query starts with AND

* enhancement: add kql docs and support for date and time only dateTimeRestriction queries

* enhancement: add the ability to decide how kql nodes get connected

connecting nodes (with edges) seem straight forward when not using group, the default connection for nodes with the same node is always OR. THis only applies for first level nodes, for grouped nodes it is defined differently. The KQL docs are saying, nodes inside a grouped node, with the same key are connected by a AND edge.

* enhancement: explicit error handling for falsy group nodes and queries with leading binary operator

* enhancement: use optimized grammar for kql parser and toolify pigeon

* enhancement: simplify error handling

* fix: kql implicit 'AND' and 'OR' follows the ms html spec instead of the pdf spec

* enhancement: add support for natural language kql date queries

* enhancement: structure kql parser tests into logical clusters

* fix: time-range error naming

services/search/pkg/query/kql/cast.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/araddon/dateparse"
+	"github.com/jinzhu/now"
 
 	"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
 )
@@ -21,21 +21,23 @@ func toNode[T ast.Node](in interface{}) (T, error) {
 
 func toNodes[T ast.Node](in interface{}) ([]T, error) {
 	switch v := in.(type) {
-	case []T:
-		return v, nil
 	case T:
 		return []T{v}, nil
+	case []T:
+		return v, nil
+	case []*ast.OperatorNode, []*ast.DateTimeNode:
+		return toNodes[T](v)
 	case []interface{}:
-		var ts []T
-		for _, inter := range v {
-			n, err := toNodes[T](inter)
+		var nodes []T
+		for _, el := range v {
+			node, err := toNodes[T](el)
 			if err != nil {
 				return nil, err
 			}
 
-			ts = append(ts, n...)
+			nodes = append(nodes, node...)
 		}
-		return ts, nil
+		return nodes, nil
 	case nil:
 		return nil, nil
 	default:
@@ -74,5 +76,53 @@ func toTime(in interface{}) (time.Time, error) {
 		return time.Time{}, err
 	}
 
-	return dateparse.ParseLocal(ts)
+	return now.Parse(ts)
+}
+
+func toTimeRange(in interface{}) (*time.Time, *time.Time, error) {
+	var from, to time.Time
+
+	value, err := toString(in)
+	if err != nil {
+		return &from, &to, UnsupportedTimeRangeError{}
+	}
+
+	c := &now.Config{
+		WeekStartDay: time.Monday,
+	}
+
+	n := c.With(timeNow())
+
+	switch value {
+	case "today":
+		from = n.BeginningOfDay()
+		to = n.EndOfDay()
+	case "yesterday":
+		yesterday := n.With(n.AddDate(0, 0, -1))
+		from = yesterday.BeginningOfDay()
+		to = yesterday.EndOfDay()
+	case "this week":
+		from = n.BeginningOfWeek()
+		to = n.EndOfWeek()
+	case "this month":
+		from = n.BeginningOfMonth()
+		to = n.EndOfMonth()
+	case "last month":
+		lastMonth := n.With(n.AddDate(0, -1, 0))
+		from = lastMonth.BeginningOfMonth()
+		to = lastMonth.EndOfMonth()
+	case "this year":
+		from = n.BeginningOfYear()
+		to = n.EndOfYear()
+	case "last year":
+		lastYear := n.With(n.AddDate(-1, 0, 0))
+		from = lastYear.BeginningOfYear()
+		to = lastYear.EndOfYear()
+	}
+
+	if from.IsZero() || to.IsZero() {
+		return nil, nil, UnsupportedTimeRangeError{}
+	}
+
+	return &from, &to, nil
 }

services/search/pkg/query/kql/dictionary.peg

@@ -61,6 +61,12 @@ DateTimeRestrictionNode <-
         FullTime
     ) '"'? {
         return buildDateTimeNode(k, o, v, c.text, c.pos)
+    } /
+    k:Char+ (
+        OperatorEqualNode /
+        OperatorColonNode
+    ) '"'? v:NaturalLanguageDateTime '"'? {
+        return buildNaturalLanguageDateTimeNodes(k, v, c.text, c.pos)
     }
 
 TextPropertyRestrictionNode <-
@@ -185,11 +191,22 @@ FullTime <-
         return c.text, nil
     }
 
-DateTime
-  = FullDate "T" FullTime {
+DateTime  <-
+    FullDate "T" FullTime {
       return c.text, nil
     }
 
+NaturalLanguageDateTime  <-
+    "today" /
+    "yesterday" /
+    "this week" /
+    "this month" /
+    "last month" /
+    "this year" /
+    "last year"  {
+        return c.text, nil
+    }
+
 ////////////////////////////////////////////////////////
 // misc
 ////////////////////////////////////////////////////////

services/search/pkg/query/kql/doc.go

@@ -10,7 +10,6 @@ The following spec parts are supported and tested:
   - 2.1.8 OR Operator
   - 2.1.12 Parentheses
   - 2.3.5 Date Tokens
-  - Human tokens not implemented
   - 3.1.11 Implicit Operator
   - 3.1.12 Parentheses
   - 3.1.2 AND Operator

services/search/pkg/query/kql/engine_suite_test.go

@@ -0,0 +1,11 @@
+package kql
+
+import (
+	"time"
+)
+
+// PatchTimeNow is here to path the package time now func,
+// this only exists for the tests context
+func PatchTimeNow(t func() time.Time) {
+	timeNow = t
+}

services/search/pkg/query/kql/error.go

@@ -28,3 +28,12 @@ func (e NamedGroupInvalidNodesError) Error() string {
 		ast.NodeValue(e.Node),
 	).Error()
 }
+
+// UnsupportedTimeRangeError records an error and the value that caused it.
+type UnsupportedTimeRangeError struct {
+	Value interface{}
+}
+
+func (e UnsupportedTimeRangeError) Error() string {
+	return fmt.Sprintf("unable to convert '%v' to a time range", e.Value)
+}

services/search/pkg/query/kql/factory.go

@@ -101,6 +101,39 @@ func buildDateTimeNode(k, o, v interface{}, text []byte, pos position) (*ast.Dat
 		Value:    value,
 	}, nil
 }
+func buildNaturalLanguageDateTimeNodes(k, v interface{}, text []byte, pos position) ([]ast.Node, error) {
+	b, err := base(text, pos)
+	if err != nil {
+		return nil, err
+	}
+
+	key, err := toString(k)
+	if err != nil {
+		return nil, err
+	}
+
+	from, to, err := toTimeRange(v)
+	if err != nil {
+		return nil, err
+	}
+
+	return []ast.Node{
+		&ast.DateTimeNode{
+			Base:     b,
+			Value:    *from,
+			Key:      key,
+			Operator: &ast.OperatorNode{Value: ">="},
+		},
+		&ast.OperatorNode{Value: BoolAND},
+		&ast.DateTimeNode{
+			Base:     b,
+			Value:    *to,
+			Key:      key,
+			Operator: &ast.OperatorNode{Value: "<="},
+		},
+	}, nil
+
+}
 
 func buildBooleanNode(k, v interface{}, text []byte, pos position) (*ast.BooleanNode, error) {
 	b, err := base(text, pos)

services/search/pkg/query/kql/kql.go

@@ -3,6 +3,7 @@ package kql
 
 import (
 	"errors"
+	"time"
 
 	"github.com/owncloud/ocis/v2/services/search/pkg/query/ast"
 )
@@ -42,3 +43,7 @@ func (b Builder) Build(q string) (*ast.Ast, error) {
 
 	return f.(*ast.Ast), nil
 }
+
+// timeNow mirrors time.Now by default, the only reason why this exists
+// is to monkey patch it from the tests. See PatchTimeNow
+var timeNow = time.Now