diff --git a/services/search/pkg/bleve/backend.go b/services/search/pkg/bleve/backend.go index d8efce8175..57e4e9078b 100644 --- a/services/search/pkg/bleve/backend.go +++ b/services/search/pkg/bleve/backend.go @@ -2,6 +2,7 @@ package bleve import ( "context" + "fmt" "math" "strconv" "time" @@ -97,7 +98,11 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques } for _, agg := range sir.GetAggregations() { - bleveReq.AddFacet(agg.GetField(), newBleveFacetRequest(agg)) + fr, err := newBleveFacetRequest(agg) + if err != nil { + return nil, err + } + bleveReq.AddFacet(agg.GetField(), fr) } // Sub-aggregations need the matched hit set, not just count facets: widen @@ -184,18 +189,70 @@ func needsSubAggScan(aggs []*searchService.AggregationOption) bool { // after cross-space merge. const defaultFacetSize = 1000 -func newBleveFacetRequest(agg *searchService.AggregationOption) *bleve.FacetRequest { +func newBleveFacetRequest(agg *searchService.AggregationOption) (*bleve.FacetRequest, error) { size := int(agg.GetSize()) if size <= 0 { size = defaultFacetSize } fr := bleve.NewFacetRequest(agg.GetField(), size) - for _, r := range aggregationRanges(agg) { + ranges := aggregationRanges(agg) + if rangesAreDates(ranges) { + // bleve facets cannot mix numeric and date ranges, so one date-looking + // bound switches the whole aggregation to date mode. + for _, r := range ranges { + start, err := parseRangeTime(r.GetFrom()) + if err != nil { + return nil, fmt.Errorf("invalid date range bound %q on field %q", r.GetFrom(), agg.GetField()) + } + end, err := parseRangeTime(r.GetTo()) + if err != nil { + return nil, fmt.Errorf("invalid date range bound %q on field %q", r.GetTo(), agg.GetField()) + } + fr.AddDateTimeRange(rangeBucketKey(r), start, end) + } + return fr, nil + } + for _, r := range ranges { minP := parseFloatPtr(r.GetFrom()) maxP := parseFloatPtr(r.GetTo()) fr.AddNumericRange(rangeBucketKey(r), minP, maxP) } - return fr + return fr, nil +} + +// rangeTimeLayouts are the accepted formats for date range bounds, tried in order. +var rangeTimeLayouts = []string{time.RFC3339, "2006-01-02"} + +// rangesAreDates reports whether the ranges should be treated as datetime +// ranges: at least one bound parses as a date rather than a number. +func rangesAreDates(ranges []*searchService.BucketRange) bool { + for _, r := range ranges { + for _, s := range []string{r.GetFrom(), r.GetTo()} { + if s == "" { + continue + } + if _, err := strconv.ParseFloat(s, 64); err == nil { + continue + } + if _, err := parseRangeTime(s); err == nil { + return true + } + } + } + return false +} + +// parseRangeTime parses a range bound; the zero time marks an open bound. +func parseRangeTime(s string) (time.Time, error) { + if s == "" { + return time.Time{}, nil + } + for _, layout := range rangeTimeLayouts { + if t, err := time.Parse(layout, s); err == nil { + return t, nil + } + } + return time.Time{}, fmt.Errorf("unsupported time format %q", s) } func extractBleveAggregations(res *bleve.SearchResult, aggs []*searchService.AggregationOption) []*searchService.AggregationResult { @@ -216,6 +273,12 @@ func extractBleveAggregations(res *bleve.SearchResult, aggs []*searchService.Agg Count: int64(nr.Count), }) } + for _, dr := range fr.DateRanges { + buckets = append(buckets, &searchService.Bucket{ + Key: dr.Name, + Count: int64(dr.Count), + }) + } } else { for _, t := range fr.Terms.Terms() { buckets = append(buckets, &searchService.Bucket{ diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go index 1e030e1730..43acfe17d3 100644 --- a/services/search/pkg/bleve/backend_test.go +++ b/services/search/pkg/bleve/backend_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "time" bleveSearch "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/index/scorch" @@ -280,5 +281,99 @@ var _ = Describe("Bleve", func() { Expect(counts).To(HaveKeyWithValue("2000-", int64(3))) }) }) + + Describe("date range aggregations", func() { + upsertWithTakenDateTime := func(id, name, taken string) { + t, err := time.Parse(time.RFC3339, taken) + Expect(err).ToNot(HaveOccurred()) + r := search.Resource{ + ID: id, + ParentID: rootResource.ID, + RootID: rootResource.ID, + Path: "./" + name, + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{ + Name: name, + MimeType: "image/jpeg", + Photo: &libregraph.Photo{TakenDateTime: &t}, + }, + } + Expect(eng.Upsert(r.ID, r)).To(Succeed()) + } + + BeforeEach(func() { + upsertWithTakenDateTime("1$2!3001", "a.jpg", "2018-08-11T09:15:00Z") + upsertWithTakenDateTime("1$2!3002", "b.jpg", "2018-08-11T19:42:00Z") + upsertWithTakenDateTime("1$2!3003", "c.jpg", "2018-09-01T12:00:00Z") + upsertWithTakenDateTime("1$2!3004", "d.jpg", "2021-08-11T08:00:00Z") + }) + + It("returns buckets per date range", func() { + res := searchWithAggs("mediatype:image", + &searchsvc.AggregationOption{ + Field: "photo.takenDateTime", + BucketDefinition: &searchsvc.BucketDefinition{ + Ranges: []*searchsvc.BucketRange{ + {From: "2018-08-11T00:00:00Z", To: "2018-08-12T00:00:00Z"}, + {From: "2018-08-01", To: "2018-09-01"}, + {From: "2021-01-01", To: "2022-01-01"}, + {From: "2023-01-01", To: "2024-01-01"}, + }, + }, + }, + ) + Expect(res.Aggregations).To(HaveLen(1)) + counts := map[string]int64{} + for _, b := range res.Aggregations[0].Buckets { + counts[b.Key] = b.Count + } + Expect(counts).To(HaveKeyWithValue("2018-08-11T00:00:00Z-2018-08-12T00:00:00Z", int64(2))) + Expect(counts).To(HaveKeyWithValue("2018-08-01-2018-09-01", int64(2))) + Expect(counts).To(HaveKeyWithValue("2021-01-01-2022-01-01", int64(1))) + }) + + It("supports open-ended date ranges", func() { + res := searchWithAggs("mediatype:image", + &searchsvc.AggregationOption{ + Field: "photo.takenDateTime", + BucketDefinition: &searchsvc.BucketDefinition{ + Ranges: []*searchsvc.BucketRange{ + {To: "2019-01-01"}, + {From: "2019-01-01"}, + }, + }, + }, + ) + Expect(res.Aggregations).To(HaveLen(1)) + counts := map[string]int64{} + for _, b := range res.Aggregations[0].Buckets { + counts[b.Key] = b.Count + } + Expect(counts).To(HaveKeyWithValue("-2019-01-01", int64(3))) + Expect(counts).To(HaveKeyWithValue("2019-01-01-", int64(1))) + }) + + It("rejects malformed date range bounds", func() { + rID, err := storagespace.ParseID(rootResource.ID) + Expect(err).ToNot(HaveOccurred()) + _, err = eng.Search(context.Background(), &searchsvc.SearchIndexRequest{ + Query: "mediatype:image", + Ref: &searchmsg.Reference{ResourceId: &searchmsg.ResourceID{ + StorageId: rID.StorageId, SpaceId: rID.SpaceId, OpaqueId: rID.OpaqueId, + }}, + Aggregations: []*searchsvc.AggregationOption{ + { + Field: "photo.takenDateTime", + BucketDefinition: &searchsvc.BucketDefinition{ + Ranges: []*searchsvc.BucketRange{ + {From: "2018-08-11T00:00:00Z", To: "not-a-date"}, + }, + }, + }, + }, + }) + Expect(err).To(HaveOccurred()) + }) + }) }) })