diff --git a/services/search/pkg/bleve/aggregations.go b/services/search/pkg/bleve/aggregations.go new file mode 100644 index 0000000000..c18d7da013 --- /dev/null +++ b/services/search/pkg/bleve/aggregations.go @@ -0,0 +1,489 @@ +package bleve + +import ( + "context" + "errors" + "fmt" + "sort" + "strconv" + "time" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/numeric" + bleveSearch "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/search/collector" + index "github.com/blevesearch/bleve_index_api" + + searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" +) + +// Bleve facets count one field and cannot nest, so metrics and +// sub-aggregations are folded from doc values by aggCollector, hooked into +// the collector walk through bleve's document-match-handler context key. +// Loading hits for them instead costs a stored-document decode per match. + +// defaultFacetSize is used when no size is requested; the service layer trims +// after cross-space merge. +const defaultFacetSize = 1000 + +func collected(agg *searchService.AggregationOption) bool { + return agg.GetMetricKind() != searchService.MetricKind_METRIC_KIND_UNSPECIFIED || len(agg.GetSubAggregations()) > 0 +} + +func newBleveFacetRequest(agg *searchService.AggregationOption) (*bleve.FacetRequest, error) { + size := int(agg.GetSize()) + if size <= 0 { + size = defaultFacetSize + } + fr := bleve.NewFacetRequest(agg.GetField(), size) + ranges := aggregationRanges(agg) + if rangesAreDates(ranges) { + // bleve facets cannot mix numeric and date ranges, so one date-looking + // bound switches the whole aggregation to date mode. + for _, r := range ranges { + start, end, err := parseDateRange(agg.GetField(), r) + if err != nil { + return nil, err + } + fr.AddDateTimeRange(rangeBucketKey(r), start, end) + } + return fr, nil + } + for _, r := range ranges { + minP := parseFloatPtr(r.GetFrom()) + maxP := parseFloatPtr(r.GetTo()) + fr.AddNumericRange(rangeBucketKey(r), minP, maxP) + } + return fr, nil +} + +var rangeTimeLayouts = []string{time.RFC3339, "2006-01-02"} + +func rangesAreDates(ranges []*searchService.BucketRange) bool { + for _, r := range ranges { + for _, s := range []string{r.GetFrom(), r.GetTo()} { + if s == "" { + continue + } + if _, err := strconv.ParseFloat(s, 64); err == nil { + continue + } + if _, err := parseRangeTime(s); err == nil { + return true + } + } + } + return false +} + +// The zero time marks an open bound. +func parseRangeTime(s string) (time.Time, error) { + if s == "" { + return time.Time{}, nil + } + for _, layout := range rangeTimeLayouts { + if t, err := time.Parse(layout, s); err == nil { + return t, nil + } + } + return time.Time{}, fmt.Errorf("unsupported time format %q", s) +} + +func parseDateRange(field string, r *searchService.BucketRange) (time.Time, time.Time, error) { + start, err := parseRangeTime(r.GetFrom()) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("invalid date range bound %q on field %q", r.GetFrom(), field) + } + end, err := parseRangeTime(r.GetTo()) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("invalid date range bound %q on field %q", r.GetTo(), field) + } + return start, end, nil +} + +func aggregationRanges(agg *searchService.AggregationOption) []*searchService.BucketRange { + bd := agg.GetBucketDefinition() + if bd == nil { + return nil + } + return bd.GetRanges() +} + +// rangeBucketKey formats a range as "from-to" for stable merge keys; open sides +// render as "-N" or "N-". +func rangeBucketKey(r *searchService.BucketRange) string { + return r.GetFrom() + "-" + r.GetTo() +} + +func parseFloatPtr(s string) *float64 { + if s == "" { + return nil + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil + } + return &v +} + +func facetBuckets(fr *bleveSearch.FacetResult, agg *searchService.AggregationOption) []*searchService.Bucket { + buckets := make([]*searchService.Bucket, 0) + if len(aggregationRanges(agg)) > 0 { + for _, nr := range fr.NumericRanges { + buckets = append(buckets, &searchService.Bucket{Key: nr.Name, Count: int64(nr.Count)}) + } + for _, dr := range fr.DateRanges { + buckets = append(buckets, &searchService.Bucket{Key: dr.Name, Count: int64(dr.Count)}) + } + return buckets + } + for _, t := range fr.Terms.Terms() { + buckets = append(buckets, &searchService.Bucket{Key: t.Term, Count: int64(t.Count)}) + } + return buckets +} + +type levelKind int + +const ( + levelTerms levelKind = iota + levelNumericRange + levelDateRange + levelMetric +) + +type numericRange struct { + name string + min, max *float64 +} + +type dateRange struct { + name string + start, end time.Time +} + +type aggLevel struct { + opt *searchService.AggregationOption + kind levelKind + numeric []numericRange + dates []dateRange + children []*aggLevel +} + +func newAggLevel(opt *searchService.AggregationOption) (*aggLevel, error) { + l := &aggLevel{opt: opt} + switch { + case opt.GetMetricKind() != searchService.MetricKind_METRIC_KIND_UNSPECIFIED: + l.kind = levelMetric + case len(aggregationRanges(opt)) > 0: + ranges := aggregationRanges(opt) + if rangesAreDates(ranges) { + l.kind = levelDateRange + for _, r := range ranges { + start, end, err := parseDateRange(opt.GetField(), r) + if err != nil { + return nil, err + } + l.dates = append(l.dates, dateRange{name: rangeBucketKey(r), start: start, end: end}) + } + } else { + l.kind = levelNumericRange + for _, r := range ranges { + l.numeric = append(l.numeric, numericRange{name: rangeBucketKey(r), min: parseFloatPtr(r.GetFrom()), max: parseFloatPtr(r.GetTo())}) + } + } + default: + l.kind = levelTerms + } + for _, sub := range opt.GetSubAggregations() { + child, err := newAggLevel(sub) + if err != nil { + return nil, err + } + l.children = append(l.children, child) + } + return l, nil +} + +// fieldValues is per-document scratch, reused across documents. +type fieldValues struct { + asTerms bool + asNumbers bool + terms []string + numbers []int64 +} + +type bucketAcc struct { + counts map[string]int64 + subs map[string][]*bucketAcc + + value float64 // SUM/MIN/MAX + sum float64 // AVG numerator + count int64 // AVG denominator + seen bool +} + +func newBucketAcc(l *aggLevel) *bucketAcc { + a := &bucketAcc{} + if l.kind != levelMetric { + a.counts = map[string]int64{} + if len(l.children) > 0 { + a.subs = map[string][]*bucketAcc{} + } + } + return a +} + +// aggCollector serves one search; bleve's collector is single-threaded. +type aggCollector struct { + fields map[string]*fieldValues + fieldNames []string + roots map[int]*aggRoot // by position in the request's aggregations +} + +type aggRoot struct { + level *aggLevel + acc *bucketAcc +} + +func newAggCollector(aggs []*searchService.AggregationOption) (*aggCollector, error) { + c := &aggCollector{fields: map[string]*fieldValues{}, roots: map[int]*aggRoot{}} + for i, agg := range aggs { + if !collected(agg) { + continue + } + l, err := newAggLevel(agg) + if err != nil { + return nil, err + } + c.register(l) + c.roots[i] = &aggRoot{level: l, acc: newBucketAcc(l)} + } + if len(c.roots) == 0 { + return nil, nil + } + return c, nil +} + +func (c *aggCollector) register(l *aggLevel) { + fv, ok := c.fields[l.opt.GetField()] + if !ok { + fv = &fieldValues{} + c.fields[l.opt.GetField()] = fv + c.fieldNames = append(c.fieldNames, l.opt.GetField()) + } + if l.kind == levelTerms { + fv.asTerms = true + } else { + fv.asNumbers = true + } + for _, child := range l.children { + c.register(child) + } +} + +// The handler runs for every match, before the top-n cut. +func (c *aggCollector) withContext(ctx context.Context) context.Context { + maker := bleveSearch.MakeDocumentMatchHandler(func(sc *bleveSearch.SearchContext) (bleveSearch.DocumentMatchHandler, bool, error) { + inner, loadID, err := collector.MakeTopNDocumentMatchHandler(sc) + if err != nil { + return nil, false, err + } + if inner == nil { + return nil, false, errors.New("aggregations need the top-n collector") + } + dvr, err := sc.IndexReader.DocValueReader(c.fieldNames) + if err != nil { + return nil, false, err + } + return func(d *bleveSearch.DocumentMatch) error { + if d != nil { + if err := c.collect(sc.IndexReader, dvr, d); err != nil { + return err + } + } + return inner(d) + }, loadID, nil + }) + return context.WithValue(ctx, bleveSearch.MakeDocumentMatchHandlerKey, maker) +} + +func (c *aggCollector) collect(reader index.IndexReader, dvr index.DocValueReader, d *bleveSearch.DocumentMatch) error { + if d.IndexInternalID == nil { + id, err := reader.InternalID(d.ID) + if err != nil { + return err + } + d.IndexInternalID = id + } + for _, fv := range c.fields { + fv.terms = fv.terms[:0] + fv.numbers = fv.numbers[:0] + } + if err := dvr.VisitDocValues(d.IndexInternalID, c.visit); err != nil { + return err + } + for _, root := range c.roots { + c.fold(root.acc, root.level) + } + return nil +} + +// Numeric and date doc values are prefix-coded at several precisions; only +// shift 0 carries the exact value. +func (c *aggCollector) visit(field string, term []byte) { + fv, ok := c.fields[field] + if !ok { + return + } + if fv.asTerms { + fv.terms = append(fv.terms, string(term)) + } + if fv.asNumbers { + pc := numeric.PrefixCoded(term) + if shift, err := pc.Shift(); err == nil && shift == 0 { + if v, err := pc.Int64(); err == nil { + fv.numbers = append(fv.numbers, v) + } + } + } +} + +func (c *aggCollector) fold(a *bucketAcc, l *aggLevel) { + fv := c.fields[l.opt.GetField()] + switch l.kind { + case levelMetric: + for _, raw := range fv.numbers { + a.addMetric(l.opt.GetMetricKind(), numeric.Int64ToFloat64(raw)) + } + case levelTerms: + for _, term := range fv.terms { + if term != "" { + c.foldBucket(a, l, term) + } + } + case levelNumericRange: + for _, raw := range fv.numbers { + v := numeric.Int64ToFloat64(raw) + for _, r := range l.numeric { + if (r.min == nil || v >= *r.min) && (r.max == nil || v < *r.max) { + c.foldBucket(a, l, r.name) + } + } + } + case levelDateRange: + for _, raw := range fv.numbers { + t := time.Unix(0, raw) + for _, r := range l.dates { + if (r.start.IsZero() || !t.Before(r.start)) && (r.end.IsZero() || t.Before(r.end)) { + c.foldBucket(a, l, r.name) + } + } + } + } +} + +func (c *aggCollector) foldBucket(a *bucketAcc, l *aggLevel, key string) { + a.counts[key]++ + a.seen = true + if len(l.children) == 0 { + return + } + subs, ok := a.subs[key] + if !ok { + subs = make([]*bucketAcc, len(l.children)) + for i, child := range l.children { + subs[i] = newBucketAcc(child) + } + a.subs[key] = subs + } + for i, child := range l.children { + c.fold(subs[i], child) + } +} + +func (a *bucketAcc) addMetric(kind searchService.MetricKind, v float64) { + switch kind { + case searchService.MetricKind_METRIC_KIND_SUM: + a.value += v + case searchService.MetricKind_METRIC_KIND_MIN: + if !a.seen || v < a.value { + a.value = v + } + case searchService.MetricKind_METRIC_KIND_MAX: + if !a.seen || v > a.value { + a.value = v + } + case searchService.MetricKind_METRIC_KIND_AVG: + a.sum += v + a.count++ + } + a.seen = true +} + +func (a *bucketAcc) result(l *aggLevel) *searchService.AggregationResult { + if l.kind == levelMetric { + if !a.seen { + return nil + } + r := &searchService.AggregationResult{Field: l.opt.GetField(), MetricKind: l.opt.GetMetricKind()} + if l.opt.GetMetricKind() == searchService.MetricKind_METRIC_KIND_AVG { + r.Sum = a.sum + r.Count = a.count + } else { + r.Value = a.value + } + return r + } + + buckets := make([]*searchService.Bucket, 0, len(a.counts)) + for key, count := range a.counts { + b := &searchService.Bucket{Key: key, Count: count} + for i, child := range l.children { + if sub := a.subs[key][i].result(child); sub != nil { + b.SubAggregations = append(b.SubAggregations, sub) + } + } + buckets = append(buckets, b) + } + // same order as a bleve terms facet + sort.Slice(buckets, func(i, j int) bool { + if buckets[i].Count == buckets[j].Count { + return buckets[i].Key < buckets[j].Key + } + return buckets[i].Count > buckets[j].Count + }) + if size := int(l.opt.GetSize()); size > 0 && len(buckets) > size { + buckets = buckets[:size] + } + return &searchService.AggregationResult{Field: l.opt.GetField(), Buckets: buckets} +} + +func extractBleveAggregations(res *bleve.SearchResult, aggs []*searchService.AggregationOption, c *aggCollector) []*searchService.AggregationResult { + if len(aggs) == 0 { + return nil + } + out := make([]*searchService.AggregationResult, 0, len(aggs)) + for i, agg := range aggs { + if collected(agg) { + if c == nil { + continue + } + if root, ok := c.roots[i]; ok { + if r := root.acc.result(root.level); r != nil { + out = append(out, r) + } + } + continue + } + fr, ok := res.Facets[agg.GetField()] + if !ok { + continue + } + out = append(out, &searchService.AggregationResult{ + Field: agg.GetField(), + Buckets: facetBuckets(fr, agg), + }) + } + return out +} diff --git a/services/search/pkg/bleve/backend.go b/services/search/pkg/bleve/backend.go index 57539b1073..a0ac33e0fc 100644 --- a/services/search/pkg/bleve/backend.go +++ b/services/search/pkg/bleve/backend.go @@ -2,13 +2,10 @@ package bleve import ( "context" - "fmt" "math" - "strconv" "time" "github.com/blevesearch/bleve/v2" - bleveSearch "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/query" storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/opencloud-eu/reva/v2/pkg/errtypes" @@ -45,7 +42,7 @@ func NewBackend(index bleve.Index, queryCreator searchQuery.Creator[query.Query] // Search executes a search request operation within the index. // Returns a SearchIndexResponse object or an error. -func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) { +func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) { createdQuery, err := b.queryCreator.CreateWithFilters(sir.Query, sir.GetAggregationFilters()) if err != nil { if kql.IsValidationError(err) { @@ -102,9 +99,7 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques } for _, agg := range sir.GetAggregations() { - // Top-level metrics are computed by scanning the matched hits, they - // have no facet representation. - if agg.GetMetricKind() != searchService.MetricKind_METRIC_KIND_UNSPECIFIED { + if collected(agg) { continue } fr, err := newBleveFacetRequest(agg) @@ -113,16 +108,16 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques } bleveReq.AddFacet(agg.GetField(), fr) } - - // Sub-aggregations and top-level metrics need the matched hit set, not just - // count facets: widen the page so the emulator has enough docs. The caller's - // larger PageSize wins. - if needsSubAggScan(sir.GetAggregations()) && bleveReq.Size < subAggScanSize { - bleveReq.Size = subAggScanSize + aggs, err := newAggCollector(sir.GetAggregations()) + if err != nil { + return nil, err + } + if aggs != nil { + ctx = aggs.withContext(ctx) } bleveReq.Fields = []string{"*"} - res, err := b.index.Search(bleveReq) + res, err := b.index.SearchInContext(ctx, bleveReq) if err != nil { return nil, err } @@ -178,356 +173,10 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques return &searchService.SearchIndexResponse{ Matches: matches, TotalMatches: int32(totalMatches), - Aggregations: extractBleveAggregations(res, sir.GetAggregations()), + Aggregations: extractBleveAggregations(res, sir.GetAggregations(), aggs), }, nil } -// subAggScanSize caps how many hits we walk when emulating sub-aggregations; -// math.MaxInt returns everything. -const subAggScanSize = math.MaxInt - -func needsSubAggScan(aggs []*searchService.AggregationOption) bool { - for _, agg := range aggs { - if len(agg.GetSubAggregations()) > 0 { - return true - } - if agg.GetMetricKind() != searchService.MetricKind_METRIC_KIND_UNSPECIFIED { - return true - } - } - return false -} - -// defaultFacetSize is used when no size is requested; the service layer trims -// after cross-space merge. -const defaultFacetSize = 1000 - -func newBleveFacetRequest(agg *searchService.AggregationOption) (*bleve.FacetRequest, error) { - size := int(agg.GetSize()) - if size <= 0 { - size = defaultFacetSize - } - fr := bleve.NewFacetRequest(agg.GetField(), size) - ranges := aggregationRanges(agg) - if rangesAreDates(ranges) { - // bleve facets cannot mix numeric and date ranges, so one date-looking - // bound switches the whole aggregation to date mode. - for _, r := range ranges { - start, err := parseRangeTime(r.GetFrom()) - if err != nil { - return nil, fmt.Errorf("invalid date range bound %q on field %q", r.GetFrom(), agg.GetField()) - } - end, err := parseRangeTime(r.GetTo()) - if err != nil { - return nil, fmt.Errorf("invalid date range bound %q on field %q", r.GetTo(), agg.GetField()) - } - fr.AddDateTimeRange(rangeBucketKey(r), start, end) - } - return fr, nil - } - for _, r := range ranges { - minP := parseFloatPtr(r.GetFrom()) - maxP := parseFloatPtr(r.GetTo()) - fr.AddNumericRange(rangeBucketKey(r), minP, maxP) - } - return fr, nil -} - -// rangeTimeLayouts are the accepted formats for date range bounds, tried in order. -var rangeTimeLayouts = []string{time.RFC3339, "2006-01-02"} - -// rangesAreDates reports whether the ranges should be treated as datetime -// ranges: at least one bound parses as a date rather than a number. -func rangesAreDates(ranges []*searchService.BucketRange) bool { - for _, r := range ranges { - for _, s := range []string{r.GetFrom(), r.GetTo()} { - if s == "" { - continue - } - if _, err := strconv.ParseFloat(s, 64); err == nil { - continue - } - if _, err := parseRangeTime(s); err == nil { - return true - } - } - } - return false -} - -// parseRangeTime parses a range bound; the zero time marks an open bound. -func parseRangeTime(s string) (time.Time, error) { - if s == "" { - return time.Time{}, nil - } - for _, layout := range rangeTimeLayouts { - if t, err := time.Parse(layout, s); err == nil { - return t, nil - } - } - return time.Time{}, fmt.Errorf("unsupported time format %q", s) -} - -func extractBleveAggregations(res *bleve.SearchResult, aggs []*searchService.AggregationOption) []*searchService.AggregationResult { - if len(aggs) == 0 { - return nil - } - out := make([]*searchService.AggregationResult, 0, len(aggs)) - for _, agg := range aggs { - // Top-level metric: fold the matched hits through the sub-agg - // accumulator, there is no facet to read from. - if agg.GetMetricKind() != searchService.MetricKind_METRIC_KIND_UNSPECIFIED { - acc := newSubAcc(agg) - for _, hit := range res.Hits { - accumulateHit(acc, agg, hit) - } - if r := emitAcc(acc, agg); r != nil { - out = append(out, r) - } - continue - } - fr, ok := res.Facets[agg.GetField()] - if !ok { - continue - } - buckets := make([]*searchService.Bucket, 0) - if len(aggregationRanges(agg)) > 0 { - for _, nr := range fr.NumericRanges { - buckets = append(buckets, &searchService.Bucket{ - Key: nr.Name, - Count: int64(nr.Count), - }) - } - for _, dr := range fr.DateRanges { - buckets = append(buckets, &searchService.Bucket{ - Key: dr.Name, - Count: int64(dr.Count), - }) - } - } else { - for _, t := range fr.Terms.Terms() { - buckets = append(buckets, &searchService.Bucket{ - Key: t.Term, - Count: int64(t.Count), - }) - } - } - if subAggs := agg.GetSubAggregations(); len(subAggs) > 0 { - attachSubAggregations(res, agg.GetField(), subAggs, buckets) - } - out = append(out, &searchService.AggregationResult{ - Field: agg.GetField(), - Buckets: buckets, - }) - } - return out -} - -// subAcc is the recursive accumulator emulating composite aggregations: one -// node per sub-aggregation under a parent bucket. -type subAcc struct { - // terms: count + recursive accumulators per child value - termCount map[string]int64 - termSubs map[string][]*subAcc - - // metric - metricVal float64 // SUM/MIN/MAX - sum float64 // AVG transport: numerator - count int64 // AVG transport: denominator - - seen bool // at least one hit contributed -} - -// newSubAcc allocates an accumulator for the given sub-agg. -func newSubAcc(sa *searchService.AggregationOption) *subAcc { - a := &subAcc{} - if sa.GetMetricKind() == searchService.MetricKind_METRIC_KIND_UNSPECIFIED { - a.termCount = map[string]int64{} - if len(sa.GetSubAggregations()) > 0 { - a.termSubs = map[string][]*subAcc{} - } - } - return a -} - -// accumulateHit folds one hit into a sub-agg accumulator, recursing into -// grand-sub-aggregations. -func accumulateHit(a *subAcc, sa *searchService.AggregationOption, hit *bleveSearch.DocumentMatch) { - switch sa.GetMetricKind() { - case searchService.MetricKind_METRIC_KIND_UNSPECIFIED: - val, ok := hit.Fields[sa.GetField()].(string) - if !ok || val == "" { - return - } - a.termCount[val]++ - a.seen = true - if subs := sa.GetSubAggregations(); len(subs) > 0 { - childAccs, ok := a.termSubs[val] - if !ok { - childAccs = make([]*subAcc, len(subs)) - for i, ssa := range subs { - childAccs[i] = newSubAcc(ssa) - } - a.termSubs[val] = childAccs - } - for i, ssa := range subs { - accumulateHit(childAccs[i], ssa, hit) - } - } - default: - v, ok := numericFieldValue(hit.Fields[sa.GetField()]) - if !ok { - return - } - switch sa.GetMetricKind() { - case searchService.MetricKind_METRIC_KIND_SUM: - a.metricVal += v - case searchService.MetricKind_METRIC_KIND_MIN: - if !a.seen || v < a.metricVal { - a.metricVal = v - } - case searchService.MetricKind_METRIC_KIND_MAX: - if !a.seen || v > a.metricVal { - a.metricVal = v - } - case searchService.MetricKind_METRIC_KIND_AVG: - a.sum += v - a.count++ - } - a.seen = true - } -} - -// emitAcc materialises a sub-agg accumulator into the proto result. -func emitAcc(a *subAcc, sa *searchService.AggregationOption) *searchService.AggregationResult { - if sa.GetMetricKind() != searchService.MetricKind_METRIC_KIND_UNSPECIFIED { - if !a.seen { - return nil - } - r := &searchService.AggregationResult{ - Field: sa.GetField(), - MetricKind: sa.GetMetricKind(), - } - if sa.GetMetricKind() == searchService.MetricKind_METRIC_KIND_AVG { - r.Sum = a.sum - r.Count = a.count - } else { - r.Value = a.metricVal - } - return r - } - - subs := sa.GetSubAggregations() - childBuckets := make([]*searchService.Bucket, 0, len(a.termCount)) - for term, count := range a.termCount { - b := &searchService.Bucket{Key: term, Count: count} - if len(subs) > 0 { - if childAccs, ok := a.termSubs[term]; ok { - for i, ssa := range subs { - if sub := emitAcc(childAccs[i], ssa); sub != nil { - b.SubAggregations = append(b.SubAggregations, sub) - } - } - } - } - childBuckets = append(childBuckets, b) - } - if sz := int(sa.GetSize()); sz > 0 && len(childBuckets) > sz { - childBuckets = childBuckets[:sz] - } - return &searchService.AggregationResult{ - Field: sa.GetField(), - Buckets: childBuckets, - } -} - -// attachSubAggregations folds the matched hits into nested aggregation results -// per parent bucket, via a single hit walk dispatched through the accumulator tree. -func attachSubAggregations(res *bleve.SearchResult, parentField string, subAggs []*searchService.AggregationOption, buckets []*searchService.Bucket) { - bucketByKey := make(map[string]*searchService.Bucket, len(buckets)) - for _, b := range buckets { - bucketByKey[b.GetKey()] = b - } - - perParent := make(map[string][]*subAcc, len(buckets)) - for _, b := range buckets { - accs := make([]*subAcc, len(subAggs)) - for i, sa := range subAggs { - accs[i] = newSubAcc(sa) - } - perParent[b.GetKey()] = accs - } - - for _, hit := range res.Hits { - parentVal, ok := hit.Fields[parentField].(string) - if !ok || parentVal == "" { - continue - } - accs, ok := perParent[parentVal] - if !ok { - continue - } - for i, sa := range subAggs { - accumulateHit(accs[i], sa, hit) - } - } - - for key, accs := range perParent { - b := bucketByKey[key] - for i, sa := range subAggs { - if r := emitAcc(accs[i], sa); r != nil { - b.SubAggregations = append(b.SubAggregations, r) - } - } - } -} - -// numericFieldValue coerces a bleve stored value to float64 (also accepts -// string forms). -func numericFieldValue(raw interface{}) (float64, bool) { - switch v := raw.(type) { - case float64: - return v, true - case int64: - return float64(v), true - case int32: - return float64(v), true - case string: - f, err := strconv.ParseFloat(v, 64) - if err != nil { - return 0, false - } - return f, true - default: - return 0, false - } -} - -func aggregationRanges(agg *searchService.AggregationOption) []*searchService.BucketRange { - bd := agg.GetBucketDefinition() - if bd == nil { - return nil - } - return bd.GetRanges() -} - -// rangeBucketKey formats a range as "from-to" for stable merge keys; open sides -// render as "-N" or "N-". -func rangeBucketKey(r *searchService.BucketRange) string { - return r.GetFrom() + "-" + r.GetTo() -} - -func parseFloatPtr(s string) *float64 { - if s == "" { - return nil - } - v, err := strconv.ParseFloat(s, 64) - if err != nil { - return nil - } - return &v -} - func (b *Backend) DocCount() (uint64, error) { return b.index.DocCount() } diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index b9122cd3d2..9c11b4be49 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -723,3 +723,10 @@ Fixtures: | AGG-07 | `mediatype:image` reads `photo.takenDateTime buckets per date range` | photo.take...-01-01=1, photo.take...-09-01=2, photo.take...00:00Z=2 | photo.take...-01-01=1, photo.take...-09-01=2, photo.take...00:00Z=2 | photo.take...-01-01=1, photo.take...-09-01=2, photo.take...00:00Z=2 | ✅ | | AGG-08 | `mediatype:image` reads `open-ended date ranges` | photo.take...-01-01=3, photo.take...01-01-=1 | photo.take...-01-01=3, photo.take...01-01-=1 | photo.take...-01-01=3, photo.take...01-01-=1 | ✅ | | AGG-09 | `mediatype:image` reads `malformed date range bound` | error | error | error | ✅ | +| AGG-10 | `mediatype:audio` reads `album buckets nested in artist buckets` | audio.arti... Floyd=2, audio.arti...Bomber=2, audio.arti...Spades=1, audio.arti...e Wall=2, audio.arti...¶rhead=3 | audio.arti... Floyd=2, audio.arti...Bomber=2, audio.arti...Spades=1, audio.arti...e Wall=2, audio.arti...¶rhead=3 | audio.arti... Floyd=2, audio.arti...Bomber=2, audio.arti...Spades=1, audio.arti...e Wall=2, audio.arti...¶rhead=3 | ✅ | +| AGG-11 | `mediatype:audio` reads `sum and avg of audio.year per artist` | audio.arti... Floyd=2, audio.arti... count=2, audio.arti... count=3, audio.arti...sum=3946, audio.arti...sum=5982, audio.arti...¶rhead=3 | audio.arti... Floyd=2, audio.arti... count=2, audio.arti... count=3, audio.arti...sum=3946, audio.arti...sum=5982, audio.arti...¶rhead=3 | audio.arti... Floyd=2, audio.arti... count=2, audio.arti... count=3, audio.arti...sum=3946, audio.arti...sum=5982, audio.arti...¶rhead=3 | ✅ | +| AGG-12 | `mediatype:audio` reads `artist buckets nested in audio.year decades` | audio.year 1970-1980=2, audio.year 1980-1990=1, audio.year 1990-2000=1, audio.year 2000-2010=3, audio.year... Floyd=2, audio.year...¶rhead=1, audio.year...¶rhead=1, audio.year...¶rhead=1 | audio.year 1970-1980=2, audio.year 1980-1990=1, audio.year 1990-2000=1, audio.year 2000-2010=3, audio.year... Floyd=2, audio.year...¶rhead=1, audio.year...¶rhead=1, audio.year...¶rhead=1 | audio.year 1970-1980=2, audio.year 1980-1990=1, audio.year 1990-2000=1, audio.year 2000-2010=3, audio.year... Floyd=2, audio.year...¶rhead=1, audio.year...¶rhead=1, audio.year...¶rhead=1 | ✅ | +| AGG-13 | `mediatype:audio` reads `max audio.year per album per artist, three levels` | audio.arti... Floyd=2, audio.arti...Bomber=2, audio.arti...Spades=1, audio.arti...e Wall=2, audio.arti...max=1975, audio.arti...max=1999, audio.arti...max=2001, audio.arti...¶rhead=3 | audio.arti... Floyd=2, audio.arti...Bomber=2, audio.arti...Spades=1, audio.arti...e Wall=2, audio.arti...max=1975, audio.arti...max=1999, audio.arti...max=2001, audio.arti...¶rhead=3 | audio.arti... Floyd=2, audio.arti...Bomber=2, audio.arti...Spades=1, audio.arti...e Wall=2, audio.arti...max=1975, audio.arti...max=1999, audio.arti...max=2001, audio.arti...¶rhead=3 | ✅ | +| AGG-14 | `mediatype:image` reads `MimeType buckets nested in open-ended date ranges` | photo.take...-01-01=3, photo.take...01-01-=1, photo.take...e/jpeg=1, photo.take...e/jpeg=3 | photo.take...-01-01=3, photo.take...01-01-=1, photo.take...e/jpeg=1, photo.take...e/jpeg=3 | photo.take...-01-01=3, photo.take...01-01-=1, photo.take...e/jpeg=1, photo.take...e/jpeg=3 | ✅ | +| AGG-15 | `mediatype:audio` reads `nested aggregations cover every match on a page of one` | 1 of 7 matches, audio.arti... Floyd=2, audio.arti...Bomber=2, audio.arti...Spades=1, audio.arti...e Wall=2, audio.arti...¶rhead=3, audio.year sum=13942 | 1 of 7 matches, audio.arti... Floyd=2, audio.arti...Bomber=2, audio.arti...Spades=1, audio.arti...e Wall=2, audio.arti...¶rhead=3, audio.year sum=13942 | 1 of 7 matches, audio.arti... Floyd=2, audio.arti...Bomber=2, audio.arti...Spades=1, audio.arti...e Wall=2, audio.arti...¶rhead=3, audio.year sum=13942 | ✅ | +| AGG-16 | `mediatype:audio` reads `malformed date range bound in a nested aggregation` | error | error | error | ✅ | diff --git a/services/search/pkg/parity/aggregations_test.go b/services/search/pkg/parity/aggregations_test.go index f708027562..f68cebcec0 100644 --- a/services/search/pkg/parity/aggregations_test.go +++ b/services/search/pkg/parity/aggregations_test.go @@ -25,6 +25,8 @@ type aggCase struct { want []string wantError bool engineOverrides map[string]override + // pageSize, when set, prefixes the answer with " of matches" + pageSize int32 } func (c aggCase) label() string { return fmt.Sprintf("AGG-%02d", c.id) } @@ -127,11 +129,77 @@ func aggregationCases() []aggCase { &searchService.BucketRange{From: "2018-08-11T00:00:00Z", To: "not-a-date"}, )}}, wantError: true, want: []string{"error"}}, + {id: 10, query: "mediatype:audio", reads: "album buckets nested in artist buckets", + aggs: []*searchService.AggregationOption{{Field: "audio.artist", SubAggregations: []*searchService.AggregationOption{{Field: "audio.album"}}}}, + want: []string{ + "audio.artist Pink Floyd=2", "audio.artist Pink Floyd=2 / audio.album The Wall=2", + "audio.artist Motörhead=3", "audio.artist Motörhead=3 / audio.album Bomber=2", "audio.artist Motörhead=3 / audio.album Ace of Spades=1", + }}, + {id: 11, query: "mediatype:audio", reads: "sum and avg of audio.year per artist", + aggs: []*searchService.AggregationOption{{Field: "audio.artist", SubAggregations: []*searchService.AggregationOption{ + {Field: "audio.year", MetricKind: searchService.MetricKind_METRIC_KIND_SUM}, + {Field: "audio.year", MetricKind: searchService.MetricKind_METRIC_KIND_AVG}, + }}}, + want: []string{ + "audio.artist Pink Floyd=2", "audio.artist Pink Floyd=2 / audio.year sum=3946", "audio.artist Pink Floyd=2 / audio.year avg sum=3946 count=2", + "audio.artist Motörhead=3", "audio.artist Motörhead=3 / audio.year sum=5982", "audio.artist Motörhead=3 / audio.year avg sum=5982 count=3", + }}, + {id: 12, query: "mediatype:audio", reads: "artist buckets nested in audio.year decades", + aggs: []*searchService.AggregationOption{{Field: "audio.year", BucketDefinition: ranges( + &searchService.BucketRange{From: "1970", To: "1980"}, + &searchService.BucketRange{From: "1980", To: "1990"}, + &searchService.BucketRange{From: "1990", To: "2000"}, + &searchService.BucketRange{From: "2000", To: "2010"}, + ), SubAggregations: []*searchService.AggregationOption{{Field: "audio.artist"}}}}, + want: []string{ + "audio.year 1970-1980=2", "audio.year 1970-1980=2 / audio.artist Pink Floyd=2", + "audio.year 1980-1990=1", "audio.year 1980-1990=1 / audio.artist Motörhead=1", + "audio.year 1990-2000=1", "audio.year 1990-2000=1 / audio.artist Motörhead=1", + "audio.year 2000-2010=3", "audio.year 2000-2010=3 / audio.artist Motörhead=1", + }}, + {id: 13, query: "mediatype:audio", reads: "max audio.year per album per artist, three levels", + aggs: []*searchService.AggregationOption{{Field: "audio.artist", SubAggregations: []*searchService.AggregationOption{ + {Field: "audio.album", SubAggregations: []*searchService.AggregationOption{ + {Field: "audio.year", MetricKind: searchService.MetricKind_METRIC_KIND_MAX}, + }}, + }}}, + want: []string{ + "audio.artist Pink Floyd=2", "audio.artist Pink Floyd=2 / audio.album The Wall=2", "audio.artist Pink Floyd=2 / audio.album The Wall=2 / audio.year max=1975", + "audio.artist Motörhead=3", "audio.artist Motörhead=3 / audio.album Bomber=2", "audio.artist Motörhead=3 / audio.album Bomber=2 / audio.year max=1999", + "audio.artist Motörhead=3 / audio.album Ace of Spades=1", "audio.artist Motörhead=3 / audio.album Ace of Spades=1 / audio.year max=2001", + }}, + {id: 14, query: "mediatype:image", reads: "MimeType buckets nested in open-ended date ranges", + aggs: []*searchService.AggregationOption{{Field: "photo.takenDateTime", BucketDefinition: ranges( + &searchService.BucketRange{To: "2019-01-01"}, + &searchService.BucketRange{From: "2019-01-01"}, + ), SubAggregations: []*searchService.AggregationOption{{Field: "MimeType"}}}}, + want: []string{ + "photo.takenDateTime -2019-01-01=3", "photo.takenDateTime -2019-01-01=3 / MimeType image/jpeg=3", + "photo.takenDateTime 2019-01-01-=1", "photo.takenDateTime 2019-01-01-=1 / MimeType image/jpeg=1", + }}, + {id: 15, query: "mediatype:audio", reads: "nested aggregations cover every match on a page of one", + pageSize: 1, + aggs: []*searchService.AggregationOption{ + {Field: "audio.artist", SubAggregations: []*searchService.AggregationOption{{Field: "audio.album"}}}, + {Field: "audio.year", MetricKind: searchService.MetricKind_METRIC_KIND_SUM}, + }, + want: []string{ + "1 of 7 matches", + "audio.artist Pink Floyd=2", "audio.artist Pink Floyd=2 / audio.album The Wall=2", + "audio.artist Motörhead=3", "audio.artist Motörhead=3 / audio.album Bomber=2", "audio.artist Motörhead=3 / audio.album Ace of Spades=1", + "audio.year sum=13942", + }}, + {id: 16, query: "mediatype:audio", reads: "malformed date range bound in a nested aggregation", + aggs: []*searchService.AggregationOption{{Field: "audio.artist", SubAggregations: []*searchService.AggregationOption{ + {Field: "photo.takenDateTime", BucketDefinition: ranges(&searchService.BucketRange{From: "2018-08-11T00:00:00Z", To: "not-a-date"})}, + }}}, + wantError: true, want: []string{"error"}}, } } // renderAggregations flattens an answer into comparable strings; buckets with // no hits are dropped, the engines differ in whether they emit them at all. +// A nested result renders under its bucket, joined by " / ". func renderAggregations(resp *searchService.SearchIndexResponse, err error) []string { if err != nil { return []string{"error"} @@ -139,21 +207,30 @@ func renderAggregations(resp *searchService.SearchIndexResponse, err error) []st out := []string{} for _, a := range resp.Aggregations { - if a.MetricKind != searchService.MetricKind_METRIC_KIND_UNSPECIFIED { - kind := strings.ToLower(strings.TrimPrefix(a.MetricKind.String(), "METRIC_KIND_")) - if a.MetricKind == searchService.MetricKind_METRIC_KIND_AVG { - out = append(out, fmt.Sprintf("%s avg sum=%v count=%d", a.Field, a.Sum, a.Count)) - continue - } - out = append(out, fmt.Sprintf("%s %s=%v", a.Field, kind, a.Value)) + out = append(out, renderAggregation("", a)...) + } + + return out +} + +func renderAggregation(prefix string, a *searchService.AggregationResult) []string { + if a.MetricKind != searchService.MetricKind_METRIC_KIND_UNSPECIFIED { + kind := strings.ToLower(strings.TrimPrefix(a.MetricKind.String(), "METRIC_KIND_")) + if a.MetricKind == searchService.MetricKind_METRIC_KIND_AVG { + return []string{prefix + fmt.Sprintf("%s avg sum=%v count=%d", a.Field, a.Sum, a.Count)} + } + return []string{prefix + fmt.Sprintf("%s %s=%v", a.Field, kind, a.Value)} + } + + out := []string{} + for _, b := range a.Buckets { + if b.Count == 0 { continue } - - for _, b := range a.Buckets { - if b.Count == 0 { - continue - } - out = append(out, fmt.Sprintf("%s %s=%d", a.Field, b.Key, b.Count)) + line := prefix + fmt.Sprintf("%s %s=%d", a.Field, b.Key, b.Count) + out = append(out, line) + for _, sub := range b.SubAggregations { + out = append(out, renderAggregation(line+" / ", sub)...) } } @@ -188,9 +265,13 @@ var _ = Describe("Aggregations", func() { resp, err := e.backend.Search(context.Background(), &searchService.SearchIndexRequest{ Query: c.query, + PageSize: c.pageSize, Aggregations: c.aggs, }) answer := renderAggregations(resp, err) + if c.pageSize > 0 && err == nil { + answer = append([]string{fmt.Sprintf("%d of %d matches", len(resp.Matches), resp.TotalMatches)}, answer...) + } recordAnswer(row, name, answer) _, overridden := c.engineOverrides[name]