feat(search): terms aggregations

Term buckets on both engines: bleve facets, OpenSearch terms aggregations, buckets merged across spaces and post-processed per BucketDefinition (minimum count, sort, size). Terms aggregations on numeric fields are rejected at the graph endpoint. Pinned in the parity suite as AGG-01 to AGG-03. Ranges, metrics and sub-aggregations are declined until the engines evaluate them.
This commit is contained in:
Dominik Schmidt committed 2026-09-12 16:34:01 +00:00
1 parent ec899aefd9
commit ec40f05943
16 files changed
+751

No files matched your search

@@ -2,6 +2,7 @@ package svc
import (
"context"
"fmt"
"net/http"
"path"
"slices"
@@ -18,6 +19,7 @@ import (
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
// SearchQuery runs the search requests and returns results grouped by request
@@ -35,6 +37,13 @@ func (g Graph) SearchQuery(w http.ResponseWriter, r *http.Request) {
return
}
for _, sr := range req.Requests {
if err := validateAggregations(sr.Aggregations); err != nil {
errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, err.Error())
return
}
}
th := r.Header.Get(revaCtx.TokenHeader)
ctx := revaCtx.ContextSetToken(r.Context(), th)
ctx = metadata.Set(ctx, revaCtx.TokenHeader, th)
@@ -138,6 +147,23 @@ func clampPagination(fromP, sizeP *int32) (int32, int32) {
return from, size
}
// validateAggregations rejects terms aggregations on numeric/time fields: bleve
// indexes them as prefix-coded binary, so term buckets are meaningless.
// Classification via search.IsNumericField.
func validateAggregations(aggs []libregraph.AggregationOption) error {
for _, a := range aggs {
if !search.IsNumericField(a.Field) {
continue
}
if a.LibreGraphMetricDefinition != nil {
// metrics reduce numeric values, no term buckets involved
continue
}
return fmt.Errorf("terms aggregation is not supported on numeric field %q", a.Field)
}
return nil
}
func libregraphAggregationsToSearch(in []libregraph.AggregationOption) []*searchsvc.AggregationOption {
if len(in) == 0 {
return nil
@@ -197,4 +197,22 @@ var _ = ginkgo.Describe("SearchQuery", func() {
ginkgo.Entry("oversized size clamps to max", int32Ptr(0), int32Ptr(1000), int32(0), int32(500)),
ginkgo.Entry("from+size overflow collapses", int32Ptr(1<<31-1), int32Ptr(500), int32(1<<31-1-500), int32(500)),
)
ginkgo.It("rejects a terms aggregation on a numeric field with 400", func() {
g := graphWithSearch(stubSearchService{
search: func(*searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) {
ginkgo.Fail("search service must not be called when validation fails")
return nil, nil
},
})
rr := postSearchQuery(g, `{
"requests": [{
"entityTypes": ["driveItem"],
"query": {"queryString": "mediatype:audio"},
"size": 0,
"aggregations": [{"field": "audio.year"}]
}]
}`)
Expect(rr.Code).To(Equal(http.StatusBadRequest), rr.Body.String())
})
})
+52
View File
@@ -0,0 +1,52 @@
package bleve
import (
"github.com/blevesearch/bleve/v2"
bleveSearch "github.com/blevesearch/bleve/v2/search"
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
)
// defaultFacetSize is used when no size is requested; the service layer trims
// after cross-space merge.
const defaultFacetSize = 1000
// collected reports an aggregation bleve facets cannot answer: a metric or one
// with sub-aggregations. Facets count one field and cannot nest.
func collected(agg *searchService.AggregationOption) bool {
return agg.GetMetricKind() != searchService.MetricKind_METRIC_KIND_UNSPECIFIED || len(agg.GetSubAggregations()) > 0
}
func newBleveFacetRequest(agg *searchService.AggregationOption) *bleve.FacetRequest {
size := int(agg.GetSize())
if size <= 0 {
size = defaultFacetSize
}
return bleve.NewFacetRequest(agg.GetField(), size)
}
func facetBuckets(fr *bleveSearch.FacetResult) []*searchService.Bucket {
buckets := make([]*searchService.Bucket, 0)
for _, t := range fr.Terms.Terms() {
buckets = append(buckets, &searchService.Bucket{Key: t.Term, Count: int64(t.Count)})
}
return buckets
}
func extractBleveAggregations(res *bleve.SearchResult, aggs []*searchService.AggregationOption) []*searchService.AggregationResult {
if len(aggs) == 0 {
return nil
}
out := make([]*searchService.AggregationResult, 0, len(aggs))
for _, agg := range aggs {
fr, ok := res.Facets[agg.GetField()]
if !ok {
continue
}
out = append(out, &searchService.AggregationResult{
Field: agg.GetField(),
Buckets: facetBuckets(fr),
})
}
return out
}
+9
View File
@@ -2,6 +2,7 @@ package bleve
import (
"context"
"errors"
"math"
"time"
@@ -94,6 +95,13 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
bleveReq.Size = int(sir.PageSize)
}
for _, agg := range sir.GetAggregations() {
if collected(agg) {
return nil, errors.New("metric and nested aggregations are not supported by bleve yet")
}
bleveReq.AddFacet(agg.GetField(), newBleveFacetRequest(agg))
}
bleveReq.Fields = []string{"*"}
res, err := b.index.Search(bleveReq)
if err != nil {
@@ -151,6 +159,7 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
return &searchService.SearchIndexResponse{
Matches: matches,
TotalMatches: int32(totalMatches),
Aggregations: extractBleveAggregations(res, sir.GetAggregations()),
}, nil
}
+13
View File
@@ -17,6 +17,7 @@ import (
"github.com/opencloud-eu/opencloud/pkg/log"
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/aggs"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
@@ -122,6 +123,11 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
searchParams.Size = conversions.ToPointer(int(sir.PageSize))
}
builtAggs, err := aggs.Build(sir.GetAggregations())
if err != nil {
return nil, err
}
req, err := osu.BuildSearchReq(&opensearchgoAPI.SearchReq{
Indices: []string{b.index},
Params: searchParams,
@@ -140,6 +146,7 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
},
},
},
Aggs: builtAggs,
},
)
if err != nil {
@@ -162,9 +169,15 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
matches = append(matches, match)
}
aggResults, err := aggs.Parse(resp.Aggregations, sir.GetAggregations())
if err != nil {
return nil, fmt.Errorf("failed to parse aggregations: %w", err)
}
return &searchService.SearchIndexResponse{
Matches: matches,
TotalMatches: int32(totalMatches),
Aggregations: aggResults,
}, nil
}
@@ -0,0 +1,147 @@
// Package aggs translates proto aggregation options into the OpenSearch
// aggregation DSL and parses the response. Internal subpackage so its unit
// tests skip the parent package's Docker OpenSearch container.
package aggs
import (
"encoding/json"
"fmt"
"strconv"
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
)
// DefaultFacetSize matches the bleve backend: pull a generous bucket count per
// space, the service layer trims to top N after cross-space merge.
const DefaultFacetSize = 1000
// Build translates AggregationOptions into the OpenSearch aggregation DSL
// (terms). Entries get an index-derived
// name so repeated aggs on one field don't collide. A range bound that is
// neither a number nor a date is an error.
func Build(opts []*searchsvc.AggregationOption) (map[string]any, error) {
return buildLevel(opts, "a")
}
func buildLevel(opts []*searchsvc.AggregationOption, prefix string) (map[string]any, error) {
if len(opts) == 0 {
return nil, nil
}
aggs := map[string]any{}
for i, opt := range opts {
name := fmt.Sprintf("%s_%d", prefix, i)
entry, err := buildOne(opt)
if err != nil {
return nil, err
}
if entry != nil {
aggs[name] = entry
}
}
if len(aggs) == 0 {
return nil, nil
}
return aggs, nil
}
func buildOne(opt *searchsvc.AggregationOption) (map[string]any, error) {
field := opt.GetField()
size := int(opt.GetSize())
if size <= 0 {
size = DefaultFacetSize
}
return map[string]any{
"terms": map[string]any{
"field": field,
"size": size,
},
}, nil
}
func Parse(raw json.RawMessage, opts []*searchsvc.AggregationOption) ([]*searchsvc.AggregationResult, error) {
if len(raw) == 0 || len(opts) == 0 {
return nil, nil
}
node, err := parseNode(raw)
if err != nil {
return nil, err
}
return parseLevel(node, opts, "a"), nil
}
// aggNode is a lazily-decoded cursor over one level of the aggs response.
type aggNode map[string]json.RawMessage
func parseNode(raw json.RawMessage) (aggNode, error) {
var m aggNode
if err := json.Unmarshal(raw, &m); err != nil {
return nil, fmt.Errorf("decode opensearch aggregations: %w", err)
}
return m, nil
}
func parseLevel(node aggNode, opts []*searchsvc.AggregationOption, prefix string) []*searchsvc.AggregationResult {
out := make([]*searchsvc.AggregationResult, 0, len(opts))
for i, opt := range opts {
name := fmt.Sprintf("%s_%d", prefix, i)
raw, ok := node[name]
if !ok {
continue
}
if res := parseOne(raw, opt); res != nil {
out = append(out, res)
}
}
return out
}
func parseOne(raw json.RawMessage, opt *searchsvc.AggregationOption) *searchsvc.AggregationResult {
field := opt.GetField()
var body struct {
Buckets []json.RawMessage `json:"buckets"`
}
if err := json.Unmarshal(raw, &body); err != nil {
return nil
}
buckets := make([]*searchsvc.Bucket, 0, len(body.Buckets))
for _, b := range body.Buckets {
if bucket := parseBucket(b); bucket != nil {
buckets = append(buckets, bucket)
}
}
return &searchsvc.AggregationResult{
Field: field,
Buckets: buckets,
}
}
func parseBucket(raw json.RawMessage) *searchsvc.Bucket {
var head struct {
Key any `json:"key"`
DocCount int64 `json:"doc_count"`
}
if err := json.Unmarshal(raw, &head); err != nil {
return nil
}
b := &searchsvc.Bucket{
Key: bucketKeyToString(head.Key),
Count: head.DocCount,
}
return b
}
func bucketKeyToString(v any) string {
switch x := v.(type) {
case string:
return x
case float64:
// format without trailing zeros so keys match filter values
return strconv.FormatFloat(x, 'f', -1, 64)
case bool:
return strconv.FormatBool(x)
case nil:
return ""
default:
return ""
}
}
@@ -0,0 +1,13 @@
package aggs_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestAggs(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Aggs Suite")
}
@@ -0,0 +1,61 @@
package aggs_test
import (
"encoding/json"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/aggs"
)
var _ = Describe("Build", func() {
build := func(opts []*searchsvc.AggregationOption) map[string]any {
res, err := aggs.Build(opts)
Expect(err).ToNot(HaveOccurred())
return res
}
It("builds a terms aggregation", func() {
res := build([]*searchsvc.AggregationOption{
{Field: "audio.artist", Size: 10},
})
Expect(res).ToNot(BeNil())
entry, ok := res["a_0"].(map[string]any)
Expect(ok).To(BeTrue())
terms, ok := entry["terms"].(map[string]any)
Expect(ok).To(BeTrue())
Expect(terms["field"]).To(Equal("audio.artist"))
Expect(terms["size"]).To(Equal(10))
})
})
var _ = Describe("Parse", func() {
It("parses term buckets, stringifying numeric keys", func() {
raw := json.RawMessage(`{
"a_0": {"buckets": [
{"key": "Pink Floyd", "doc_count": 42},
{"key": "Motörhead", "doc_count": 35}
]},
"a_1": {"buckets": [
{"key": 9, "doc_count": 3}
]}
}`)
out, err := aggs.Parse(raw, []*searchsvc.AggregationOption{
{Field: "audio.artist"},
{Field: "audio.track"},
})
Expect(err).ToNot(HaveOccurred())
Expect(out).To(HaveLen(2))
Expect(out[0].Field).To(Equal("audio.artist"))
Expect(out[0].Buckets).To(HaveLen(2))
Expect(out[0].Buckets[0].Key).To(Equal("Pink Floyd"))
Expect(out[0].Buckets[0].Count).To(Equal(int64(42)))
// numeric term key stringified without trailing zeros
Expect(out[1].Buckets[0].Key).To(Equal("9"))
})
})
@@ -95,6 +95,7 @@ func BuildSearchReq(req *opensearchgoAPI.SearchReq, q Builder, p ...SearchBodyPa
type SearchBodyParams struct {
Highlight *BodyParamHighlight `json:"highlight,omitempty"`
Aggs map[string]any `json:"aggs,omitempty"`
}
//----------------------------------------------------------------------------//
+24
View File
@@ -703,3 +703,27 @@ Fixtures:
| METADATA-01 | `*song*` reads `Audio` | all 16 fields unchanged | all 16 fields unchanged | all 16 fields unchanged | ✅ |
| METADATA-02 | `*team*` reads `Location` | all 3 fields unchanged | all 3 fields unchanged | all 3 fields unchanged | ✅ |
| METADATA-03 | `*team*` reads `Audio` | none | none | none | ✅ |
## Aggregations
### aggregations
Fixtures:
- `a.mp3`, MimeType = audio/mpeg
- `b.mp3`, MimeType = audio/mpeg
- `c.mp3`, MimeType = audio/mpeg
- `d.mp3`, MimeType = audio/mpeg
- `e.mp3`, MimeType = audio/mpeg
- `f.mp3`, MimeType = audio/mpeg
- `g.mp3`, MimeType = audio/mpeg
- `a.jpg`, MimeType = image/jpeg
- `b.jpg`, MimeType = image/jpeg
- `c.jpg`, MimeType = image/jpeg
- `d.jpg`, MimeType = image/jpeg
| Case | Query | expected | bleve | OpenSearch | same? |
|---|---|---|---|---|---|
| AGG-01 | `mediatype:audio` reads `term buckets on audio.artist` | audio.arti... Floyd=2, audio.arti...¶rhead=3 | audio.arti... Floyd=2, audio.arti...¶rhead=3 | audio.arti... Floyd=2, audio.arti...¶rhead=3 | ✅ |
| AGG-02 | `mediatype:audio` reads `no aggregations requested` | no match | no match | no match | ✅ |
| AGG-03 | `mediatype:audio` reads `artist and album buckets in one request` | audio.albu...Spades=1, audio.album Bomber=2, audio.album The Wall=2, audio.arti... Floyd=2, audio.arti...¶rhead=3 | audio.albu...Spades=1, audio.album Bomber=2, audio.album The Wall=2, audio.arti... Floyd=2, audio.arti...¶rhead=3 | audio.albu...Spades=1, audio.album Bomber=2, audio.album The Wall=2, audio.arti... Floyd=2, audio.arti...¶rhead=3 | ✅ |
@@ -0,0 +1,168 @@
package parity
import (
"context"
"fmt"
"strings"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
// aggCase is one aggregation request both engines have to answer alike. The
// answer is rendered to strings (one per non-empty bucket or metric), so the
// matrix machinery can carry it like any query answer.
type aggCase struct {
id int
query string
aggs []*searchService.AggregationOption
reads string
want []string
wantError bool
engineOverrides map[string]override
}
func (c aggCase) label() string { return fmt.Sprintf("AGG-%02d", c.id) }
func withYear(name string, year int32) search.Resource {
return fixtureDoc(name, withMime("audio/mpeg"), withAudio(&libregraph.Audio{Year: libregraph.PtrInt32(year)}))
}
func withTaken(name, taken string) search.Resource {
t, err := time.Parse(time.RFC3339, taken)
if err != nil {
panic(err)
}
return fixtureDoc(name, withMime("image/jpeg"), withPhoto(&libregraph.Photo{TakenDateTime: &t}))
}
func song(name, artist, album string, year int32) search.Resource {
return fixtureDoc(name, withMime("audio/mpeg"), withAudio(&libregraph.Audio{
Artist: libregraph.PtrString(artist),
Album: libregraph.PtrString(album),
Year: libregraph.PtrInt32(year),
}))
}
func aggregationFixtures() []search.Resource {
return []search.Resource{
// years: 1971, 1975, 1982, 1999, 2001, 2005, 2009
song("a.mp3", "Pink Floyd", "The Wall", 1971),
song("b.mp3", "Pink Floyd", "The Wall", 1975),
song("c.mp3", "Motörhead", "Bomber", 1982),
song("d.mp3", "Motörhead", "Bomber", 1999),
song("e.mp3", "Motörhead", "Ace of Spades", 2001),
withYear("f.mp3", 2005),
withYear("g.mp3", 2009),
withTaken("a.jpg", "2018-08-11T09:15:00Z"),
withTaken("b.jpg", "2018-08-11T19:42:00Z"),
withTaken("c.jpg", "2018-09-01T12:00:00Z"),
withTaken("d.jpg", "2021-08-11T08:00:00Z"),
}
}
func aggregationCases() []aggCase {
return []aggCase{
{id: 1, query: "mediatype:audio", reads: "term buckets on audio.artist",
aggs: []*searchService.AggregationOption{{Field: "audio.artist", Size: 10}},
want: []string{"audio.artist Pink Floyd=2", "audio.artist Motörhead=3"}},
{id: 2, query: "mediatype:audio", reads: "no aggregations requested"},
{id: 3, query: "mediatype:audio", reads: "artist and album buckets in one request",
aggs: []*searchService.AggregationOption{{Field: "audio.artist"}, {Field: "audio.album"}},
want: []string{
"audio.artist Pink Floyd=2", "audio.artist Motörhead=3",
"audio.album The Wall=2", "audio.album Bomber=2", "audio.album Ace of Spades=1",
}},
}
}
// renderAggregations flattens an answer into comparable strings; buckets with
// no hits are dropped, the engines differ in whether they emit them at all.
// A nested result renders under its bucket, joined by " / ".
func renderAggregations(resp *searchService.SearchIndexResponse, err error) []string {
if err != nil {
return []string{"error"}
}
out := []string{}
for _, a := range resp.Aggregations {
out = append(out, renderAggregation("", a)...)
}
return out
}
func renderAggregation(prefix string, a *searchService.AggregationResult) []string {
if a.MetricKind != searchService.MetricKind_METRIC_KIND_UNSPECIFIED {
kind := strings.ToLower(strings.TrimPrefix(a.MetricKind.String(), "METRIC_KIND_"))
if a.MetricKind == searchService.MetricKind_METRIC_KIND_AVG {
return []string{prefix + fmt.Sprintf("%s avg sum=%v count=%d", a.Field, a.Sum, a.Count)}
}
return []string{prefix + fmt.Sprintf("%s %s=%v", a.Field, kind, a.Value)}
}
out := []string{}
for _, b := range a.Buckets {
if b.Count == 0 {
continue
}
line := prefix + fmt.Sprintf("%s %s=%d", a.Field, b.Key, b.Count)
out = append(out, line)
for _, sub := range b.SubAggregations {
out = append(out, renderAggregation(line+" / ", sub)...)
}
}
return out
}
var _ = Describe("Aggregations", func() {
Describe("aggregations", Ordered, ContinueOnFailure, func() {
var engines []testEngine
BeforeAll(func() {
engines = newEngines("opencloud-test-engine-parity-aggregations", aggregationFixtures())
})
for caseAt, c := range aggregationCases() {
row := matrixRow{
Section: "Aggregations", Group: "aggregations", ID: c.label(),
Query: c.query, Reads: c.reads,
Want: c.want, Overrides: renderOverrides(c.engineOverrides),
GroupAt: 100, CaseAt: caseAt,
}
planRow(row)
Describe(c.label()+" "+c.reads, func() {
for _, name := range engineNames {
It("on "+name, func() {
e := engineNamed(engines, name)
if e.unavailable != "" {
recordSkip(row, name)
Skip(e.unavailable)
}
resp, err := e.backend.Search(context.Background(), &searchService.SearchIndexRequest{
Query: c.query,
Aggregations: c.aggs,
})
answer := renderAggregations(resp, err)
recordAnswer(row, name, answer)
_, overridden := c.engineOverrides[name]
if !overridden && !c.wantError {
Expect(err).NotTo(HaveOccurred(), "the aggregation has to answer")
}
expectAnswer(name, answer, override{want: c.want}, c.engineOverrides)
})
}
})
}
})
})
@@ -50,6 +50,10 @@ func withAudio(audio *libregraph.Audio) fixtureOption {
return func(r *search.Resource) { r.Audio = audio }
}
func withPhoto(photo *libregraph.Photo) fixtureOption {
return func(r *search.Resource) { r.Photo = photo }
}
func withLocation(location *libregraph.GeoCoordinates) fixtureOption {
return func(r *search.Resource) { r.Location = location }
}
@@ -278,6 +278,10 @@ func matrixFixtures(group string) string {
fixtures = g.fixtures
}
if group == "aggregations" {
fixtures = aggregationFixtures()
}
if len(fixtures) == 0 {
return "Fixtures: none"
}
+71
View File
@@ -0,0 +1,71 @@
package search
import (
"reflect"
"strings"
"time"
)
// IsNumericField reports whether the indexed field at the dotted path holds a
// numeric or time value. Terms aggregations on those are rejected: bleve stores
// them as prefix-coded binary, so term buckets are meaningless. The set is built
// by walking the Resource type, so new facet fields are picked up automatically.
func IsNumericField(dottedPath string) bool {
return numericFields[dottedPath]
}
var numericFields = buildNumericFieldSet()
var timeType = reflect.TypeOf(time.Time{})
func buildNumericFieldSet() map[string]bool {
out := map[string]bool{}
walkStruct(out, "", reflect.TypeOf(Resource{}))
return out
}
func walkStruct(out map[string]bool, prefix string, t reflect.Type) {
for t.Kind() == reflect.Ptr {
t = t.Elem()
}
if t.Kind() != reflect.Struct {
return
}
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
if !f.IsExported() {
continue
}
if f.Anonymous {
// embedded: promote fields into the current prefix, like encoding/json.
walkStruct(out, prefix, f.Type)
continue
}
path := prefix + jsonFieldName(f)
ft := f.Type
for ft.Kind() == reflect.Ptr {
ft = ft.Elem()
}
switch ft.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
reflect.Float32, reflect.Float64:
out[path] = true
case reflect.Struct:
if ft == timeType {
// time.Time round-trips as RFC3339; treat as numeric.
out[path] = true
continue
}
walkStruct(out, path+".", ft)
}
}
}
func jsonFieldName(f reflect.StructField) string {
tag := f.Tag.Get("json")
if tag == "" {
return f.Name
}
return strings.Split(tag, ",")[0]
}
+46
View File
@@ -0,0 +1,46 @@
package search_test
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
var _ = Describe("IsNumericField", func() {
DescribeTable("reports whether a field maps to a numeric type",
func(field string, numeric bool) {
Expect(search.IsNumericField(field)).To(Equal(numeric))
},
// top-level numeric fields on Resource / Document
Entry("Size (uint64 via embedded Document)", "Size", true),
Entry("Type (uint64 on Resource)", "Type", true),
// top-level string fields
Entry("Name", "Name", false),
Entry("Path", "Path", false),
Entry("MimeType", "MimeType", false),
// nested audio
Entry("audio.artist", "audio.artist", false),
Entry("audio.album", "audio.album", false),
Entry("audio.year", "audio.year", true),
Entry("audio.bitrate", "audio.bitrate", true),
Entry("audio.track", "audio.track", true),
Entry("audio.hasDrm (bool, not numeric)", "audio.hasDrm", false),
// nested image
Entry("image.width", "image.width", true),
Entry("image.height", "image.height", true),
// nested photo
Entry("photo.cameraMake", "photo.cameraMake", false),
Entry("photo.iso", "photo.iso", true),
Entry("photo.focalLength (float32)", "photo.focalLength", true),
Entry("photo.exposureDenominator (float32)", "photo.exposureDenominator", true),
Entry("photo.takenDateTime (time.Time, treated as numeric)", "photo.takenDateTime", true),
// nested location
Entry("location.altitude", "location.altitude", true),
Entry("location.latitude", "location.latitude", true),
Entry("location.longitude", "location.longitude", true),
// unknown fields the caller may still aggregate on
Entry("nonexistent", "nonexistent", false),
Entry("audio.nonexistent", "audio.nonexistent", false),
)
})
+94
View File
@@ -97,6 +97,16 @@ func NewService(gatewaySelector pool.Selectable[gateway.GatewayAPIClient], eng E
// Search processes a search request and passes it down to the engine.
func (s *Service) Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error) {
// terms aggregations only for now: the engines do not evaluate ranges,
// metrics and sub-aggregations yet
for _, opt := range req.GetAggregations() {
switch {
case len(opt.GetBucketDefinition().GetRanges()) > 0:
return nil, errtypes.BadRequest("range aggregations are not supported yet")
case opt.GetMetricKind() != searchsvc.MetricKind_METRIC_KIND_UNSPECIFIED || len(opt.GetSubAggregations()) > 0:
return nil, errtypes.BadRequest("metric and nested aggregations are not supported yet")
}
}
s.logger.Debug().Str("query", req.Query).Msg("performing a search")
// collect metrics
@@ -287,6 +297,7 @@ func (s *Service) Search(ctx context.Context, req *searchsvc.SearchRequest) (*se
return nil, err
}
mergedAggregations := map[string]map[string]*searchmsgBucket{}
for _, res := range responses {
if res == nil {
continue
@@ -295,6 +306,22 @@ func (s *Service) Search(ctx context.Context, req *searchsvc.SearchRequest) (*se
for _, match := range res.Matches {
matches = append(matches, match)
}
for _, agg := range res.GetAggregations() {
field := agg.GetField()
if _, ok := mergedAggregations[field]; !ok {
mergedAggregations[field] = map[string]*searchmsgBucket{}
}
for _, b := range agg.GetBuckets() {
if existing, ok := mergedAggregations[field][b.GetKey()]; ok {
existing.Count += b.GetCount()
continue
}
mergedAggregations[field][b.GetKey()] = &searchsvc.Bucket{
Key: b.GetKey(),
Count: b.GetCount(),
}
}
}
}
// compile one sorted list of matches from all spaces and apply the limit if needed
@@ -307,13 +334,80 @@ func (s *Service) Search(ctx context.Context, req *searchsvc.SearchRequest) (*se
matches = matches[0:limit]
}
aggregations := make([]*searchsvc.AggregationResult, 0, len(req.GetAggregations()))
for _, opt := range req.GetAggregations() {
field := opt.GetField()
bucketMap := mergedAggregations[field]
buckets := make([]*searchsvc.Bucket, 0, len(bucketMap))
for _, b := range bucketMap {
buckets = append(buckets, b)
}
aggregations = append(aggregations, &searchsvc.AggregationResult{
Field: field,
Buckets: postProcessBuckets(buckets, opt),
})
}
success = true
return &searchsvc.SearchResponse{
Matches: matches,
TotalMatches: total,
Aggregations: aggregations,
}, nil
}
// searchmsgBucket aliases the bucket type for the map-of-maps below.
type searchmsgBucket = searchsvc.Bucket
// postProcessBuckets applies the BucketDefinition (minimumCount filter, sort by
// count/keyAsString/keyAsNumber, trim to Size). Defaults to count-descending.
func postProcessBuckets(buckets []*searchsvc.Bucket, opt *searchsvc.AggregationOption) []*searchsvc.Bucket {
bd := opt.GetBucketDefinition()
sortBy := "count"
desc := true
var minCount int64
if bd != nil {
if bd.GetSortBy() != "" {
sortBy = bd.GetSortBy()
}
desc = bd.GetIsDescending()
minCount = int64(bd.GetMinimumCount())
}
if minCount > 0 {
filtered := buckets[:0]
for _, b := range buckets {
if b.GetCount() >= minCount {
filtered = append(filtered, b)
}
}
buckets = filtered
}
sort.SliceStable(buckets, func(i, j int) bool {
less := false
switch sortBy {
case "keyAsString":
less = buckets[i].GetKey() < buckets[j].GetKey()
case "keyAsNumber":
iv, _ := strconv.ParseFloat(buckets[i].GetKey(), 64)
jv, _ := strconv.ParseFloat(buckets[j].GetKey(), 64)
less = iv < jv
default: // "count"
less = buckets[i].GetCount() < buckets[j].GetCount()
}
if desc {
return !less
}
return less
})
if size := opt.GetSize(); size > 0 && int32(len(buckets)) > size {
buckets = buckets[:size]
}
return buckets
}
func (s *Service) searchIndex(ctx context.Context, req *searchsvc.SearchRequest, space *provider.StorageSpace, mountpointID string) (*searchsvc.SearchIndexResponse, error) {
if req.Ref != nil &&
(req.Ref.ResourceId.StorageId != space.Root.StorageId ||