From d7bdce848efd4863743f74d16ca8988c0723fd58 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Fri, 11 Sep 2026 14:33:49 +0000 Subject: [PATCH] feat(search): geohash aggregation on the bleve backend A geohash_precision aggregation runs as a terms facet on the geohash sibling of the geopoint, restricted to the terms tagged with the requested precision; nested, it folds the same terms from doc values. The cells match OpenSearch's geohash_grid, pinned in the parity suite. --- services/search/pkg/bleve/aggregations.go | 61 ++++++++++++++++--- services/search/pkg/parity/README.md | 6 +- .../search/pkg/parity/aggregations_test.go | 23 +++++++ services/search/pkg/query/resolver.go | 24 +++++--- 4 files changed, 98 insertions(+), 16 deletions(-) diff --git a/services/search/pkg/bleve/aggregations.go b/services/search/pkg/bleve/aggregations.go index c18d7da013..63bdb5d1be 100644 --- a/services/search/pkg/bleve/aggregations.go +++ b/services/search/pkg/bleve/aggregations.go @@ -6,6 +6,7 @@ import ( "fmt" "sort" "strconv" + "strings" "time" "github.com/blevesearch/bleve/v2" @@ -15,6 +16,7 @@ import ( index "github.com/blevesearch/bleve_index_api" searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + searchQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query" ) // Bleve facets count one field and cannot nest, so metrics and @@ -30,11 +32,36 @@ func collected(agg *searchService.AggregationOption) bool { return agg.GetMetricKind() != searchService.MetricKind_METRIC_KIND_UNSPECIFIED || len(agg.GetSubAggregations()) > 0 } +// geohashLevel resolves a geohash aggregation to the geohash sibling field of +// its geopoint and the term prefix of the requested precision: the sibling +// holds one depth-tagged term per precision (see geohash.go), so the terms +// with the prefix "/" are the cells of that precision. +func geohashLevel(agg *searchService.AggregationOption) (field, prefix string, err error) { + p := int(agg.GetGeohashPrecision()) + if p < 1 || p > geohashPrecision { + return "", "", fmt.Errorf("geohash precision %d out of range 1-%d", p, geohashPrecision) + } + base, ok := searchQuery.ResolveGeopointField(agg.GetField()) + if !ok { + return "", "", fmt.Errorf("geohash aggregation on non-geo field %q", agg.GetField()) + } + return base + geohashSuffix, strconv.Itoa(p) + "/", nil +} + func newBleveFacetRequest(agg *searchService.AggregationOption) (*bleve.FacetRequest, error) { size := int(agg.GetSize()) if size <= 0 { size = defaultFacetSize } + if agg.GetGeohashPrecision() != 0 { + field, prefix, err := geohashLevel(agg) + if err != nil { + return nil, err + } + fr := bleve.NewFacetRequest(field, size) + fr.TermPrefix = prefix + return fr, nil + } fr := bleve.NewFacetRequest(agg.GetField(), size) ranges := aggregationRanges(agg) if rangesAreDates(ranges) { @@ -137,8 +164,13 @@ func facetBuckets(fr *bleveSearch.FacetResult, agg *searchService.AggregationOpt } return buckets } + // a geohash facet carries the depth tag in every term, the cell is the rest + var prefix string + if agg.GetGeohashPrecision() != 0 { + prefix = strconv.Itoa(int(agg.GetGeohashPrecision())) + "/" + } for _, t := range fr.Terms.Terms() { - buckets = append(buckets, &searchService.Bucket{Key: t.Term, Count: int64(t.Count)}) + buckets = append(buckets, &searchService.Bucket{Key: strings.TrimPrefix(t.Term, prefix), Count: int64(t.Count)}) } return buckets } @@ -147,6 +179,7 @@ type levelKind int const ( levelTerms levelKind = iota + levelGeohash levelNumericRange levelDateRange levelMetric @@ -164,17 +197,25 @@ type dateRange struct { type aggLevel struct { opt *searchService.AggregationOption + field string // the indexed field the doc values are read from kind levelKind + prefix string // geohash: the depth tag of the requested precision numeric []numericRange dates []dateRange children []*aggLevel } func newAggLevel(opt *searchService.AggregationOption) (*aggLevel, error) { - l := &aggLevel{opt: opt} + l := &aggLevel{opt: opt, field: opt.GetField()} switch { case opt.GetMetricKind() != searchService.MetricKind_METRIC_KIND_UNSPECIFIED: l.kind = levelMetric + case opt.GetGeohashPrecision() != 0: + field, prefix, err := geohashLevel(opt) + if err != nil { + return nil, err + } + l.kind, l.field, l.prefix = levelGeohash, field, prefix case len(aggregationRanges(opt)) > 0: ranges := aggregationRanges(opt) if rangesAreDates(ranges) { @@ -266,13 +307,13 @@ func newAggCollector(aggs []*searchService.AggregationOption) (*aggCollector, er } func (c *aggCollector) register(l *aggLevel) { - fv, ok := c.fields[l.opt.GetField()] + fv, ok := c.fields[l.field] if !ok { fv = &fieldValues{} - c.fields[l.opt.GetField()] = fv - c.fieldNames = append(c.fieldNames, l.opt.GetField()) + c.fields[l.field] = fv + c.fieldNames = append(c.fieldNames, l.field) } - if l.kind == levelTerms { + if l.kind == levelTerms || l.kind == levelGeohash { fv.asTerms = true } else { fv.asNumbers = true @@ -350,7 +391,7 @@ func (c *aggCollector) visit(field string, term []byte) { } func (c *aggCollector) fold(a *bucketAcc, l *aggLevel) { - fv := c.fields[l.opt.GetField()] + fv := c.fields[l.field] switch l.kind { case levelMetric: for _, raw := range fv.numbers { @@ -362,6 +403,12 @@ func (c *aggCollector) fold(a *bucketAcc, l *aggLevel) { c.foldBucket(a, l, term) } } + case levelGeohash: + for _, term := range fv.terms { + if cell, ok := strings.CutPrefix(term, l.prefix); ok { + c.foldBucket(a, l, cell) + } + } case levelNumericRange: for _, raw := range fv.numbers { v := numeric.Int64ToFloat64(raw) diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index 7135d7ec34..c00374abed 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -720,7 +720,7 @@ Fixtures: - `a.jpg`, MimeType = image/jpeg - `b.jpg`, MimeType = image/jpeg - `c.jpg`, MimeType = image/jpeg -- `d.jpg`, MimeType = image/jpeg +- ... and 4 more of the same | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| @@ -740,3 +740,7 @@ Fixtures: | AGG-14 | `mediatype:image` reads `MimeType buckets nested in open-ended date ranges` | photo.take...-01-01=3, photo.take...01-01-=1, photo.take...e/jpeg=1, photo.take...e/jpeg=3 | photo.take...-01-01=3, photo.take...01-01-=1, photo.take...e/jpeg=1, photo.take...e/jpeg=3 | photo.take...-01-01=3, photo.take...01-01-=1, photo.take...e/jpeg=1, photo.take...e/jpeg=3 | ✅ | | AGG-15 | `mediatype:audio` reads `nested aggregations cover every match on a page of one` | 1 of 7 matches, audio.arti... Floyd=2, audio.arti...Bomber=2, audio.arti...Spades=1, audio.arti...e Wall=2, audio.arti...¶rhead=3, audio.year sum=13942 | 1 of 7 matches, audio.arti... Floyd=2, audio.arti...Bomber=2, audio.arti...Spades=1, audio.arti...e Wall=2, audio.arti...¶rhead=3, audio.year sum=13942 | 1 of 7 matches, audio.arti... Floyd=2, audio.arti...Bomber=2, audio.arti...Spades=1, audio.arti...e Wall=2, audio.arti...¶rhead=3, audio.year sum=13942 | ✅ | | AGG-16 | `mediatype:audio` reads `malformed date range bound in a nested aggregation` | error | error | error | ✅ | +| AGG-17 | `mediatype:image` reads `geohash cells at precision 5` | location u33dc=1, location u4pru=2 | location u33dc=1, location u4pru=2 | location u33dc=1, location u4pru=2 | ✅ | +| AGG-18 | `mediatype:image` reads `MimeType buckets nested in geohash cells at precision 3` | location u...e/jpeg=1, location u...e/jpeg=2, location u33=1, location u4p=2 | location u...e/jpeg=1, location u...e/jpeg=2, location u33=1, location u4p=2 | location u...e/jpeg=1, location u...e/jpeg=2, location u33=1, location u4p=2 | ✅ | +| AGG-19 | `mediatype:image` reads `geohash aggregation on a field that is no geopoint` | error | error | error | ✅ | +| AGG-20 | `mediatype:image` reads `geohash precision beyond 12` | error | error | error | ✅ | diff --git a/services/search/pkg/parity/aggregations_test.go b/services/search/pkg/parity/aggregations_test.go index f68cebcec0..694dd99c37 100644 --- a/services/search/pkg/parity/aggregations_test.go +++ b/services/search/pkg/parity/aggregations_test.go @@ -65,9 +65,17 @@ func aggregationFixtures() []search.Resource { withTaken("b.jpg", "2018-08-11T19:42:00Z"), withTaken("c.jpg", "2018-09-01T12:00:00Z"), withTaken("d.jpg", "2021-08-11T08:00:00Z"), + // two in one precision-5 cell (u4pru), one in another (u33dc) + withGeo("skagen-a.jpg", 57.64911, 10.40744), + withGeo("skagen-b.jpg", 57.6495, 10.4090), + withGeo("berlin.jpg", 52.52, 13.405), } } +func withGeo(name string, lat, lon float64) search.Resource { + return fixtureDoc(name, withMime("image/jpeg"), withLocation(&libregraph.GeoCoordinates{Latitude: &lat, Longitude: &lon})) +} + func aggregationCases() []aggCase { ranges := func(rs ...*searchService.BucketRange) *searchService.BucketDefinition { return &searchService.BucketDefinition{Ranges: rs} @@ -194,6 +202,21 @@ func aggregationCases() []aggCase { {Field: "photo.takenDateTime", BucketDefinition: ranges(&searchService.BucketRange{From: "2018-08-11T00:00:00Z", To: "not-a-date"})}, }}}, wantError: true, want: []string{"error"}}, + {id: 17, query: "mediatype:image", reads: "geohash cells at precision 5", + aggs: []*searchService.AggregationOption{{Field: "location", GeohashPrecision: 5}}, + want: []string{"location u4pru=2", "location u33dc=1"}}, + {id: 18, query: "mediatype:image", reads: "MimeType buckets nested in geohash cells at precision 3", + aggs: []*searchService.AggregationOption{{Field: "location", GeohashPrecision: 3, SubAggregations: []*searchService.AggregationOption{{Field: "MimeType"}}}}, + want: []string{ + "location u4p=2", "location u4p=2 / MimeType image/jpeg=2", + "location u33=1", "location u33=1 / MimeType image/jpeg=1", + }}, + {id: 19, query: "mediatype:image", reads: "geohash aggregation on a field that is no geopoint", + aggs: []*searchService.AggregationOption{{Field: "MimeType", GeohashPrecision: 5}}, + wantError: true, want: []string{"error"}}, + {id: 20, query: "mediatype:image", reads: "geohash precision beyond 12", + aggs: []*searchService.AggregationOption{{Field: "location", GeohashPrecision: 13}}, + wantError: true, want: []string{"error"}}, } } diff --git a/services/search/pkg/query/resolver.go b/services/search/pkg/query/resolver.go index b5eafe51c6..cfab46a84b 100644 --- a/services/search/pkg/query/resolver.go +++ b/services/search/pkg/query/resolver.go @@ -103,23 +103,31 @@ func FieldIsWordBroken(field string) bool { return siblingFields()[field].Words } -// geopointFields maps a lowercased KQL key to its indexed geopoint sibling field, -// derived from the TypeGeopoint entries in the resource field overrides (e.g. -// "location" -> "location_geopoint", "journey.start" -> "journey.start_geopoint"). +// geopointFields maps a lowercased KQL key to the field name of every +// TypeGeopoint entry in the resource field overrides (e.g. "location" -> +// "location", "journey.start" -> "journey.start"); the engines derive their +// sibling fields from it. var geopointFields = sync.OnceValue(func() map[string]string { out := map[string]string{} for key, opts := range (search.Resource{}).SearchFieldOverrides() { if opts.Type == mapping.TypeGeopoint { - out[strings.ToLower(key)] = key + mapping.GeopointSuffix + out[strings.ToLower(key)] = key } } return out }) -// ResolveGeoField maps a KQL key to its indexed geopoint field name. ok is false -// when the key is not a geopoint field, so callers can reject geo predicates on -// non-geo fields. -func ResolveGeoField(name string) (string, bool) { +// ResolveGeopointField maps a KQL key to the name of the geopoint field it +// addresses. ok is false when the key is not a geopoint field, so callers can +// reject geo predicates on non-geo fields. +func ResolveGeopointField(name string) (string, bool) { f, ok := geopointFields()[strings.ToLower(name)] return f, ok } + +// ResolveGeoField maps a KQL key to its indexed geopoint sibling field name +// (e.g. "location" -> "location_geopoint"). +func ResolveGeoField(name string) (string, bool) { + f, ok := ResolveGeopointField(name) + return f + mapping.GeopointSuffix, ok +}