feat(search): geohash aggregation on the bleve backend

A geohash_precision aggregation runs as a terms facet on the geohash sibling of the geopoint, restricted to the terms tagged with the requested precision; nested, it folds the same terms from doc values. The cells match OpenSearch's geohash_grid, pinned in the parity suite.

services/search/pkg/bleve/aggregations.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"sort"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/blevesearch/bleve/v2"
@@ -15,6 +16,7 @@ import (
 	index "github.com/blevesearch/bleve_index_api"
 
 	searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
+	searchQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query"
 )
 
 // Bleve facets count one field and cannot nest, so metrics and
@@ -30,11 +32,36 @@ func collected(agg *searchService.AggregationOption) bool {
 	return agg.GetMetricKind() != searchService.MetricKind_METRIC_KIND_UNSPECIFIED || len(agg.GetSubAggregations()) > 0
 }
 
+// geohashLevel resolves a geohash aggregation to the geohash sibling field of
+// its geopoint and the term prefix of the requested precision: the sibling
+// holds one depth-tagged term per precision (see geohash.go), so the terms
+// with the prefix "<precision>/" are the cells of that precision.
+func geohashLevel(agg *searchService.AggregationOption) (field, prefix string, err error) {
+	p := int(agg.GetGeohashPrecision())
+	if p < 1 || p > geohashPrecision {
+		return "", "", fmt.Errorf("geohash precision %d out of range 1-%d", p, geohashPrecision)
+	}
+	base, ok := searchQuery.ResolveGeopointField(agg.GetField())
+	if !ok {
+		return "", "", fmt.Errorf("geohash aggregation on non-geo field %q", agg.GetField())
+	}
+	return base + geohashSuffix, strconv.Itoa(p) + "/", nil
+}
+
 func newBleveFacetRequest(agg *searchService.AggregationOption) (*bleve.FacetRequest, error) {
 	size := int(agg.GetSize())
 	if size <= 0 {
 		size = defaultFacetSize
 	}
+	if agg.GetGeohashPrecision() != 0 {
+		field, prefix, err := geohashLevel(agg)
+		if err != nil {
+			return nil, err
+		}
+		fr := bleve.NewFacetRequest(field, size)
+		fr.TermPrefix = prefix
+		return fr, nil
+	}
 	fr := bleve.NewFacetRequest(agg.GetField(), size)
 	ranges := aggregationRanges(agg)
 	if rangesAreDates(ranges) {
@@ -137,8 +164,13 @@ func facetBuckets(fr *bleveSearch.FacetResult, agg *searchService.AggregationOpt
 		}
 		return buckets
 	}
+	// a geohash facet carries the depth tag in every term, the cell is the rest
+	var prefix string
+	if agg.GetGeohashPrecision() != 0 {
+		prefix = strconv.Itoa(int(agg.GetGeohashPrecision())) + "/"
+	}
 	for _, t := range fr.Terms.Terms() {
-		buckets = append(buckets, &searchService.Bucket{Key: t.Term, Count: int64(t.Count)})
+		buckets = append(buckets, &searchService.Bucket{Key: strings.TrimPrefix(t.Term, prefix), Count: int64(t.Count)})
 	}
 	return buckets
 }
@@ -147,6 +179,7 @@ type levelKind int
 
 const (
 	levelTerms levelKind = iota
+	levelGeohash
 	levelNumericRange
 	levelDateRange
 	levelMetric
@@ -164,17 +197,25 @@ type dateRange struct {
 
 type aggLevel struct {
 	opt      *searchService.AggregationOption
+	field    string // the indexed field the doc values are read from
 	kind     levelKind
+	prefix   string // geohash: the depth tag of the requested precision
 	numeric  []numericRange
 	dates    []dateRange
 	children []*aggLevel
 }
 
 func newAggLevel(opt *searchService.AggregationOption) (*aggLevel, error) {
-	l := &aggLevel{opt: opt}
+	l := &aggLevel{opt: opt, field: opt.GetField()}
 	switch {
 	case opt.GetMetricKind() != searchService.MetricKind_METRIC_KIND_UNSPECIFIED:
 		l.kind = levelMetric
+	case opt.GetGeohashPrecision() != 0:
+		field, prefix, err := geohashLevel(opt)
+		if err != nil {
+			return nil, err
+		}
+		l.kind, l.field, l.prefix = levelGeohash, field, prefix
 	case len(aggregationRanges(opt)) > 0:
 		ranges := aggregationRanges(opt)
 		if rangesAreDates(ranges) {
@@ -266,13 +307,13 @@ func newAggCollector(aggs []*searchService.AggregationOption) (*aggCollector, er
 }
 
 func (c *aggCollector) register(l *aggLevel) {
-	fv, ok := c.fields[l.opt.GetField()]
+	fv, ok := c.fields[l.field]
 	if !ok {
 		fv = &fieldValues{}
-		c.fields[l.opt.GetField()] = fv
-		c.fieldNames = append(c.fieldNames, l.opt.GetField())
+		c.fields[l.field] = fv
+		c.fieldNames = append(c.fieldNames, l.field)
 	}
-	if l.kind == levelTerms {
+	if l.kind == levelTerms || l.kind == levelGeohash {
 		fv.asTerms = true
 	} else {
 		fv.asNumbers = true
@@ -350,7 +391,7 @@ func (c *aggCollector) visit(field string, term []byte) {
 }
 
 func (c *aggCollector) fold(a *bucketAcc, l *aggLevel) {
-	fv := c.fields[l.opt.GetField()]
+	fv := c.fields[l.field]
 	switch l.kind {
 	case levelMetric:
 		for _, raw := range fv.numbers {
@@ -362,6 +403,12 @@ func (c *aggCollector) fold(a *bucketAcc, l *aggLevel) {
 				c.foldBucket(a, l, term)
 			}
 		}
+	case levelGeohash:
+		for _, term := range fv.terms {
+			if cell, ok := strings.CutPrefix(term, l.prefix); ok {
+				c.foldBucket(a, l, cell)
+			}
+		}
 	case levelNumericRange:
 		for _, raw := range fv.numbers {
 			v := numeric.Int64ToFloat64(raw)

services/search/pkg/parity/README.md

@@ -720,7 +720,7 @@ Fixtures:
 - `a.jpg`, MimeType = image/jpeg
 - `b.jpg`, MimeType = image/jpeg
 - `c.jpg`, MimeType = image/jpeg
-- `d.jpg`, MimeType = image/jpeg
+- ... and 4 more of the same
 
 | Case | Query | expected | bleve | OpenSearch | same? |
 |---|---|---|---|---|---|
@@ -740,3 +740,7 @@ Fixtures:
 | AGG-14 | `mediatype:image` reads `MimeType buckets nested in open-ended date ranges` | photo.take...-01-01=3, photo.take...01-01-=1, photo.take...e/jpeg=1, photo.take...e/jpeg=3 | photo.take...-01-01=3, photo.take...01-01-=1, photo.take...e/jpeg=1, photo.take...e/jpeg=3 | photo.take...-01-01=3, photo.take...01-01-=1, photo.take...e/jpeg=1, photo.take...e/jpeg=3 | ✅ |
 | AGG-15 | `mediatype:audio` reads `nested aggregations cover every match on a page of one` | 1 of 7 matches, audio.arti... Floyd=2, audio.arti...Bomber=2, audio.arti...Spades=1, audio.arti...e Wall=2, audio.arti...¶rhead=3, audio.year sum=13942 | 1 of 7 matches, audio.arti... Floyd=2, audio.arti...Bomber=2, audio.arti...Spades=1, audio.arti...e Wall=2, audio.arti...¶rhead=3, audio.year sum=13942 | 1 of 7 matches, audio.arti... Floyd=2, audio.arti...Bomber=2, audio.arti...Spades=1, audio.arti...e Wall=2, audio.arti...¶rhead=3, audio.year sum=13942 | ✅ |
 | AGG-16 | `mediatype:audio` reads `malformed date range bound in a nested aggregation` | error | error | error | ✅ |
+| AGG-17 | `mediatype:image` reads `geohash cells at precision 5` | location u33dc=1, location u4pru=2 | location u33dc=1, location u4pru=2 | location u33dc=1, location u4pru=2 | ✅ |
+| AGG-18 | `mediatype:image` reads `MimeType buckets nested in geohash cells at precision 3` | location u...e/jpeg=1, location u...e/jpeg=2, location u33=1, location u4p=2 | location u...e/jpeg=1, location u...e/jpeg=2, location u33=1, location u4p=2 | location u...e/jpeg=1, location u...e/jpeg=2, location u33=1, location u4p=2 | ✅ |
+| AGG-19 | `mediatype:image` reads `geohash aggregation on a field that is no geopoint` | error | error | error | ✅ |
+| AGG-20 | `mediatype:image` reads `geohash precision beyond 12` | error | error | error | ✅ |

services/search/pkg/parity/aggregations_test.go

@@ -65,9 +65,17 @@ func aggregationFixtures() []search.Resource {
 		withTaken("b.jpg", "2018-08-11T19:42:00Z"),
 		withTaken("c.jpg", "2018-09-01T12:00:00Z"),
 		withTaken("d.jpg", "2021-08-11T08:00:00Z"),
+		// two in one precision-5 cell (u4pru), one in another (u33dc)
+		withGeo("skagen-a.jpg", 57.64911, 10.40744),
+		withGeo("skagen-b.jpg", 57.6495, 10.4090),
+		withGeo("berlin.jpg", 52.52, 13.405),
 	}
 }
 
+func withGeo(name string, lat, lon float64) search.Resource {
+	return fixtureDoc(name, withMime("image/jpeg"), withLocation(&libregraph.GeoCoordinates{Latitude: &lat, Longitude: &lon}))
+}
+
 func aggregationCases() []aggCase {
 	ranges := func(rs ...*searchService.BucketRange) *searchService.BucketDefinition {
 		return &searchService.BucketDefinition{Ranges: rs}
@@ -194,6 +202,21 @@ func aggregationCases() []aggCase {
 				{Field: "photo.takenDateTime", BucketDefinition: ranges(&searchService.BucketRange{From: "2018-08-11T00:00:00Z", To: "not-a-date"})},
 			}}},
 			wantError: true, want: []string{"error"}},
+		{id: 17, query: "mediatype:image", reads: "geohash cells at precision 5",
+			aggs: []*searchService.AggregationOption{{Field: "location", GeohashPrecision: 5}},
+			want: []string{"location u4pru=2", "location u33dc=1"}},
+		{id: 18, query: "mediatype:image", reads: "MimeType buckets nested in geohash cells at precision 3",
+			aggs: []*searchService.AggregationOption{{Field: "location", GeohashPrecision: 3, SubAggregations: []*searchService.AggregationOption{{Field: "MimeType"}}}},
+			want: []string{
+				"location u4p=2", "location u4p=2 / MimeType image/jpeg=2",
+				"location u33=1", "location u33=1 / MimeType image/jpeg=1",
+			}},
+		{id: 19, query: "mediatype:image", reads: "geohash aggregation on a field that is no geopoint",
+			aggs:      []*searchService.AggregationOption{{Field: "MimeType", GeohashPrecision: 5}},
+			wantError: true, want: []string{"error"}},
+		{id: 20, query: "mediatype:image", reads: "geohash precision beyond 12",
+			aggs:      []*searchService.AggregationOption{{Field: "location", GeohashPrecision: 13}},
+			wantError: true, want: []string{"error"}},
 	}
 }
 

services/search/pkg/query/resolver.go

@@ -103,23 +103,31 @@ func FieldIsWordBroken(field string) bool {
 	return siblingFields()[field].Words
 }
 
-// geopointFields maps a lowercased KQL key to its indexed geopoint sibling field,
-// derived from the TypeGeopoint entries in the resource field overrides (e.g.
-// "location" -> "location_geopoint", "journey.start" -> "journey.start_geopoint").
+// geopointFields maps a lowercased KQL key to the field name of every
+// TypeGeopoint entry in the resource field overrides (e.g. "location" ->
+// "location", "journey.start" -> "journey.start"); the engines derive their
+// sibling fields from it.
 var geopointFields = sync.OnceValue(func() map[string]string {
 	out := map[string]string{}
 	for key, opts := range (search.Resource{}).SearchFieldOverrides() {
 		if opts.Type == mapping.TypeGeopoint {
-			out[strings.ToLower(key)] = key + mapping.GeopointSuffix
+			out[strings.ToLower(key)] = key
 		}
 	}
 	return out
 })
 
-// ResolveGeoField maps a KQL key to its indexed geopoint field name. ok is false
-// when the key is not a geopoint field, so callers can reject geo predicates on
-// non-geo fields.
-func ResolveGeoField(name string) (string, bool) {
+// ResolveGeopointField maps a KQL key to the name of the geopoint field it
+// addresses. ok is false when the key is not a geopoint field, so callers can
+// reject geo predicates on non-geo fields.
+func ResolveGeopointField(name string) (string, bool) {
 	f, ok := geopointFields()[strings.ToLower(name)]
 	return f, ok
 }
+
+// ResolveGeoField maps a KQL key to its indexed geopoint sibling field name
+// (e.g. "location" -> "location_geopoint").
+func ResolveGeoField(name string) (string, bool) {
+	f, ok := ResolveGeopointField(name)
+	return f + mapping.GeopointSuffix, ok
+}