diff --git a/protogen/gen/opencloud/services/search/v0/search.pb.go b/protogen/gen/opencloud/services/search/v0/search.pb.go index 102555bc44..553e39b6ef 100644 --- a/protogen/gen/opencloud/services/search/v0/search.pb.go +++ b/protogen/gen/opencloud/services/search/v0/search.pb.go @@ -441,7 +441,7 @@ type AggregationOption struct { MetricKind MetricKind `protobuf:"varint,5,opt,name=metric_kind,json=metricKind,proto3,enum=opencloud.services.search.v0.MetricKind" json:"metric_kind,omitempty"` // Optional. When > 0, this is a geohash-grid aggregation over `field` (which // must resolve to a geo-point field) at the given precision (1-12). Buckets - // carry the geohash cell as key and its doc count. OpenSearch backend only. + // carry the geohash cell as key and its doc count. GeohashPrecision int32 `protobuf:"varint,6,opt,name=geohash_precision,json=geohashPrecision,proto3" json:"geohash_precision,omitempty"` } diff --git a/protogen/gen/opencloud/services/search/v0/search.swagger.json b/protogen/gen/opencloud/services/search/v0/search.swagger.json index 44cea6e68c..61c7e8d898 100644 --- a/protogen/gen/opencloud/services/search/v0/search.swagger.json +++ b/protogen/gen/opencloud/services/search/v0/search.swagger.json @@ -197,7 +197,7 @@ "geohashPrecision": { "type": "integer", "format": "int32", - "description": "Optional. When \u003e 0, this is a geohash-grid aggregation over `field` (which\nmust resolve to a geo-point field) at the given precision (1-12). Buckets\ncarry the geohash cell as key and its doc count. OpenSearch backend only." + "description": "Optional. When \u003e 0, this is a geohash-grid aggregation over `field` (which\nmust resolve to a geo-point field) at the given precision (1-12). Buckets\ncarry the geohash cell as key and its doc count." } } }, diff --git a/protogen/proto/opencloud/services/search/v0/search.proto b/protogen/proto/opencloud/services/search/v0/search.proto index 671150dff5..992db365e0 100644 --- a/protogen/proto/opencloud/services/search/v0/search.proto +++ b/protogen/proto/opencloud/services/search/v0/search.proto @@ -147,7 +147,7 @@ message AggregationOption { MetricKind metric_kind = 5 [(google.api.field_behavior) = OPTIONAL]; // Optional. When > 0, this is a geohash-grid aggregation over `field` (which // must resolve to a geo-point field) at the given precision (1-12). Buckets - // carry the geohash cell as key and its doc count. OpenSearch backend only. + // carry the geohash cell as key and its doc count. int32 geohash_precision = 6 [(google.api.field_behavior) = OPTIONAL]; } diff --git a/services/search/pkg/bleve/backend.go b/services/search/pkg/bleve/backend.go index 57539b1073..afb722e4f9 100644 --- a/services/search/pkg/bleve/backend.go +++ b/services/search/pkg/bleve/backend.go @@ -207,6 +207,15 @@ func newBleveFacetRequest(agg *searchService.AggregationOption) (*bleve.FacetReq if size <= 0 { size = defaultFacetSize } + // Geohash: terms facet on the precomputed geohash-prefix sibling field of + // the requested precision (bleve has no native geohash-grid aggregation). + if p := int(agg.GetGeohashPrecision()); p > 0 { + field, ok := searchQuery.ResolveGeohashField(agg.GetField(), p) + if !ok { + return nil, fmt.Errorf("geohash aggregation on non-geo field %q", agg.GetField()) + } + return bleve.NewFacetRequest(field, size), nil + } fr := bleve.NewFacetRequest(agg.GetField(), size) ranges := aggregationRanges(agg) if rangesAreDates(ranges) { diff --git a/services/search/pkg/bleve/testdata/mapping.golden.json b/services/search/pkg/bleve/testdata/mapping.golden.json index 61ea6474f1..d7777c7001 100644 --- a/services/search/pkg/bleve/testdata/mapping.golden.json +++ b/services/search/pkg/bleve/testdata/mapping.golden.json @@ -820,6 +820,174 @@ } } }, + "location_geohash_1": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "keyword", + "store": true, + "index": true, + "include_term_vectors": true, + "docvalues": true + } + ] + }, + "location_geohash_10": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "keyword", + "store": true, + "index": true, + "include_term_vectors": true, + "docvalues": true + } + ] + }, + "location_geohash_11": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "keyword", + "store": true, + "index": true, + "include_term_vectors": true, + "docvalues": true + } + ] + }, + "location_geohash_12": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "keyword", + "store": true, + "index": true, + "include_term_vectors": true, + "docvalues": true + } + ] + }, + "location_geohash_2": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "keyword", + "store": true, + "index": true, + "include_term_vectors": true, + "docvalues": true + } + ] + }, + "location_geohash_3": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "keyword", + "store": true, + "index": true, + "include_term_vectors": true, + "docvalues": true + } + ] + }, + "location_geohash_4": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "keyword", + "store": true, + "index": true, + "include_term_vectors": true, + "docvalues": true + } + ] + }, + "location_geohash_5": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "keyword", + "store": true, + "index": true, + "include_term_vectors": true, + "docvalues": true + } + ] + }, + "location_geohash_6": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "keyword", + "store": true, + "index": true, + "include_term_vectors": true, + "docvalues": true + } + ] + }, + "location_geohash_7": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "keyword", + "store": true, + "index": true, + "include_term_vectors": true, + "docvalues": true + } + ] + }, + "location_geohash_8": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "keyword", + "store": true, + "index": true, + "include_term_vectors": true, + "docvalues": true + } + ] + }, + "location_geohash_9": { + "enabled": true, + "dynamic": true, + "fields": [ + { + "type": "text", + "analyzer": "keyword", + "store": true, + "index": true, + "include_term_vectors": true, + "docvalues": true + } + ] + }, "location_geopoint": { "enabled": true, "dynamic": true, diff --git a/services/search/pkg/mapping/bleve.go b/services/search/pkg/mapping/bleve.go index 0a3a55c45b..767954c666 100644 --- a/services/search/pkg/mapping/bleve.go +++ b/services/search/pkg/mapping/bleve.go @@ -56,6 +56,13 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix } doc.AddSubDocumentMapping(fi.Name, subDoc) doc.AddFieldMappingsAt(fi.Name+GeopointSuffix, bleve.NewGeoPointFieldMapping()) + // Geohash prefix siblings power the terms-based geohash aggregation + // (bleve has no native geohash-grid). + for p := 1; p <= MaxGeohashPrecision; p++ { + gh := bleve.NewKeywordFieldMapping() + gh.IncludeInAll = false + doc.AddFieldMappingsAt(GeohashField(fi.Name, p), gh) + } return nil } diff --git a/services/search/pkg/mapping/geo.go b/services/search/pkg/mapping/geo.go index 3d0adf61a7..ba53bed1ca 100644 --- a/services/search/pkg/mapping/geo.go +++ b/services/search/pkg/mapping/geo.go @@ -1,6 +1,9 @@ package mapping -import "strings" +import ( + "strconv" + "strings" +) // GeopointSuffix is appended to a field's name to produce the sibling key // that carries the geo_point / bleve-geopoint representation of the @@ -49,3 +52,93 @@ func addGeopointSibling(m map[string]any, dottedPath string) { } parent[leaf+GeopointSuffix] = map[string]any{"lat": lat, "lon": lon} } + +// GeohashSuffix + a precision produce the sibling keyword field carrying the +// geohash prefix of a geopoint at that precision (e.g. "location_geohash_6"). +// bleve has no native geohash-grid aggregation, so a terms aggregation on the +// field of the requested precision reproduces it. OpenSearch runs geohash_grid +// on the _geopoint field directly and ignores these. +const GeohashSuffix = "_geohash_" + +// MaxGeohashPrecision is the finest geohash length indexed as a sibling field. +const MaxGeohashPrecision = 12 + +// GeohashField returns the sibling field name carrying base's geohash at the +// given precision, e.g. GeohashField("location", 6) == "location_geohash_6". +func GeohashField(base string, precision int) string { + return base + GeohashSuffix + strconv.Itoa(precision) +} + +const geohashBase32 = "0123456789bcdefghjkmnpqrstuvwxyz" + +// encodeGeohash returns the standard geohash of (lat, lon) at the given length, +// matching Lucene/OpenSearch so both backends bucket points into the same cells. +func encodeGeohash(lat, lon float64, precision int) string { + latMin, latMax := -90.0, 90.0 + lonMin, lonMax := -180.0, 180.0 + var b strings.Builder + even := true + bit, ch := 0, 0 + for b.Len() < precision { + if even { + mid := (lonMin + lonMax) / 2 + if lon >= mid { + ch |= 1 << (4 - bit) + lonMin = mid + } else { + lonMax = mid + } + } else { + mid := (latMin + latMax) / 2 + if lat >= mid { + ch |= 1 << (4 - bit) + latMin = mid + } else { + latMax = mid + } + } + even = !even + if bit < 4 { + bit++ + } else { + b.WriteByte(geohashBase32[ch]) + bit, ch = 0, 0 + } + } + return b.String() +} + +// addGeohashSiblings writes, for each geopoint override, the geohash prefix of +// the point at every precision 1..MaxGeohashPrecision under the suffixed keys. +func addGeohashSiblings(m map[string]any, overrides map[string]FieldOpts) { + for key, opts := range overrides { + if opts.Type == TypeGeopoint { + addGeohashSibling(m, key) + } + } +} + +func addGeohashSibling(m map[string]any, dottedPath string) { + parts := strings.Split(dottedPath, ".") + parent := m + for _, p := range parts[:len(parts)-1] { + next, ok := parent[p].(map[string]any) + if !ok { + return + } + parent = next + } + leaf := parts[len(parts)-1] + obj, ok := parent[leaf].(map[string]any) + if !ok { + return + } + lon, hasLon := obj["longitude"].(float64) + lat, hasLat := obj["latitude"].(float64) + if !hasLon || !hasLat { + return + } + for p := 1; p <= MaxGeohashPrecision; p++ { + parent[GeohashField(leaf, p)] = encodeGeohash(lat, lon, p) + } +} diff --git a/services/search/pkg/mapping/geo_test.go b/services/search/pkg/mapping/geo_test.go index c0eac54fef..cb2906aa7f 100644 --- a/services/search/pkg/mapping/geo_test.go +++ b/services/search/pkg/mapping/geo_test.go @@ -141,3 +141,42 @@ var _ = Describe("PrepareForIndex geopoint", func() { Expect(endGp["lon"]).To(Equal(elon)) }) }) + +var _ = Describe("geohash", func() { + // Wikipedia's canonical reference point. + const refLat, refLon = 57.64911, 10.40744 + + It("encodes the canonical geohash vector (matches Lucene/OpenSearch)", func() { + Expect(encodeGeohash(refLat, refLon, 11)).To(Equal("u4pruydqqvj")) + }) + + It("is prefix-consistent across precisions", func() { + full := encodeGeohash(refLat, refLon, 11) + for p := 1; p <= 11; p++ { + Expect(encodeGeohash(refLat, refLon, p)).To(Equal(full[:p]), + "precision %d must be the %d-char prefix", p, p) + } + }) + + It("adds geohash prefix siblings for a geopoint at every precision", func() { + type geoDoc struct { + Location *struct { + Longitude *float64 `json:"longitude,omitempty"` + Latitude *float64 `json:"latitude,omitempty"` + } `json:"location,omitempty"` + } + lon, lat := refLon, refLat + doc := geoDoc{Location: &struct { + Longitude *float64 `json:"longitude,omitempty"` + Latitude *float64 `json:"latitude,omitempty"` + }{Longitude: &lon, Latitude: &lat}} + + m, err := PrepareForIndex(doc, map[string]FieldOpts{"location": {Type: TypeGeopoint}}) + Expect(err).ToNot(HaveOccurred()) + for p := 1; p <= MaxGeohashPrecision; p++ { + Expect(m[GeohashField("location", p)]).To(Equal(encodeGeohash(lat, lon, p)), + "location_geohash_%d", p) + } + Expect(m[GeohashField("location", 5)]).To(Equal("u4pru")) + }) +}) diff --git a/services/search/pkg/mapping/opensearch.go b/services/search/pkg/mapping/opensearch.go index ddd6249b6f..2da6f76131 100644 --- a/services/search/pkg/mapping/opensearch.go +++ b/services/search/pkg/mapping/opensearch.go @@ -54,6 +54,12 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p } props[fi.Name] = map[string]any{"properties": subProps} props[fi.Name+GeopointSuffix] = map[string]any{"type": "geo_point"} + // Mirror the bleve geohash prefix siblings so the document shape + // stays consistent across backends (OpenSearch aggregates via + // geohash_grid on _geopoint and does not query these). + for p := 1; p <= MaxGeohashPrecision; p++ { + props[GeohashField(fi.Name, p)] = map[string]any{"type": "keyword"} + } return nil } diff --git a/services/search/pkg/mapping/serialize.go b/services/search/pkg/mapping/serialize.go index e115e95e65..54c14833cc 100644 --- a/services/search/pkg/mapping/serialize.go +++ b/services/search/pkg/mapping/serialize.go @@ -21,5 +21,6 @@ func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, err } addGeopointSiblings(out, overrides) addSearchSiblings(out, deref(reflect.TypeOf(v)), overrides) + addGeohashSiblings(out, overrides) return out, nil } diff --git a/services/search/pkg/query/resolver.go b/services/search/pkg/query/resolver.go index b5eafe51c6..bf3c2c931e 100644 --- a/services/search/pkg/query/resolver.go +++ b/services/search/pkg/query/resolver.go @@ -123,3 +123,28 @@ func ResolveGeoField(name string) (string, bool) { f, ok := geopointFields()[strings.ToLower(name)] return f, ok } + +// geopointBaseFields maps a lowercased KQL key to the base field name of every +// geopoint field (e.g. "location" -> "location"), used to derive geohash prefix +// sibling fields. +var geopointBaseFields = sync.OnceValue(func() map[string]string { + out := map[string]string{} + for key, opts := range (search.Resource{}).SearchFieldOverrides() { + if opts.Type == mapping.TypeGeopoint { + out[strings.ToLower(key)] = key + } + } + return out +}) + +// ResolveGeohashField maps a KQL key + geohash precision to the indexed geohash +// prefix sibling field (e.g. "location", 6 -> "location_geohash_6"). ok is false +// when the key is not a geopoint field, so callers can reject geohash +// aggregations on non-geo fields. +func ResolveGeohashField(name string, precision int) (string, bool) { + base, ok := geopointBaseFields()[strings.ToLower(name)] + if !ok { + return "", false + } + return mapping.GeohashField(base, precision), true +}