feat(search): index Location as a geopoint on both backends

Add a TypeGeopoint field type. The libregraph Location facet is kept as an
object (retrieval / numeric queries) and a sibling <name>_geopoint field
carries the {lat,lon} form for geo-distance / bbox / polygon queries,
uniform across bleve and OpenSearch via the shared mapping. PrepareForIndex
splices the sibling in at write time.
This commit is contained in:
Dominik Schmidt committed 2026-08-31 13:40:42 +02:00
1 parent e78cad1cbe
commit 29cfdb01a0
11 files changed
+516 -2

No files matched your search

@@ -0,0 +1,178 @@
package bleve_test
import (
"sort"
"testing"
bleveSearch "github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/search/query"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
// geoFixture builds an in-memory bleve index with a single resource that
// carries the given lon/lat/alt. Used by the search tests below.
func geoFixture(t *testing.T, lon, lat, alt float64) bleveSearch.Index {
t.Helper()
idxMapping, err := bleve.NewMapping()
if err != nil {
t.Fatalf("NewMapping: %v", err)
}
idx, err := bleveSearch.NewMemOnly(idxMapping)
if err != nil {
t.Fatalf("NewMemOnly: %v", err)
}
r := search.Resource{
ID: "x",
Document: content.Document{
Name: "team.jpg",
Location: &libregraph.GeoCoordinates{
Longitude: &lon,
Latitude: &lat,
Altitude: &alt,
},
},
}
doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
if err != nil {
t.Fatalf("PrepareForIndex: %v", err)
}
if err := idx.Index(r.ID, doc); err != nil {
t.Fatalf("Index: %v", err)
}
return idx
}
// TestLocationAltitudeRoundTrip proves that every subfield of Location
// (including altitude) ends up in hit.Fields when a Resource is indexed
// through the full bleve pipeline. This is the invariant the Move /
// Delete / Restore round-trip depends on.
func TestLocationAltitudeRoundTrip(t *testing.T) {
idx := geoFixture(t, 11.103870357204285, 49.48675890884328, 1047.7)
req := bleveSearch.NewSearchRequest(bleveSearch.NewMatchAllQuery())
req.Fields = []string{"*"}
res, err := idx.Search(req)
if err != nil {
t.Fatalf("Search: %v", err)
}
if len(res.Hits) == 0 {
t.Fatal("no hits")
}
keys := make([]string, 0, len(res.Hits[0].Fields))
for k := range res.Hits[0].Fields {
keys = append(keys, k)
}
sort.Strings(keys)
t.Logf("hit.Fields keys: %v", keys)
for _, k := range []string{"location.longitude", "location.latitude", "location.altitude"} {
if _, ok := res.Hits[0].Fields[k]; !ok {
t.Errorf("missing %q in hit.Fields (got %v)", k, keys)
}
}
}
func TestLocationLatitudeRangeQueryMatches(t *testing.T) {
idx := geoFixture(t, 11.1, 49.48, 1000)
// numeric range on the sub-field
min, max := 49.0, 50.0
incl := true
q := query.NewNumericRangeInclusiveQuery(&min, &max, &incl, &incl)
q.SetField("location.latitude")
res, err := idx.Search(bleveSearch.NewSearchRequest(q))
if err != nil {
t.Fatalf("Search: %v", err)
}
if len(res.Hits) != 1 {
t.Fatalf("latitude range: got %d hits, want 1", len(res.Hits))
}
// same range excluding the indexed latitude => no match
lowMin, lowMax := 0.0, 10.0
q2 := query.NewNumericRangeInclusiveQuery(&lowMin, &lowMax, &incl, &incl)
q2.SetField("location.latitude")
res, err = idx.Search(bleveSearch.NewSearchRequest(q2))
if err != nil {
t.Fatalf("Search: %v", err)
}
if len(res.Hits) != 0 {
t.Errorf("latitude range outside value: got %d hits, want 0", len(res.Hits))
}
}
func TestLocationLongitudeRangeQueryMatches(t *testing.T) {
idx := geoFixture(t, 11.1, 49.48, 1000)
min, max := 11.0, 12.0
incl := true
q := query.NewNumericRangeInclusiveQuery(&min, &max, &incl, &incl)
q.SetField("location.longitude")
res, err := idx.Search(bleveSearch.NewSearchRequest(q))
if err != nil {
t.Fatalf("Search: %v", err)
}
if len(res.Hits) != 1 {
t.Fatalf("longitude range: got %d hits, want 1", len(res.Hits))
}
}
func TestLocationAltitudeRangeQueryMatches(t *testing.T) {
idx := geoFixture(t, 11.1, 49.48, 1047.7)
min := 1000.0
incl := true
q := query.NewNumericRangeInclusiveQuery(&min, nil, &incl, nil)
q.SetField("location.altitude")
res, err := idx.Search(bleveSearch.NewSearchRequest(q))
if err != nil {
t.Fatalf("Search: %v", err)
}
if len(res.Hits) != 1 {
t.Fatalf("altitude >= 1000: got %d hits, want 1", len(res.Hits))
}
// altitude floor above the indexed value => no hits
highMin := 2000.0
q2 := query.NewNumericRangeInclusiveQuery(&highMin, nil, &incl, nil)
q2.SetField("location.altitude")
res, err = idx.Search(bleveSearch.NewSearchRequest(q2))
if err != nil {
t.Fatalf("Search: %v", err)
}
if len(res.Hits) != 0 {
t.Errorf("altitude >= 2000 against 1047.7: got %d hits, want 0", len(res.Hits))
}
}
func TestLocationGeoDistanceQueryMatches(t *testing.T) {
// Nuremberg-ish coordinates.
idx := geoFixture(t, 11.103870357204285, 49.48675890884328, 1047.7)
// 10 km radius around the indexed point should match.
near := query.NewGeoDistanceQuery(11.103870357204285, 49.48675890884328, "10km")
near.SetField("location" + mapping.GeopointSuffix)
res, err := idx.Search(bleveSearch.NewSearchRequest(near))
if err != nil {
t.Fatalf("Search (near): %v", err)
}
if len(res.Hits) != 1 {
t.Fatalf("geo distance near: got %d hits, want 1", len(res.Hits))
}
// Far away (Berlin, ~400 km) with a 10 km radius should miss.
far := query.NewGeoDistanceQuery(13.404954, 52.520008, "10km")
far.SetField("location" + mapping.GeopointSuffix)
res, err = idx.Search(bleveSearch.NewSearchRequest(far))
if err != nil {
t.Fatalf("Search (far): %v", err)
}
if len(res.Hits) != 0 {
t.Errorf("geo distance far: got %d hits, want 0", len(res.Hits))
}
}
+17
View File
@@ -46,6 +46,21 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix
return nil
}
if fieldType == TypeGeopoint {
// Keep the facet object, add a sibling _geopoint field (see GeopointSuffix).
sub := structType(fi.GoField.Type)
if sub == nil {
return fmt.Errorf("mapping: geopoint type on non-struct field %q", key)
}
subDoc, err := buildBleveDocMapping(sub, overrides, key)
if err != nil {
return err
}
doc.AddSubDocumentMapping(fi.Name, subDoc)
doc.AddFieldMappingsAt(fi.Name+GeopointSuffix, bleve.NewGeoPointFieldMapping())
return nil
}
fm, err := bleveFieldMapping(fieldType, opts)
if err != nil {
return fmt.Errorf("mapping: field %q: %w", key, err)
@@ -85,6 +100,8 @@ func bleveFieldMapping(fieldType string, opts FieldOpts) (*bleveMapping.FieldMap
return bleve.NewBooleanFieldMapping(), nil
case TypeDatetime:
return bleve.NewDateTimeFieldMapping(), nil
case TypeGeopoint:
return bleve.NewGeoPointFieldMapping(), nil
case "":
return nil, fmt.Errorf("no type inferred and no override")
}
+44
View File
@@ -114,3 +114,47 @@ func TestBleveBuildMappingOverrides(t *testing.T) {
t.Errorf("Tags IncludeInAll should honor the explicit false override")
}
}
func TestBleveBuildMappingGeopoint(t *testing.T) {
type geoDoc struct {
Location *struct {
Lon *float64 `json:"longitude,omitempty"`
Lat *float64 `json:"latitude,omitempty"`
Alt *float64 `json:"altitude,omitempty"`
} `json:"location,omitempty"`
}
dm, err := BleveBuildMapping(reflect.TypeFor[geoDoc](), map[string]FieldOpts{
"location": {Type: TypeGeopoint},
})
if err != nil {
t.Fatalf("BleveBuildMapping: %v", err)
}
// Original facet stays as an object sub-document with numeric
// sub-properties - for data retrieval via hit.Fields and ordinary
// numeric queries.
loc := dm.Properties["location"]
if loc == nil {
t.Fatalf("location sub-document missing: %#v", dm.Properties)
}
if len(loc.Fields) != 0 {
t.Errorf("location should not carry field mappings directly, got %#v", loc.Fields)
}
for _, sub := range []string{"longitude", "latitude", "altitude"} {
prop, ok := loc.Properties[sub]
if !ok {
t.Errorf("missing sub-field %q under location (properties: %v)", sub, loc.Properties)
continue
}
if len(prop.Fields) == 0 || prop.Fields[0].Type != "number" {
t.Errorf("location.%s Fields: %#v, want [number]", sub, prop.Fields)
}
}
// Sibling geopoint at "<name>_geopoint" for geo-distance queries.
sibling := dm.Properties["location"+GeopointSuffix]
if sibling == nil {
t.Fatalf("location%s missing: %#v", GeopointSuffix, dm.Properties)
}
if len(sibling.Fields) == 0 || sibling.Fields[0].Type != "geopoint" {
t.Errorf("location%s Fields: %#v, want [geopoint]", GeopointSuffix, sibling.Fields)
}
}
+51
View File
@@ -0,0 +1,51 @@
package mapping
import "strings"
// GeopointSuffix is appended to a field's name to produce the sibling key
// that carries the geo_point / bleve-geopoint representation of the
// original facet. For example, a libregraph "location" object with
// longitude / latitude / altitude is preserved as-is under "location" (for
// data retrieval and numeric queries) while "location_geopoint" carries
// the {lat, lon} form the geo indices understand.
const GeopointSuffix = "_geopoint"
// addGeopointSiblings walks the overrides; for each TypeGeopoint entry at
// a dotted path (e.g. "location" or "journey.start") it writes a sibling
// under the suffixed key with the {lat, lon} form both bleve's
// ExtractGeoPoint and OpenSearch's geo_point parser accept. The original
// facet object stays untouched so downstream code still sees the full
// libregraph shape (including altitude).
func addGeopointSiblings(m map[string]any, overrides map[string]FieldOpts) {
for key, opts := range overrides {
if opts.Type == TypeGeopoint {
addGeopointSibling(m, key)
}
}
}
// addGeopointSibling resolves dottedPath within m and, if the target is a
// libregraph-shaped geo object (with numeric "longitude" and "latitude"),
// writes the `{lat, lon}` sibling at the same level under the suffixed key.
func addGeopointSibling(m map[string]any, dottedPath string) {
parts := strings.Split(dottedPath, ".")
parent := m
for _, p := range parts[:len(parts)-1] {
next, ok := parent[p].(map[string]any)
if !ok {
return
}
parent = next
}
leaf := parts[len(parts)-1]
obj, ok := parent[leaf].(map[string]any)
if !ok {
return
}
lon, hasLon := obj["longitude"].(float64)
lat, hasLat := obj["latitude"].(float64)
if !hasLon || !hasLat {
return
}
parent[leaf+GeopointSuffix] = map[string]any{"lat": lat, "lon": lon}
}
+160
View File
@@ -0,0 +1,160 @@
package mapping
import (
"reflect"
"testing"
)
// build-mapping error paths: an override that doesn't fit the Go field.
func TestBuildMappingErrors(t *testing.T) {
type doc struct {
Name string `json:"name"`
}
// Geopoint on a non-struct field must error on both backends.
if _, err := BleveBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"name": {Type: TypeGeopoint}}); err == nil {
t.Error("bleve: expected error for geopoint on string field")
}
if _, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"name": {Type: TypeGeopoint}}); err == nil {
t.Error("opensearch: expected error for geopoint on string field")
}
}
func TestAddGeopointSiblingMissingIntermediate(t *testing.T) {
m := map[string]any{"journey": "not-a-map"}
addGeopointSibling(m, "journey.start") // must bail, not panic
if _, ok := m["journey.start"+GeopointSuffix]; ok {
t.Error("no sibling should be written when the path can't be resolved")
}
}
func TestPrepareForIndexAddsGeopointSibling(t *testing.T) {
type geoDoc struct {
Location *struct {
Longitude *float64 `json:"longitude,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
Altitude *float64 `json:"altitude,omitempty"`
} `json:"location,omitempty"`
}
lon, lat, alt := 11.1, 49.4, 1047.7
doc := geoDoc{Location: &struct {
Longitude *float64 `json:"longitude,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
Altitude *float64 `json:"altitude,omitempty"`
}{Longitude: &lon, Latitude: &lat, Altitude: &alt}}
m, err := PrepareForIndex(doc, map[string]FieldOpts{
"location": {Type: TypeGeopoint},
})
if err != nil {
t.Fatalf("PrepareForIndex: %v", err)
}
// Original location object stays untouched (full libregraph shape).
orig, ok := m["location"].(map[string]any)
if !ok {
t.Fatalf("expected location object preserved, got %T", m["location"])
}
if orig["longitude"] != lon || orig["latitude"] != lat || orig["altitude"] != alt {
t.Errorf("location object: %#v", orig)
}
// Sibling location_geopoint has {lat, lon} for the geo indices.
gp, ok := m["location"+GeopointSuffix].(map[string]any)
if !ok {
t.Fatalf("expected location_geopoint sibling, got %T", m["location"+GeopointSuffix])
}
if gp["lat"] != lat || gp["lon"] != lon {
t.Errorf("sibling: %#v", gp)
}
}
func TestPrepareForIndexSkipsIncompleteGeopoint(t *testing.T) {
type geoDoc struct {
Location *struct {
Altitude *float64 `json:"altitude,omitempty"`
} `json:"location,omitempty"`
}
alt := 100.0
doc := geoDoc{Location: &struct {
Altitude *float64 `json:"altitude,omitempty"`
}{Altitude: &alt}}
m, err := PrepareForIndex(doc, map[string]FieldOpts{
"location": {Type: TypeGeopoint},
})
if err != nil {
t.Fatalf("PrepareForIndex: %v", err)
}
// Original stays (altitude alone is still useful metadata).
if _, ok := m["location"]; !ok {
t.Error("location should still be present when only altitude is set")
}
// No sibling without both lon and lat.
if _, ok := m["location"+GeopointSuffix]; ok {
t.Errorf("no sibling expected, got %#v", m["location"+GeopointSuffix])
}
}
func TestPrepareForIndexWithoutOverrideNoSibling(t *testing.T) {
type geoDoc struct {
Location *struct {
Longitude *float64 `json:"longitude,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
} `json:"location,omitempty"`
}
lon, lat := 11.1, 49.4
doc := geoDoc{Location: &struct {
Longitude *float64 `json:"longitude,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
}{Longitude: &lon, Latitude: &lat}}
m, err := PrepareForIndex(doc, nil)
if err != nil {
t.Fatalf("PrepareForIndex: %v", err)
}
if _, ok := m["location"+GeopointSuffix]; ok {
t.Errorf("no sibling expected without override, got %#v", m["location"+GeopointSuffix])
}
}
func TestPrepareForIndexHandlesNestedGeopoint(t *testing.T) {
// journey.start and journey.end - two geopoints in the same facet,
// demonstrating the dotted-path walker.
type geo struct {
Longitude *float64 `json:"longitude,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
}
type journey struct {
Start *geo `json:"start,omitempty"`
End *geo `json:"end,omitempty"`
}
type doc struct {
Journey *journey `json:"journey,omitempty"`
}
slon, slat := 11.0, 49.0
elon, elat := 13.4, 52.5
d := doc{Journey: &journey{
Start: &geo{Longitude: &slon, Latitude: &slat},
End: &geo{Longitude: &elon, Latitude: &elat},
}}
m, err := PrepareForIndex(d, map[string]FieldOpts{
"journey.start": {Type: TypeGeopoint},
"journey.end": {Type: TypeGeopoint},
})
if err != nil {
t.Fatalf("PrepareForIndex: %v", err)
}
j, ok := m["journey"].(map[string]any)
if !ok {
t.Fatalf("journey not an object: %T", m["journey"])
}
startGp, ok := j["start"+GeopointSuffix].(map[string]any)
if !ok || startGp["lat"] != slat || startGp["lon"] != slon {
t.Errorf("journey.start sibling: %#v", j["start"+GeopointSuffix])
}
endGp, ok := j["end"+GeopointSuffix].(map[string]any)
if !ok || endGp["lat"] != elat || endGp["lon"] != elon {
t.Errorf("journey.end sibling: %#v", j["end"+GeopointSuffix])
}
}
+17
View File
@@ -41,6 +41,21 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p
return nil
}
if fieldType == TypeGeopoint {
// Keep the facet object, add a sibling _geopoint field (see GeopointSuffix).
sub := structType(fi.GoField.Type)
if sub == nil {
return fmt.Errorf("mapping: geopoint type on non-struct field %q", key)
}
subProps, err := buildOpenSearchProperties(sub, overrides, key)
if err != nil {
return err
}
props[fi.Name] = map[string]any{"properties": subProps}
props[fi.Name+GeopointSuffix] = map[string]any{"type": "geo_point"}
return nil
}
fm, err := openSearchFieldMapping(fieldType, opts, fi.GoField.Type)
if err != nil {
return fmt.Errorf("mapping: field %q: %w", key, err)
@@ -88,6 +103,8 @@ func openSearchFieldMapping(fieldType string, opts FieldOpts, goType reflect.Typ
return map[string]any{"type": "boolean"}, nil
case TypeDatetime:
return map[string]any{"type": "date"}, nil
case TypeGeopoint:
return map[string]any{"type": "geo_point"}, nil
case "":
return nil, fmt.Errorf("no type inferred and no override")
}
@@ -130,3 +130,42 @@ func TestOpenSearchBuildMappingOverrides(t *testing.T) {
t.Errorf("MimeType: %#v", mime)
}
}
func TestOpenSearchBuildMappingGeopoint(t *testing.T) {
type doc struct {
Location *struct {
Lon float64 `json:"longitude"`
Lat float64 `json:"latitude"`
Alt float64 `json:"altitude"`
} `json:"location,omitempty"`
}
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{
"location": {Type: TypeGeopoint},
})
if err != nil {
t.Fatalf("OpenSearchBuildMapping: %v", err)
}
// Object for libregraph-shape data retrieval.
loc, ok := props["location"].(map[string]any)
if !ok {
t.Fatalf("location: %#v", props["location"])
}
sub, ok := loc["properties"].(map[string]any)
if !ok {
t.Fatalf("location should have numeric sub-properties, got %#v", loc)
}
for _, k := range []string{"longitude", "latitude", "altitude"} {
prop, ok := sub[k].(map[string]any)
if !ok || prop["type"] != "double" {
t.Errorf("location.%s: %#v", k, sub[k])
}
}
// Sibling geo_point for spatial queries.
gp, ok := props["location"+GeopointSuffix].(map[string]any)
if !ok {
t.Fatalf("location%s: %#v", GeopointSuffix, props["location"+GeopointSuffix])
}
if gp["type"] != "geo_point" {
t.Errorf("location%s.type: %v", GeopointSuffix, gp["type"])
}
}
+1
View File
@@ -14,6 +14,7 @@ const (
TypeDatetime = "datetime"
TypeBool = "bool"
TypeObject = "object"
TypeGeopoint = "geopoint"
)
// FieldOpts overrides the default type inference for a struct field. Keys in
+7 -2
View File
@@ -7,12 +7,17 @@ import (
)
// PrepareForIndex converts v to the flat map[string]any the backend index
// clients expect, via a json round-trip (conversions.To). overrides is
// reserved for type-specific adaptations wired in by follow-up features.
// clients expect: a json round-trip (conversions.To) plus type-specific
// adaptations (currently geopoint siblings). Pass the same overrides as the
// *BuildMapping calls so the document and the mapping stay in sync.
func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, error) {
out, err := conversions.To[map[string]any](v)
if err != nil {
return nil, fmt.Errorf("mapping: prepare %T: %w", v, err)
}
if out == nil {
return out, nil
}
addGeopointSiblings(out, overrides)
return out, nil
}
@@ -24,6 +24,7 @@ func TestValidateAccepts(t *testing.T) {
"Name": {Analyzer: "lowercaseKeyword"},
"audio": {Type: TypeObject},
"audio.artist": {Analyzer: "lowercaseKeyword"},
"location": {Type: TypeGeopoint},
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
+1
View File
@@ -72,6 +72,7 @@ var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts
"Content": {Type: mapping.TypeFulltext},
"Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll},
"Favorites": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll},
"location": {Type: mapping.TypeGeopoint},
// Mtime is stored as an RFC3339 string; type it as a date so mtime:>...
// range queries are chronological on both backends (bleve DateRangeQuery
// / OpenSearch date range), not a lexicographic keyword compare.