mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-09-09 12:19:08 -04:00
Compare commits
15
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cbd196218b | ||
|
|
00341ce31b | ||
|
|
368d86f159 | ||
|
|
733bee58dc | ||
|
|
7c15f197ff | ||
|
|
c359a84e2d | ||
|
|
53ef1cabc4 | ||
|
|
8fd96d506c | ||
|
|
21b298fc53 | ||
|
|
599022e40f | ||
|
|
431c97e712 | ||
|
|
22f46dd9d4 | ||
|
|
9ebf755cfd | ||
|
|
00468734f6 | ||
|
|
ac86d9b907 |
No files matched your search
@@ -9,9 +9,7 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
@@ -28,6 +26,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
)
|
||||
|
||||
// CreateUploadSession create an upload session to allow your app to upload files up to the maximum file size.
|
||||
@@ -452,130 +451,20 @@ func cs3ResourceToDriveItem(logger *log.Logger, publicBaseURL *url.URL, res *sto
|
||||
}
|
||||
}
|
||||
|
||||
if res.GetArbitraryMetadata() != nil {
|
||||
driveItem.Audio = cs3ResourceToDriveItemAudioFacet(logger, res)
|
||||
driveItem.Image = cs3ResourceToDriveItemImageFacet(logger, res)
|
||||
driveItem.Location = cs3ResourceToDriveItemLocationFacet(logger, res)
|
||||
driveItem.Photo = cs3ResourceToDriveItemPhotoFacet(logger, res)
|
||||
if metadata := res.GetArbitraryMetadata().GetMetadata(); metadata != nil {
|
||||
driveItem.Audio = metadataToFacet[libregraph.Audio](metadata, "audio")
|
||||
driveItem.Image = metadataToFacet[libregraph.Image](metadata, "image")
|
||||
driveItem.Location = metadataToFacet[libregraph.GeoCoordinates](metadata, "location")
|
||||
driveItem.Photo = metadataToFacet[libregraph.Photo](metadata, "photo")
|
||||
}
|
||||
|
||||
return driveItem, nil
|
||||
}
|
||||
|
||||
func cs3ResourceToDriveItemAudioFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Audio {
|
||||
if !strings.HasPrefix(res.GetMimeType(), "audio/") {
|
||||
return nil
|
||||
}
|
||||
|
||||
k := res.GetArbitraryMetadata().GetMetadata()
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var audio = &libregraph.Audio{}
|
||||
if ok := unmarshalStringMap(logger, audio, k, "libre.graph.audio."); ok {
|
||||
return audio
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func cs3ResourceToDriveItemImageFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Image {
|
||||
k := res.GetArbitraryMetadata().GetMetadata()
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var image = &libregraph.Image{}
|
||||
if ok := unmarshalStringMap(logger, image, k, "libre.graph.image."); ok {
|
||||
return image
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func cs3ResourceToDriveItemLocationFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.GeoCoordinates {
|
||||
k := res.GetArbitraryMetadata().GetMetadata()
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var location = &libregraph.GeoCoordinates{}
|
||||
if ok := unmarshalStringMap(logger, location, k, "libre.graph.location."); ok {
|
||||
return location
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func cs3ResourceToDriveItemPhotoFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Photo {
|
||||
k := res.GetArbitraryMetadata().GetMetadata()
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var photo = &libregraph.Photo{}
|
||||
if ok := unmarshalStringMap(logger, photo, k, "libre.graph.photo."); ok {
|
||||
return photo
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getFieldName(structField reflect.StructField) string {
|
||||
tag := structField.Tag.Get("json")
|
||||
if tag == "" {
|
||||
return structField.Name
|
||||
}
|
||||
|
||||
return strings.Split(tag, ",")[0]
|
||||
}
|
||||
|
||||
func unmarshalStringMap(logger *log.Logger, out any, flatMap map[string]string, prefix string) bool {
|
||||
nonEmpty := false
|
||||
obj := reflect.ValueOf(out).Elem()
|
||||
timeKind := reflect.TypeOf(&time.Time{}).Elem().Kind()
|
||||
for i := 0; i < obj.NumField(); i++ {
|
||||
field := obj.Field(i)
|
||||
structField := obj.Type().Field(i)
|
||||
mapKey := prefix + getFieldName(structField)
|
||||
|
||||
if value, ok := flatMap[mapKey]; ok {
|
||||
if field.Kind() == reflect.Ptr {
|
||||
newValue := reflect.New(field.Type().Elem())
|
||||
var tmp any
|
||||
var err error
|
||||
switch t := newValue.Type().Elem().Kind(); t {
|
||||
case reflect.String:
|
||||
tmp = value
|
||||
case reflect.Int32:
|
||||
tmp, err = strconv.ParseInt(value, 10, 32)
|
||||
case reflect.Int64:
|
||||
tmp, err = strconv.ParseInt(value, 10, 64)
|
||||
case reflect.Float32:
|
||||
tmp, err = strconv.ParseFloat(value, 32)
|
||||
case reflect.Float64:
|
||||
tmp, err = strconv.ParseFloat(value, 64)
|
||||
case reflect.Bool:
|
||||
tmp, err = strconv.ParseBool(value)
|
||||
case timeKind:
|
||||
tmp, err = time.Parse(time.RFC3339, value)
|
||||
default:
|
||||
err = errors.New("unsupported type")
|
||||
logger.Error().Err(err).Str("type", t.String()).Str("mapKey", mapKey).Msg("target field type for value of mapKey is not supported")
|
||||
}
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Str("mapKey", mapKey).Msg("unmarshalling failed")
|
||||
continue
|
||||
}
|
||||
newValue.Elem().Set(reflect.ValueOf(tmp).Convert(field.Type().Elem()))
|
||||
field.Set(newValue)
|
||||
nonEmpty = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nonEmpty
|
||||
// metadataToFacet builds a DriveItem facet *T from CS3 arbitrary metadata under
|
||||
// the "libre.graph.<facet>." key prefix. Nil when no such keys are present.
|
||||
func metadataToFacet[T any](metadata map[string]string, facet string) *T {
|
||||
return mapping.DeserializeStringsAt[T](metadata, "libre.graph."+facet+".")
|
||||
}
|
||||
|
||||
func cs3ResourceToRemoteItem(res *storageprovider.ResourceInfo) (*libregraph.RemoteItem, error) {
|
||||
|
||||
@@ -136,10 +136,10 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
|
||||
Tags: getFieldSliceValue[string](hit.Fields, "Tags"),
|
||||
Favorites: getFieldSliceValue[string](hit.Fields, "Favorites"),
|
||||
Highlights: getFragmentValue(hit.Fragments, "Content", 0),
|
||||
Audio: getAudioValue[searchMessage.Audio](hit.Fields),
|
||||
Image: getImageValue[searchMessage.Image](hit.Fields),
|
||||
Location: getLocationValue[searchMessage.GeoCoordinates](hit.Fields),
|
||||
Photo: getPhotoValue[searchMessage.Photo](hit.Fields),
|
||||
Audio: hitToFacet[searchMessage.Audio](hit.Fields, "audio"),
|
||||
Image: hitToFacet[searchMessage.Image](hit.Fields, "image"),
|
||||
Location: hitToFacet[searchMessage.GeoCoordinates](hit.Fields, "location"),
|
||||
Photo: hitToFacet[searchMessage.Photo](hit.Fields, "photo"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
@@ -36,10 +37,21 @@ func NewBatch(index bleve.Index, size int) (*Batch, error) {
|
||||
|
||||
func (b *Batch) Upsert(id string, r search.Resource) error {
|
||||
return b.withSizeLimit(func() error {
|
||||
return b.batch.Index(id, r)
|
||||
return b.indexResource(id, r)
|
||||
})
|
||||
}
|
||||
|
||||
// indexResource prepares r for bleve (resolving json tags and splicing in
|
||||
// type-specific adaptations via the mapping package) and appends it to the
|
||||
// batch under id.
|
||||
func (b *Batch) indexResource(id string, r search.Resource) error {
|
||||
doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return b.batch.Index(id, doc)
|
||||
}
|
||||
|
||||
func (b *Batch) Move(id, parentID, location string) error {
|
||||
return b.withSizeLimit(func() error {
|
||||
rootResource, err := searchResourceByID(id, b.index)
|
||||
@@ -68,7 +80,7 @@ func (b *Batch) Move(id, parentID, location string) error {
|
||||
}
|
||||
|
||||
for _, resource := range resources {
|
||||
if err := b.batch.Index(resource.ID, resource); err != nil {
|
||||
if err := b.indexResource(resource.ID, *resource); err != nil {
|
||||
return err
|
||||
}
|
||||
if b.batch.Size() >= b.size {
|
||||
@@ -90,7 +102,7 @@ func (b *Batch) Delete(id string) error {
|
||||
}
|
||||
|
||||
for _, resource := range affectedResources {
|
||||
if err := b.batch.Index(resource.ID, resource); err != nil {
|
||||
if err := b.indexResource(resource.ID, *resource); err != nil {
|
||||
return err
|
||||
}
|
||||
if b.batch.Size() >= b.size {
|
||||
@@ -112,7 +124,7 @@ func (b *Batch) Restore(id string) error {
|
||||
}
|
||||
|
||||
for _, resource := range affectedResources {
|
||||
if err := b.batch.Index(resource.ID, resource); err != nil {
|
||||
if err := b.indexResource(resource.ID, *resource); err != nil {
|
||||
return err
|
||||
}
|
||||
if b.batch.Size() >= b.size {
|
||||
|
||||
@@ -1,18 +1,13 @@
|
||||
package bleve
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
bleveSearch "github.com/blevesearch/bleve/v2/search"
|
||||
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
@@ -75,131 +70,19 @@ func getFragmentValue(m bleveSearch.FieldFragmentMap, key string, idx int) strin
|
||||
return val[idx]
|
||||
}
|
||||
|
||||
func getAudioValue[T any](fields map[string]any) *T {
|
||||
if !strings.HasPrefix(getFieldValue[string](fields, "MimeType"), "audio/") {
|
||||
return nil
|
||||
}
|
||||
|
||||
var audio = newPointerOfType[T]()
|
||||
if ok := unmarshalInterfaceMap(audio, fields, "audio."); ok {
|
||||
return audio
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getImageValue[T any](fields map[string]any) *T {
|
||||
var image = newPointerOfType[T]()
|
||||
if ok := unmarshalInterfaceMap(image, fields, "image."); ok {
|
||||
return image
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getLocationValue[T any](fields map[string]any) *T {
|
||||
var location = newPointerOfType[T]()
|
||||
if ok := unmarshalInterfaceMap(location, fields, "location."); ok {
|
||||
return location
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getPhotoValue[T any](fields map[string]any) *T {
|
||||
var photo = newPointerOfType[T]()
|
||||
if ok := unmarshalInterfaceMap(photo, fields, "photo."); ok {
|
||||
return photo
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func newPointerOfType[T any]() *T {
|
||||
t := reflect.TypeOf((*T)(nil)).Elem()
|
||||
ptr := reflect.New(t).Interface()
|
||||
return ptr.(*T)
|
||||
}
|
||||
|
||||
func unmarshalInterfaceMap(out any, flatMap map[string]any, prefix string) bool {
|
||||
nonEmpty := false
|
||||
obj := reflect.ValueOf(out).Elem()
|
||||
for i := 0; i < obj.NumField(); i++ {
|
||||
field := obj.Field(i)
|
||||
structField := obj.Type().Field(i)
|
||||
mapKey := prefix + getFieldName(structField)
|
||||
|
||||
if value, ok := flatMap[mapKey]; ok {
|
||||
if field.Kind() == reflect.Ptr {
|
||||
alloc := reflect.New(field.Type().Elem())
|
||||
elemType := field.Type().Elem()
|
||||
|
||||
// convert time strings from index for search requests
|
||||
if elemType == reflect.TypeOf(timestamppb.Timestamp{}) {
|
||||
if strValue, ok := value.(string); ok {
|
||||
if parsedTime, err := time.Parse(time.RFC3339, strValue); err == nil {
|
||||
alloc.Elem().Set(reflect.ValueOf(*timestamppb.New(parsedTime)))
|
||||
field.Set(alloc)
|
||||
nonEmpty = true
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// convert time strings from index for libregraph structs when updating resources
|
||||
if elemType == reflect.TypeOf(time.Time{}) {
|
||||
if strValue, ok := value.(string); ok {
|
||||
if parsedTime, err := time.Parse(time.RFC3339, strValue); err == nil {
|
||||
alloc.Elem().Set(reflect.ValueOf(parsedTime))
|
||||
field.Set(alloc)
|
||||
nonEmpty = true
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
alloc.Elem().Set(reflect.ValueOf(value).Convert(elemType))
|
||||
field.Set(alloc)
|
||||
nonEmpty = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nonEmpty
|
||||
}
|
||||
|
||||
func getFieldName(structField reflect.StructField) string {
|
||||
tag := structField.Tag.Get("json")
|
||||
if tag == "" {
|
||||
return structField.Name
|
||||
}
|
||||
|
||||
return strings.Split(tag, ",")[0]
|
||||
// hitToFacet builds a search Entity facet *T from a bleve hit's fields under the
|
||||
// given key prefix. Nil when the hit has no such fields.
|
||||
func hitToFacet[T any](fields map[string]any, prefix string) *T {
|
||||
return mapping.DeserializeAt[T](fields, prefix)
|
||||
}
|
||||
|
||||
// matchToResource reconstructs a search.Resource from a bleve hit. Used by
|
||||
// the Move / Delete / Restore / Purge paths that round-trip a record through
|
||||
// the index. Always returns a non-nil *Resource: Deserialize is fail-soft
|
||||
// for per-field parse errors, so corrupted hit values surface as zero
|
||||
// values on individual fields instead of dropping the whole record.
|
||||
func matchToResource(match *bleveSearch.DocumentMatch) *search.Resource {
|
||||
return &search.Resource{
|
||||
ID: getFieldValue[string](match.Fields, "ID"),
|
||||
RootID: getFieldValue[string](match.Fields, "RootID"),
|
||||
Path: getFieldValue[string](match.Fields, "Path"),
|
||||
ParentID: getFieldValue[string](match.Fields, "ParentID"),
|
||||
Type: uint64(getFieldValue[float64](match.Fields, "Type")),
|
||||
Deleted: getFieldValue[bool](match.Fields, "Deleted"),
|
||||
Document: content.Document{
|
||||
Name: getFieldValue[string](match.Fields, "Name"),
|
||||
Title: getFieldValue[string](match.Fields, "Title"),
|
||||
Size: uint64(getFieldValue[float64](match.Fields, "Size")),
|
||||
Mtime: getFieldValue[string](match.Fields, "Mtime"),
|
||||
MimeType: getFieldValue[string](match.Fields, "MimeType"),
|
||||
Content: getFieldValue[string](match.Fields, "Content"),
|
||||
Tags: getFieldSliceValue[string](match.Fields, "Tags"),
|
||||
Favorites: getFieldSliceValue[string](match.Fields, "Favorites"),
|
||||
Audio: getAudioValue[libregraph.Audio](match.Fields),
|
||||
Image: getImageValue[libregraph.Image](match.Fields),
|
||||
Location: getLocationValue[libregraph.GeoCoordinates](match.Fields),
|
||||
Photo: getPhotoValue[libregraph.Photo](match.Fields),
|
||||
},
|
||||
}
|
||||
return mapping.Deserialize[search.Resource](match.Fields)
|
||||
}
|
||||
|
||||
func escapeQuery(s string) string {
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
package bleve_test
|
||||
|
||||
import (
|
||||
bleveSearch "github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/search/query"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
// geoFixture builds an in-memory bleve index with a single resource carrying
|
||||
// the given lon/lat/alt, indexed through the full bleve pipeline.
|
||||
func geoFixture(lon, lat, alt float64) bleveSearch.Index {
|
||||
idxMapping, err := bleve.NewMapping()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
idx, err := bleveSearch.NewMemOnly(idxMapping)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
r := search.Resource{
|
||||
ID: "x",
|
||||
Document: content.Document{
|
||||
Name: "team.jpg",
|
||||
Location: &libregraph.GeoCoordinates{
|
||||
Longitude: &lon,
|
||||
Latitude: &lat,
|
||||
Altitude: &alt,
|
||||
},
|
||||
},
|
||||
}
|
||||
doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(idx.Index(r.ID, doc)).To(Succeed())
|
||||
return idx
|
||||
}
|
||||
|
||||
var _ = Describe("Location geo queries", func() {
|
||||
// Every Location subfield (including altitude) must end up in hit.Fields
|
||||
// when a Resource is indexed through the full bleve pipeline. This is the
|
||||
// invariant the Move / Delete / Restore round-trip depends on.
|
||||
It("round-trips every Location subfield into hit.Fields", func() {
|
||||
idx := geoFixture(11.103870357204285, 49.48675890884328, 1047.7)
|
||||
|
||||
req := bleveSearch.NewSearchRequest(bleveSearch.NewMatchAllQuery())
|
||||
req.Fields = []string{"*"}
|
||||
res, err := idx.Search(req)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits).ToNot(BeEmpty())
|
||||
|
||||
for _, k := range []string{"location.longitude", "location.latitude", "location.altitude"} {
|
||||
Expect(res.Hits[0].Fields).To(HaveKey(k))
|
||||
}
|
||||
})
|
||||
|
||||
It("matches a latitude numeric range and misses outside it", func() {
|
||||
idx := geoFixture(11.1, 49.48, 1000)
|
||||
|
||||
min, max := 49.0, 50.0
|
||||
incl := true
|
||||
q := query.NewNumericRangeInclusiveQuery(&min, &max, &incl, &incl)
|
||||
q.SetField("location.latitude")
|
||||
res, err := idx.Search(bleveSearch.NewSearchRequest(q))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits).To(HaveLen(1))
|
||||
|
||||
lowMin, lowMax := 0.0, 10.0
|
||||
q2 := query.NewNumericRangeInclusiveQuery(&lowMin, &lowMax, &incl, &incl)
|
||||
q2.SetField("location.latitude")
|
||||
res, err = idx.Search(bleveSearch.NewSearchRequest(q2))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("matches a longitude numeric range", func() {
|
||||
idx := geoFixture(11.1, 49.48, 1000)
|
||||
|
||||
min, max := 11.0, 12.0
|
||||
incl := true
|
||||
q := query.NewNumericRangeInclusiveQuery(&min, &max, &incl, &incl)
|
||||
q.SetField("location.longitude")
|
||||
res, err := idx.Search(bleveSearch.NewSearchRequest(q))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits).To(HaveLen(1))
|
||||
})
|
||||
|
||||
It("matches an altitude lower-bound range and misses above it", func() {
|
||||
idx := geoFixture(11.1, 49.48, 1047.7)
|
||||
|
||||
min := 1000.0
|
||||
incl := true
|
||||
q := query.NewNumericRangeInclusiveQuery(&min, nil, &incl, nil)
|
||||
q.SetField("location.altitude")
|
||||
res, err := idx.Search(bleveSearch.NewSearchRequest(q))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits).To(HaveLen(1))
|
||||
|
||||
highMin := 2000.0
|
||||
q2 := query.NewNumericRangeInclusiveQuery(&highMin, nil, &incl, nil)
|
||||
q2.SetField("location.altitude")
|
||||
res, err = idx.Search(bleveSearch.NewSearchRequest(q2))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("matches a geo-distance query near the point and misses far away", func() {
|
||||
// Nuremberg-ish coordinates.
|
||||
idx := geoFixture(11.103870357204285, 49.48675890884328, 1047.7)
|
||||
|
||||
// 10 km radius around the indexed point should match.
|
||||
near := query.NewGeoDistanceQuery(11.103870357204285, 49.48675890884328, "10km")
|
||||
near.SetField("location" + mapping.GeopointSuffix)
|
||||
res, err := idx.Search(bleveSearch.NewSearchRequest(near))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits).To(HaveLen(1))
|
||||
|
||||
// Far away (Berlin, ~400 km) with a 10 km radius should miss.
|
||||
far := query.NewGeoDistanceQuery(13.404954, 52.520008, "10km")
|
||||
far.SetField("location" + mapping.GeopointSuffix)
|
||||
res, err = idx.Search(bleveSearch.NewSearchRequest(far))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits).To(BeEmpty())
|
||||
})
|
||||
})
|
||||
@@ -1,9 +1,15 @@
|
||||
package bleve
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"math"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
|
||||
@@ -15,50 +21,174 @@ import (
|
||||
"github.com/blevesearch/bleve/v2/mapping"
|
||||
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
|
||||
searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
func NewIndex(root string) (bleve.Index, error) {
|
||||
// bolt_timeout makes a second process on the same datapath fail after 5s
|
||||
// instead of blocking forever on the file lock.
|
||||
var openRuntimeConfig = map[string]interface{}{"bolt_timeout": "5s"}
|
||||
|
||||
// NewIndex opens (or creates) the bleve index at root and classifies the
|
||||
// stored schema against NewMapping(). Breaking changes refuse with
|
||||
// ErrManualActionRequired, additive ones are persisted into the index (the
|
||||
// bleve analogue of PUT _mapping); the caller must warn that pre-upgrade
|
||||
// documents lack the Classification.NewFields until re-indexed.
|
||||
func NewIndex(root string) (bleve.Index, searchmapping.Classification, error) {
|
||||
destination := filepath.Join(root, "bleve")
|
||||
index, err := bleve.Open(destination)
|
||||
if errors.Is(bleve.ErrorIndexPathDoesNotExist, err) {
|
||||
index, err := bleve.OpenUsing(destination, openRuntimeConfig)
|
||||
if errors.Is(err, bleve.ErrorIndexPathDoesNotExist) {
|
||||
indexMapping, err := NewMapping()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, searchmapping.Classification{}, err
|
||||
}
|
||||
index, err = bleve.New(destination, indexMapping)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, searchmapping.Classification{}, err
|
||||
}
|
||||
|
||||
return index, nil
|
||||
return index, searchmapping.Classification{Verdict: searchmapping.VerdictEqual}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, searchmapping.Classification{}, err
|
||||
}
|
||||
|
||||
return index, err
|
||||
classification, codeB, err := classifyStoredMapping(index)
|
||||
if err != nil {
|
||||
_ = index.Close()
|
||||
return nil, searchmapping.Classification{}, err
|
||||
}
|
||||
|
||||
switch classification.Verdict {
|
||||
case searchmapping.VerdictBreaking:
|
||||
_ = index.Close()
|
||||
return nil, classification, searchmapping.ManualActionRequiredError(destination, "delete the index directory "+destination, classification.Reasons)
|
||||
case searchmapping.VerdictAdditive:
|
||||
// Safe: everything else is identical and the new fields hold no data.
|
||||
// Reopen so the live mapping picks the change up; otherwise the fields
|
||||
// get indexed dynamically and flip to breaking on the next start.
|
||||
// return the classification even on errors: the mapping may already be
|
||||
// persisted, so the next start classifies equal and the caller's
|
||||
// warning is the only chance to surface the new fields
|
||||
if err := index.SetInternal([]byte("_mapping"), codeB); err != nil {
|
||||
_ = index.Close()
|
||||
return nil, classification, fmt.Errorf("failed to store the updated index mapping: %w", err)
|
||||
}
|
||||
if err := index.Close(); err != nil {
|
||||
return nil, classification, err
|
||||
}
|
||||
index, err = bleve.OpenUsing(destination, openRuntimeConfig)
|
||||
if err != nil {
|
||||
return nil, classification, err
|
||||
}
|
||||
}
|
||||
|
||||
return index, classification, nil
|
||||
}
|
||||
|
||||
// classifyStoredMapping diffs the stored mapping against NewMapping() and
|
||||
// returns the marshaled code mapping. New-in-code fields that already hold
|
||||
// data (previously indexed dynamically) are breaking. The compare is only
|
||||
// stable within one bleve version: a changed marshaling default fails towards
|
||||
// breaking, normalize the affected key here if that ever fires.
|
||||
func classifyStoredMapping(index bleve.Index) (searchmapping.Classification, []byte, error) {
|
||||
storedB, err := index.GetInternal([]byte("_mapping"))
|
||||
if err != nil {
|
||||
return searchmapping.Classification{}, nil, fmt.Errorf("failed to read the stored index mapping: %w", err)
|
||||
}
|
||||
codeMapping, err := NewMapping()
|
||||
if err != nil {
|
||||
return searchmapping.Classification{}, nil, err
|
||||
}
|
||||
codeB, err := json.Marshal(codeMapping)
|
||||
if err != nil {
|
||||
return searchmapping.Classification{}, nil, err
|
||||
}
|
||||
|
||||
var stored, code map[string]any
|
||||
if err := json.Unmarshal(storedB, &stored); err != nil {
|
||||
return searchmapping.Classification{}, nil, fmt.Errorf("failed to parse the stored index mapping: %w", err)
|
||||
}
|
||||
if err := json.Unmarshal(codeB, &code); err != nil {
|
||||
return searchmapping.Classification{}, nil, err
|
||||
}
|
||||
|
||||
fields, err := index.Fields()
|
||||
if err != nil {
|
||||
return searchmapping.Classification{}, nil, fmt.Errorf("failed to list the indexed fields: %w", err)
|
||||
}
|
||||
indexedFields := make(map[string]struct{}, len(fields))
|
||||
for _, f := range fields {
|
||||
if !strings.HasPrefix(f, "_") { // skip bleve-internal fields like _all
|
||||
indexedFields[f] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
storedDM, _ := stored["default_mapping"].(map[string]any)
|
||||
codeDM, _ := code["default_mapping"].(map[string]any)
|
||||
storedProps, _ := storedDM["properties"].(map[string]any)
|
||||
codeProps, _ := codeDM["properties"].(map[string]any)
|
||||
|
||||
classification := searchmapping.Classify(storedProps, codeProps, func(path string) bool {
|
||||
if _, ok := indexedFields[path]; ok {
|
||||
return true
|
||||
}
|
||||
nested := path + "."
|
||||
for f := range indexedFields {
|
||||
if strings.HasPrefix(f, nested) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
})
|
||||
|
||||
// everything outside default_mapping.properties (analyzer definitions,
|
||||
// default analyzer, dynamic flags, ...) must match exactly
|
||||
var reasons []string
|
||||
compareKeysExcept(stored, code, "default_mapping", "", &reasons)
|
||||
compareKeysExcept(storedDM, codeDM, "properties", "default_mapping.", &reasons)
|
||||
if len(reasons) > 0 {
|
||||
classification.Verdict = searchmapping.VerdictBreaking
|
||||
classification.Reasons = append(reasons, classification.Reasons...)
|
||||
}
|
||||
|
||||
return classification, codeB, nil
|
||||
}
|
||||
|
||||
// compareKeysExcept deep-compares all keys present on either side except skip.
|
||||
func compareKeysExcept(stored, code map[string]any, skip, prefix string, reasons *[]string) {
|
||||
keys := slices.Collect(maps.Keys(stored))
|
||||
for k := range code {
|
||||
if _, ok := stored[k]; !ok {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
slices.Sort(keys)
|
||||
for _, k := range keys {
|
||||
if k == skip {
|
||||
continue
|
||||
}
|
||||
if !reflect.DeepEqual(stored[k], code[k]) {
|
||||
*reasons = append(*reasons, fmt.Sprintf("%s%s changed", prefix, k))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func NewMapping() (mapping.IndexMapping, error) {
|
||||
nameMapping := bleve.NewTextFieldMapping()
|
||||
nameMapping.Analyzer = "lowercaseKeyword"
|
||||
|
||||
lowercaseMapping := bleve.NewTextFieldMapping()
|
||||
lowercaseMapping.IncludeInAll = false
|
||||
lowercaseMapping.Analyzer = "lowercaseKeyword"
|
||||
|
||||
fulltextFieldMapping := bleve.NewTextFieldMapping()
|
||||
fulltextFieldMapping.Analyzer = "fulltext"
|
||||
fulltextFieldMapping.IncludeInAll = false
|
||||
|
||||
docMapping := bleve.NewDocumentMapping()
|
||||
docMapping.AddFieldMappingsAt("Name", nameMapping)
|
||||
docMapping.AddFieldMappingsAt("Tags", lowercaseMapping)
|
||||
docMapping.AddFieldMappingsAt("Favorites", lowercaseMapping)
|
||||
docMapping.AddFieldMappingsAt("Content", fulltextFieldMapping)
|
||||
resourceType := reflect.TypeFor[search.Resource]()
|
||||
overrides := search.Resource{}.SearchFieldOverrides()
|
||||
if err := searchmapping.Validate(resourceType, overrides); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
docMapping, err := searchmapping.BleveBuildMapping(resourceType, overrides)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
indexMapping := bleve.NewIndexMapping()
|
||||
indexMapping.DefaultAnalyzer = keyword.Name
|
||||
indexMapping.DefaultMapping = docMapping
|
||||
err := indexMapping.AddCustomAnalyzer("lowercaseKeyword",
|
||||
err = indexMapping.AddCustomAnalyzer("lowercaseKeyword",
|
||||
map[string]any{
|
||||
"type": custom.Name,
|
||||
"tokenizer": single.Name,
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
package bleve_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
bleveSearch "github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
|
||||
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
|
||||
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
|
||||
bleveMapping "github.com/blevesearch/bleve/v2/mapping"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
|
||||
searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
)
|
||||
|
||||
var _ = Describe("NewIndex", func() {
|
||||
var root string
|
||||
|
||||
BeforeEach(func() {
|
||||
root = GinkgoT().TempDir()
|
||||
})
|
||||
|
||||
codeMapping := func() *bleveMapping.IndexMappingImpl {
|
||||
m, err := bleve.NewMapping()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
impl, ok := m.(*bleveMapping.IndexMappingImpl)
|
||||
Expect(ok).To(BeTrue())
|
||||
return impl
|
||||
}
|
||||
|
||||
// buildIndex simulates an index left behind by an older release
|
||||
buildIndex := func(m bleveMapping.IndexMapping, docs map[string]map[string]any) {
|
||||
idx, err := bleveSearch.New(filepath.Join(root, "bleve"), m)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
for id, doc := range docs {
|
||||
Expect(idx.Index(id, doc)).To(Succeed())
|
||||
}
|
||||
Expect(idx.Close()).To(Succeed())
|
||||
}
|
||||
|
||||
It("creates a fresh index", func() {
|
||||
idx, classification, err := bleve.NewIndex(root)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(classification.Verdict).To(Equal(searchmapping.VerdictEqual))
|
||||
Expect(idx.Close()).To(Succeed())
|
||||
})
|
||||
|
||||
It("opens an index with an identical schema", func() {
|
||||
buildIndex(codeMapping(), nil)
|
||||
|
||||
idx, classification, err := bleve.NewIndex(root)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(classification.Verdict).To(Equal(searchmapping.VerdictEqual))
|
||||
Expect(classification.NewFields).To(BeEmpty())
|
||||
Expect(idx.Close()).To(Succeed())
|
||||
})
|
||||
|
||||
It("treats a genuinely new field as additive", func() {
|
||||
old := codeMapping()
|
||||
Expect(old.DefaultMapping.Properties).To(HaveKey("Title"))
|
||||
delete(old.DefaultMapping.Properties, "Title")
|
||||
buildIndex(old, nil)
|
||||
|
||||
idx, classification, err := bleve.NewIndex(root)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(classification.Verdict).To(Equal(searchmapping.VerdictAdditive))
|
||||
Expect(classification.NewFields).To(ConsistOf("Title"))
|
||||
Expect(idx.Index("1", map[string]any{"Title": "hello"})).To(Succeed())
|
||||
Expect(idx.Close()).To(Succeed())
|
||||
})
|
||||
|
||||
It("treats a new nested field as additive", func() {
|
||||
old := codeMapping()
|
||||
photo := old.DefaultMapping.Properties["photo"]
|
||||
Expect(photo).ToNot(BeNil())
|
||||
Expect(photo.Properties).To(HaveKey("cameraMake"))
|
||||
delete(photo.Properties, "cameraMake")
|
||||
buildIndex(old, nil)
|
||||
|
||||
idx, classification, err := bleve.NewIndex(root)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(classification.Verdict).To(Equal(searchmapping.VerdictAdditive))
|
||||
Expect(classification.NewFields).To(ConsistOf("photo.cameraMake"))
|
||||
Expect(idx.Close()).To(Succeed())
|
||||
})
|
||||
|
||||
It("persists an additive schema change so later startups classify it as equal", func() {
|
||||
old := codeMapping()
|
||||
delete(old.DefaultMapping.Properties, "Title")
|
||||
buildIndex(old, nil)
|
||||
|
||||
idx, classification, err := bleve.NewIndex(root)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(classification.Verdict).To(Equal(searchmapping.VerdictAdditive))
|
||||
Expect(idx.Index("1", map[string]any{"Title": "hello"})).To(Succeed())
|
||||
Expect(idx.Close()).To(Succeed())
|
||||
|
||||
idx, classification, err = bleve.NewIndex(root)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(classification.Verdict).To(Equal(searchmapping.VerdictEqual))
|
||||
Expect(idx.Close()).To(Succeed())
|
||||
})
|
||||
|
||||
It("refuses when a new field already has data in the index", func() {
|
||||
old := codeMapping()
|
||||
Expect(old.DefaultMapping.Properties).To(HaveKey("Mtime"))
|
||||
delete(old.DefaultMapping.Properties, "Mtime")
|
||||
buildIndex(old, map[string]map[string]any{"1": {"Mtime": "2026-01-02T03:04:05Z"}})
|
||||
|
||||
idx, _, err := bleve.NewIndex(root)
|
||||
Expect(err).To(MatchError(searchmapping.ErrManualActionRequired))
|
||||
Expect(idx).To(BeNil())
|
||||
})
|
||||
|
||||
It("refuses when a new object field already has nested data in the index", func() {
|
||||
old := codeMapping()
|
||||
Expect(old.DefaultMapping.Properties).To(HaveKey("photo"))
|
||||
delete(old.DefaultMapping.Properties, "photo")
|
||||
buildIndex(old, map[string]map[string]any{"1": {"photo": map[string]any{"cameraMake": "ACME"}}})
|
||||
|
||||
_, _, err := bleve.NewIndex(root)
|
||||
Expect(err).To(MatchError(searchmapping.ErrManualActionRequired))
|
||||
})
|
||||
|
||||
It("refuses on a changed field definition", func() {
|
||||
old := codeMapping()
|
||||
name := old.DefaultMapping.Properties["Name"]
|
||||
Expect(name).ToNot(BeNil())
|
||||
Expect(name.Fields).ToNot(BeEmpty())
|
||||
name.Fields[0].Analyzer = "fulltext"
|
||||
buildIndex(old, nil)
|
||||
|
||||
_, _, err := bleve.NewIndex(root)
|
||||
Expect(err).To(MatchError(searchmapping.ErrManualActionRequired))
|
||||
})
|
||||
|
||||
It("refuses when a stored field was removed from the code schema", func() {
|
||||
old := codeMapping()
|
||||
old.DefaultMapping.AddFieldMappingsAt("Legacy", bleveSearch.NewTextFieldMapping())
|
||||
buildIndex(old, nil)
|
||||
|
||||
_, _, err := bleve.NewIndex(root)
|
||||
Expect(err).To(MatchError(searchmapping.ErrManualActionRequired))
|
||||
})
|
||||
|
||||
It("refuses when a default_mapping attribute changed", func() {
|
||||
old := codeMapping()
|
||||
old.DefaultMapping.Dynamic = false
|
||||
buildIndex(old, nil)
|
||||
|
||||
_, _, err := bleve.NewIndex(root)
|
||||
Expect(err).To(MatchError(searchmapping.ErrManualActionRequired))
|
||||
})
|
||||
|
||||
It("refuses on a changed analyzer definition", func() {
|
||||
old := codeMapping()
|
||||
Expect(old.CustomAnalysis.Analyzers).To(HaveKey("fulltext"))
|
||||
old.CustomAnalysis.Analyzers["fulltext"] = map[string]any{
|
||||
"type": custom.Name,
|
||||
"tokenizer": unicode.Name,
|
||||
"token_filters": []string{lowercase.Name},
|
||||
}
|
||||
buildIndex(old, nil)
|
||||
|
||||
_, _, err := bleve.NewIndex(root)
|
||||
Expect(err).To(MatchError(searchmapping.ErrManualActionRequired))
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("NewMapping", func() {
|
||||
It("only references registered analyzers", func() {
|
||||
m, err := bleve.NewMapping()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
impl, ok := m.(*bleveMapping.IndexMappingImpl)
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(impl.Validate()).To(Succeed())
|
||||
})
|
||||
|
||||
// A diff here means existing indexes will classify as breaking (schema or
|
||||
// bleve marshaling changed); update the golden file only deliberately.
|
||||
It("matches the committed golden mapping", func() {
|
||||
m, err := bleve.NewMapping()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
b, err := json.Marshal(m)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
var got, golden map[string]any
|
||||
Expect(json.Unmarshal(b, &got)).To(Succeed())
|
||||
|
||||
goldenB, err := os.ReadFile("testdata/mapping.golden.json")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(json.Unmarshal(goldenB, &golden)).To(Succeed())
|
||||
|
||||
Expect(got).To(Equal(golden))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,41 @@
|
||||
package bleve_test
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/conversions"
|
||||
|
||||
bleveSearch "github.com/blevesearch/bleve/v2"
|
||||
bquery "github.com/blevesearch/bleve/v2/search/query"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
// Mtime is typed as a date, so range queries are chronological, not a
|
||||
// lexicographic keyword compare.
|
||||
var _ = Describe("Mtime date range", func() {
|
||||
It("compares chronologically, not lexicographically", func() {
|
||||
m, err := bleve.NewMapping()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
idx, err := bleveSearch.NewMemOnly(m)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
r := search.Resource{ID: "x", Document: content.Document{Name: "f", Mtime: conversions.ToPointer(time.Date(2026, 3, 15, 12, 0, 0, 123456789, time.UTC))}}
|
||||
doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(idx.Index(r.ID, doc)).To(Succeed())
|
||||
|
||||
hits := func(qs string) uint64 {
|
||||
res, err := idx.Search(bleveSearch.NewSearchRequest(bquery.NewQueryStringQuery(qs)))
|
||||
Expect(err).ToNot(HaveOccurred(), qs)
|
||||
return res.Total
|
||||
}
|
||||
Expect(hits(`Mtime:>"2026-01-01T00:00:00Z"`)).To(Equal(uint64(1)), "in-range")
|
||||
Expect(hits(`Mtime:>"2026-06-01T00:00:00Z"`)).To(Equal(uint64(0)), "out-of-range")
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,677 @@
|
||||
{
|
||||
"default_mapping": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"properties": {
|
||||
"Content": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"analyzer": "fulltext",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"Deleted": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "boolean",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"Favorites": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"analyzer": "lowercaseKeyword",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"Hidden": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "boolean",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"ID": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"MimeType": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"Mtime": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "datetime",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"Name": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"analyzer": "lowercaseKeyword",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"ParentID": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"Path": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"RootID": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"Size": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"Tags": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"analyzer": "lowercaseKeyword",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"Title": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"Type": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"audio": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"properties": {
|
||||
"album": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"albumArtist": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"artist": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"bitrate": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"composers": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"copyright": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"disc": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"discCount": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"duration": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"genre": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"hasDrm": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "boolean",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"isVariableBitrate": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "boolean",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"title": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"track": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"trackCount": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"year": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"image": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"properties": {
|
||||
"height": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"width": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"location": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"properties": {
|
||||
"altitude": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"latitude": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"longitude": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"location_geopoint": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "geopoint",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"photo": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"properties": {
|
||||
"cameraMake": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"cameraModel": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "text",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_term_vectors": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"exposureDenominator": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"exposureNumerator": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"fNumber": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"focalLength": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"iso": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"orientation": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "number",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"takenDateTime": {
|
||||
"enabled": true,
|
||||
"dynamic": true,
|
||||
"fields": [
|
||||
{
|
||||
"type": "datetime",
|
||||
"store": true,
|
||||
"index": true,
|
||||
"include_in_all": true,
|
||||
"docvalues": true
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"type_field": "_type",
|
||||
"default_type": "_default",
|
||||
"default_analyzer": "keyword",
|
||||
"default_datetime_parser": "dateTimeOptional",
|
||||
"default_field": "_all",
|
||||
"store_dynamic": true,
|
||||
"index_dynamic": true,
|
||||
"docvalues_dynamic": true,
|
||||
"analysis": {
|
||||
"analyzers": {
|
||||
"fulltext": {
|
||||
"token_filters": [
|
||||
"to_lower",
|
||||
"stemmer_porter"
|
||||
],
|
||||
"tokenizer": "unicode",
|
||||
"type": "custom"
|
||||
},
|
||||
"lowercaseKeyword": {
|
||||
"token_filters": [
|
||||
"to_lower"
|
||||
],
|
||||
"tokenizer": "single",
|
||||
"type": "custom"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"time"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
|
||||
"github.com/opencloud-eu/opencloud/pkg/generators"
|
||||
@@ -20,6 +21,7 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/config"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/config/parser"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/metrics"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
|
||||
bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve"
|
||||
@@ -71,7 +73,14 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
var eng search.Engine
|
||||
switch cfg.Engine.Type {
|
||||
case "bleve":
|
||||
idx, err := bleve.NewIndex(cfg.Engine.Bleve.Datapath)
|
||||
idx, classification, err := bleve.NewIndex(cfg.Engine.Bleve.Datapath)
|
||||
// warn before the error check: the new mapping may already be
|
||||
// persisted, then later startups classify equal and stay silent
|
||||
if classification.Verdict == searchmapping.VerdictAdditive {
|
||||
logger.Warn().
|
||||
Strs("fields", classification.NewFields).
|
||||
Msgf("the bleve index at %s was built with an older schema; the new fields were added to the index schema, but documents indexed before the upgrade do not contain them and queries on these fields will miss those documents until they are re-indexed; to re-index everything run: opencloud search index --all-spaces --force-rescan", cfg.Engine.Bleve.Datapath)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -119,7 +128,10 @@ func Server(cfg *config.Config) *cobra.Command {
|
||||
return fmt.Errorf("failed to create OpenSearch client: %w", err)
|
||||
}
|
||||
|
||||
openSearchBackend, err := opensearch.NewBackend(cfg.Engine.OpenSearch.ResourceIndex.Name, client)
|
||||
// a hung cluster must fail the start, not block it forever
|
||||
startupCtx, cancelStartup := context.WithTimeout(ctx, time.Minute)
|
||||
openSearchBackend, err := opensearch.NewBackend(startupCtx, cfg.Engine.OpenSearch.ResourceIndex.Name, client, logger)
|
||||
cancelStartup()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create OpenSearch backend: %w", err)
|
||||
}
|
||||
|
||||
@@ -3,9 +3,9 @@ package content
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/conversions"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/tags"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
@@ -54,7 +54,7 @@ func (b Basic) Extract(_ context.Context, ri *storageProvider.ResourceInfo) (Doc
|
||||
}
|
||||
|
||||
if ri.Mtime != nil {
|
||||
doc.Mtime = utils.TSToTime(ri.Mtime).UTC().Format(time.RFC3339Nano)
|
||||
doc.Mtime = conversions.ToPointer(utils.TSToTime(ri.Mtime).UTC())
|
||||
}
|
||||
|
||||
return doc, nil
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
package content_test
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/conversions"
|
||||
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
@@ -69,11 +73,11 @@ var _ = Describe("Basic", func() {
|
||||
It("RFC3339 mtime", func() {
|
||||
for _, data := range []struct {
|
||||
second uint64
|
||||
expect string
|
||||
expect *time.Time
|
||||
}{
|
||||
{second: 4000, expect: "1970-01-01T01:06:40Z"},
|
||||
{second: 3000, expect: "1970-01-01T00:50:00Z"},
|
||||
{expect: ""},
|
||||
{second: 4000, expect: conversions.ToPointer(time.Unix(4000, 0).UTC())},
|
||||
{second: 3000, expect: conversions.ToPointer(time.Unix(3000, 0).UTC())},
|
||||
{},
|
||||
} {
|
||||
ri := &storageProvider.ResourceInfo{}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package content
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/bbalet/stopwords"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
@@ -14,14 +15,14 @@ func init() {
|
||||
// Document wraps all resource meta fields,
|
||||
// it is used as a content extraction result.
|
||||
type Document struct {
|
||||
Title string
|
||||
Name string
|
||||
Content string
|
||||
Size uint64
|
||||
Mtime string
|
||||
MimeType string
|
||||
Tags []string
|
||||
Favorites []string
|
||||
Title string `json:"Title"`
|
||||
Name string `json:"Name"`
|
||||
Content string `json:"Content"`
|
||||
Size uint64 `json:"Size"`
|
||||
Mtime *time.Time `json:"Mtime"`
|
||||
MimeType string `json:"MimeType"`
|
||||
Tags []string `json:"Tags"`
|
||||
Favorites []string `json:"Favorites"`
|
||||
Audio *libregraph.Audio `json:"audio,omitempty"`
|
||||
Image *libregraph.Image `json:"image,omitempty"`
|
||||
Location *libregraph.GeoCoordinates `json:"location,omitempty"`
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
bleveMapping "github.com/blevesearch/bleve/v2/mapping"
|
||||
)
|
||||
|
||||
// BleveBuildMapping builds a bleve DocumentMapping for t by walking the
|
||||
// struct via reflection. Field names come from json tags; overrides are
|
||||
// keyed by those names (or dotted paths for nested fields).
|
||||
//
|
||||
// The returned mapping references analyzer names (Analyzer field on the
|
||||
// FieldOpts, plus "fulltext" / "path_hierarchy" for the corresponding Types);
|
||||
// the caller must register every referenced analyzer on the enclosing
|
||||
// IndexMapping (IndexMapping.Validate catches missing ones).
|
||||
func BleveBuildMapping(t reflect.Type, overrides map[string]FieldOpts) (*bleveMapping.DocumentMapping, error) {
|
||||
return buildBleveDocMapping(t, overrides, "")
|
||||
}
|
||||
|
||||
func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix string) (*bleveMapping.DocumentMapping, error) {
|
||||
doc := bleve.NewDocumentMapping()
|
||||
err := walkFields(t, func(fi fieldInfo) error {
|
||||
key := fi.Name
|
||||
if prefix != "" {
|
||||
key = prefix + "." + fi.Name
|
||||
}
|
||||
opts := overrides[key]
|
||||
fieldType := opts.Type
|
||||
if fieldType == "" {
|
||||
fieldType = inferType(fi.GoField.Type)
|
||||
}
|
||||
|
||||
if fieldType == TypeObject {
|
||||
sub := structType(fi.GoField.Type)
|
||||
if sub == nil {
|
||||
return fmt.Errorf("mapping: object type on non-struct field %q", key)
|
||||
}
|
||||
subDoc, err := buildBleveDocMapping(sub, overrides, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
doc.AddSubDocumentMapping(fi.Name, subDoc)
|
||||
return nil
|
||||
}
|
||||
|
||||
if fieldType == TypeGeopoint {
|
||||
// Keep the facet object, add a sibling _geopoint field (see GeopointSuffix).
|
||||
sub := structType(fi.GoField.Type)
|
||||
if sub == nil {
|
||||
return fmt.Errorf("mapping: geopoint type on non-struct field %q", key)
|
||||
}
|
||||
subDoc, err := buildBleveDocMapping(sub, overrides, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
doc.AddSubDocumentMapping(fi.Name, subDoc)
|
||||
doc.AddFieldMappingsAt(fi.Name+GeopointSuffix, bleve.NewGeoPointFieldMapping())
|
||||
return nil
|
||||
}
|
||||
|
||||
fm, err := bleveFieldMapping(fieldType, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mapping: field %q: %w", key, err)
|
||||
}
|
||||
doc.AddFieldMappingsAt(fi.Name, fm)
|
||||
return nil
|
||||
})
|
||||
return doc, err
|
||||
}
|
||||
|
||||
func bleveFieldMapping(fieldType string, opts FieldOpts) (*bleveMapping.FieldMapping, error) {
|
||||
switch fieldType {
|
||||
case TypeWildcard:
|
||||
// bleve has no wildcard type; fall back to keyword-ish text.
|
||||
fieldType = TypeKeyword
|
||||
fallthrough
|
||||
case TypeKeyword, TypeFulltext, TypePath:
|
||||
fm := bleve.NewTextFieldMapping()
|
||||
switch {
|
||||
case opts.Analyzer != "":
|
||||
fm.Analyzer = opts.Analyzer
|
||||
case fieldType == TypeFulltext:
|
||||
fm.Analyzer = "fulltext"
|
||||
case fieldType == TypePath:
|
||||
fm.Analyzer = "path_hierarchy"
|
||||
}
|
||||
switch {
|
||||
case opts.IncludeInAll != nil:
|
||||
fm.IncludeInAll = *opts.IncludeInAll
|
||||
case fieldType == TypeFulltext, fieldType == TypePath:
|
||||
fm.IncludeInAll = false
|
||||
}
|
||||
return fm, nil
|
||||
case TypeNumeric:
|
||||
return bleve.NewNumericFieldMapping(), nil
|
||||
case TypeBool:
|
||||
return bleve.NewBooleanFieldMapping(), nil
|
||||
case TypeDatetime:
|
||||
return bleve.NewDateTimeFieldMapping(), nil
|
||||
case TypeGeopoint:
|
||||
return bleve.NewGeoPointFieldMapping(), nil
|
||||
case "":
|
||||
return nil, fmt.Errorf("no type inferred and no override")
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported type %q", fieldType)
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
type bleveDoc struct {
|
||||
Name string `json:"Name"`
|
||||
Content string `json:"Content"`
|
||||
Tags []string `json:"Tags"`
|
||||
Size uint64 `json:"Size"`
|
||||
Deleted bool `json:"Deleted"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
Nested *nested `json:"nested,omitempty"`
|
||||
}
|
||||
|
||||
type nested struct {
|
||||
Artist string `json:"artist"`
|
||||
Year int `json:"year"`
|
||||
}
|
||||
|
||||
var _ = Describe("BleveBuildMapping", func() {
|
||||
It("falls back to text for wildcard fields", func() {
|
||||
// bleve wildcard falls back to keyword-ish text (bleve has no wildcard type).
|
||||
type doc struct {
|
||||
Mime string `json:"mime"`
|
||||
}
|
||||
dm, err := BleveBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"mime": {Type: TypeWildcard}})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
fms := dm.Properties["mime"].Fields
|
||||
Expect(fms).To(HaveLen(1))
|
||||
Expect(fms[0].Type).To(Equal("text"), "wildcard should map to text")
|
||||
})
|
||||
|
||||
DescribeTable("infers field types",
|
||||
func(field, wantType string) {
|
||||
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
prop := dm.Properties[field]
|
||||
Expect(prop).ToNot(BeNil(), "missing property %q", field)
|
||||
Expect(prop.Fields).ToNot(BeEmpty(), "%q: no field mappings", field)
|
||||
Expect(prop.Fields[0].Type).To(Equal(wantType), "%q type", field)
|
||||
},
|
||||
Entry("Name", "Name", "text"),
|
||||
Entry("Content", "Content", "text"),
|
||||
Entry("Tags", "Tags", "text"),
|
||||
Entry("Size", "Size", "number"),
|
||||
Entry("Deleted", "Deleted", "boolean"),
|
||||
Entry("CreatedAt", "CreatedAt", "datetime"),
|
||||
)
|
||||
|
||||
It("maps a nested struct as a sub-document", func() {
|
||||
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
sub := dm.Properties["nested"]
|
||||
Expect(sub).ToNot(BeNil(), "missing nested sub-document")
|
||||
Expect(sub.Properties["artist"]).ToNot(BeNil())
|
||||
Expect(sub.Properties["year"]).ToNot(BeNil())
|
||||
Expect(sub.Properties["artist"].Fields[0].Type).To(Equal("text"), "nested.artist")
|
||||
Expect(sub.Properties["year"].Fields[0].Type).To(Equal("number"), "nested.year")
|
||||
})
|
||||
|
||||
It("applies field overrides", func() {
|
||||
includeInAllFalse := false
|
||||
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{
|
||||
"Name": {Analyzer: "lowercaseKeyword"},
|
||||
"Content": {Type: TypeFulltext},
|
||||
"Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &includeInAllFalse},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
nameField := dm.Properties["Name"].Fields[0]
|
||||
Expect(nameField.Analyzer).To(Equal("lowercaseKeyword"), "Name analyzer")
|
||||
Expect(nameField.IncludeInAll).To(BeTrue(), "Name IncludeInAll should stay default-true when not overridden")
|
||||
contentField := dm.Properties["Content"].Fields[0]
|
||||
Expect(contentField.Analyzer).To(Equal("fulltext"), "Content analyzer")
|
||||
Expect(contentField.IncludeInAll).To(BeFalse(), "Content IncludeInAll should default to false for fulltext type")
|
||||
tagsField := dm.Properties["Tags"].Fields[0]
|
||||
Expect(tagsField.IncludeInAll).To(BeFalse(), "Tags IncludeInAll should honor the explicit false override")
|
||||
})
|
||||
|
||||
It("builds an object sub-document plus a geopoint sibling", func() {
|
||||
type geoDoc struct {
|
||||
Location *struct {
|
||||
Lon *float64 `json:"longitude,omitempty"`
|
||||
Lat *float64 `json:"latitude,omitempty"`
|
||||
Alt *float64 `json:"altitude,omitempty"`
|
||||
} `json:"location,omitempty"`
|
||||
}
|
||||
dm, err := BleveBuildMapping(reflect.TypeFor[geoDoc](), map[string]FieldOpts{
|
||||
"location": {Type: TypeGeopoint},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// Original facet stays as an object sub-document with numeric
|
||||
// sub-properties - for data retrieval via hit.Fields and ordinary
|
||||
// numeric queries.
|
||||
loc := dm.Properties["location"]
|
||||
Expect(loc).ToNot(BeNil(), "location sub-document missing")
|
||||
Expect(loc.Fields).To(BeEmpty(), "location should not carry field mappings directly")
|
||||
for _, sub := range []string{"longitude", "latitude", "altitude"} {
|
||||
prop, ok := loc.Properties[sub]
|
||||
Expect(ok).To(BeTrue(), "missing sub-field %q under location", sub)
|
||||
Expect(prop.Fields).ToNot(BeEmpty(), "location.%s Fields", sub)
|
||||
Expect(prop.Fields[0].Type).To(Equal("number"), "location.%s type", sub)
|
||||
}
|
||||
// Sibling geopoint at "<name>_geopoint" for geo-distance queries.
|
||||
sibling := dm.Properties["location"+GeopointSuffix]
|
||||
Expect(sibling).ToNot(BeNil(), "location%s missing", GeopointSuffix)
|
||||
Expect(sibling.Fields).ToNot(BeEmpty(), "location%s Fields", GeopointSuffix)
|
||||
Expect(sibling.Fields[0].Type).To(Equal("geopoint"), "location%s type", GeopointSuffix)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,168 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ErrManualActionRequired marks schema changes that cannot be applied in place.
|
||||
var ErrManualActionRequired = errors.New("manual action required")
|
||||
|
||||
// ManualActionRequiredError builds the operator-facing error for a breaking
|
||||
// schema change. index names the index, deleteStep is the engine-specific
|
||||
// instruction to remove it.
|
||||
func ManualActionRequiredError(index, deleteStep string, reasons []string) error {
|
||||
return fmt.Errorf(
|
||||
"%w: search index %s was built with a different schema:\n - %s\n"+
|
||||
"There is no in-place migration: with the OpenCloud search service stopped, %s, "+
|
||||
"then start it again (an empty index with the new schema is created), "+
|
||||
"then rebuild the content by running: opencloud search index --all-spaces. "+
|
||||
"To bring the instance up without search until a maintenance window, "+
|
||||
"set OC_EXCLUDE_RUN_SERVICES=search; until the service is back, search "+
|
||||
"and features built on it (e.g. the search bar and the tag list) are "+
|
||||
"unavailable",
|
||||
ErrManualActionRequired, index, strings.Join(reasons, "\n - "), deleteStep,
|
||||
)
|
||||
}
|
||||
|
||||
type Verdict string
|
||||
|
||||
const (
|
||||
VerdictEqual Verdict = "equal"
|
||||
VerdictAdditive Verdict = "additive"
|
||||
VerdictBreaking Verdict = "breaking"
|
||||
)
|
||||
|
||||
// Classification is the outcome of diffing a stored index schema against the
|
||||
// schema generated from code.
|
||||
type Classification struct {
|
||||
Verdict Verdict
|
||||
// NewFields are dotted paths of fields that only exist in the code schema.
|
||||
NewFields []string
|
||||
// Reasons are human-readable breaking differences.
|
||||
Reasons []string
|
||||
}
|
||||
|
||||
// Classify recursively compares a stored `properties` tree against the one
|
||||
// generated from code. Both sides must be generic JSON-decoded values, not
|
||||
// marshaled Go structs. dataFields reports whether the index holds data at or
|
||||
// below a dotted field path absent from the stored schema (bleve dynamic
|
||||
// fields); engines without that blind spot pass nil.
|
||||
func Classify(stored, code map[string]any, dataFields func(path string) bool) Classification {
|
||||
c := Classification{Verdict: VerdictEqual}
|
||||
classifyProperties(stored, code, dataFields, "", &c)
|
||||
if c.Verdict == VerdictEqual && len(c.NewFields) > 0 {
|
||||
c.Verdict = VerdictAdditive
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func classifyProperties(stored, code map[string]any, dataFields func(string) bool, prefix string, c *Classification) {
|
||||
for _, k := range slices.Sorted(maps.Keys(stored)) {
|
||||
path := joinPath(prefix, k)
|
||||
codeNode, ok := code[k]
|
||||
if !ok {
|
||||
c.breaking(fmt.Sprintf("field %s exists in the index but not in the code schema (removed or renamed)", path))
|
||||
continue
|
||||
}
|
||||
classifyNode(stored[k], codeNode, dataFields, path, c)
|
||||
}
|
||||
|
||||
for _, k := range slices.Sorted(maps.Keys(code)) {
|
||||
if _, ok := stored[k]; ok {
|
||||
continue
|
||||
}
|
||||
path := joinPath(prefix, k)
|
||||
if dataFields != nil && dataFields(path) {
|
||||
c.breaking(fmt.Sprintf("field %s is explicitly mapped now, but the index already holds data that was indexed dynamically for it, of an unknown type", path))
|
||||
continue
|
||||
}
|
||||
c.NewFields = append(c.NewFields, leafPaths(code[k], path)...)
|
||||
}
|
||||
}
|
||||
|
||||
func classifyNode(stored, code any, dataFields func(string) bool, path string, c *Classification) {
|
||||
storedMap, sOK := stored.(map[string]any)
|
||||
codeMap, cOK := code.(map[string]any)
|
||||
if !sOK || !cOK {
|
||||
if !reflect.DeepEqual(stored, code) {
|
||||
c.breaking(fmt.Sprintf("field %s changed: index %s, code %s", path, compactJSON(stored), compactJSON(code)))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
for _, k := range sortedUnionKeys(storedMap, codeMap) {
|
||||
if k == "properties" {
|
||||
continue
|
||||
}
|
||||
sv, sHas := storedMap[k]
|
||||
cv, cHas := codeMap[k]
|
||||
if sHas && cHas && reflect.DeepEqual(sv, cv) {
|
||||
continue
|
||||
}
|
||||
c.breaking(fmt.Sprintf("field %s: %s changed: index %s, code %s", path, k, optJSON(sv, sHas), optJSON(cv, cHas)))
|
||||
}
|
||||
|
||||
storedProps, _ := storedMap["properties"].(map[string]any)
|
||||
codeProps, _ := codeMap["properties"].(map[string]any)
|
||||
if len(storedProps) > 0 || len(codeProps) > 0 {
|
||||
classifyProperties(storedProps, codeProps, dataFields, path, c)
|
||||
}
|
||||
}
|
||||
|
||||
// leafPaths lists the dotted paths of all leaf fields at or below node.
|
||||
func leafPaths(node any, path string) []string {
|
||||
if nodeMap, ok := node.(map[string]any); ok {
|
||||
if props, ok := nodeMap["properties"].(map[string]any); ok && len(props) > 0 {
|
||||
var leaves []string
|
||||
for _, k := range slices.Sorted(maps.Keys(props)) {
|
||||
leaves = append(leaves, leafPaths(props[k], path+"."+k)...)
|
||||
}
|
||||
return leaves
|
||||
}
|
||||
}
|
||||
return []string{path}
|
||||
}
|
||||
|
||||
func (c *Classification) breaking(reason string) {
|
||||
c.Verdict = VerdictBreaking
|
||||
c.Reasons = append(c.Reasons, reason)
|
||||
}
|
||||
|
||||
func joinPath(prefix, k string) string {
|
||||
if prefix == "" {
|
||||
return k
|
||||
}
|
||||
return prefix + "." + k
|
||||
}
|
||||
|
||||
func sortedUnionKeys(a, b map[string]any) []string {
|
||||
keys := slices.Collect(maps.Keys(a))
|
||||
for k := range b {
|
||||
if _, ok := a[k]; !ok {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
slices.Sort(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func optJSON(v any, present bool) string {
|
||||
if !present {
|
||||
return "(unset)"
|
||||
}
|
||||
return compactJSON(v)
|
||||
}
|
||||
|
||||
func compactJSON(v any) string {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("%v", v)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("Classify", func() {
|
||||
code := `{
|
||||
"Name": {"type": "keyword"},
|
||||
"Size": {"type": "long"},
|
||||
"photo": {"properties": {"cameraMake": {"type": "keyword"}, "cameraModel": {"type": "keyword"}}}
|
||||
}`
|
||||
|
||||
parse := func(s string) map[string]any {
|
||||
var m map[string]any
|
||||
Expect(json.Unmarshal([]byte(s), &m)).To(Succeed())
|
||||
return m
|
||||
}
|
||||
|
||||
hasData := func(fields ...string) func(string) bool {
|
||||
return func(path string) bool { return slices.Contains(fields, path) }
|
||||
}
|
||||
|
||||
It("classifies identical schemas as equal", func() {
|
||||
c := Classify(parse(code), parse(code), nil)
|
||||
Expect(c.Verdict).To(Equal(VerdictEqual))
|
||||
Expect(c.NewFields).To(BeEmpty())
|
||||
Expect(c.Reasons).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("classifies a new top-level field as additive", func() {
|
||||
stored := parse(code)
|
||||
delete(stored, "Size")
|
||||
|
||||
c := Classify(stored, parse(code), nil)
|
||||
Expect(c.Verdict).To(Equal(VerdictAdditive))
|
||||
Expect(c.NewFields).To(ConsistOf("Size"))
|
||||
Expect(c.Reasons).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("classifies a new nested field as additive", func() {
|
||||
stored := parse(code)
|
||||
delete(stored["photo"].(map[string]any)["properties"].(map[string]any), "cameraMake")
|
||||
|
||||
c := Classify(stored, parse(code), nil)
|
||||
Expect(c.Verdict).To(Equal(VerdictAdditive))
|
||||
Expect(c.NewFields).To(ConsistOf("photo.cameraMake"))
|
||||
})
|
||||
|
||||
It("lists every leaf of a new subtree", func() {
|
||||
stored := parse(code)
|
||||
delete(stored, "photo")
|
||||
|
||||
c := Classify(stored, parse(code), nil)
|
||||
Expect(c.Verdict).To(Equal(VerdictAdditive))
|
||||
Expect(c.NewFields).To(ConsistOf("photo.cameraMake", "photo.cameraModel"))
|
||||
})
|
||||
|
||||
It("breaks when a new field already has data in the index", func() {
|
||||
stored := parse(code)
|
||||
delete(stored, "Size")
|
||||
|
||||
c := Classify(stored, parse(code), hasData("Size"))
|
||||
Expect(c.Verdict).To(Equal(VerdictBreaking))
|
||||
Expect(c.Reasons).To(ConsistOf(ContainSubstring("Size")))
|
||||
})
|
||||
|
||||
It("breaks when a new nested field already has data in the index", func() {
|
||||
stored := parse(code)
|
||||
delete(stored["photo"].(map[string]any)["properties"].(map[string]any), "cameraMake")
|
||||
|
||||
c := Classify(stored, parse(code), hasData("photo.cameraMake"))
|
||||
Expect(c.Verdict).To(Equal(VerdictBreaking))
|
||||
Expect(c.Reasons).To(ConsistOf(ContainSubstring("photo.cameraMake")))
|
||||
})
|
||||
|
||||
It("breaks when a new subtree already has data below it", func() {
|
||||
stored := parse(code)
|
||||
delete(stored, "photo")
|
||||
|
||||
// the callback is consulted with the subtree root
|
||||
c := Classify(stored, parse(code), hasData("photo"))
|
||||
Expect(c.Verdict).To(Equal(VerdictBreaking))
|
||||
Expect(c.Reasons).To(ConsistOf(ContainSubstring("photo")))
|
||||
})
|
||||
|
||||
It("breaks on a changed field definition", func() {
|
||||
stored := parse(code)
|
||||
stored["Size"].(map[string]any)["type"] = "keyword"
|
||||
|
||||
c := Classify(stored, parse(code), nil)
|
||||
Expect(c.Verdict).To(Equal(VerdictBreaking))
|
||||
Expect(c.Reasons).To(ConsistOf(ContainSubstring("Size")))
|
||||
})
|
||||
|
||||
It("breaks on a field that was removed from the code schema", func() {
|
||||
reduced := parse(code)
|
||||
delete(reduced, "Size")
|
||||
|
||||
c := Classify(parse(code), reduced, nil)
|
||||
Expect(c.Verdict).To(Equal(VerdictBreaking))
|
||||
Expect(c.Reasons).To(ConsistOf(ContainSubstring("removed or renamed")))
|
||||
})
|
||||
|
||||
It("breaks on a changed object attribute", func() {
|
||||
stored := parse(code)
|
||||
stored["photo"].(map[string]any)["dynamic"] = true
|
||||
|
||||
c := Classify(stored, parse(code), nil)
|
||||
Expect(c.Verdict).To(Equal(VerdictBreaking))
|
||||
Expect(c.Reasons).To(ConsistOf(ContainSubstring("dynamic")))
|
||||
})
|
||||
|
||||
It("lets breaking win over additive", func() {
|
||||
stored := parse(code)
|
||||
delete(stored, "Size")
|
||||
stored["Name"].(map[string]any)["type"] = "text"
|
||||
|
||||
c := Classify(stored, parse(code), nil)
|
||||
Expect(c.Verdict).To(Equal(VerdictBreaking))
|
||||
Expect(c.NewFields).To(ConsistOf("Size"))
|
||||
Expect(c.Reasons).To(ConsistOf(ContainSubstring("Name")))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,172 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
// Deserialize builds a *T from bleve's flat hit.Fields map (json-tag keys,
|
||||
// "parent.child" for nested pointers). Used to rebuild a search Resource from a
|
||||
// hit. Fail-soft: an unparseable field stays at its zero value.
|
||||
func Deserialize[T any](fields map[string]any) *T {
|
||||
t := reflect.TypeFor[T]()
|
||||
if t.Kind() != reflect.Struct {
|
||||
panic(fmt.Sprintf("mapping: Deserialize requires a struct type, got %v", t))
|
||||
}
|
||||
out := reflect.New(t)
|
||||
fillStruct(out.Elem(), fields, "", setValue)
|
||||
return out.Interface().(*T)
|
||||
}
|
||||
|
||||
// DeserializeAt is Deserialize scoped to a dotted prefix, used to rebuild one
|
||||
// search-result facet (e.g. "audio"). Returns nil when nothing matched, so the
|
||||
// caller can leave the enclosing pointer nil.
|
||||
func DeserializeAt[T any](fields map[string]any, prefix string) *T {
|
||||
t := reflect.TypeFor[T]()
|
||||
if t.Kind() != reflect.Struct {
|
||||
panic(fmt.Sprintf("mapping: DeserializeAt requires a struct type, got %v", t))
|
||||
}
|
||||
out := reflect.New(t)
|
||||
if !fillStruct(out.Elem(), fields, prefix, setValue) {
|
||||
return nil
|
||||
}
|
||||
return out.Interface().(*T)
|
||||
}
|
||||
|
||||
// fillStruct walks v's exported fields, reading values from the flat fields map
|
||||
// (json-tag keys, "parent.child" for nested pointers). setLeaf converts each raw
|
||||
// value into a leaf, which is what lets the any-valued and string-valued
|
||||
// deserializers share one walker. Returns true if any leaf was populated.
|
||||
func fillStruct[V any](v reflect.Value, fields map[string]V, prefix string, setLeaf func(reflect.Value, V) error) bool {
|
||||
t := v.Type()
|
||||
touched := false
|
||||
for i := 0; i < t.NumField(); i++ {
|
||||
fi := resolveField(t.Field(i))
|
||||
if fi.Skip {
|
||||
continue
|
||||
}
|
||||
fv := v.Field(i)
|
||||
|
||||
if fi.Embedded {
|
||||
// Embedded *struct: allocate, recurse, keep the pointer only if set.
|
||||
if fv.Kind() == reflect.Ptr {
|
||||
if fv.Type().Elem().Kind() != reflect.Struct || !fv.CanSet() {
|
||||
continue
|
||||
}
|
||||
alloc := reflect.New(fv.Type().Elem())
|
||||
if fillStruct(alloc.Elem(), fields, prefix, setLeaf) {
|
||||
fv.Set(alloc)
|
||||
touched = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if fv.Kind() == reflect.Struct && fillStruct(fv, fields, prefix, setLeaf) {
|
||||
touched = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
key := fi.Name
|
||||
if prefix != "" {
|
||||
key = prefix + "." + fi.Name
|
||||
}
|
||||
|
||||
// Pointer to a nested (non-time) struct: recurse, keep the pointer only
|
||||
// if a field was populated.
|
||||
if fv.Kind() == reflect.Ptr {
|
||||
if elem := fv.Type().Elem(); elem.Kind() == reflect.Struct && elem != timeType && elem != timestampType {
|
||||
alloc := reflect.New(elem)
|
||||
if fillStruct(alloc.Elem(), fields, key, setLeaf) {
|
||||
fv.Set(alloc)
|
||||
touched = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if raw, ok := fields[key]; ok && setLeaf(fv, raw) == nil {
|
||||
touched = true
|
||||
}
|
||||
}
|
||||
return touched
|
||||
}
|
||||
|
||||
// setValue writes raw (an any value from a bleve hit) into v, converting to the
|
||||
// field's type. Returns an error on nil or a type mismatch.
|
||||
func setValue(v reflect.Value, raw any) error {
|
||||
if v.Kind() == reflect.Ptr {
|
||||
alloc := reflect.New(v.Type().Elem())
|
||||
if err := setValue(alloc.Elem(), raw); err != nil {
|
||||
return err
|
||||
}
|
||||
v.Set(alloc)
|
||||
return nil
|
||||
}
|
||||
if v.Type() == timeType || v.Type() == timestampType {
|
||||
t, ok := parseTime(raw)
|
||||
if !ok {
|
||||
return fmt.Errorf("not an RFC3339 time: %v", raw)
|
||||
}
|
||||
setParsedTime(v, t)
|
||||
return nil
|
||||
}
|
||||
if v.Kind() == reflect.Slice {
|
||||
return setSlice(v, raw)
|
||||
}
|
||||
rv := reflect.ValueOf(raw)
|
||||
if !rv.IsValid() {
|
||||
return fmt.Errorf("nil value for %s", v.Type())
|
||||
}
|
||||
if !rv.Type().ConvertibleTo(v.Type()) {
|
||||
return fmt.Errorf("cannot convert %s to %s", rv.Type(), v.Type())
|
||||
}
|
||||
v.Set(rv.Convert(v.Type()))
|
||||
return nil
|
||||
}
|
||||
|
||||
func setSlice(v reflect.Value, raw any) error {
|
||||
items, ok := raw.([]any)
|
||||
if !ok {
|
||||
// bleve unwraps single-element slices; re-wrap here.
|
||||
items = []any{raw}
|
||||
}
|
||||
// Compact in place with a single MakeSlice: unparseable elements are
|
||||
// dropped, Slice(0, j) trims the tail.
|
||||
out := reflect.MakeSlice(v.Type(), len(items), len(items))
|
||||
j := 0
|
||||
for _, item := range items {
|
||||
if setValue(out.Index(j), item) == nil {
|
||||
j++
|
||||
}
|
||||
}
|
||||
if j == 0 {
|
||||
return fmt.Errorf("no slice elements set from %T", raw)
|
||||
}
|
||||
v.Set(out.Slice(0, j))
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseTime(raw any) (time.Time, bool) {
|
||||
s, ok := raw.(string)
|
||||
if !ok {
|
||||
return time.Time{}, false
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return t, true
|
||||
}
|
||||
|
||||
// setParsedTime writes t into v, which must be a time.Time or
|
||||
// timestamppb.Timestamp field.
|
||||
func setParsedTime(v reflect.Value, t time.Time) {
|
||||
if v.Type() == timestampType {
|
||||
v.Set(reflect.ValueOf(*timestamppb.New(t)))
|
||||
return
|
||||
}
|
||||
v.Set(reflect.ValueOf(t))
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DeserializeStringsAt is DeserializeAt for a string-valued map (e.g. CS3
|
||||
// ArbitraryMetadata), parsing each string into the field's Go type via strconv/
|
||||
// time.Parse. Used to build a graph DriveItem facet. Returns nil when nothing
|
||||
// under the prefix matched.
|
||||
func DeserializeStringsAt[T any](fields map[string]string, prefix string) *T {
|
||||
t := reflect.TypeFor[T]()
|
||||
if t.Kind() != reflect.Struct {
|
||||
panic(fmt.Sprintf("mapping: DeserializeStringsAt requires a struct type, got %v", t))
|
||||
}
|
||||
out := reflect.New(t)
|
||||
// Callers pass a flat-key prefix with a trailing dot (e.g.
|
||||
// "libre.graph.audio."); fillStruct joins segments with ".", so drop it.
|
||||
if !fillStruct(out.Elem(), fields, strings.TrimSuffix(prefix, "."), setValueFromString) {
|
||||
return nil
|
||||
}
|
||||
return out.Interface().(*T)
|
||||
}
|
||||
|
||||
// setValueFromString parses the string raw into v's Go type via strconv/
|
||||
// time.Parse, returning a descriptive error on failure.
|
||||
func setValueFromString(v reflect.Value, raw string) error {
|
||||
if v.Kind() == reflect.Ptr {
|
||||
alloc := reflect.New(v.Type().Elem())
|
||||
if err := setValueFromString(alloc.Elem(), raw); err != nil {
|
||||
return err
|
||||
}
|
||||
v.Set(alloc)
|
||||
return nil
|
||||
}
|
||||
if v.Type() == timeType || v.Type() == timestampType {
|
||||
t, err := time.Parse(time.RFC3339, raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse time %q: %w", raw, err)
|
||||
}
|
||||
setParsedTime(v, t)
|
||||
return nil
|
||||
}
|
||||
switch v.Kind() {
|
||||
case reflect.String:
|
||||
v.SetString(raw)
|
||||
return nil
|
||||
case reflect.Bool:
|
||||
b, err := strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse bool %q: %w", raw, err)
|
||||
}
|
||||
v.SetBool(b)
|
||||
return nil
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
n, err := strconv.ParseInt(raw, 10, v.Type().Bits())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse int %q: %w", raw, err)
|
||||
}
|
||||
v.SetInt(n)
|
||||
return nil
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
n, err := strconv.ParseUint(raw, 10, v.Type().Bits())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse uint %q: %w", raw, err)
|
||||
}
|
||||
v.SetUint(n)
|
||||
return nil
|
||||
case reflect.Float32, reflect.Float64:
|
||||
f, err := strconv.ParseFloat(raw, v.Type().Bits())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse float %q: %w", raw, err)
|
||||
}
|
||||
v.SetFloat(f)
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("unsupported target kind %s", v.Kind())
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
type stringFacet struct {
|
||||
Artist *string `json:"artist,omitempty"`
|
||||
Year *int32 `json:"year,omitempty"`
|
||||
Duration *int64 `json:"duration,omitempty"`
|
||||
Rating *float64 `json:"rating,omitempty"`
|
||||
Explicit *bool `json:"explicit,omitempty"`
|
||||
Taken *time.Time `json:"takenDateTime,omitempty"`
|
||||
}
|
||||
|
||||
var _ = Describe("DeserializeStringsAt", func() {
|
||||
It("errors on an unsupported target kind", func() {
|
||||
v := reflect.New(reflect.TypeFor[[]int]()).Elem() // settable slice
|
||||
Expect(setValueFromString(v, "x")).To(HaveOccurred(), "expected error for unsupported target kind (slice)")
|
||||
})
|
||||
|
||||
It("parses basic types", func() {
|
||||
r := DeserializeStringsAt[stringFacet](map[string]string{
|
||||
"libre.graph.audio.artist": "Queen",
|
||||
"libre.graph.audio.year": "1975",
|
||||
"libre.graph.audio.duration": "354000",
|
||||
"libre.graph.audio.rating": "4.9",
|
||||
"libre.graph.audio.explicit": "true",
|
||||
"libre.graph.audio.takenDateTime": "2024-01-02T03:04:05Z",
|
||||
}, "libre.graph.audio.")
|
||||
Expect(r).ToNot(BeNil())
|
||||
Expect(r.Artist).ToNot(BeNil())
|
||||
Expect(*r.Artist).To(Equal("Queen"))
|
||||
Expect(r.Year).ToNot(BeNil())
|
||||
Expect(*r.Year).To(Equal(int32(1975)))
|
||||
Expect(r.Duration).ToNot(BeNil())
|
||||
Expect(*r.Duration).To(Equal(int64(354000)))
|
||||
Expect(r.Rating).ToNot(BeNil())
|
||||
Expect(*r.Rating).To(Equal(4.9))
|
||||
Expect(r.Explicit).ToNot(BeNil())
|
||||
Expect(*r.Explicit).To(BeTrue())
|
||||
Expect(r.Taken).ToNot(BeNil())
|
||||
Expect(r.Taken.Equal(time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC))).To(BeTrue(), "Taken: %#v", r.Taken)
|
||||
})
|
||||
|
||||
It("returns nil when nothing matches the prefix", func() {
|
||||
r := DeserializeStringsAt[stringFacet](map[string]string{
|
||||
"libre.graph.image.width": "1200",
|
||||
}, "libre.graph.audio.")
|
||||
Expect(r).To(BeNil())
|
||||
})
|
||||
|
||||
It("parses into a timestamppb.Timestamp", func() {
|
||||
type photoFacet struct {
|
||||
Taken *timestamppb.Timestamp `json:"takenDateTime,omitempty"`
|
||||
}
|
||||
r := DeserializeStringsAt[photoFacet](map[string]string{
|
||||
"libre.graph.photo.takenDateTime": "2024-05-06T07:08:09Z",
|
||||
}, "libre.graph.photo.")
|
||||
Expect(r).ToNot(BeNil())
|
||||
Expect(r.Taken).ToNot(BeNil())
|
||||
want := time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC)
|
||||
Expect(r.Taken.AsTime().Equal(want)).To(BeTrue(), "Taken: got %v, want %v", r.Taken.AsTime(), want)
|
||||
})
|
||||
|
||||
It("is fail-soft on malformed fields", func() {
|
||||
// A single malformed field (year is unparseable as int) must not drop
|
||||
// the whole facet. The bad field stays at zero value, the rest of the
|
||||
// facet still populates. Mirrors the bleve-hit Deserialize behavior.
|
||||
r := DeserializeStringsAt[stringFacet](map[string]string{
|
||||
"libre.graph.audio.artist": "Iron Maiden",
|
||||
"libre.graph.audio.year": "not-a-number",
|
||||
"libre.graph.audio.duration": "354000",
|
||||
"libre.graph.audio.explicit": "not-a-bool",
|
||||
"libre.graph.audio.rating": "4.9",
|
||||
}, "libre.graph.audio.")
|
||||
Expect(r).ToNot(BeNil())
|
||||
Expect(r.Artist).ToNot(BeNil())
|
||||
Expect(*r.Artist).To(Equal("Iron Maiden"), "Artist should still be populated")
|
||||
Expect(r.Duration).ToNot(BeNil())
|
||||
Expect(*r.Duration).To(Equal(int64(354000)), "Duration should still be populated")
|
||||
Expect(r.Rating).ToNot(BeNil())
|
||||
Expect(*r.Rating).To(Equal(4.9), "Rating should still be populated")
|
||||
Expect(r.Year).To(BeNil(), "Year should stay nil for bad int")
|
||||
Expect(r.Explicit).To(BeNil(), "Explicit should stay nil for bad bool")
|
||||
})
|
||||
|
||||
It("panics for a non-struct T", func() {
|
||||
Expect(func() {
|
||||
DeserializeStringsAt[int](nil, "")
|
||||
}).To(Panic())
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,118 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
type Leaf struct {
|
||||
Name string `json:"Name"`
|
||||
Size uint64 `json:"Size"`
|
||||
Deleted bool `json:"Deleted"`
|
||||
Tags []string `json:"Tags"`
|
||||
Favorites []string `json:"Favorites"`
|
||||
}
|
||||
|
||||
type audio struct {
|
||||
Artist *string `json:"artist,omitempty"`
|
||||
Year *int32 `json:"year,omitempty"`
|
||||
}
|
||||
|
||||
type photo struct {
|
||||
Taken *timestamppb.Timestamp `json:"takenDateTime,omitempty"`
|
||||
Mtime *time.Time `json:"mtime,omitempty"`
|
||||
}
|
||||
|
||||
type embedded struct {
|
||||
Leaf
|
||||
Audio *audio `json:"audio,omitempty"`
|
||||
Photo *photo `json:"photo,omitempty"`
|
||||
}
|
||||
|
||||
var _ = Describe("Deserialize", func() {
|
||||
It("panics for a non-struct type at a prefix", func() {
|
||||
Expect(func() {
|
||||
_ = DeserializeAt[int](map[string]any{}, "")
|
||||
}).To(Panic())
|
||||
})
|
||||
|
||||
It("panics for a non-struct T", func() {
|
||||
Expect(func() {
|
||||
Deserialize[int](nil)
|
||||
}).To(Panic())
|
||||
})
|
||||
|
||||
It("deserializes leaf fields", func() {
|
||||
r := Deserialize[Leaf](map[string]any{
|
||||
"Name": "n",
|
||||
"Size": float64(42),
|
||||
"Deleted": true,
|
||||
})
|
||||
Expect(r.Name).To(Equal("n"))
|
||||
Expect(r.Size).To(Equal(uint64(42)))
|
||||
Expect(r.Deleted).To(BeTrue())
|
||||
})
|
||||
|
||||
It("coerces a scalar into a slice field", func() {
|
||||
r := Deserialize[Leaf](map[string]any{
|
||||
"Tags": "single",
|
||||
"Favorites": []any{"a", "b"},
|
||||
})
|
||||
Expect(r.Tags).To(Equal([]string{"single"}))
|
||||
Expect(r.Favorites).To(Equal([]string{"a", "b"}))
|
||||
})
|
||||
|
||||
It("parses timestamps", func() {
|
||||
r := Deserialize[embedded](map[string]any{
|
||||
"photo.takenDateTime": "2024-01-02T03:04:05Z",
|
||||
"photo.mtime": "2024-05-06T07:08:09Z",
|
||||
})
|
||||
Expect(r.Photo).ToNot(BeNil())
|
||||
Expect(r.Photo.Taken).ToNot(BeNil())
|
||||
expected := time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
Expect(r.Photo.Taken.AsTime().Equal(expected)).To(BeTrue(), "Taken: got %v, want %v", r.Photo.Taken.AsTime(), expected)
|
||||
Expect(r.Photo.Mtime).ToNot(BeNil())
|
||||
Expect(r.Photo.Mtime.Equal(time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC))).To(BeTrue(), "Mtime: %v", r.Photo.Mtime)
|
||||
})
|
||||
|
||||
It("is fail-soft on malformed values", func() {
|
||||
// Malformed values (type mismatch, unparseable time) leave the
|
||||
// affected field at its zero value instead of dropping the whole
|
||||
// record. Matches the pre-refactor getFieldValue behavior so
|
||||
// matchToResource never returns nil on a corrupted hit.
|
||||
r := Deserialize[embedded](map[string]any{
|
||||
"Name": "n",
|
||||
"Size": "not-a-number", // wrong type
|
||||
"Deleted": true,
|
||||
"photo.takenDateTime": "not-an-rfc3339-time",
|
||||
"photo.mtime": "2024-05-06T07:08:09Z",
|
||||
})
|
||||
Expect(r).ToNot(BeNil())
|
||||
Expect(r.Name).To(Equal("n"))
|
||||
Expect(r.Size).To(Equal(uint64(0)), "Size should stay zero on mismatch")
|
||||
Expect(r.Deleted).To(BeTrue())
|
||||
Expect(r.Photo).ToNot(BeNil(), "Photo should be populated because Mtime parsed ok")
|
||||
Expect(r.Photo.Taken).To(BeNil(), "Taken should stay nil for unparseable time")
|
||||
Expect(r.Photo.Mtime).ToNot(BeNil(), "Mtime should be parsed")
|
||||
})
|
||||
|
||||
It("returns nil when nothing matches the prefix", func() {
|
||||
r := DeserializeAt[audio](map[string]any{"Name": "n"}, "audio")
|
||||
Expect(r).To(BeNil())
|
||||
})
|
||||
|
||||
It("returns a value when the prefix matches", func() {
|
||||
r := DeserializeAt[audio](map[string]any{
|
||||
"audio.artist": "A",
|
||||
"audio.year": float64(2024), // setValue: pointer + numeric convert
|
||||
}, "audio")
|
||||
Expect(r).ToNot(BeNil())
|
||||
Expect(r.Artist).ToNot(BeNil())
|
||||
Expect(*r.Artist).To(Equal("A"))
|
||||
Expect(r.Year).ToNot(BeNil())
|
||||
Expect(*r.Year).To(Equal(int32(2024)))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,106 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
// The walker is shared by both deserializers, so its structural behavior
|
||||
// (flattening embedded structs, recursing into nested pointers, joining the
|
||||
// prefix, keeping a pointer only when a field was set, fail-soft skipping) is
|
||||
// tested here once against a trivial string setter instead of twice through
|
||||
// Deserialize and DeserializeStringsAt.
|
||||
|
||||
// FsEmbVal/FsEmbPtr are exported so their embedded field is exported; an
|
||||
// unexported embedded type would be skipped by resolveField.
|
||||
type FsEmbVal struct {
|
||||
EV string `json:"ev"`
|
||||
}
|
||||
|
||||
type FsEmbPtr struct {
|
||||
EP string `json:"ep"`
|
||||
}
|
||||
|
||||
type fsNested struct {
|
||||
N string `json:"n"`
|
||||
}
|
||||
|
||||
type fsRoot struct {
|
||||
FsEmbVal // embedded value struct: fields promoted
|
||||
*FsEmbPtr // embedded pointer struct: allocated on demand
|
||||
Leaf string `json:"leaf"`
|
||||
Nested *fsNested `json:"nested"` // nested pointer: recursed under "nested."
|
||||
}
|
||||
|
||||
var errBadLeaf = errors.New("bad leaf")
|
||||
|
||||
// fsSet writes raw into a string field; the "BAD" sentinel simulates a parse
|
||||
// failure so the fail-soft path can be exercised.
|
||||
func fsSet(v reflect.Value, raw string) error {
|
||||
if raw == "BAD" {
|
||||
return errBadLeaf
|
||||
}
|
||||
v.SetString(raw)
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ = Describe("fillStruct", func() {
|
||||
fill := func(fields map[string]string, prefix string) (fsRoot, bool) {
|
||||
var root fsRoot
|
||||
touched := fillStruct(reflect.ValueOf(&root).Elem(), fields, prefix, fsSet)
|
||||
return root, touched
|
||||
}
|
||||
|
||||
It("flattens embedded and recurses nested", func() {
|
||||
root, touched := fill(map[string]string{
|
||||
"leaf": "L",
|
||||
"ev": "EV",
|
||||
"ep": "EP",
|
||||
"nested.n": "N",
|
||||
}, "")
|
||||
Expect(touched).To(BeTrue())
|
||||
Expect(root.Leaf).To(Equal("L"))
|
||||
Expect(root.EV).To(Equal("EV"), "embedded value not promoted")
|
||||
Expect(root.FsEmbPtr).ToNot(BeNil(), "embedded pointer not allocated")
|
||||
Expect(root.EP).To(Equal("EP"))
|
||||
Expect(root.Nested).ToNot(BeNil(), "nested pointer not populated")
|
||||
Expect(root.Nested.N).To(Equal("N"))
|
||||
})
|
||||
|
||||
It("leaves touched false and pointers nil when nothing matches", func() {
|
||||
root, touched := fill(map[string]string{"other": "x"}, "")
|
||||
Expect(touched).To(BeFalse())
|
||||
Expect(root.Nested).To(BeNil(), "Nested should stay nil")
|
||||
Expect(root.FsEmbPtr).To(BeNil(), "embedded pointer should stay nil")
|
||||
})
|
||||
|
||||
It("joins the prefix arg with the field name", func() {
|
||||
root, touched := fill(map[string]string{
|
||||
"pre.leaf": "L",
|
||||
"pre.nested.n": "N",
|
||||
}, "pre")
|
||||
Expect(touched).To(BeTrue())
|
||||
Expect(root.Leaf).To(Equal("L"))
|
||||
Expect(root.Nested).ToNot(BeNil())
|
||||
Expect(root.Nested.N).To(Equal("N"))
|
||||
})
|
||||
|
||||
It("is fail-soft: errored leaf stays zero, walk continues", func() {
|
||||
root, touched := fill(map[string]string{
|
||||
"leaf": "BAD",
|
||||
"ev": "EV",
|
||||
}, "")
|
||||
Expect(touched).To(BeTrue(), "expected touched because ev was set")
|
||||
Expect(root.Leaf).To(BeEmpty(), "errored leaf should stay zero")
|
||||
Expect(root.EV).To(Equal("EV"), "walk should continue past the error")
|
||||
})
|
||||
|
||||
It("drops the embedded pointer when its only field errors", func() {
|
||||
root, touched := fill(map[string]string{"ep": "BAD"}, "")
|
||||
Expect(touched).To(BeFalse())
|
||||
Expect(root.FsEmbPtr).To(BeNil(), "embedded pointer should stay nil on error")
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,51 @@
|
||||
package mapping
|
||||
|
||||
import "strings"
|
||||
|
||||
// GeopointSuffix is appended to a field's name to produce the sibling key
|
||||
// that carries the geo_point / bleve-geopoint representation of the
|
||||
// original facet. For example, a libregraph "location" object with
|
||||
// longitude / latitude / altitude is preserved as-is under "location" (for
|
||||
// data retrieval and numeric queries) while "location_geopoint" carries
|
||||
// the {lat, lon} form the geo indices understand.
|
||||
const GeopointSuffix = "_geopoint"
|
||||
|
||||
// addGeopointSiblings walks the overrides; for each TypeGeopoint entry at
|
||||
// a dotted path (e.g. "location" or "journey.start") it writes a sibling
|
||||
// under the suffixed key with the {lat, lon} form both bleve's
|
||||
// ExtractGeoPoint and OpenSearch's geo_point parser accept. The original
|
||||
// facet object stays untouched so downstream code still sees the full
|
||||
// libregraph shape (including altitude).
|
||||
func addGeopointSiblings(m map[string]any, overrides map[string]FieldOpts) {
|
||||
for key, opts := range overrides {
|
||||
if opts.Type == TypeGeopoint {
|
||||
addGeopointSibling(m, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// addGeopointSibling resolves dottedPath within m and, if the target is a
|
||||
// libregraph-shaped geo object (with numeric "longitude" and "latitude"),
|
||||
// writes the `{lat, lon}` sibling at the same level under the suffixed key.
|
||||
func addGeopointSibling(m map[string]any, dottedPath string) {
|
||||
parts := strings.Split(dottedPath, ".")
|
||||
parent := m
|
||||
for _, p := range parts[:len(parts)-1] {
|
||||
next, ok := parent[p].(map[string]any)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
parent = next
|
||||
}
|
||||
leaf := parts[len(parts)-1]
|
||||
obj, ok := parent[leaf].(map[string]any)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
lon, hasLon := obj["longitude"].(float64)
|
||||
lat, hasLat := obj["latitude"].(float64)
|
||||
if !hasLon || !hasLat {
|
||||
return
|
||||
}
|
||||
parent[leaf+GeopointSuffix] = map[string]any{"lat": lat, "lon": lon}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("BuildMapping geopoint errors", func() {
|
||||
// build-mapping error paths: an override that doesn't fit the Go field.
|
||||
It("errors for geopoint on a non-struct field on both backends", func() {
|
||||
type doc struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
// Geopoint on a non-struct field must error on both backends.
|
||||
_, err := BleveBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"name": {Type: TypeGeopoint}})
|
||||
Expect(err).To(HaveOccurred(), "bleve: expected error for geopoint on string field")
|
||||
_, err = OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"name": {Type: TypeGeopoint}})
|
||||
Expect(err).To(HaveOccurred(), "opensearch: expected error for geopoint on string field")
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("addGeopointSibling", func() {
|
||||
It("bails without panicking when the intermediate is missing", func() {
|
||||
m := map[string]any{"journey": "not-a-map"}
|
||||
addGeopointSibling(m, "journey.start") // must bail, not panic
|
||||
Expect(m).ToNot(HaveKey("journey.start" + GeopointSuffix))
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("PrepareForIndex geopoint", func() {
|
||||
It("adds a geopoint sibling", func() {
|
||||
type geoDoc struct {
|
||||
Location *struct {
|
||||
Longitude *float64 `json:"longitude,omitempty"`
|
||||
Latitude *float64 `json:"latitude,omitempty"`
|
||||
Altitude *float64 `json:"altitude,omitempty"`
|
||||
} `json:"location,omitempty"`
|
||||
}
|
||||
lon, lat, alt := 11.1, 49.4, 1047.7
|
||||
doc := geoDoc{Location: &struct {
|
||||
Longitude *float64 `json:"longitude,omitempty"`
|
||||
Latitude *float64 `json:"latitude,omitempty"`
|
||||
Altitude *float64 `json:"altitude,omitempty"`
|
||||
}{Longitude: &lon, Latitude: &lat, Altitude: &alt}}
|
||||
|
||||
m, err := PrepareForIndex(doc, map[string]FieldOpts{
|
||||
"location": {Type: TypeGeopoint},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Original location object stays untouched (full libregraph shape).
|
||||
orig, ok := m["location"].(map[string]any)
|
||||
Expect(ok).To(BeTrue(), "expected location object preserved, got %T", m["location"])
|
||||
Expect(orig["longitude"]).To(Equal(lon))
|
||||
Expect(orig["latitude"]).To(Equal(lat))
|
||||
Expect(orig["altitude"]).To(Equal(alt))
|
||||
|
||||
// Sibling location_geopoint has {lat, lon} for the geo indices.
|
||||
gp, ok := m["location"+GeopointSuffix].(map[string]any)
|
||||
Expect(ok).To(BeTrue(), "expected location_geopoint sibling, got %T", m["location"+GeopointSuffix])
|
||||
Expect(gp["lat"]).To(Equal(lat))
|
||||
Expect(gp["lon"]).To(Equal(lon))
|
||||
})
|
||||
|
||||
It("skips incomplete geopoints", func() {
|
||||
type geoDoc struct {
|
||||
Location *struct {
|
||||
Altitude *float64 `json:"altitude,omitempty"`
|
||||
} `json:"location,omitempty"`
|
||||
}
|
||||
alt := 100.0
|
||||
doc := geoDoc{Location: &struct {
|
||||
Altitude *float64 `json:"altitude,omitempty"`
|
||||
}{Altitude: &alt}}
|
||||
|
||||
m, err := PrepareForIndex(doc, map[string]FieldOpts{
|
||||
"location": {Type: TypeGeopoint},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// Original stays (altitude alone is still useful metadata).
|
||||
Expect(m).To(HaveKey("location"), "location should still be present when only altitude is set")
|
||||
// No sibling without both lon and lat.
|
||||
Expect(m).ToNot(HaveKey("location"+GeopointSuffix), "no sibling expected")
|
||||
})
|
||||
|
||||
It("writes no sibling without the geopoint override", func() {
|
||||
type geoDoc struct {
|
||||
Location *struct {
|
||||
Longitude *float64 `json:"longitude,omitempty"`
|
||||
Latitude *float64 `json:"latitude,omitempty"`
|
||||
} `json:"location,omitempty"`
|
||||
}
|
||||
lon, lat := 11.1, 49.4
|
||||
doc := geoDoc{Location: &struct {
|
||||
Longitude *float64 `json:"longitude,omitempty"`
|
||||
Latitude *float64 `json:"latitude,omitempty"`
|
||||
}{Longitude: &lon, Latitude: &lat}}
|
||||
|
||||
m, err := PrepareForIndex(doc, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(m).ToNot(HaveKey("location"+GeopointSuffix), "no sibling expected without override")
|
||||
})
|
||||
|
||||
It("handles nested geopoints via the dotted-path walker", func() {
|
||||
// journey.start and journey.end - two geopoints in the same facet,
|
||||
// demonstrating the dotted-path walker.
|
||||
type geo struct {
|
||||
Longitude *float64 `json:"longitude,omitempty"`
|
||||
Latitude *float64 `json:"latitude,omitempty"`
|
||||
}
|
||||
type journey struct {
|
||||
Start *geo `json:"start,omitempty"`
|
||||
End *geo `json:"end,omitempty"`
|
||||
}
|
||||
type doc struct {
|
||||
Journey *journey `json:"journey,omitempty"`
|
||||
}
|
||||
slon, slat := 11.0, 49.0
|
||||
elon, elat := 13.4, 52.5
|
||||
d := doc{Journey: &journey{
|
||||
Start: &geo{Longitude: &slon, Latitude: &slat},
|
||||
End: &geo{Longitude: &elon, Latitude: &elat},
|
||||
}}
|
||||
|
||||
m, err := PrepareForIndex(d, map[string]FieldOpts{
|
||||
"journey.start": {Type: TypeGeopoint},
|
||||
"journey.end": {Type: TypeGeopoint},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
j, ok := m["journey"].(map[string]any)
|
||||
Expect(ok).To(BeTrue(), "journey not an object: %T", m["journey"])
|
||||
startGp, ok := j["start"+GeopointSuffix].(map[string]any)
|
||||
Expect(ok).To(BeTrue(), "journey.start sibling: %#v", j["start"+GeopointSuffix])
|
||||
Expect(startGp["lat"]).To(Equal(slat))
|
||||
Expect(startGp["lon"]).To(Equal(slon))
|
||||
endGp, ok := j["end"+GeopointSuffix].(map[string]any)
|
||||
Expect(ok).To(BeTrue(), "journey.end sibling: %#v", j["end"+GeopointSuffix])
|
||||
Expect(endGp["lat"]).To(Equal(elat))
|
||||
Expect(endGp["lon"]).To(Equal(elon))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,116 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
var (
|
||||
timeType = reflect.TypeFor[time.Time]()
|
||||
timestampType = reflect.TypeFor[timestamppb.Timestamp]()
|
||||
)
|
||||
|
||||
// deref unwraps pointer and slice types to their element type.
|
||||
func deref(t reflect.Type) reflect.Type {
|
||||
for t.Kind() == reflect.Ptr || t.Kind() == reflect.Slice {
|
||||
t = t.Elem()
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// inferType returns the mapping type for a Go type. Pointers and slices are
|
||||
// unwrapped to their element type. time.Time and timestamppb.Timestamp become
|
||||
// datetime; other structs become object.
|
||||
func inferType(t reflect.Type) string {
|
||||
t = deref(t)
|
||||
switch t.Kind() {
|
||||
case reflect.String:
|
||||
return TypeKeyword
|
||||
case reflect.Bool:
|
||||
return TypeBool
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
|
||||
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
|
||||
reflect.Float32, reflect.Float64:
|
||||
return TypeNumeric
|
||||
case reflect.Struct:
|
||||
if t == timeType || t == timestampType {
|
||||
return TypeDatetime
|
||||
}
|
||||
return TypeObject
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// fieldInfo is the resolved metadata for one struct field.
|
||||
type fieldInfo struct {
|
||||
Name string
|
||||
GoField reflect.StructField
|
||||
Skip bool
|
||||
Embedded bool
|
||||
}
|
||||
|
||||
// resolveField resolves a struct field's json-tag name and skip/embed state.
|
||||
func resolveField(sf reflect.StructField) fieldInfo {
|
||||
if !sf.IsExported() {
|
||||
return fieldInfo{Skip: true}
|
||||
}
|
||||
name := sf.Name
|
||||
tag := sf.Tag.Get("json")
|
||||
if tag != "" {
|
||||
first, _, _ := strings.Cut(tag, ",")
|
||||
if first == "-" {
|
||||
return fieldInfo{Skip: true}
|
||||
}
|
||||
if first != "" {
|
||||
name = first
|
||||
}
|
||||
}
|
||||
return fieldInfo{
|
||||
Name: name,
|
||||
GoField: sf,
|
||||
Embedded: sf.Anonymous,
|
||||
}
|
||||
}
|
||||
|
||||
// walkFields visits exported leaf fields of t, flattening embedded structs
|
||||
// onto the enclosing level. It returns the first error returned by fn.
|
||||
func walkFields(t reflect.Type, fn func(fi fieldInfo) error) error {
|
||||
for t.Kind() == reflect.Ptr {
|
||||
t = t.Elem()
|
||||
}
|
||||
if t.Kind() != reflect.Struct {
|
||||
return nil
|
||||
}
|
||||
for i := 0; i < t.NumField(); i++ {
|
||||
fi := resolveField(t.Field(i))
|
||||
if fi.Skip {
|
||||
continue
|
||||
}
|
||||
if fi.Embedded {
|
||||
if err := walkFields(fi.GoField.Type, fn); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := fn(fi); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// structType returns the underlying struct type, unwrapping pointers and
|
||||
// slices. Returns nil when t is not a walkable struct (e.g. time.Time).
|
||||
func structType(t reflect.Type) reflect.Type {
|
||||
t = deref(t)
|
||||
if t.Kind() != reflect.Struct {
|
||||
return nil
|
||||
}
|
||||
if t == timeType || t == timestampType {
|
||||
return nil
|
||||
}
|
||||
return t
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
var _ = Describe("inferType", func() {
|
||||
It("returns empty for unsupported kinds", func() {
|
||||
Expect(inferType(reflect.TypeFor[map[string]int]())).To(BeEmpty(), "map")
|
||||
Expect(inferType(reflect.TypeFor[chan int]())).To(BeEmpty(), "chan")
|
||||
})
|
||||
|
||||
DescribeTable("infers the mapping type",
|
||||
func(in any, want string) {
|
||||
Expect(inferType(reflect.TypeOf(in))).To(Equal(want))
|
||||
},
|
||||
Entry("string", "", TypeKeyword),
|
||||
Entry("*string", (*string)(nil), TypeKeyword),
|
||||
Entry("[]string", []string(nil), TypeKeyword),
|
||||
Entry("bool", false, TypeBool),
|
||||
Entry("int", int(0), TypeNumeric),
|
||||
Entry("int64", int64(0), TypeNumeric),
|
||||
Entry("uint64", uint64(0), TypeNumeric),
|
||||
Entry("float64", float64(0), TypeNumeric),
|
||||
Entry("time.Time", time.Time{}, TypeDatetime),
|
||||
Entry("*time.Time", (*time.Time)(nil), TypeDatetime),
|
||||
Entry("*timestamppb.Timestamp", (*timestamppb.Timestamp)(nil), TypeDatetime),
|
||||
Entry("struct", struct{ X int }{}, TypeObject),
|
||||
Entry("*struct", (*struct{ X int })(nil), TypeObject),
|
||||
)
|
||||
})
|
||||
|
||||
var _ = Describe("resolveField", func() {
|
||||
type S struct {
|
||||
Exported string `json:"exp"`
|
||||
Renamed string `json:"renamed,omitempty"`
|
||||
NoTag string
|
||||
OmitOnly string `json:",omitempty"`
|
||||
Skipped string `json:"-"`
|
||||
unexported string //nolint:unused
|
||||
}
|
||||
st := reflect.TypeFor[S]()
|
||||
|
||||
DescribeTable("resolves the field name and skip flag",
|
||||
func(fieldIdx int, wantName string, wantSkip bool) {
|
||||
fi := resolveField(st.Field(fieldIdx))
|
||||
Expect(fi.Skip).To(Equal(wantSkip), "field %d skip", fieldIdx)
|
||||
if !wantSkip {
|
||||
Expect(fi.Name).To(Equal(wantName), "field %d name", fieldIdx)
|
||||
}
|
||||
},
|
||||
Entry("exported json tag", 0, "exp", false),
|
||||
Entry("renamed with omitempty", 1, "renamed", false),
|
||||
Entry("no tag", 2, "NoTag", false),
|
||||
Entry("omitempty only", 3, "OmitOnly", false),
|
||||
Entry("json:- skipped", 4, "", true),
|
||||
Entry("unexported skipped", 5, "", true),
|
||||
)
|
||||
})
|
||||
|
||||
var _ = Describe("walkFields", func() {
|
||||
It("flattens embedded structs", func() {
|
||||
type Inner struct {
|
||||
A string `json:"a"`
|
||||
B int `json:"b"`
|
||||
}
|
||||
type Outer struct {
|
||||
Inner
|
||||
C bool `json:"c"`
|
||||
}
|
||||
var names []string
|
||||
err := walkFields(reflect.TypeFor[Outer](), func(fi fieldInfo) error {
|
||||
names = append(names, fi.Name)
|
||||
return nil
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(names).To(Equal([]string{"a", "b", "c"}))
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("structType", func() {
|
||||
type S struct{ X int }
|
||||
|
||||
DescribeTable("resolves struct-ish types",
|
||||
func(in reflect.Type, wantNil bool) {
|
||||
got := structType(in)
|
||||
Expect(got == nil).To(Equal(wantNil))
|
||||
},
|
||||
Entry("struct", reflect.TypeFor[S](), false),
|
||||
Entry("*struct", reflect.TypeFor[*S](), false),
|
||||
Entry("[]struct", reflect.TypeFor[[]S](), false),
|
||||
Entry("time.Time", reflect.TypeFor[time.Time](), true),
|
||||
Entry("string", reflect.TypeFor[string](), true),
|
||||
)
|
||||
})
|
||||
@@ -0,0 +1,13 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestMapping(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Mapping Suite")
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
// OpenSearchBuildMapping builds the OpenSearch "properties" map (the value
|
||||
// of mappings.properties) for type t by walking the struct via reflection.
|
||||
// Field names come from json tags; overrides are keyed by those names.
|
||||
//
|
||||
// The returned map contains plain JSON-friendly values (strings, bools,
|
||||
// nested maps) and can be marshalled directly.
|
||||
func OpenSearchBuildMapping(t reflect.Type, overrides map[string]FieldOpts) (map[string]any, error) {
|
||||
return buildOpenSearchProperties(t, overrides, "")
|
||||
}
|
||||
|
||||
func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, prefix string) (map[string]any, error) {
|
||||
props := map[string]any{}
|
||||
err := walkFields(t, func(fi fieldInfo) error {
|
||||
key := fi.Name
|
||||
if prefix != "" {
|
||||
key = prefix + "." + fi.Name
|
||||
}
|
||||
opts := overrides[key]
|
||||
fieldType := opts.Type
|
||||
if fieldType == "" {
|
||||
fieldType = inferType(fi.GoField.Type)
|
||||
}
|
||||
|
||||
if fieldType == TypeObject {
|
||||
sub := structType(fi.GoField.Type)
|
||||
if sub == nil {
|
||||
return fmt.Errorf("mapping: object type on non-struct field %q", key)
|
||||
}
|
||||
subProps, err := buildOpenSearchProperties(sub, overrides, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
props[fi.Name] = map[string]any{"properties": subProps}
|
||||
return nil
|
||||
}
|
||||
|
||||
if fieldType == TypeGeopoint {
|
||||
// Keep the facet object, add a sibling _geopoint field (see GeopointSuffix).
|
||||
sub := structType(fi.GoField.Type)
|
||||
if sub == nil {
|
||||
return fmt.Errorf("mapping: geopoint type on non-struct field %q", key)
|
||||
}
|
||||
subProps, err := buildOpenSearchProperties(sub, overrides, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
props[fi.Name] = map[string]any{"properties": subProps}
|
||||
props[fi.Name+GeopointSuffix] = map[string]any{"type": "geo_point"}
|
||||
return nil
|
||||
}
|
||||
|
||||
fm, err := openSearchFieldMapping(fieldType, opts, fi.GoField.Type)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mapping: field %q: %w", key, err)
|
||||
}
|
||||
props[fi.Name] = fm
|
||||
return nil
|
||||
})
|
||||
return props, err
|
||||
}
|
||||
|
||||
func openSearchFieldMapping(fieldType string, opts FieldOpts, goType reflect.Type) (map[string]any, error) {
|
||||
switch fieldType {
|
||||
case TypeKeyword:
|
||||
m := map[string]any{"type": "keyword"}
|
||||
if opts.Analyzer != "" {
|
||||
m["type"] = "text"
|
||||
m["analyzer"] = opts.Analyzer
|
||||
}
|
||||
return m, nil
|
||||
case TypeFulltext:
|
||||
m := map[string]any{
|
||||
"type": "text",
|
||||
"term_vector": "with_positions_offsets",
|
||||
}
|
||||
if opts.Analyzer != "" {
|
||||
m["analyzer"] = opts.Analyzer
|
||||
}
|
||||
return m, nil
|
||||
case TypePath:
|
||||
m := map[string]any{"type": "text"}
|
||||
if opts.Analyzer != "" {
|
||||
m["analyzer"] = opts.Analyzer
|
||||
} else {
|
||||
m["analyzer"] = "path_hierarchy"
|
||||
}
|
||||
return m, nil
|
||||
case TypeWildcard:
|
||||
// OpenSearch stores wildcard fields with doc_values=false by
|
||||
// default, so emit it explicitly to keep local and remote
|
||||
// mappings in sync for the Apply comparison.
|
||||
return map[string]any{"type": "wildcard", "doc_values": false}, nil
|
||||
case TypeNumeric:
|
||||
return map[string]any{"type": openSearchNumericType(goType)}, nil
|
||||
case TypeBool:
|
||||
return map[string]any{"type": "boolean"}, nil
|
||||
case TypeDatetime:
|
||||
return map[string]any{"type": "date"}, nil
|
||||
case TypeGeopoint:
|
||||
return map[string]any{"type": "geo_point"}, nil
|
||||
case "":
|
||||
return nil, fmt.Errorf("no type inferred and no override")
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported type %q", fieldType)
|
||||
}
|
||||
|
||||
// openSearchNumericType maps a Go numeric type to an OpenSearch numeric
|
||||
// field type.
|
||||
func openSearchNumericType(t reflect.Type) string {
|
||||
t = deref(t)
|
||||
switch t.Kind() {
|
||||
case reflect.Float32:
|
||||
return "float"
|
||||
case reflect.Float64:
|
||||
return "double"
|
||||
case reflect.Int8, reflect.Uint8, reflect.Int16, reflect.Uint16:
|
||||
return "short"
|
||||
case reflect.Int32, reflect.Uint32:
|
||||
return "integer"
|
||||
}
|
||||
return "long"
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
type osDoc struct {
|
||||
ID string `json:"ID"`
|
||||
Size uint64 `json:"Size"`
|
||||
Deleted bool `json:"Deleted"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
Rating float64 `json:"Rating"`
|
||||
Nested *struct {
|
||||
Artist string `json:"artist"`
|
||||
Year int32 `json:"year"`
|
||||
} `json:"nested,omitempty"`
|
||||
}
|
||||
|
||||
var _ = Describe("OpenSearchBuildMapping", func() {
|
||||
DescribeTable("maps numeric Go types to OpenSearch types",
|
||||
func(field, wantType string) {
|
||||
type doc struct {
|
||||
A int8 `json:"a"`
|
||||
B int16 `json:"b"`
|
||||
C int32 `json:"c"`
|
||||
D int64 `json:"d"`
|
||||
E uint8 `json:"e"`
|
||||
F uint64 `json:"f"`
|
||||
G float32 `json:"g"`
|
||||
H float64 `json:"h"`
|
||||
}
|
||||
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(props[field].(map[string]any)["type"]).To(Equal(wantType), "%s type", field)
|
||||
},
|
||||
Entry("int8 -> short", "a", "short"),
|
||||
Entry("int16 -> short", "b", "short"),
|
||||
Entry("int32 -> integer", "c", "integer"),
|
||||
Entry("int64 -> long", "d", "long"),
|
||||
Entry("uint8 -> short", "e", "short"),
|
||||
Entry("uint64 -> long", "f", "long"),
|
||||
Entry("float32 -> float", "g", "float"),
|
||||
Entry("float64 -> double", "h", "double"),
|
||||
)
|
||||
|
||||
DescribeTable("infers field types",
|
||||
func(field, wantType string) {
|
||||
props, err := OpenSearchBuildMapping(reflect.TypeFor[osDoc](), nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
m, ok := props[field].(map[string]any)
|
||||
Expect(ok).To(BeTrue(), "%s: missing or not a map: %#v", field, props[field])
|
||||
Expect(m["type"]).To(Equal(wantType), "%s type", field)
|
||||
},
|
||||
Entry("ID -> keyword", "ID", "keyword"),
|
||||
Entry("Size -> long", "Size", "long"),
|
||||
Entry("Deleted -> boolean", "Deleted", "boolean"),
|
||||
Entry("CreatedAt -> date", "CreatedAt", "date"),
|
||||
Entry("Rating -> double", "Rating", "double"),
|
||||
)
|
||||
|
||||
It("maps nested structs with their sub-properties", func() {
|
||||
props, err := OpenSearchBuildMapping(reflect.TypeFor[osDoc](), nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
nested, ok := props["nested"].(map[string]any)
|
||||
Expect(ok).To(BeTrue(), "nested: not a map: %#v", props["nested"])
|
||||
sub, ok := nested["properties"].(map[string]any)
|
||||
Expect(ok).To(BeTrue(), "nested.properties: missing: %#v", nested)
|
||||
artist, ok := sub["artist"].(map[string]any)
|
||||
Expect(ok).To(BeTrue(), "nested.artist: %#v", sub)
|
||||
Expect(artist["type"]).To(Equal("keyword"), "nested.artist.type")
|
||||
year, ok := sub["year"].(map[string]any)
|
||||
Expect(ok).To(BeTrue(), "nested.year: %#v", sub)
|
||||
Expect(year["type"]).To(Equal("integer"), "nested.year.type (int32 -> integer expected)")
|
||||
})
|
||||
|
||||
It("applies field overrides", func() {
|
||||
type doc struct {
|
||||
Name string `json:"Name"`
|
||||
Content string `json:"Content"`
|
||||
Path string `json:"Path"`
|
||||
MimeType string `json:"MimeType"`
|
||||
}
|
||||
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{
|
||||
"Name": {Analyzer: "lowercaseKeyword"},
|
||||
"Content": {Type: TypeFulltext},
|
||||
"Path": {Type: TypePath},
|
||||
"MimeType": {Type: TypeWildcard},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
name := props["Name"].(map[string]any)
|
||||
Expect(name["type"]).To(Equal("text"), "Name: %#v", name)
|
||||
Expect(name["analyzer"]).To(Equal("lowercaseKeyword"), "Name: %#v", name)
|
||||
content := props["Content"].(map[string]any)
|
||||
Expect(content["type"]).To(Equal("text"), "Content: %#v", content)
|
||||
Expect(content["term_vector"]).To(Equal("with_positions_offsets"), "Content: %#v", content)
|
||||
_, ok := content["analyzer"]
|
||||
Expect(ok).To(BeFalse(), "Content should leave analyzer unset (use OpenSearch default)")
|
||||
path := props["Path"].(map[string]any)
|
||||
Expect(path["type"]).To(Equal("text"), "Path: %#v", path)
|
||||
Expect(path["analyzer"]).To(Equal("path_hierarchy"), "Path: %#v", path)
|
||||
mime := props["MimeType"].(map[string]any)
|
||||
Expect(mime["type"]).To(Equal("wildcard"), "MimeType: %#v", mime)
|
||||
})
|
||||
|
||||
It("builds an object plus a geo_point sibling for geopoints", func() {
|
||||
type doc struct {
|
||||
Location *struct {
|
||||
Lon float64 `json:"longitude"`
|
||||
Lat float64 `json:"latitude"`
|
||||
Alt float64 `json:"altitude"`
|
||||
} `json:"location,omitempty"`
|
||||
}
|
||||
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{
|
||||
"location": {Type: TypeGeopoint},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
// Object for libregraph-shape data retrieval.
|
||||
loc, ok := props["location"].(map[string]any)
|
||||
Expect(ok).To(BeTrue(), "location: %#v", props["location"])
|
||||
sub, ok := loc["properties"].(map[string]any)
|
||||
Expect(ok).To(BeTrue(), "location should have numeric sub-properties, got %#v", loc)
|
||||
for _, k := range []string{"longitude", "latitude", "altitude"} {
|
||||
prop, ok := sub[k].(map[string]any)
|
||||
Expect(ok).To(BeTrue(), "location.%s: %#v", k, sub[k])
|
||||
Expect(prop["type"]).To(Equal("double"), "location.%s: %#v", k, sub[k])
|
||||
}
|
||||
// Sibling geo_point for spatial queries.
|
||||
gp, ok := props["location"+GeopointSuffix].(map[string]any)
|
||||
Expect(ok).To(BeTrue(), "location%s: %#v", GeopointSuffix, props["location"+GeopointSuffix])
|
||||
Expect(gp["type"]).To(Equal("geo_point"), "location%s.type", GeopointSuffix)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,35 @@
|
||||
// Package mapping builds search index mappings for bleve and OpenSearch from
|
||||
// a Go struct via reflection. Field names come from json tags; the caller
|
||||
// provides overrides for fields that need a specific type or analyzer.
|
||||
package mapping
|
||||
|
||||
// Field type constants used in FieldOpts.Type. An empty Type means the type
|
||||
// is inferred from the Go field via reflection.
|
||||
const (
|
||||
TypeKeyword = "keyword"
|
||||
TypeFulltext = "fulltext"
|
||||
TypePath = "path"
|
||||
TypeWildcard = "wildcard"
|
||||
TypeNumeric = "numeric"
|
||||
TypeDatetime = "datetime"
|
||||
TypeBool = "bool"
|
||||
TypeObject = "object"
|
||||
TypeGeopoint = "geopoint"
|
||||
)
|
||||
|
||||
// FieldOpts overrides the default type inference for a struct field. Keys in
|
||||
// the override map are json-tag names (e.g. "Name", "location", "audio.artist"),
|
||||
// not Go field names.
|
||||
type FieldOpts struct {
|
||||
// Type is one of the Type* constants. Empty means "infer from Go type".
|
||||
Type string
|
||||
|
||||
// Analyzer is the name of a custom analyzer registered on the bleve
|
||||
// IndexMapping (e.g. "lowercaseKeyword", "fulltext"). For OpenSearch it
|
||||
// becomes the analyzer attribute on the field.
|
||||
Analyzer string
|
||||
|
||||
// IncludeInAll controls bleve's _all field inclusion. Nil means "use the
|
||||
// bleve default for this field type". Has no effect on OpenSearch.
|
||||
IncludeInAll *bool
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/conversions"
|
||||
)
|
||||
|
||||
// PrepareForIndex converts v to the flat map[string]any the backend index
|
||||
// clients expect: a json round-trip (conversions.To) plus type-specific
|
||||
// adaptations (currently geopoint siblings). Pass the same overrides as the
|
||||
// *BuildMapping calls so the document and the mapping stay in sync.
|
||||
func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, error) {
|
||||
out, err := conversions.To[map[string]any](v)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("mapping: prepare %T: %w", v, err)
|
||||
}
|
||||
if out == nil {
|
||||
return out, nil
|
||||
}
|
||||
addGeopointSiblings(out, overrides)
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("PrepareForIndex serialization", func() {
|
||||
It("errors for a non-marshallable value", func() {
|
||||
// a func field can't be json-marshalled -> conversions.To errors
|
||||
type bad struct {
|
||||
F func() `json:"f"`
|
||||
}
|
||||
_, err := PrepareForIndex(bad{}, nil)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
|
||||
It("returns nil map for a typed nil pointer", func() {
|
||||
// a typed nil pointer marshals to null -> nil map, no error, no panic
|
||||
out, err := PrepareForIndex((*struct{})(nil), nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(out).To(BeNil())
|
||||
})
|
||||
|
||||
It("flattens embedded structs", func() {
|
||||
type inner struct {
|
||||
Name string `json:"Name"`
|
||||
Size uint64 `json:"Size"`
|
||||
}
|
||||
type outer struct {
|
||||
inner
|
||||
ID string `json:"ID"`
|
||||
}
|
||||
m, err := PrepareForIndex(outer{inner: inner{Name: "a", Size: 7}, ID: "x"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
want := map[string]any{"Name": "a", "Size": float64(7), "ID": "x"}
|
||||
Expect(m).To(Equal(want))
|
||||
})
|
||||
|
||||
It("omits nil fields tagged omitempty", func() {
|
||||
type facet struct {
|
||||
Artist string `json:"artist"`
|
||||
}
|
||||
type doc struct {
|
||||
Name string `json:"Name"`
|
||||
Audio *facet `json:"audio,omitempty"`
|
||||
}
|
||||
m, err := PrepareForIndex(doc{Name: "n"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(m).ToNot(HaveKey("audio"))
|
||||
Expect(m["Name"]).To(Equal("n"))
|
||||
})
|
||||
|
||||
It("includes nested facets when set", func() {
|
||||
type facet struct {
|
||||
Artist string `json:"artist"`
|
||||
}
|
||||
type doc struct {
|
||||
Audio *facet `json:"audio,omitempty"`
|
||||
}
|
||||
m, err := PrepareForIndex(doc{Audio: &facet{Artist: "A"}}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
nested, ok := m["audio"].(map[string]any)
|
||||
Expect(ok).To(BeTrue(), "audio should be a nested map: %#v", m["audio"])
|
||||
Expect(nested["artist"]).To(Equal("A"))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,49 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Validate returns an error if any override key does not match a known field
|
||||
// name in t. Top-level fields are identified by their json-tag name; nested
|
||||
// named struct fields are reachable as "parent.child". Embedded (anonymous)
|
||||
// structs are flattened, so their fields sit at the parent level (as with
|
||||
// encoding/json).
|
||||
func Validate(t reflect.Type, overrides map[string]FieldOpts) error {
|
||||
if len(overrides) == 0 {
|
||||
return nil
|
||||
}
|
||||
names := collectNames(t, "")
|
||||
var unknown []string
|
||||
for k := range overrides {
|
||||
if _, ok := names[k]; !ok {
|
||||
unknown = append(unknown, k)
|
||||
}
|
||||
}
|
||||
if len(unknown) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Strings(unknown)
|
||||
return fmt.Errorf("mapping: unknown override keys: %s", strings.Join(unknown, ", "))
|
||||
}
|
||||
|
||||
func collectNames(t reflect.Type, prefix string) map[string]struct{} {
|
||||
out := map[string]struct{}{}
|
||||
_ = walkFields(t, func(fi fieldInfo) error {
|
||||
key := fi.Name
|
||||
if prefix != "" {
|
||||
key = prefix + "." + fi.Name
|
||||
}
|
||||
out[key] = struct{}{}
|
||||
if sub := structType(fi.GoField.Type); sub != nil {
|
||||
for k := range collectNames(sub, key) {
|
||||
out[k] = struct{}{}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
type inner struct {
|
||||
Artist string `json:"artist"`
|
||||
}
|
||||
|
||||
type sample struct {
|
||||
Name string `json:"Name"`
|
||||
Audio *inner `json:"audio,omitempty"`
|
||||
Location *struct { //nolint:unused
|
||||
Lon float64 `json:"longitude"`
|
||||
Lat float64 `json:"latitude"`
|
||||
} `json:"location,omitempty"`
|
||||
}
|
||||
|
||||
var _ = Describe("Validate", func() {
|
||||
It("accepts known override keys", func() {
|
||||
err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{
|
||||
"Name": {Analyzer: "lowercaseKeyword"},
|
||||
"audio": {Type: TypeObject},
|
||||
"audio.artist": {Analyzer: "lowercaseKeyword"},
|
||||
"location": {Type: TypeGeopoint},
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("rejects unknown override keys", func() {
|
||||
err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{
|
||||
"nope": {},
|
||||
"audio.zzz": {},
|
||||
})
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("nope"))
|
||||
Expect(err.Error()).To(ContainSubstring("audio.zzz"))
|
||||
})
|
||||
|
||||
It("accepts empty overrides", func() {
|
||||
Expect(Validate(reflect.TypeFor[sample](), nil)).To(Succeed())
|
||||
})
|
||||
})
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/conversions"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
|
||||
@@ -31,8 +32,8 @@ type Backend struct {
|
||||
client *opensearchgoAPI.Client
|
||||
}
|
||||
|
||||
func NewBackend(index string, client *opensearchgoAPI.Client) (*Backend, error) {
|
||||
pingResp, err := client.Ping(context.TODO(), &opensearchgoAPI.PingReq{})
|
||||
func NewBackend(ctx context.Context, index string, client *opensearchgoAPI.Client, logger log.Logger) (*Backend, error) {
|
||||
pingResp, err := client.Ping(ctx, &opensearchgoAPI.PingReq{})
|
||||
switch {
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("%w, failed to ping opensearch: %w", ErrUnhealthyCluster, err)
|
||||
@@ -41,13 +42,13 @@ func NewBackend(index string, client *opensearchgoAPI.Client) (*Backend, error)
|
||||
}
|
||||
|
||||
// apply the index template
|
||||
if err := IndexManagerLatest.Apply(context.TODO(), index, client); err != nil {
|
||||
if err := IndexManagerLatest.Apply(ctx, index, client, logger); err != nil {
|
||||
return nil, fmt.Errorf("failed to apply index template: %w", err)
|
||||
}
|
||||
|
||||
// first check if the cluster is healthy
|
||||
|
||||
resp, err := client.Cluster.Health(context.TODO(), &opensearchgoAPI.ClusterHealthReq{
|
||||
resp, err := client.Cluster.Health(ctx, &opensearchgoAPI.ClusterHealthReq{
|
||||
Indices: []string{index},
|
||||
Params: opensearchgoAPI.ClusterHealthParams{
|
||||
Local: opensearchgoAPI.ToPointer(true),
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
|
||||
@@ -24,7 +25,7 @@ func TestNewBackend(t *testing.T) {
|
||||
})
|
||||
require.NoError(t, err, "failed to create OpenSearch client")
|
||||
|
||||
backend, err := opensearch.NewBackend("test-engine-new-engine", client)
|
||||
backend, err := opensearch.NewBackend(t.Context(), "test-engine-new-engine", client, log.NopLogger())
|
||||
require.Nil(t, backend)
|
||||
require.ErrorIs(t, err, opensearch.ErrUnhealthyCluster)
|
||||
})
|
||||
@@ -38,7 +39,7 @@ func TestEngine_Search(t *testing.T) {
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
backend, err := opensearch.NewBackend(t.Context(), indexName, tc.Client(), log.NopLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
@@ -81,7 +82,7 @@ func TestEngine_Upsert(t *testing.T) {
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
backend, err := opensearch.NewBackend(t.Context(), indexName, tc.Client(), log.NopLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("upsert with full document", func(t *testing.T) {
|
||||
@@ -90,6 +91,16 @@ func TestEngine_Upsert(t *testing.T) {
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 1)
|
||||
})
|
||||
|
||||
t.Run("upsert without mtime", func(t *testing.T) {
|
||||
// content.Extract leaves Mtime nil when the resource info carries none
|
||||
document := opensearchtest.Testdata.Resources.File
|
||||
document.ID = "1$1!4"
|
||||
document.Mtime = nil
|
||||
require.NoError(t, backend.Upsert(document.ID, document))
|
||||
|
||||
tc.Require.IndicesCount([]string{indexName}, nil, 2)
|
||||
})
|
||||
}
|
||||
|
||||
func TestEngine_Move(t *testing.T) {
|
||||
@@ -100,7 +111,7 @@ func TestEngine_Move(t *testing.T) {
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
backend, err := opensearch.NewBackend(t.Context(), indexName, tc.Client(), log.NopLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("moves the document to a new path", func(t *testing.T) {
|
||||
@@ -137,7 +148,7 @@ func TestEngine_Delete(t *testing.T) {
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
backend, err := opensearch.NewBackend(t.Context(), indexName, tc.Client(), log.NopLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("mark document as deleted", func(t *testing.T) {
|
||||
@@ -170,7 +181,7 @@ func TestEngine_Restore(t *testing.T) {
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
backend, err := opensearch.NewBackend(t.Context(), indexName, tc.Client(), log.NopLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("mark document as not deleted", func(t *testing.T) {
|
||||
@@ -204,7 +215,7 @@ func TestEngine_Purge(t *testing.T) {
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
backend, err := opensearch.NewBackend(t.Context(), indexName, tc.Client(), log.NopLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("purge with full document", func(t *testing.T) {
|
||||
@@ -256,7 +267,7 @@ func TestEngine_DocCount(t *testing.T) {
|
||||
|
||||
defer tc.Require.IndicesDelete([]string{indexName})
|
||||
|
||||
backend, err := opensearch.NewBackend(indexName, tc.Client())
|
||||
backend, err := opensearch.NewBackend(t.Context(), indexName, tc.Client(), log.NopLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("ignore deleted documents", func(t *testing.T) {
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/conversions"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
@@ -43,7 +44,7 @@ func NewBatch(client *opensearchgoAPI.Client, index string, size int) (*Batch, e
|
||||
|
||||
func (b *Batch) Upsert(id string, r search.Resource) error {
|
||||
return b.withSizeLimit(func() error {
|
||||
body, err := conversions.To[map[string]any](r)
|
||||
body, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal resource: %w", err)
|
||||
}
|
||||
|
||||
@@ -3,29 +3,36 @@ package opensearch
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"maps"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"github.com/go-jose/go-jose/v3/json"
|
||||
opensearchgo "github.com/opensearch-project/opensearch-go/v4"
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
"github.com/tidwall/gjson"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrManualActionRequired = errors.New("manual action required")
|
||||
// ErrManualActionRequired is the shared sentinel, see the mapping package.
|
||||
ErrManualActionRequired = searchmapping.ErrManualActionRequired
|
||||
IndexManagerLatest = IndexIndexManagerResourceV2
|
||||
IndexIndexManagerResourceV1 IndexManager = "resource_v1.json"
|
||||
IndexIndexManagerResourceV2 IndexManager = "resource_v2.json"
|
||||
IndexIndexManagerResourceV2 IndexManager = "resource_v2"
|
||||
)
|
||||
|
||||
//go:embed internal/indexes/*.json
|
||||
var indexes embed.FS
|
||||
|
||||
type IndexManager string
|
||||
|
||||
// indexGenerators dispatches each IndexManager variant to its builder.
|
||||
var indexGenerators = map[IndexManager]func() ([]byte, error){
|
||||
IndexIndexManagerResourceV2: buildResourceV2Mapping,
|
||||
}
|
||||
|
||||
func (m IndexManager) String() string {
|
||||
b, err := m.MarshalJSON()
|
||||
if err != nil {
|
||||
@@ -36,108 +43,184 @@ func (m IndexManager) String() string {
|
||||
}
|
||||
|
||||
func (m IndexManager) MarshalJSON() ([]byte, error) {
|
||||
filePath := string(m)
|
||||
body, err := indexes.ReadFile(path.Join("./internal/indexes", filePath))
|
||||
switch {
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("failed to read index file %s: %w", filePath, err)
|
||||
case len(body) <= 0:
|
||||
return nil, fmt.Errorf("index file %s is empty", filePath)
|
||||
gen, ok := indexGenerators[m]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown index manager %q", string(m))
|
||||
}
|
||||
|
||||
return body, nil
|
||||
return gen()
|
||||
}
|
||||
|
||||
func (m IndexManager) Apply(ctx context.Context, name string, client *opensearchgoAPI.Client) error {
|
||||
// buildResourceV2Mapping renders the OpenSearch index template for a
|
||||
// search.Resource from the shared SearchFieldOverrides. OpenSearch-specific
|
||||
// tweaks (wildcard MimeType, path_hierarchy Path) are applied on top.
|
||||
func buildResourceV2Mapping() ([]byte, error) {
|
||||
resourceType := reflect.TypeFor[search.Resource]()
|
||||
overrides := maps.Clone(search.Resource{}.SearchFieldOverrides())
|
||||
overrides["MimeType"] = searchmapping.FieldOpts{Type: searchmapping.TypeWildcard}
|
||||
overrides["Path"] = searchmapping.FieldOpts{Type: searchmapping.TypePath}
|
||||
if err := searchmapping.Validate(resourceType, overrides); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
props, err := searchmapping.OpenSearchBuildMapping(resourceType, overrides)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
index := map[string]any{
|
||||
"settings": map[string]any{
|
||||
"number_of_shards": "1",
|
||||
"number_of_replicas": "1",
|
||||
"analysis": map[string]any{
|
||||
"analyzer": map[string]any{
|
||||
"path_hierarchy": map[string]any{
|
||||
"type": "custom",
|
||||
"tokenizer": "path_hierarchy",
|
||||
"filter": []string{"lowercase"},
|
||||
},
|
||||
"lowercaseKeyword": map[string]any{
|
||||
"type": "custom",
|
||||
"tokenizer": "keyword",
|
||||
"filter": []string{"lowercase"},
|
||||
},
|
||||
},
|
||||
"tokenizer": map[string]any{
|
||||
"path_hierarchy": map[string]any{"type": "path_hierarchy"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"mappings": map[string]any{
|
||||
"properties": props,
|
||||
},
|
||||
}
|
||||
return json.Marshal(index)
|
||||
}
|
||||
|
||||
// Apply ensures the index exists and matches the schema generated from code:
|
||||
// created if missing, additive changes applied via PUT _mapping, breaking ones
|
||||
// refused with ErrManualActionRequired. The classifier judges, PUT _mapping
|
||||
// only applies (its merge semantics hide removals and renames).
|
||||
func (m IndexManager) Apply(ctx context.Context, name string, client *opensearchgoAPI.Client, logger log.Logger) error {
|
||||
localIndexB, err := m.MarshalJSON()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal index %s: %w", name, err)
|
||||
}
|
||||
|
||||
// Exists first: a pre-provisioned index must not require create privileges
|
||||
indicesExistsResp, err := client.Indices.Exists(ctx, opensearchgoAPI.IndicesExistsReq{
|
||||
Indices: []string{name},
|
||||
})
|
||||
switch {
|
||||
case indicesExistsResp != nil && indicesExistsResp.StatusCode == 404:
|
||||
break
|
||||
createResp, createErr := client.Indices.Create(ctx, opensearchgoAPI.IndicesCreateReq{
|
||||
Index: name,
|
||||
Body: bytes.NewReader(localIndexB),
|
||||
})
|
||||
var structErr *opensearchgo.StructError
|
||||
switch {
|
||||
case createErr == nil && createResp.Acknowledged:
|
||||
return nil
|
||||
case createErr == nil:
|
||||
return fmt.Errorf("failed to create index %s: not acknowledged", name)
|
||||
case !errors.As(createErr, &structErr) || structErr.Err.Type != "resource_already_exists_exception":
|
||||
// transport errors, disk-full etc. stay plain fatal, the restart policy retries
|
||||
return fmt.Errorf("failed to create index %s: %w", name, createErr)
|
||||
}
|
||||
// lost the creation race to another instance, compare against its index
|
||||
case err != nil:
|
||||
return fmt.Errorf("failed to check if index %s exists: %w", name, err)
|
||||
case indicesExistsResp == nil:
|
||||
return fmt.Errorf("indicesExistsResp is nil for index %s", name)
|
||||
}
|
||||
|
||||
if indicesExistsResp.StatusCode == 200 {
|
||||
resp, err := client.Indices.Get(ctx, opensearchgoAPI.IndicesGetReq{
|
||||
Indices: []string{name},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get index %s: %w", name, err)
|
||||
}
|
||||
|
||||
remoteIndex, ok := resp.Indices[name]
|
||||
if !ok {
|
||||
return fmt.Errorf("index %s not found in response", name)
|
||||
}
|
||||
remoteIndexB, err := json.Marshal(remoteIndex)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal index %s: %w", name, err)
|
||||
}
|
||||
|
||||
localIndexJson := gjson.ParseBytes(localIndexB)
|
||||
remoteIndexJson := gjson.ParseBytes(remoteIndexB)
|
||||
|
||||
compare := func(lvPath, rvPath string) (any, any, bool) {
|
||||
lv := localIndexJson.Get(lvPath).Raw
|
||||
rv := remoteIndexJson.Get(rvPath).Raw
|
||||
|
||||
var lvv, rvv any
|
||||
if err := json.Unmarshal([]byte(lv), &lvv); err != nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
|
||||
if err := json.Unmarshal([]byte(rv), &rvv); err != nil {
|
||||
return nil, nil, false
|
||||
}
|
||||
|
||||
return lv, rv, reflect.DeepEqual(lvv, rvv)
|
||||
}
|
||||
|
||||
var errs []error
|
||||
|
||||
for k := range localIndexJson.Get("settings").Map() {
|
||||
if lv, rv, ok := compare("settings."+k, "settings.index."+k); !ok {
|
||||
errs = append(errs, fmt.Errorf("settings.%s local %s, remote %s", k, lv, rv))
|
||||
}
|
||||
}
|
||||
|
||||
for k := range localIndexJson.Get("mappings.properties").Map() {
|
||||
if _, _, ok := compare("mappings.properties."+k, "mappings.properties."+k); !ok {
|
||||
errs = append(errs, fmt.Errorf("mappings.properties.%s", k))
|
||||
}
|
||||
}
|
||||
|
||||
if errs != nil {
|
||||
return fmt.Errorf(
|
||||
"index %s already exists and is different from the requested version, %w: %w",
|
||||
name,
|
||||
ErrManualActionRequired,
|
||||
errors.Join(errs...),
|
||||
)
|
||||
}
|
||||
|
||||
return nil // Index is already up to date, no action needed
|
||||
}
|
||||
|
||||
createResp, err := client.Indices.Create(ctx, opensearchgoAPI.IndicesCreateReq{
|
||||
Index: name,
|
||||
Body: bytes.NewReader(localIndexB),
|
||||
// the index exists: compare settings and classify the mapping diff
|
||||
resp, err := client.Indices.Get(ctx, opensearchgoAPI.IndicesGetReq{
|
||||
Indices: []string{name},
|
||||
})
|
||||
switch {
|
||||
case err != nil:
|
||||
return fmt.Errorf("failed to create index %s: %w", name, err)
|
||||
case !createResp.Acknowledged:
|
||||
return fmt.Errorf("failed to create index %s: not acknowledged", name)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get index %s: %w", name, err)
|
||||
}
|
||||
|
||||
remoteIndex, ok := resp.Indices[name]
|
||||
if !ok {
|
||||
return fmt.Errorf("index %s not found in response", name)
|
||||
}
|
||||
remoteIndexB, err := json.Marshal(remoteIndex)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal index %s: %w", name, err)
|
||||
}
|
||||
|
||||
localIndexJson := gjson.ParseBytes(localIndexB)
|
||||
remoteIndexJson := gjson.ParseBytes(remoteIndexB)
|
||||
|
||||
var reasons []string
|
||||
for k := range localIndexJson.Get("settings").Map() {
|
||||
if k == "number_of_replicas" {
|
||||
continue // runtime-tunable via PUT _settings, drift needs no rebuild
|
||||
}
|
||||
lv := localIndexJson.Get("settings." + k).Raw
|
||||
rv := remoteIndexJson.Get("settings.index." + k).Raw
|
||||
if !jsonEqual(lv, rv) {
|
||||
reasons = append(reasons, fmt.Sprintf("settings.%s changed: index %s, code %s", k, rawOrUnset(rv), rawOrUnset(lv)))
|
||||
}
|
||||
}
|
||||
|
||||
classification := searchmapping.Classify(
|
||||
propertiesMap(remoteIndexJson.Get("mappings.properties").Raw),
|
||||
propertiesMap(localIndexJson.Get("mappings.properties").Raw),
|
||||
nil,
|
||||
)
|
||||
reasons = append(reasons, classification.Reasons...)
|
||||
if len(reasons) > 0 {
|
||||
return searchmapping.ManualActionRequiredError(name, fmt.Sprintf("delete the index (DELETE /%s)", name), reasons)
|
||||
}
|
||||
if len(classification.NewFields) == 0 {
|
||||
return nil // schema is up to date
|
||||
}
|
||||
|
||||
// additive: the classifier guarantees every existing field matches the
|
||||
// remote state, so putting the full code properties can only add fields
|
||||
putResp, err := client.Indices.Mapping.Put(ctx, opensearchgoAPI.MappingPutReq{
|
||||
Indices: []string{name},
|
||||
Body: strings.NewReader(localIndexJson.Get("mappings").Raw),
|
||||
})
|
||||
var putErr *opensearchgo.StructError
|
||||
switch {
|
||||
case err != nil && errors.As(err, &putErr) && putErr.Err.Type == "illegal_argument_exception" &&
|
||||
(strings.Contains(putErr.Err.Reason, "cannot be changed") || strings.Contains(putErr.Err.Reason, "Cannot update parameter")):
|
||||
// backstop, should be unreachable after the classification above
|
||||
return searchmapping.ManualActionRequiredError(name, fmt.Sprintf("delete the index (DELETE /%s)", name), []string{putErr.Err.Reason})
|
||||
case err != nil:
|
||||
return fmt.Errorf("failed to update mapping of index %s: %w", name, err)
|
||||
case !putResp.Acknowledged:
|
||||
return fmt.Errorf("failed to update mapping of index %s: not acknowledged", name)
|
||||
}
|
||||
|
||||
logger.Warn().Strs("fields", classification.NewFields).Str("index", name).Msg("extended the search index mapping with new fields; documents indexed before the upgrade do not contain them and queries on these fields will miss those documents until they are re-indexed; to re-index everything run: opencloud search index --all-spaces --force-rescan")
|
||||
return nil
|
||||
}
|
||||
|
||||
func jsonEqual(a, b string) bool {
|
||||
var av, bv any
|
||||
if err := json.Unmarshal([]byte(a), &av); err != nil {
|
||||
return false
|
||||
}
|
||||
if err := json.Unmarshal([]byte(b), &bv); err != nil {
|
||||
return false
|
||||
}
|
||||
return reflect.DeepEqual(av, bv)
|
||||
}
|
||||
|
||||
// propertiesMap parses a raw mappings.properties object; missing or empty
|
||||
// input yields an empty map, which classifies as purely additive.
|
||||
func propertiesMap(raw string) map[string]any {
|
||||
props := map[string]any{}
|
||||
_ = json.Unmarshal([]byte(raw), &props)
|
||||
return props
|
||||
}
|
||||
|
||||
func rawOrUnset(raw string) string {
|
||||
if raw == "" {
|
||||
return "(unset)"
|
||||
}
|
||||
return raw
|
||||
}
|
||||
@@ -4,9 +4,13 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
opensearchgo "github.com/opensearch-project/opensearch-go/v4"
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/tidwall/gjson"
|
||||
"github.com/tidwall/sjson"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
|
||||
)
|
||||
@@ -31,7 +35,7 @@ func TestIndexManager(t *testing.T) {
|
||||
require.NotEmpty(t, body)
|
||||
require.NotEmpty(t, test.Got.String())
|
||||
require.JSONEq(t, test.Got.String(), string(body))
|
||||
require.NoError(t, test.Got.Apply(t.Context(), indexName, tc.Client()))
|
||||
require.NoError(t, test.Got.Apply(t.Context(), indexName, tc.Client(), log.NopLogger()))
|
||||
})
|
||||
}
|
||||
})
|
||||
@@ -44,7 +48,7 @@ func TestIndexManager(t *testing.T) {
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
tc.Require.IndicesCreate(indexName, strings.NewReader(indexManager.String()))
|
||||
|
||||
require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client()))
|
||||
require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client(), log.NopLogger()))
|
||||
})
|
||||
|
||||
t.Run("fails to create index if it already exists but is not up to date", func(t *testing.T) {
|
||||
@@ -58,6 +62,108 @@ func TestIndexManager(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
tc.Require.IndicesCreate(indexName, strings.NewReader(body))
|
||||
|
||||
require.ErrorIs(t, indexManager.Apply(t.Context(), indexName, tc.Client()), opensearch.ErrManualActionRequired)
|
||||
require.ErrorIs(t, indexManager.Apply(t.Context(), indexName, tc.Client(), log.NopLogger()), opensearch.ErrManualActionRequired)
|
||||
})
|
||||
|
||||
t.Run("tolerates replica drift", func(t *testing.T) {
|
||||
indexManager := opensearch.IndexManagerLatest
|
||||
indexName := "opencloud-test-resource"
|
||||
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
|
||||
body, err := sjson.Set(indexManager.String(), "settings.number_of_replicas", "2")
|
||||
require.NoError(t, err)
|
||||
tc.Require.IndicesCreate(indexName, strings.NewReader(body))
|
||||
|
||||
require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client(), log.NopLogger()))
|
||||
})
|
||||
|
||||
t.Run("is idempotent", func(t *testing.T) {
|
||||
indexManager := opensearch.IndexManagerLatest
|
||||
indexName := "opencloud-test-resource"
|
||||
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
|
||||
require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client(), log.NopLogger()))
|
||||
require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client(), log.NopLogger()))
|
||||
})
|
||||
|
||||
t.Run("adds a new field to an existing index in place", func(t *testing.T) {
|
||||
indexManager := opensearch.IndexManagerLatest
|
||||
indexName := "opencloud-test-resource"
|
||||
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
|
||||
body, err := sjson.Delete(indexManager.String(), "mappings.properties.Title")
|
||||
require.NoError(t, err)
|
||||
tc.Require.IndicesCreate(indexName, strings.NewReader(body))
|
||||
|
||||
require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client(), log.NopLogger()))
|
||||
|
||||
resp, err := tc.Client().Indices.Mapping.Get(t.Context(), &opensearchgoAPI.MappingGetReq{Indices: []string{indexName}})
|
||||
require.NoError(t, err)
|
||||
require.True(t, gjson.GetBytes(resp.Indices[indexName].Mappings, "properties.Title").Exists())
|
||||
})
|
||||
|
||||
t.Run("adds a new nested field to an existing index in place", func(t *testing.T) {
|
||||
indexManager := opensearch.IndexManagerLatest
|
||||
indexName := "opencloud-test-resource"
|
||||
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
|
||||
body, err := sjson.Delete(indexManager.String(), "mappings.properties.photo.properties.cameraMake")
|
||||
require.NoError(t, err)
|
||||
tc.Require.IndicesCreate(indexName, strings.NewReader(body))
|
||||
|
||||
require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client(), log.NopLogger()))
|
||||
|
||||
resp, err := tc.Client().Indices.Mapping.Get(t.Context(), &opensearchgoAPI.MappingGetReq{Indices: []string{indexName}})
|
||||
require.NoError(t, err)
|
||||
require.True(t, gjson.GetBytes(resp.Indices[indexName].Mappings, "properties.photo.properties.cameraMake").Exists())
|
||||
})
|
||||
|
||||
t.Run("fails when an existing field changed its definition", func(t *testing.T) {
|
||||
indexManager := opensearch.IndexManagerLatest
|
||||
indexName := "opencloud-test-resource"
|
||||
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
|
||||
body, err := sjson.Set(indexManager.String(), "mappings.properties.Deleted.type", "keyword")
|
||||
require.NoError(t, err)
|
||||
tc.Require.IndicesCreate(indexName, strings.NewReader(body))
|
||||
|
||||
require.ErrorIs(t, indexManager.Apply(t.Context(), indexName, tc.Client(), log.NopLogger()), opensearch.ErrManualActionRequired)
|
||||
})
|
||||
|
||||
t.Run("fails when the index contains a field the code schema does not know", func(t *testing.T) {
|
||||
indexManager := opensearch.IndexManagerLatest
|
||||
indexName := "opencloud-test-resource"
|
||||
|
||||
tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client)
|
||||
tc.Require.IndicesReset([]string{indexName})
|
||||
|
||||
body, err := sjson.Set(indexManager.String(), "mappings.properties.legacyField.type", "keyword")
|
||||
require.NoError(t, err)
|
||||
tc.Require.IndicesCreate(indexName, strings.NewReader(body))
|
||||
|
||||
require.ErrorIs(t, indexManager.Apply(t.Context(), indexName, tc.Client(), log.NopLogger()), opensearch.ErrManualActionRequired)
|
||||
})
|
||||
|
||||
t.Run("transport errors do not demand manual action", func(t *testing.T) {
|
||||
client, err := opensearchgoAPI.NewClient(opensearchgoAPI.Config{
|
||||
Client: opensearchgo.Config{
|
||||
Addresses: []string{"http://localhost:1025"},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
err = opensearch.IndexManagerLatest.Apply(t.Context(), "opencloud-test-resource", client, log.NopLogger())
|
||||
require.Error(t, err)
|
||||
require.NotErrorIs(t, err, opensearch.ErrManualActionRequired)
|
||||
})
|
||||
}
|
||||
@@ -3,7 +3,6 @@ package convert
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
@@ -15,6 +14,17 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
// copyFacet converts a typed pointer from the indexed shape (libregraph) to
|
||||
// the protobuf shape via conversions.To. Returns nil when src is nil so the
|
||||
// enclosing Match.Entity field stays nil.
|
||||
func copyFacet[Dst, Src any](src *Src) *Dst {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
dst, _ := conversions.To[*Dst](src)
|
||||
return dst
|
||||
}
|
||||
|
||||
func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match, error) {
|
||||
resource, err := conversions.To[search.Resource](hit.Source)
|
||||
if err != nil {
|
||||
@@ -68,31 +78,15 @@ func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match,
|
||||
|
||||
return strings.Join(contentHighlights[:], "; ")
|
||||
}(),
|
||||
Audio: func() *searchMessage.Audio {
|
||||
if !strings.HasPrefix(resource.MimeType, "audio/") {
|
||||
return nil
|
||||
}
|
||||
|
||||
audio, _ := conversions.To[*searchMessage.Audio](resource.Audio)
|
||||
return audio
|
||||
}(),
|
||||
Image: func() *searchMessage.Image {
|
||||
image, _ := conversions.To[*searchMessage.Image](resource.Image)
|
||||
return image
|
||||
}(),
|
||||
Location: func() *searchMessage.GeoCoordinates {
|
||||
geoCoordinates, _ := conversions.To[*searchMessage.GeoCoordinates](resource.Location)
|
||||
return geoCoordinates
|
||||
}(),
|
||||
Photo: func() *searchMessage.Photo {
|
||||
photo, _ := conversions.To[*searchMessage.Photo](resource.Photo)
|
||||
return photo
|
||||
}(),
|
||||
Audio: copyFacet[searchMessage.Audio](resource.Audio),
|
||||
Image: copyFacet[searchMessage.Image](resource.Image),
|
||||
Location: copyFacet[searchMessage.GeoCoordinates](resource.Location),
|
||||
Photo: copyFacet[searchMessage.Photo](resource.Photo),
|
||||
},
|
||||
}
|
||||
|
||||
if mtime, err := time.Parse(time.RFC3339, resource.Mtime); err == nil {
|
||||
match.Entity.LastModifiedTime = ×tamppb.Timestamp{Seconds: mtime.Unix(), Nanos: int32(mtime.Nanosecond())}
|
||||
if resource.Mtime != nil {
|
||||
match.Entity.LastModifiedTime = timestamppb.New(*resource.Mtime)
|
||||
}
|
||||
|
||||
return match, nil
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
{
|
||||
"settings": {
|
||||
"number_of_shards": "1",
|
||||
"number_of_replicas": "1",
|
||||
"analysis": {
|
||||
"analyzer": {
|
||||
"path_hierarchy": {
|
||||
"filter": [
|
||||
"lowercase"
|
||||
],
|
||||
"tokenizer": "path_hierarchy",
|
||||
"type": "custom"
|
||||
}
|
||||
},
|
||||
"tokenizer": {
|
||||
"path_hierarchy": {
|
||||
"type": "path_hierarchy"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"ID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"ParentID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"RootID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"MimeType": {
|
||||
"type": "wildcard",
|
||||
"doc_values": false
|
||||
},
|
||||
"Path": {
|
||||
"type": "text",
|
||||
"analyzer": "path_hierarchy"
|
||||
},
|
||||
"Deleted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"Hidden": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
{
|
||||
"settings": {
|
||||
"number_of_shards": "1",
|
||||
"number_of_replicas": "1",
|
||||
"analysis": {
|
||||
"analyzer": {
|
||||
"path_hierarchy": {
|
||||
"filter": [
|
||||
"lowercase"
|
||||
],
|
||||
"tokenizer": "path_hierarchy",
|
||||
"type": "custom"
|
||||
}
|
||||
},
|
||||
"tokenizer": {
|
||||
"path_hierarchy": {
|
||||
"type": "path_hierarchy"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"Content": {
|
||||
"type": "text",
|
||||
"term_vector": "with_positions_offsets"
|
||||
},
|
||||
"ID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"ParentID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"RootID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"MimeType": {
|
||||
"type": "wildcard",
|
||||
"doc_values": false
|
||||
},
|
||||
"Path": {
|
||||
"type": "text",
|
||||
"analyzer": "path_hierarchy"
|
||||
},
|
||||
"Deleted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"Hidden": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"Favorites": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@
|
||||
"Name" : "dummy name",
|
||||
"Content" : "dummy content",
|
||||
"Size" : 42,
|
||||
"Mtime" : "2025-07-24 15:15:01.324093 +0200 CEST m=+0.000056251",
|
||||
"Mtime" : "2025-07-24T15:15:01.324093+02:00",
|
||||
"MimeType" : "image/jpeg",
|
||||
"Tags" : [ "dummy" ],
|
||||
"Deleted" : false,
|
||||
|
||||
@@ -3,5 +3,6 @@
|
||||
"RootID" : "1$1!1",
|
||||
"ParentID" : "1$1!1",
|
||||
"Path" : "./parent d!r",
|
||||
"Type" : 2
|
||||
"Type" : 2,
|
||||
"Mtime" : "2025-07-24T15:15:01.324093+02:00"
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
{
|
||||
"ID" : "1$1!1",
|
||||
"RootID" : "1$1!1",
|
||||
"Path" : "."
|
||||
"Path" : ".",
|
||||
"Mtime" : "2025-07-24T15:15:01.324093+02:00"
|
||||
}
|
||||
@@ -8,17 +8,25 @@ import (
|
||||
bleveQuery "github.com/blevesearch/bleve/v2/search/query"
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/pkg/kql"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
// lowercaseFields lists the bleve fields whose index mapping uses a
|
||||
// lowercasing analyzer. Values bound to these fields are pre-lowercased
|
||||
// so query-side matching stays consistent with the index.
|
||||
// Keep in sync with services/search/pkg/bleve/index.go NewMapping.
|
||||
var lowercaseFields = map[string]struct{}{
|
||||
"Name": {},
|
||||
"Tags": {},
|
||||
"Favorites": {},
|
||||
"Content": {},
|
||||
// lowercaseFields is derived from Resource.SearchFieldOverrides(): any
|
||||
// field whose override picks a lowercasing analyzer (`lowercaseKeyword`)
|
||||
// or the fulltext type (which uses a lowercasing analyzer under the hood)
|
||||
// gets its query-side value pre-lowercased so compile-time matches the
|
||||
// index-time tokenization. Anything else keeps its original casing.
|
||||
var lowercaseFields = buildLowercaseFields()
|
||||
|
||||
func buildLowercaseFields() map[string]struct{} {
|
||||
out := map[string]struct{}{}
|
||||
for key, opts := range (search.Resource{}).SearchFieldOverrides() {
|
||||
if opts.Analyzer == "lowercaseKeyword" || opts.Type == mapping.TypeFulltext {
|
||||
out[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var _fields = map[string]string{
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
|
||||
@@ -19,6 +20,7 @@ import (
|
||||
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
)
|
||||
|
||||
var scopeRegex = regexp.MustCompile(`scope:\s*([^" "\n\r]*)`)
|
||||
@@ -51,13 +53,33 @@ type BatchOperator interface {
|
||||
type Resource struct {
|
||||
content.Document
|
||||
|
||||
ID string
|
||||
RootID string
|
||||
Path string
|
||||
ParentID string
|
||||
Type uint64
|
||||
Deleted bool
|
||||
Hidden bool
|
||||
ID string `json:"ID"`
|
||||
RootID string `json:"RootID"`
|
||||
Path string `json:"Path"`
|
||||
ParentID string `json:"ParentID"`
|
||||
Type uint64 `json:"Type"`
|
||||
Deleted bool `json:"Deleted"`
|
||||
Hidden bool `json:"Hidden"`
|
||||
}
|
||||
|
||||
// resourceFieldOverrides is built once (it never changes) and reused on hot
|
||||
// paths instead of reallocating per call.
|
||||
var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts {
|
||||
excludeFromAll := false
|
||||
return map[string]mapping.FieldOpts{
|
||||
"Name": {Analyzer: "lowercaseKeyword"},
|
||||
"Content": {Type: mapping.TypeFulltext},
|
||||
"Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll},
|
||||
"Favorites": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll},
|
||||
"location": {Type: mapping.TypeGeopoint},
|
||||
}
|
||||
})
|
||||
|
||||
// SearchFieldOverrides returns the field options the mapping package needs to
|
||||
// build per-backend index mappings for a Resource (keys are json-tag names).
|
||||
// The map is shared and read-only; clone it before mutating.
|
||||
func (Resource) SearchFieldOverrides() map[string]mapping.FieldOpts {
|
||||
return resourceFieldOverrides()
|
||||
}
|
||||
|
||||
// ResolveReference makes sure the path is relative to the space root
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -628,10 +629,10 @@ func (s *Service) doUpsertItem(ref *provider.Reference, batch BatchOperator) {
|
||||
|
||||
// determine if metadata needs to be stored in storage as well
|
||||
metadata := map[string]string{}
|
||||
addAudioMetadata(metadata, doc.Audio)
|
||||
addImageMetadata(metadata, doc.Image)
|
||||
addLocationMetadata(metadata, doc.Location)
|
||||
addPhotoMetadata(metadata, doc.Photo)
|
||||
facetToMetadata(metadata, doc.Audio, "libre.graph.audio.")
|
||||
facetToMetadata(metadata, doc.Image, "libre.graph.image.")
|
||||
facetToMetadata(metadata, doc.Location, "libre.graph.location.")
|
||||
facetToMetadata(metadata, doc.Photo, "libre.graph.photo.")
|
||||
if len(metadata) == 0 {
|
||||
return
|
||||
}
|
||||
@@ -656,43 +657,25 @@ func (s *Service) doUpsertItem(ref *provider.Reference, batch BatchOperator) {
|
||||
}
|
||||
}
|
||||
|
||||
func addAudioMetadata(metadata map[string]string, audio *libregraph.Audio) {
|
||||
if audio == nil {
|
||||
return
|
||||
// facetToMetadata flattens a libregraph facet (Audio / Image / Location / Photo
|
||||
// pointer) into the metadata map under the given prefix via the model's ToMap.
|
||||
// No-op when the facet is nil.
|
||||
func facetToMetadata[T libregraph.MappedNullable](metadata map[string]string, facet T, prefix string) {
|
||||
// Only nilable kinds can be nil; IsNil panics on a value type (some
|
||||
// libregraph models satisfy MappedNullable with a value receiver).
|
||||
switch v := reflect.ValueOf(facet); v.Kind() {
|
||||
case reflect.Ptr, reflect.Map, reflect.Slice, reflect.Interface, reflect.Chan, reflect.Func:
|
||||
if v.IsNil() {
|
||||
return
|
||||
}
|
||||
}
|
||||
marshalToStringMap(audio, metadata, "libre.graph.audio.")
|
||||
}
|
||||
|
||||
func addImageMetadata(metadata map[string]string, image *libregraph.Image) {
|
||||
if image == nil {
|
||||
return
|
||||
}
|
||||
marshalToStringMap(image, metadata, "libre.graph.image.")
|
||||
}
|
||||
|
||||
func addLocationMetadata(metadata map[string]string, location *libregraph.GeoCoordinates) {
|
||||
if location == nil {
|
||||
return
|
||||
}
|
||||
marshalToStringMap(location, metadata, "libre.graph.location.")
|
||||
}
|
||||
|
||||
func addPhotoMetadata(metadata map[string]string, photo *libregraph.Photo) {
|
||||
if photo == nil {
|
||||
return
|
||||
}
|
||||
marshalToStringMap(photo, metadata, "libre.graph.photo.")
|
||||
}
|
||||
|
||||
func marshalToStringMap[T libregraph.MappedNullable](source T, target map[string]string, prefix string) {
|
||||
// ToMap never returns a non-nil error ...
|
||||
m, _ := source.ToMap()
|
||||
|
||||
// ToMap never returns a non-nil error.
|
||||
m, _ := facet.ToMap()
|
||||
for k, v := range m {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
target[prefix+k] = valueToString(v)
|
||||
metadata[prefix+k] = valueToString(v)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user