mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-08-01 02:11:15 -04:00
refactor: reflection-based search mapping
Build the bleve and OpenSearch index mappings from the Go struct via reflection (json tags + per-field overrides) instead of hand-rolled mappings and hit deserializers. New mapping package: BleveBuildMapping, OpenSearchBuildMapping, Deserialize[T], PrepareForIndex; field decoding is fail-soft. Mtime is typed as a date so mtime ranges are chronological on both backends. Route CS3 facet parsing through mapping.DeserializeStringMap. The any-valued (bleve hit) and string-valued (CS3 metadata) deserializers share one generic fillStruct walker with a per-value setLeaf callback.
This commit is contained in:
@@ -9,9 +9,7 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
@@ -28,6 +26,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
)
|
||||
|
||||
// CreateUploadSession create an upload session to allow your app to upload files up to the maximum file size.
|
||||
@@ -452,130 +451,20 @@ func cs3ResourceToDriveItem(logger *log.Logger, publicBaseURL *url.URL, res *sto
|
||||
}
|
||||
}
|
||||
|
||||
if res.GetArbitraryMetadata() != nil {
|
||||
driveItem.Audio = cs3ResourceToDriveItemAudioFacet(logger, res)
|
||||
driveItem.Image = cs3ResourceToDriveItemImageFacet(logger, res)
|
||||
driveItem.Location = cs3ResourceToDriveItemLocationFacet(logger, res)
|
||||
driveItem.Photo = cs3ResourceToDriveItemPhotoFacet(logger, res)
|
||||
if metadata := res.GetArbitraryMetadata().GetMetadata(); metadata != nil {
|
||||
driveItem.Audio = metadataToFacet[libregraph.Audio](metadata, "audio")
|
||||
driveItem.Image = metadataToFacet[libregraph.Image](metadata, "image")
|
||||
driveItem.Location = metadataToFacet[libregraph.GeoCoordinates](metadata, "location")
|
||||
driveItem.Photo = metadataToFacet[libregraph.Photo](metadata, "photo")
|
||||
}
|
||||
|
||||
return driveItem, nil
|
||||
}
|
||||
|
||||
func cs3ResourceToDriveItemAudioFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Audio {
|
||||
if !strings.HasPrefix(res.GetMimeType(), "audio/") {
|
||||
return nil
|
||||
}
|
||||
|
||||
k := res.GetArbitraryMetadata().GetMetadata()
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var audio = &libregraph.Audio{}
|
||||
if ok := unmarshalStringMap(logger, audio, k, "libre.graph.audio."); ok {
|
||||
return audio
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func cs3ResourceToDriveItemImageFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Image {
|
||||
k := res.GetArbitraryMetadata().GetMetadata()
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var image = &libregraph.Image{}
|
||||
if ok := unmarshalStringMap(logger, image, k, "libre.graph.image."); ok {
|
||||
return image
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func cs3ResourceToDriveItemLocationFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.GeoCoordinates {
|
||||
k := res.GetArbitraryMetadata().GetMetadata()
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var location = &libregraph.GeoCoordinates{}
|
||||
if ok := unmarshalStringMap(logger, location, k, "libre.graph.location."); ok {
|
||||
return location
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func cs3ResourceToDriveItemPhotoFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Photo {
|
||||
k := res.GetArbitraryMetadata().GetMetadata()
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var photo = &libregraph.Photo{}
|
||||
if ok := unmarshalStringMap(logger, photo, k, "libre.graph.photo."); ok {
|
||||
return photo
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getFieldName(structField reflect.StructField) string {
|
||||
tag := structField.Tag.Get("json")
|
||||
if tag == "" {
|
||||
return structField.Name
|
||||
}
|
||||
|
||||
return strings.Split(tag, ",")[0]
|
||||
}
|
||||
|
||||
func unmarshalStringMap(logger *log.Logger, out any, flatMap map[string]string, prefix string) bool {
|
||||
nonEmpty := false
|
||||
obj := reflect.ValueOf(out).Elem()
|
||||
timeKind := reflect.TypeOf(&time.Time{}).Elem().Kind()
|
||||
for i := 0; i < obj.NumField(); i++ {
|
||||
field := obj.Field(i)
|
||||
structField := obj.Type().Field(i)
|
||||
mapKey := prefix + getFieldName(structField)
|
||||
|
||||
if value, ok := flatMap[mapKey]; ok {
|
||||
if field.Kind() == reflect.Ptr {
|
||||
newValue := reflect.New(field.Type().Elem())
|
||||
var tmp any
|
||||
var err error
|
||||
switch t := newValue.Type().Elem().Kind(); t {
|
||||
case reflect.String:
|
||||
tmp = value
|
||||
case reflect.Int32:
|
||||
tmp, err = strconv.ParseInt(value, 10, 32)
|
||||
case reflect.Int64:
|
||||
tmp, err = strconv.ParseInt(value, 10, 64)
|
||||
case reflect.Float32:
|
||||
tmp, err = strconv.ParseFloat(value, 32)
|
||||
case reflect.Float64:
|
||||
tmp, err = strconv.ParseFloat(value, 64)
|
||||
case reflect.Bool:
|
||||
tmp, err = strconv.ParseBool(value)
|
||||
case timeKind:
|
||||
tmp, err = time.Parse(time.RFC3339, value)
|
||||
default:
|
||||
err = errors.New("unsupported type")
|
||||
logger.Error().Err(err).Str("type", t.String()).Str("mapKey", mapKey).Msg("target field type for value of mapKey is not supported")
|
||||
}
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Str("mapKey", mapKey).Msg("unmarshalling failed")
|
||||
continue
|
||||
}
|
||||
newValue.Elem().Set(reflect.ValueOf(tmp).Convert(field.Type().Elem()))
|
||||
field.Set(newValue)
|
||||
nonEmpty = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nonEmpty
|
||||
// metadataToFacet builds a DriveItem facet *T from CS3 arbitrary metadata under
|
||||
// the "libre.graph.<facet>." key prefix. Nil when no such keys are present.
|
||||
func metadataToFacet[T any](metadata map[string]string, facet string) *T {
|
||||
return mapping.DeserializeStringsAt[T](metadata, "libre.graph."+facet+".")
|
||||
}
|
||||
|
||||
func cs3ResourceToRemoteItem(res *storageprovider.ResourceInfo) (*libregraph.RemoteItem, error) {
|
||||
|
||||
@@ -136,10 +136,10 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
|
||||
Tags: getFieldSliceValue[string](hit.Fields, "Tags"),
|
||||
Favorites: getFieldSliceValue[string](hit.Fields, "Favorites"),
|
||||
Highlights: getFragmentValue(hit.Fragments, "Content", 0),
|
||||
Audio: getAudioValue[searchMessage.Audio](hit.Fields),
|
||||
Image: getImageValue[searchMessage.Image](hit.Fields),
|
||||
Location: getLocationValue[searchMessage.GeoCoordinates](hit.Fields),
|
||||
Photo: getPhotoValue[searchMessage.Photo](hit.Fields),
|
||||
Audio: hitToFacet[searchMessage.Audio](hit.Fields, "audio"),
|
||||
Image: hitToFacet[searchMessage.Image](hit.Fields, "image"),
|
||||
Location: hitToFacet[searchMessage.GeoCoordinates](hit.Fields, "location"),
|
||||
Photo: hitToFacet[searchMessage.Photo](hit.Fields, "photo"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
@@ -36,10 +37,21 @@ func NewBatch(index bleve.Index, size int) (*Batch, error) {
|
||||
|
||||
func (b *Batch) Upsert(id string, r search.Resource) error {
|
||||
return b.withSizeLimit(func() error {
|
||||
return b.batch.Index(id, r)
|
||||
return b.indexResource(id, r)
|
||||
})
|
||||
}
|
||||
|
||||
// indexResource prepares r for bleve (resolving json tags and splicing in
|
||||
// type-specific adaptations via the mapping package) and appends it to the
|
||||
// batch under id.
|
||||
func (b *Batch) indexResource(id string, r search.Resource) error {
|
||||
doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return b.batch.Index(id, doc)
|
||||
}
|
||||
|
||||
func (b *Batch) Move(id, parentID, location string) error {
|
||||
return b.withSizeLimit(func() error {
|
||||
rootResource, err := searchResourceByID(id, b.index)
|
||||
@@ -68,7 +80,7 @@ func (b *Batch) Move(id, parentID, location string) error {
|
||||
}
|
||||
|
||||
for _, resource := range resources {
|
||||
if err := b.batch.Index(resource.ID, resource); err != nil {
|
||||
if err := b.indexResource(resource.ID, *resource); err != nil {
|
||||
return err
|
||||
}
|
||||
if b.batch.Size() >= b.size {
|
||||
@@ -90,7 +102,7 @@ func (b *Batch) Delete(id string) error {
|
||||
}
|
||||
|
||||
for _, resource := range affectedResources {
|
||||
if err := b.batch.Index(resource.ID, resource); err != nil {
|
||||
if err := b.indexResource(resource.ID, *resource); err != nil {
|
||||
return err
|
||||
}
|
||||
if b.batch.Size() >= b.size {
|
||||
@@ -112,7 +124,7 @@ func (b *Batch) Restore(id string) error {
|
||||
}
|
||||
|
||||
for _, resource := range affectedResources {
|
||||
if err := b.batch.Index(resource.ID, resource); err != nil {
|
||||
if err := b.indexResource(resource.ID, *resource); err != nil {
|
||||
return err
|
||||
}
|
||||
if b.batch.Size() >= b.size {
|
||||
|
||||
@@ -1,18 +1,13 @@
|
||||
package bleve
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
bleveSearch "github.com/blevesearch/bleve/v2/search"
|
||||
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
|
||||
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
@@ -75,131 +70,19 @@ func getFragmentValue(m bleveSearch.FieldFragmentMap, key string, idx int) strin
|
||||
return val[idx]
|
||||
}
|
||||
|
||||
func getAudioValue[T any](fields map[string]any) *T {
|
||||
if !strings.HasPrefix(getFieldValue[string](fields, "MimeType"), "audio/") {
|
||||
return nil
|
||||
}
|
||||
|
||||
var audio = newPointerOfType[T]()
|
||||
if ok := unmarshalInterfaceMap(audio, fields, "audio."); ok {
|
||||
return audio
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getImageValue[T any](fields map[string]any) *T {
|
||||
var image = newPointerOfType[T]()
|
||||
if ok := unmarshalInterfaceMap(image, fields, "image."); ok {
|
||||
return image
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getLocationValue[T any](fields map[string]any) *T {
|
||||
var location = newPointerOfType[T]()
|
||||
if ok := unmarshalInterfaceMap(location, fields, "location."); ok {
|
||||
return location
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getPhotoValue[T any](fields map[string]any) *T {
|
||||
var photo = newPointerOfType[T]()
|
||||
if ok := unmarshalInterfaceMap(photo, fields, "photo."); ok {
|
||||
return photo
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func newPointerOfType[T any]() *T {
|
||||
t := reflect.TypeOf((*T)(nil)).Elem()
|
||||
ptr := reflect.New(t).Interface()
|
||||
return ptr.(*T)
|
||||
}
|
||||
|
||||
func unmarshalInterfaceMap(out any, flatMap map[string]any, prefix string) bool {
|
||||
nonEmpty := false
|
||||
obj := reflect.ValueOf(out).Elem()
|
||||
for i := 0; i < obj.NumField(); i++ {
|
||||
field := obj.Field(i)
|
||||
structField := obj.Type().Field(i)
|
||||
mapKey := prefix + getFieldName(structField)
|
||||
|
||||
if value, ok := flatMap[mapKey]; ok {
|
||||
if field.Kind() == reflect.Ptr {
|
||||
alloc := reflect.New(field.Type().Elem())
|
||||
elemType := field.Type().Elem()
|
||||
|
||||
// convert time strings from index for search requests
|
||||
if elemType == reflect.TypeOf(timestamppb.Timestamp{}) {
|
||||
if strValue, ok := value.(string); ok {
|
||||
if parsedTime, err := time.Parse(time.RFC3339, strValue); err == nil {
|
||||
alloc.Elem().Set(reflect.ValueOf(*timestamppb.New(parsedTime)))
|
||||
field.Set(alloc)
|
||||
nonEmpty = true
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// convert time strings from index for libregraph structs when updating resources
|
||||
if elemType == reflect.TypeOf(time.Time{}) {
|
||||
if strValue, ok := value.(string); ok {
|
||||
if parsedTime, err := time.Parse(time.RFC3339, strValue); err == nil {
|
||||
alloc.Elem().Set(reflect.ValueOf(parsedTime))
|
||||
field.Set(alloc)
|
||||
nonEmpty = true
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
alloc.Elem().Set(reflect.ValueOf(value).Convert(elemType))
|
||||
field.Set(alloc)
|
||||
nonEmpty = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nonEmpty
|
||||
}
|
||||
|
||||
func getFieldName(structField reflect.StructField) string {
|
||||
tag := structField.Tag.Get("json")
|
||||
if tag == "" {
|
||||
return structField.Name
|
||||
}
|
||||
|
||||
return strings.Split(tag, ",")[0]
|
||||
// hitToFacet builds a search Entity facet *T from a bleve hit's fields under the
|
||||
// given key prefix. Nil when the hit has no such fields.
|
||||
func hitToFacet[T any](fields map[string]any, prefix string) *T {
|
||||
return mapping.DeserializeAt[T](fields, prefix)
|
||||
}
|
||||
|
||||
// matchToResource reconstructs a search.Resource from a bleve hit. Used by
|
||||
// the Move / Delete / Restore / Purge paths that round-trip a record through
|
||||
// the index. Always returns a non-nil *Resource: Deserialize is fail-soft
|
||||
// for per-field parse errors, so corrupted hit values surface as zero
|
||||
// values on individual fields instead of dropping the whole record.
|
||||
func matchToResource(match *bleveSearch.DocumentMatch) *search.Resource {
|
||||
return &search.Resource{
|
||||
ID: getFieldValue[string](match.Fields, "ID"),
|
||||
RootID: getFieldValue[string](match.Fields, "RootID"),
|
||||
Path: getFieldValue[string](match.Fields, "Path"),
|
||||
ParentID: getFieldValue[string](match.Fields, "ParentID"),
|
||||
Type: uint64(getFieldValue[float64](match.Fields, "Type")),
|
||||
Deleted: getFieldValue[bool](match.Fields, "Deleted"),
|
||||
Document: content.Document{
|
||||
Name: getFieldValue[string](match.Fields, "Name"),
|
||||
Title: getFieldValue[string](match.Fields, "Title"),
|
||||
Size: uint64(getFieldValue[float64](match.Fields, "Size")),
|
||||
Mtime: getFieldValue[string](match.Fields, "Mtime"),
|
||||
MimeType: getFieldValue[string](match.Fields, "MimeType"),
|
||||
Content: getFieldValue[string](match.Fields, "Content"),
|
||||
Tags: getFieldSliceValue[string](match.Fields, "Tags"),
|
||||
Favorites: getFieldSliceValue[string](match.Fields, "Favorites"),
|
||||
Audio: getAudioValue[libregraph.Audio](match.Fields),
|
||||
Image: getImageValue[libregraph.Image](match.Fields),
|
||||
Location: getLocationValue[libregraph.GeoCoordinates](match.Fields),
|
||||
Photo: getPhotoValue[libregraph.Photo](match.Fields),
|
||||
},
|
||||
}
|
||||
return mapping.Deserialize[search.Resource](match.Fields)
|
||||
}
|
||||
|
||||
func escapeQuery(s string) string {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"errors"
|
||||
"math"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
|
||||
@@ -15,6 +16,7 @@ import (
|
||||
"github.com/blevesearch/bleve/v2/mapping"
|
||||
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
|
||||
searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
@@ -38,27 +40,20 @@ func NewIndex(root string) (bleve.Index, error) {
|
||||
}
|
||||
|
||||
func NewMapping() (mapping.IndexMapping, error) {
|
||||
nameMapping := bleve.NewTextFieldMapping()
|
||||
nameMapping.Analyzer = "lowercaseKeyword"
|
||||
|
||||
lowercaseMapping := bleve.NewTextFieldMapping()
|
||||
lowercaseMapping.IncludeInAll = false
|
||||
lowercaseMapping.Analyzer = "lowercaseKeyword"
|
||||
|
||||
fulltextFieldMapping := bleve.NewTextFieldMapping()
|
||||
fulltextFieldMapping.Analyzer = "fulltext"
|
||||
fulltextFieldMapping.IncludeInAll = false
|
||||
|
||||
docMapping := bleve.NewDocumentMapping()
|
||||
docMapping.AddFieldMappingsAt("Name", nameMapping)
|
||||
docMapping.AddFieldMappingsAt("Tags", lowercaseMapping)
|
||||
docMapping.AddFieldMappingsAt("Favorites", lowercaseMapping)
|
||||
docMapping.AddFieldMappingsAt("Content", fulltextFieldMapping)
|
||||
resourceType := reflect.TypeFor[search.Resource]()
|
||||
overrides := search.Resource{}.SearchFieldOverrides()
|
||||
if err := searchmapping.Validate(resourceType, overrides); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
docMapping, err := searchmapping.BleveBuildMapping(resourceType, overrides)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
indexMapping := bleve.NewIndexMapping()
|
||||
indexMapping.DefaultAnalyzer = keyword.Name
|
||||
indexMapping.DefaultMapping = docMapping
|
||||
err := indexMapping.AddCustomAnalyzer("lowercaseKeyword",
|
||||
err = indexMapping.AddCustomAnalyzer("lowercaseKeyword",
|
||||
map[string]any{
|
||||
"type": custom.Name,
|
||||
"tokenizer": single.Name,
|
||||
|
||||
48
services/search/pkg/bleve/mtime_test.go
Normal file
48
services/search/pkg/bleve/mtime_test.go
Normal file
@@ -0,0 +1,48 @@
|
||||
package bleve_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
bleveSearch "github.com/blevesearch/bleve/v2"
|
||||
bquery "github.com/blevesearch/bleve/v2/search/query"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
// Mtime is typed as a date, so range queries are chronological, not a
|
||||
// lexicographic keyword compare.
|
||||
func TestMtimeDateRange(t *testing.T) {
|
||||
m, err := bleve.NewMapping()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
idx, err := bleveSearch.NewMemOnly(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := search.Resource{ID: "x", Document: content.Document{Name: "f", Mtime: "2026-03-15T12:00:00.123456789Z"}}
|
||||
doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := idx.Index(r.ID, doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
hits := func(qs string) uint64 {
|
||||
res, err := idx.Search(bleveSearch.NewSearchRequest(bquery.NewQueryStringQuery(qs)))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", qs, err)
|
||||
}
|
||||
return res.Total
|
||||
}
|
||||
if got := hits(`Mtime:>"2026-01-01T00:00:00Z"`); got != 1 {
|
||||
t.Errorf("in-range: got %d hits, want 1", got)
|
||||
}
|
||||
if got := hits(`Mtime:>"2026-06-01T00:00:00Z"`); got != 0 {
|
||||
t.Errorf("out-of-range: got %d hits, want 0", got)
|
||||
}
|
||||
}
|
||||
@@ -14,14 +14,14 @@ func init() {
|
||||
// Document wraps all resource meta fields,
|
||||
// it is used as a content extraction result.
|
||||
type Document struct {
|
||||
Title string
|
||||
Name string
|
||||
Content string
|
||||
Size uint64
|
||||
Mtime string
|
||||
MimeType string
|
||||
Tags []string
|
||||
Favorites []string
|
||||
Title string `json:"Title"`
|
||||
Name string `json:"Name"`
|
||||
Content string `json:"Content"`
|
||||
Size uint64 `json:"Size"`
|
||||
Mtime string `json:"Mtime"`
|
||||
MimeType string `json:"MimeType"`
|
||||
Tags []string `json:"Tags"`
|
||||
Favorites []string `json:"Favorites"`
|
||||
Audio *libregraph.Audio `json:"audio,omitempty"`
|
||||
Image *libregraph.Image `json:"image,omitempty"`
|
||||
Location *libregraph.GeoCoordinates `json:"location,omitempty"`
|
||||
|
||||
92
services/search/pkg/mapping/bleve.go
Normal file
92
services/search/pkg/mapping/bleve.go
Normal file
@@ -0,0 +1,92 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
bleveMapping "github.com/blevesearch/bleve/v2/mapping"
|
||||
)
|
||||
|
||||
// BleveBuildMapping builds a bleve DocumentMapping for t by walking the
|
||||
// struct via reflection. Field names come from json tags; overrides are
|
||||
// keyed by those names (or dotted paths for nested fields).
|
||||
//
|
||||
// The returned mapping references analyzer names (Analyzer field on the
|
||||
// FieldOpts, plus "fulltext" / "path_hierarchy" for the corresponding Types);
|
||||
// the caller is responsible for registering those analyzers on the enclosing
|
||||
// IndexMapping.
|
||||
func BleveBuildMapping(t reflect.Type, overrides map[string]FieldOpts) (*bleveMapping.DocumentMapping, error) {
|
||||
return buildBleveDocMapping(t, overrides, "")
|
||||
}
|
||||
|
||||
func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix string) (*bleveMapping.DocumentMapping, error) {
|
||||
doc := bleve.NewDocumentMapping()
|
||||
err := walkFields(t, func(fi fieldInfo) error {
|
||||
key := fi.Name
|
||||
if prefix != "" {
|
||||
key = prefix + "." + fi.Name
|
||||
}
|
||||
opts := overrides[key]
|
||||
fieldType := opts.Type
|
||||
if fieldType == "" {
|
||||
fieldType = inferType(fi.GoField.Type)
|
||||
}
|
||||
|
||||
if fieldType == TypeObject {
|
||||
sub := structType(fi.GoField.Type)
|
||||
if sub == nil {
|
||||
return fmt.Errorf("mapping: object type on non-struct field %q", key)
|
||||
}
|
||||
subDoc, err := buildBleveDocMapping(sub, overrides, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
doc.AddSubDocumentMapping(fi.Name, subDoc)
|
||||
return nil
|
||||
}
|
||||
|
||||
fm, err := bleveFieldMapping(fieldType, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mapping: field %q: %w", key, err)
|
||||
}
|
||||
doc.AddFieldMappingsAt(fi.Name, fm)
|
||||
return nil
|
||||
})
|
||||
return doc, err
|
||||
}
|
||||
|
||||
func bleveFieldMapping(fieldType string, opts FieldOpts) (*bleveMapping.FieldMapping, error) {
|
||||
switch fieldType {
|
||||
case TypeWildcard:
|
||||
// bleve has no wildcard type; fall back to keyword-ish text.
|
||||
fieldType = TypeKeyword
|
||||
fallthrough
|
||||
case TypeKeyword, TypeFulltext, TypePath:
|
||||
fm := bleve.NewTextFieldMapping()
|
||||
switch {
|
||||
case opts.Analyzer != "":
|
||||
fm.Analyzer = opts.Analyzer
|
||||
case fieldType == TypeFulltext:
|
||||
fm.Analyzer = "fulltext"
|
||||
case fieldType == TypePath:
|
||||
fm.Analyzer = "path_hierarchy"
|
||||
}
|
||||
switch {
|
||||
case opts.IncludeInAll != nil:
|
||||
fm.IncludeInAll = *opts.IncludeInAll
|
||||
case fieldType == TypeFulltext, fieldType == TypePath:
|
||||
fm.IncludeInAll = false
|
||||
}
|
||||
return fm, nil
|
||||
case TypeNumeric:
|
||||
return bleve.NewNumericFieldMapping(), nil
|
||||
case TypeBool:
|
||||
return bleve.NewBooleanFieldMapping(), nil
|
||||
case TypeDatetime:
|
||||
return bleve.NewDateTimeFieldMapping(), nil
|
||||
case "":
|
||||
return nil, fmt.Errorf("no type inferred and no override")
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported type %q", fieldType)
|
||||
}
|
||||
116
services/search/pkg/mapping/bleve_test.go
Normal file
116
services/search/pkg/mapping/bleve_test.go
Normal file
@@ -0,0 +1,116 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type bleveDoc struct {
|
||||
Name string `json:"Name"`
|
||||
Content string `json:"Content"`
|
||||
Tags []string `json:"Tags"`
|
||||
Size uint64 `json:"Size"`
|
||||
Deleted bool `json:"Deleted"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
Nested *nested `json:"nested,omitempty"`
|
||||
}
|
||||
|
||||
type nested struct {
|
||||
Artist string `json:"artist"`
|
||||
Year int `json:"year"`
|
||||
}
|
||||
|
||||
// bleve wildcard falls back to keyword-ish text (bleve has no wildcard type).
|
||||
func TestBleveWildcardFallback(t *testing.T) {
|
||||
type doc struct {
|
||||
Mime string `json:"mime"`
|
||||
}
|
||||
dm, err := BleveBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"mime": {Type: TypeWildcard}})
|
||||
if err != nil {
|
||||
t.Fatalf("BleveBuildMapping: %v", err)
|
||||
}
|
||||
fms := dm.Properties["mime"].Fields
|
||||
if len(fms) != 1 || fms[0].Type != "text" {
|
||||
t.Fatalf("wildcard should map to text, got %+v", fms)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBleveBuildMappingInferredTypes(t *testing.T) {
|
||||
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("BleveBuildMapping: %v", err)
|
||||
}
|
||||
cases := map[string]string{
|
||||
"Name": "text",
|
||||
"Content": "text",
|
||||
"Tags": "text",
|
||||
"Size": "number",
|
||||
"Deleted": "boolean",
|
||||
"CreatedAt": "datetime",
|
||||
}
|
||||
for field, wantType := range cases {
|
||||
prop := dm.Properties[field]
|
||||
if prop == nil {
|
||||
t.Errorf("missing property %q", field)
|
||||
continue
|
||||
}
|
||||
if len(prop.Fields) == 0 {
|
||||
t.Errorf("%q: no field mappings", field)
|
||||
continue
|
||||
}
|
||||
if got := prop.Fields[0].Type; got != wantType {
|
||||
t.Errorf("%q: got type %q, want %q", field, got, wantType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBleveBuildMappingNestedIsSubDocument(t *testing.T) {
|
||||
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("BleveBuildMapping: %v", err)
|
||||
}
|
||||
sub := dm.Properties["nested"]
|
||||
if sub == nil {
|
||||
t.Fatal("missing nested sub-document")
|
||||
}
|
||||
if sub.Properties["artist"] == nil || sub.Properties["year"] == nil {
|
||||
t.Fatalf("nested fields missing: %#v", sub.Properties)
|
||||
}
|
||||
if got := sub.Properties["artist"].Fields[0].Type; got != "text" {
|
||||
t.Errorf("nested.artist: type %q, want text", got)
|
||||
}
|
||||
if got := sub.Properties["year"].Fields[0].Type; got != "number" {
|
||||
t.Errorf("nested.year: type %q, want number", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBleveBuildMappingOverrides(t *testing.T) {
|
||||
includeInAllFalse := false
|
||||
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{
|
||||
"Name": {Analyzer: "lowercaseKeyword"},
|
||||
"Content": {Type: TypeFulltext},
|
||||
"Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &includeInAllFalse},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("BleveBuildMapping: %v", err)
|
||||
}
|
||||
nameField := dm.Properties["Name"].Fields[0]
|
||||
if nameField.Analyzer != "lowercaseKeyword" {
|
||||
t.Errorf("Name analyzer: %q, want lowercaseKeyword", nameField.Analyzer)
|
||||
}
|
||||
if !nameField.IncludeInAll {
|
||||
t.Errorf("Name IncludeInAll should stay default-true when not overridden")
|
||||
}
|
||||
contentField := dm.Properties["Content"].Fields[0]
|
||||
if contentField.Analyzer != "fulltext" {
|
||||
t.Errorf("Content analyzer: %q, want fulltext", contentField.Analyzer)
|
||||
}
|
||||
if contentField.IncludeInAll {
|
||||
t.Errorf("Content IncludeInAll should default to false for fulltext type")
|
||||
}
|
||||
tagsField := dm.Properties["Tags"].Fields[0]
|
||||
if tagsField.IncludeInAll {
|
||||
t.Errorf("Tags IncludeInAll should honor the explicit false override")
|
||||
}
|
||||
}
|
||||
172
services/search/pkg/mapping/deserialize.go
Normal file
172
services/search/pkg/mapping/deserialize.go
Normal file
@@ -0,0 +1,172 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
// Deserialize builds a *T from bleve's flat hit.Fields map (json-tag keys,
|
||||
// "parent.child" for nested pointers). Used to rebuild a search Resource from a
|
||||
// hit. Fail-soft: an unparseable field stays at its zero value.
|
||||
func Deserialize[T any](fields map[string]any) *T {
|
||||
t := reflect.TypeFor[T]()
|
||||
if t.Kind() != reflect.Struct {
|
||||
panic(fmt.Sprintf("mapping: Deserialize requires a struct type, got %v", t))
|
||||
}
|
||||
out := reflect.New(t)
|
||||
fillStruct(out.Elem(), fields, "", setValue)
|
||||
return out.Interface().(*T)
|
||||
}
|
||||
|
||||
// DeserializeAt is Deserialize scoped to a dotted prefix, used to rebuild one
|
||||
// search-result facet (e.g. "audio"). Returns nil when nothing matched, so the
|
||||
// caller can leave the enclosing pointer nil.
|
||||
func DeserializeAt[T any](fields map[string]any, prefix string) *T {
|
||||
t := reflect.TypeFor[T]()
|
||||
if t.Kind() != reflect.Struct {
|
||||
panic(fmt.Sprintf("mapping: DeserializeAt requires a struct type, got %v", t))
|
||||
}
|
||||
out := reflect.New(t)
|
||||
if !fillStruct(out.Elem(), fields, prefix, setValue) {
|
||||
return nil
|
||||
}
|
||||
return out.Interface().(*T)
|
||||
}
|
||||
|
||||
// fillStruct walks v's exported fields, reading values from the flat fields map
|
||||
// (json-tag keys, "parent.child" for nested pointers). setLeaf converts each raw
|
||||
// value into a leaf, which is what lets the any-valued and string-valued
|
||||
// deserializers share one walker. Returns true if any leaf was populated.
|
||||
func fillStruct[V any](v reflect.Value, fields map[string]V, prefix string, setLeaf func(reflect.Value, V) error) bool {
|
||||
t := v.Type()
|
||||
touched := false
|
||||
for i := 0; i < t.NumField(); i++ {
|
||||
fi := resolveField(t.Field(i))
|
||||
if fi.Skip {
|
||||
continue
|
||||
}
|
||||
fv := v.Field(i)
|
||||
|
||||
if fi.Embedded {
|
||||
// Embedded *struct: allocate, recurse, keep the pointer only if set.
|
||||
if fv.Kind() == reflect.Ptr {
|
||||
if fv.Type().Elem().Kind() != reflect.Struct || !fv.CanSet() {
|
||||
continue
|
||||
}
|
||||
alloc := reflect.New(fv.Type().Elem())
|
||||
if fillStruct(alloc.Elem(), fields, prefix, setLeaf) {
|
||||
fv.Set(alloc)
|
||||
touched = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
if fv.Kind() == reflect.Struct && fillStruct(fv, fields, prefix, setLeaf) {
|
||||
touched = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
key := fi.Name
|
||||
if prefix != "" {
|
||||
key = prefix + "." + fi.Name
|
||||
}
|
||||
|
||||
// Pointer to a nested (non-time) struct: recurse, keep the pointer only
|
||||
// if a field was populated.
|
||||
if fv.Kind() == reflect.Ptr {
|
||||
if elem := fv.Type().Elem(); elem.Kind() == reflect.Struct && elem != timeType && elem != timestampType {
|
||||
alloc := reflect.New(elem)
|
||||
if fillStruct(alloc.Elem(), fields, key, setLeaf) {
|
||||
fv.Set(alloc)
|
||||
touched = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if raw, ok := fields[key]; ok && setLeaf(fv, raw) == nil {
|
||||
touched = true
|
||||
}
|
||||
}
|
||||
return touched
|
||||
}
|
||||
|
||||
// setValue writes raw (an any value from a bleve hit) into v, converting to the
|
||||
// field's type. Returns an error on nil or a type mismatch.
|
||||
func setValue(v reflect.Value, raw any) error {
|
||||
if v.Kind() == reflect.Ptr {
|
||||
alloc := reflect.New(v.Type().Elem())
|
||||
if err := setValue(alloc.Elem(), raw); err != nil {
|
||||
return err
|
||||
}
|
||||
v.Set(alloc)
|
||||
return nil
|
||||
}
|
||||
if v.Type() == timeType || v.Type() == timestampType {
|
||||
t, ok := parseTime(raw)
|
||||
if !ok {
|
||||
return fmt.Errorf("not an RFC3339 time: %v", raw)
|
||||
}
|
||||
setParsedTime(v, t)
|
||||
return nil
|
||||
}
|
||||
if v.Kind() == reflect.Slice {
|
||||
return setSlice(v, raw)
|
||||
}
|
||||
rv := reflect.ValueOf(raw)
|
||||
if !rv.IsValid() {
|
||||
return fmt.Errorf("nil value for %s", v.Type())
|
||||
}
|
||||
if !rv.Type().ConvertibleTo(v.Type()) {
|
||||
return fmt.Errorf("cannot convert %s to %s", rv.Type(), v.Type())
|
||||
}
|
||||
v.Set(rv.Convert(v.Type()))
|
||||
return nil
|
||||
}
|
||||
|
||||
func setSlice(v reflect.Value, raw any) error {
|
||||
items, ok := raw.([]any)
|
||||
if !ok {
|
||||
// bleve unwraps single-element slices; re-wrap here.
|
||||
items = []any{raw}
|
||||
}
|
||||
// Compact in place with a single MakeSlice: unparseable elements are
|
||||
// dropped, Slice(0, j) trims the tail.
|
||||
out := reflect.MakeSlice(v.Type(), len(items), len(items))
|
||||
j := 0
|
||||
for _, item := range items {
|
||||
if setValue(out.Index(j), item) == nil {
|
||||
j++
|
||||
}
|
||||
}
|
||||
if j == 0 {
|
||||
return fmt.Errorf("no slice elements set from %T", raw)
|
||||
}
|
||||
v.Set(out.Slice(0, j))
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseTime(raw any) (time.Time, bool) {
|
||||
s, ok := raw.(string)
|
||||
if !ok {
|
||||
return time.Time{}, false
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
return t, true
|
||||
}
|
||||
|
||||
// setParsedTime writes t into v, which must be a time.Time or
|
||||
// timestamppb.Timestamp field.
|
||||
func setParsedTime(v reflect.Value, t time.Time) {
|
||||
if v.Type() == timestampType {
|
||||
v.Set(reflect.ValueOf(*timestamppb.New(t)))
|
||||
return
|
||||
}
|
||||
v.Set(reflect.ValueOf(t))
|
||||
}
|
||||
82
services/search/pkg/mapping/deserialize_string.go
Normal file
82
services/search/pkg/mapping/deserialize_string.go
Normal file
@@ -0,0 +1,82 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DeserializeStringsAt is DeserializeAt for a string-valued map (e.g. CS3
|
||||
// ArbitraryMetadata), parsing each string into the field's Go type via strconv/
|
||||
// time.Parse. Used to build a graph DriveItem facet. Returns nil when nothing
|
||||
// under the prefix matched.
|
||||
func DeserializeStringsAt[T any](fields map[string]string, prefix string) *T {
|
||||
t := reflect.TypeFor[T]()
|
||||
if t.Kind() != reflect.Struct {
|
||||
panic(fmt.Sprintf("mapping: DeserializeStringsAt requires a struct type, got %v", t))
|
||||
}
|
||||
out := reflect.New(t)
|
||||
// Callers pass a flat-key prefix with a trailing dot (e.g.
|
||||
// "libre.graph.audio."); fillStruct joins segments with ".", so drop it.
|
||||
if !fillStruct(out.Elem(), fields, strings.TrimSuffix(prefix, "."), setValueFromString) {
|
||||
return nil
|
||||
}
|
||||
return out.Interface().(*T)
|
||||
}
|
||||
|
||||
// setValueFromString parses the string raw into v's Go type via strconv/
|
||||
// time.Parse, returning a descriptive error on failure.
|
||||
func setValueFromString(v reflect.Value, raw string) error {
|
||||
if v.Kind() == reflect.Ptr {
|
||||
alloc := reflect.New(v.Type().Elem())
|
||||
if err := setValueFromString(alloc.Elem(), raw); err != nil {
|
||||
return err
|
||||
}
|
||||
v.Set(alloc)
|
||||
return nil
|
||||
}
|
||||
if v.Type() == timeType || v.Type() == timestampType {
|
||||
t, err := time.Parse(time.RFC3339, raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse time %q: %w", raw, err)
|
||||
}
|
||||
setParsedTime(v, t)
|
||||
return nil
|
||||
}
|
||||
switch v.Kind() {
|
||||
case reflect.String:
|
||||
v.SetString(raw)
|
||||
return nil
|
||||
case reflect.Bool:
|
||||
b, err := strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse bool %q: %w", raw, err)
|
||||
}
|
||||
v.SetBool(b)
|
||||
return nil
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
n, err := strconv.ParseInt(raw, 10, v.Type().Bits())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse int %q: %w", raw, err)
|
||||
}
|
||||
v.SetInt(n)
|
||||
return nil
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
n, err := strconv.ParseUint(raw, 10, v.Type().Bits())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse uint %q: %w", raw, err)
|
||||
}
|
||||
v.SetUint(n)
|
||||
return nil
|
||||
case reflect.Float32, reflect.Float64:
|
||||
f, err := strconv.ParseFloat(raw, v.Type().Bits())
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse float %q: %w", raw, err)
|
||||
}
|
||||
v.SetFloat(f)
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("unsupported target kind %s", v.Kind())
|
||||
}
|
||||
122
services/search/pkg/mapping/deserialize_string_test.go
Normal file
122
services/search/pkg/mapping/deserialize_string_test.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
type stringFacet struct {
|
||||
Artist *string `json:"artist,omitempty"`
|
||||
Year *int32 `json:"year,omitempty"`
|
||||
Duration *int64 `json:"duration,omitempty"`
|
||||
Rating *float64 `json:"rating,omitempty"`
|
||||
Explicit *bool `json:"explicit,omitempty"`
|
||||
Taken *time.Time `json:"takenDateTime,omitempty"`
|
||||
}
|
||||
|
||||
func TestSetValueFromStringUnsupportedKind(t *testing.T) {
|
||||
v := reflect.New(reflect.TypeFor[[]int]()).Elem() // settable slice
|
||||
if err := setValueFromString(v, "x"); err == nil {
|
||||
t.Error("expected error for unsupported target kind (slice)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeserializeStringsAtBasicTypes(t *testing.T) {
|
||||
r := DeserializeStringsAt[stringFacet](map[string]string{
|
||||
"libre.graph.audio.artist": "Queen",
|
||||
"libre.graph.audio.year": "1975",
|
||||
"libre.graph.audio.duration": "354000",
|
||||
"libre.graph.audio.rating": "4.9",
|
||||
"libre.graph.audio.explicit": "true",
|
||||
"libre.graph.audio.takenDateTime": "2024-01-02T03:04:05Z",
|
||||
}, "libre.graph.audio.")
|
||||
if r == nil {
|
||||
t.Fatal("expected non-nil *stringFacet")
|
||||
}
|
||||
if r.Artist == nil || *r.Artist != "Queen" {
|
||||
t.Errorf("Artist: %#v", r.Artist)
|
||||
}
|
||||
if r.Year == nil || *r.Year != 1975 {
|
||||
t.Errorf("Year: %#v", r.Year)
|
||||
}
|
||||
if r.Duration == nil || *r.Duration != 354000 {
|
||||
t.Errorf("Duration: %#v", r.Duration)
|
||||
}
|
||||
if r.Rating == nil || *r.Rating != 4.9 {
|
||||
t.Errorf("Rating: %#v", r.Rating)
|
||||
}
|
||||
if r.Explicit == nil || !*r.Explicit {
|
||||
t.Errorf("Explicit: %#v", r.Explicit)
|
||||
}
|
||||
if r.Taken == nil || !r.Taken.Equal(time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC)) {
|
||||
t.Errorf("Taken: %#v", r.Taken)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeserializeStringsAtReturnsNilWhenEmpty(t *testing.T) {
|
||||
r := DeserializeStringsAt[stringFacet](map[string]string{
|
||||
"libre.graph.image.width": "1200",
|
||||
}, "libre.graph.audio.")
|
||||
if r != nil {
|
||||
t.Fatalf("expected nil, got %#v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeserializeStringsAtTimestamppb(t *testing.T) {
|
||||
type photoFacet struct {
|
||||
Taken *timestamppb.Timestamp `json:"takenDateTime,omitempty"`
|
||||
}
|
||||
r := DeserializeStringsAt[photoFacet](map[string]string{
|
||||
"libre.graph.photo.takenDateTime": "2024-05-06T07:08:09Z",
|
||||
}, "libre.graph.photo.")
|
||||
if r == nil || r.Taken == nil {
|
||||
t.Fatalf("Taken missing: %#v", r)
|
||||
}
|
||||
want := time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC)
|
||||
if !r.Taken.AsTime().Equal(want) {
|
||||
t.Errorf("Taken: got %v, want %v", r.Taken.AsTime(), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeserializeStringsAtIsFailSoft(t *testing.T) {
|
||||
// A single malformed field (year is unparseable as int) must not drop
|
||||
// the whole facet. The bad field stays at zero value, the rest of the
|
||||
// facet still populates. Mirrors the bleve-hit Deserialize behavior.
|
||||
r := DeserializeStringsAt[stringFacet](map[string]string{
|
||||
"libre.graph.audio.artist": "Iron Maiden",
|
||||
"libre.graph.audio.year": "not-a-number",
|
||||
"libre.graph.audio.duration": "354000",
|
||||
"libre.graph.audio.explicit": "not-a-bool",
|
||||
"libre.graph.audio.rating": "4.9",
|
||||
}, "libre.graph.audio.")
|
||||
if r == nil {
|
||||
t.Fatal("expected non-nil *stringFacet despite bad fields")
|
||||
}
|
||||
if r.Artist == nil || *r.Artist != "Iron Maiden" {
|
||||
t.Errorf("Artist should still be populated, got %#v", r.Artist)
|
||||
}
|
||||
if r.Duration == nil || *r.Duration != 354000 {
|
||||
t.Errorf("Duration should still be populated, got %#v", r.Duration)
|
||||
}
|
||||
if r.Rating == nil || *r.Rating != 4.9 {
|
||||
t.Errorf("Rating should still be populated, got %#v", r.Rating)
|
||||
}
|
||||
if r.Year != nil {
|
||||
t.Errorf("Year should stay nil for bad int, got %#v", r.Year)
|
||||
}
|
||||
if r.Explicit != nil {
|
||||
t.Errorf("Explicit should stay nil for bad bool, got %#v", r.Explicit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeserializeStringsAtPanicsOnNonStruct(t *testing.T) {
|
||||
defer func() {
|
||||
if r := recover(); r == nil {
|
||||
t.Fatal("expected panic for non-struct T")
|
||||
}
|
||||
}()
|
||||
DeserializeStringsAt[int](nil, "")
|
||||
}
|
||||
155
services/search/pkg/mapping/deserialize_test.go
Normal file
155
services/search/pkg/mapping/deserialize_test.go
Normal file
@@ -0,0 +1,155 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
type Leaf struct {
|
||||
Name string `json:"Name"`
|
||||
Size uint64 `json:"Size"`
|
||||
Deleted bool `json:"Deleted"`
|
||||
Tags []string `json:"Tags"`
|
||||
Favorites []string `json:"Favorites"`
|
||||
}
|
||||
|
||||
type audio struct {
|
||||
Artist *string `json:"artist,omitempty"`
|
||||
Year *int32 `json:"year,omitempty"`
|
||||
}
|
||||
|
||||
type photo struct {
|
||||
Taken *timestamppb.Timestamp `json:"takenDateTime,omitempty"`
|
||||
Mtime *time.Time `json:"mtime,omitempty"`
|
||||
}
|
||||
|
||||
type embedded struct {
|
||||
Leaf
|
||||
Audio *audio `json:"audio,omitempty"`
|
||||
Photo *photo `json:"photo,omitempty"`
|
||||
}
|
||||
|
||||
func TestDeserializeAtNonStructPanics(t *testing.T) {
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Error("expected panic for non-struct type")
|
||||
}
|
||||
}()
|
||||
_ = DeserializeAt[int](map[string]any{}, "")
|
||||
}
|
||||
|
||||
func TestDeserializeLeafFields(t *testing.T) {
|
||||
r := Deserialize[Leaf](map[string]any{
|
||||
"Name": "n",
|
||||
"Size": float64(42),
|
||||
"Deleted": true,
|
||||
})
|
||||
if r.Name != "n" || r.Size != 42 || !r.Deleted {
|
||||
t.Fatalf("got %#v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeserializeScalarToSlice(t *testing.T) {
|
||||
r := Deserialize[Leaf](map[string]any{
|
||||
"Tags": "single",
|
||||
"Favorites": []any{"a", "b"},
|
||||
})
|
||||
if len(r.Tags) != 1 || r.Tags[0] != "single" {
|
||||
t.Errorf("Tags: %#v", r.Tags)
|
||||
}
|
||||
if len(r.Favorites) != 2 || r.Favorites[0] != "a" || r.Favorites[1] != "b" {
|
||||
t.Errorf("Favorites: %#v", r.Favorites)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeserializeTimestamp(t *testing.T) {
|
||||
r := Deserialize[embedded](map[string]any{
|
||||
"photo.takenDateTime": "2024-01-02T03:04:05Z",
|
||||
"photo.mtime": "2024-05-06T07:08:09Z",
|
||||
})
|
||||
if r.Photo == nil {
|
||||
t.Fatal("Photo is nil")
|
||||
}
|
||||
if r.Photo.Taken == nil {
|
||||
t.Fatal("Taken is nil")
|
||||
}
|
||||
expected := time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
if !r.Photo.Taken.AsTime().Equal(expected) {
|
||||
t.Errorf("Taken: got %v, want %v", r.Photo.Taken.AsTime(), expected)
|
||||
}
|
||||
if r.Photo.Mtime == nil {
|
||||
t.Fatal("Mtime is nil")
|
||||
}
|
||||
if !r.Photo.Mtime.Equal(time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC)) {
|
||||
t.Errorf("Mtime: %v", r.Photo.Mtime)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeserializeIsFailSoft(t *testing.T) {
|
||||
// Malformed values (type mismatch, unparseable time) leave the
|
||||
// affected field at its zero value instead of dropping the whole
|
||||
// record. Matches the pre-refactor getFieldValue behavior so
|
||||
// matchToResource never returns nil on a corrupted hit.
|
||||
r := Deserialize[embedded](map[string]any{
|
||||
"Name": "n",
|
||||
"Size": "not-a-number", // wrong type
|
||||
"Deleted": true,
|
||||
"photo.takenDateTime": "not-an-rfc3339-time",
|
||||
"photo.mtime": "2024-05-06T07:08:09Z",
|
||||
})
|
||||
if r == nil {
|
||||
t.Fatal("expected non-nil *embedded even with partial corruption")
|
||||
}
|
||||
if r.Name != "n" {
|
||||
t.Errorf("Name: %q", r.Name)
|
||||
}
|
||||
if r.Size != 0 {
|
||||
t.Errorf("Size should stay zero on mismatch, got %d", r.Size)
|
||||
}
|
||||
if !r.Deleted {
|
||||
t.Errorf("Deleted should still be true")
|
||||
}
|
||||
if r.Photo == nil {
|
||||
t.Fatal("Photo should be populated because Mtime parsed ok")
|
||||
}
|
||||
if r.Photo.Taken != nil {
|
||||
t.Errorf("Taken should stay nil for unparseable time, got %v", r.Photo.Taken)
|
||||
}
|
||||
if r.Photo.Mtime == nil {
|
||||
t.Error("Mtime should be parsed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeserializePanicsOnNonStruct(t *testing.T) {
|
||||
defer func() {
|
||||
if r := recover(); r == nil {
|
||||
t.Fatal("expected panic for non-struct T")
|
||||
}
|
||||
}()
|
||||
Deserialize[int](nil)
|
||||
}
|
||||
|
||||
func TestDeserializeAtReturnsNilWhenNothingMatches(t *testing.T) {
|
||||
r := DeserializeAt[audio](map[string]any{"Name": "n"}, "audio")
|
||||
if r != nil {
|
||||
t.Fatalf("expected nil, got %#v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeserializeAtReturnsValueWhenPrefixMatches(t *testing.T) {
|
||||
r := DeserializeAt[audio](map[string]any{
|
||||
"audio.artist": "A",
|
||||
"audio.year": float64(2024), // setValue: pointer + numeric convert
|
||||
}, "audio")
|
||||
if r == nil {
|
||||
t.Fatal("expected non-nil *audio")
|
||||
}
|
||||
if r.Artist == nil || *r.Artist != "A" {
|
||||
t.Errorf("Artist: %#v", r.Artist)
|
||||
}
|
||||
if r.Year == nil || *r.Year != 2024 {
|
||||
t.Errorf("Year: %#v", r.Year)
|
||||
}
|
||||
}
|
||||
127
services/search/pkg/mapping/fillstruct_test.go
Normal file
127
services/search/pkg/mapping/fillstruct_test.go
Normal file
@@ -0,0 +1,127 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The walker is shared by both deserializers, so its structural behavior
|
||||
// (flattening embedded structs, recursing into nested pointers, joining the
|
||||
// prefix, keeping a pointer only when a field was set, fail-soft skipping) is
|
||||
// tested here once against a trivial string setter instead of twice through
|
||||
// Deserialize and DeserializeStringsAt.
|
||||
|
||||
// FsEmbVal/FsEmbPtr are exported so their embedded field is exported; an
|
||||
// unexported embedded type would be skipped by resolveField.
|
||||
type FsEmbVal struct {
|
||||
EV string `json:"ev"`
|
||||
}
|
||||
|
||||
type FsEmbPtr struct {
|
||||
EP string `json:"ep"`
|
||||
}
|
||||
|
||||
type fsNested struct {
|
||||
N string `json:"n"`
|
||||
}
|
||||
|
||||
type fsRoot struct {
|
||||
FsEmbVal // embedded value struct: fields promoted
|
||||
*FsEmbPtr // embedded pointer struct: allocated on demand
|
||||
Leaf string `json:"leaf"`
|
||||
Nested *fsNested `json:"nested"` // nested pointer: recursed under "nested."
|
||||
}
|
||||
|
||||
var errBadLeaf = errors.New("bad leaf")
|
||||
|
||||
// fsSet writes raw into a string field; the "BAD" sentinel simulates a parse
|
||||
// failure so the fail-soft path can be exercised.
|
||||
func fsSet(v reflect.Value, raw string) error {
|
||||
if raw == "BAD" {
|
||||
return errBadLeaf
|
||||
}
|
||||
v.SetString(raw)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestFillStruct(t *testing.T) {
|
||||
fill := func(fields map[string]string, prefix string) (fsRoot, bool) {
|
||||
var root fsRoot
|
||||
touched := fillStruct(reflect.ValueOf(&root).Elem(), fields, prefix, fsSet)
|
||||
return root, touched
|
||||
}
|
||||
|
||||
t.Run("flattens embedded, recurses nested", func(t *testing.T) {
|
||||
root, touched := fill(map[string]string{
|
||||
"leaf": "L",
|
||||
"ev": "EV",
|
||||
"ep": "EP",
|
||||
"nested.n": "N",
|
||||
}, "")
|
||||
if !touched {
|
||||
t.Fatal("expected touched")
|
||||
}
|
||||
if root.Leaf != "L" {
|
||||
t.Errorf("Leaf: %q", root.Leaf)
|
||||
}
|
||||
if root.EV != "EV" {
|
||||
t.Errorf("embedded value not promoted: %q", root.EV)
|
||||
}
|
||||
if root.FsEmbPtr == nil || root.EP != "EP" {
|
||||
t.Errorf("embedded pointer not allocated: %+v", root.FsEmbPtr)
|
||||
}
|
||||
if root.Nested == nil || root.Nested.N != "N" {
|
||||
t.Errorf("nested pointer not populated: %+v", root.Nested)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nothing matches: touched false, pointers stay nil", func(t *testing.T) {
|
||||
root, touched := fill(map[string]string{"other": "x"}, "")
|
||||
if touched {
|
||||
t.Fatal("expected untouched")
|
||||
}
|
||||
if root.Nested != nil {
|
||||
t.Errorf("Nested should stay nil: %+v", root.Nested)
|
||||
}
|
||||
if root.FsEmbPtr != nil {
|
||||
t.Errorf("embedded pointer should stay nil: %+v", root.FsEmbPtr)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("prefix arg is joined with the field name", func(t *testing.T) {
|
||||
root, touched := fill(map[string]string{
|
||||
"pre.leaf": "L",
|
||||
"pre.nested.n": "N",
|
||||
}, "pre")
|
||||
if !touched || root.Leaf != "L" || root.Nested == nil || root.Nested.N != "N" {
|
||||
t.Fatalf("prefix not joined: leaf=%q nested=%+v", root.Leaf, root.Nested)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("fail-soft: errored leaf stays zero, walk continues", func(t *testing.T) {
|
||||
root, touched := fill(map[string]string{
|
||||
"leaf": "BAD",
|
||||
"ev": "EV",
|
||||
}, "")
|
||||
if !touched {
|
||||
t.Fatal("expected touched because ev was set")
|
||||
}
|
||||
if root.Leaf != "" {
|
||||
t.Errorf("errored leaf should stay zero, got %q", root.Leaf)
|
||||
}
|
||||
if root.EV != "EV" {
|
||||
t.Errorf("walk should continue past the error: %q", root.EV)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("embedded pointer dropped when its only field errors", func(t *testing.T) {
|
||||
root, touched := fill(map[string]string{"ep": "BAD"}, "")
|
||||
if touched {
|
||||
t.Fatal("expected untouched")
|
||||
}
|
||||
if root.FsEmbPtr != nil {
|
||||
t.Errorf("embedded pointer should stay nil on error, got %+v", root.FsEmbPtr)
|
||||
}
|
||||
})
|
||||
}
|
||||
116
services/search/pkg/mapping/infer.go
Normal file
116
services/search/pkg/mapping/infer.go
Normal file
@@ -0,0 +1,116 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
var (
|
||||
timeType = reflect.TypeFor[time.Time]()
|
||||
timestampType = reflect.TypeFor[timestamppb.Timestamp]()
|
||||
)
|
||||
|
||||
// deref unwraps pointer and slice types to their element type.
|
||||
func deref(t reflect.Type) reflect.Type {
|
||||
for t.Kind() == reflect.Ptr || t.Kind() == reflect.Slice {
|
||||
t = t.Elem()
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// inferType returns the mapping type for a Go type. Pointers and slices are
|
||||
// unwrapped to their element type. time.Time and timestamppb.Timestamp become
|
||||
// datetime; other structs become object.
|
||||
func inferType(t reflect.Type) string {
|
||||
t = deref(t)
|
||||
switch t.Kind() {
|
||||
case reflect.String:
|
||||
return TypeKeyword
|
||||
case reflect.Bool:
|
||||
return TypeBool
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
|
||||
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
|
||||
reflect.Float32, reflect.Float64:
|
||||
return TypeNumeric
|
||||
case reflect.Struct:
|
||||
if t == timeType || t == timestampType {
|
||||
return TypeDatetime
|
||||
}
|
||||
return TypeObject
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// fieldInfo is the resolved metadata for one struct field.
|
||||
type fieldInfo struct {
|
||||
Name string
|
||||
GoField reflect.StructField
|
||||
Skip bool
|
||||
Embedded bool
|
||||
}
|
||||
|
||||
// resolveField resolves a struct field's json-tag name and skip/embed state.
|
||||
func resolveField(sf reflect.StructField) fieldInfo {
|
||||
if !sf.IsExported() {
|
||||
return fieldInfo{Skip: true}
|
||||
}
|
||||
name := sf.Name
|
||||
tag := sf.Tag.Get("json")
|
||||
if tag != "" {
|
||||
first, _, _ := strings.Cut(tag, ",")
|
||||
if first == "-" {
|
||||
return fieldInfo{Skip: true}
|
||||
}
|
||||
if first != "" {
|
||||
name = first
|
||||
}
|
||||
}
|
||||
return fieldInfo{
|
||||
Name: name,
|
||||
GoField: sf,
|
||||
Embedded: sf.Anonymous,
|
||||
}
|
||||
}
|
||||
|
||||
// walkFields visits exported leaf fields of t, flattening embedded structs
|
||||
// onto the enclosing level. It returns the first error returned by fn.
|
||||
func walkFields(t reflect.Type, fn func(fi fieldInfo) error) error {
|
||||
for t.Kind() == reflect.Ptr {
|
||||
t = t.Elem()
|
||||
}
|
||||
if t.Kind() != reflect.Struct {
|
||||
return nil
|
||||
}
|
||||
for i := 0; i < t.NumField(); i++ {
|
||||
fi := resolveField(t.Field(i))
|
||||
if fi.Skip {
|
||||
continue
|
||||
}
|
||||
if fi.Embedded {
|
||||
if err := walkFields(fi.GoField.Type, fn); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := fn(fi); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// structType returns the underlying struct type, unwrapping pointers and
|
||||
// slices. Returns nil when t is not a walkable struct (e.g. time.Time).
|
||||
func structType(t reflect.Type) reflect.Type {
|
||||
t = deref(t)
|
||||
if t.Kind() != reflect.Struct {
|
||||
return nil
|
||||
}
|
||||
if t == timeType || t == timestampType {
|
||||
return nil
|
||||
}
|
||||
return t
|
||||
}
|
||||
125
services/search/pkg/mapping/infer_test.go
Normal file
125
services/search/pkg/mapping/infer_test.go
Normal file
@@ -0,0 +1,125 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
func TestInferTypeUnsupported(t *testing.T) {
|
||||
if got := inferType(reflect.TypeFor[map[string]int]()); got != "" {
|
||||
t.Errorf("map: got %q, want empty", got)
|
||||
}
|
||||
if got := inferType(reflect.TypeFor[chan int]()); got != "" {
|
||||
t.Errorf("chan: got %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInferType(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
in any
|
||||
want string
|
||||
}{
|
||||
{"string", "", TypeKeyword},
|
||||
{"*string", (*string)(nil), TypeKeyword},
|
||||
{"[]string", []string(nil), TypeKeyword},
|
||||
{"bool", false, TypeBool},
|
||||
{"int", int(0), TypeNumeric},
|
||||
{"int64", int64(0), TypeNumeric},
|
||||
{"uint64", uint64(0), TypeNumeric},
|
||||
{"float64", float64(0), TypeNumeric},
|
||||
{"time.Time", time.Time{}, TypeDatetime},
|
||||
{"*time.Time", (*time.Time)(nil), TypeDatetime},
|
||||
{"*timestamppb.Timestamp", (*timestamppb.Timestamp)(nil), TypeDatetime},
|
||||
{"struct", struct{ X int }{}, TypeObject},
|
||||
{"*struct", (*struct{ X int })(nil), TypeObject},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
got := inferType(reflect.TypeOf(c.in))
|
||||
if got != c.want {
|
||||
t.Fatalf("inferType(%s): got %q, want %q", c.name, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveField(t *testing.T) {
|
||||
type S struct {
|
||||
Exported string `json:"exp"`
|
||||
Renamed string `json:"renamed,omitempty"`
|
||||
NoTag string
|
||||
OmitOnly string `json:",omitempty"`
|
||||
Skipped string `json:"-"`
|
||||
unexported string //nolint:unused
|
||||
}
|
||||
st := reflect.TypeFor[S]()
|
||||
cases := []struct {
|
||||
fieldIdx int
|
||||
wantName string
|
||||
wantSkip bool
|
||||
}{
|
||||
{0, "exp", false},
|
||||
{1, "renamed", false},
|
||||
{2, "NoTag", false},
|
||||
{3, "OmitOnly", false},
|
||||
{4, "", true},
|
||||
{5, "", true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
fi := resolveField(st.Field(c.fieldIdx))
|
||||
if fi.Skip != c.wantSkip {
|
||||
t.Errorf("field %d: skip=%v, want %v", c.fieldIdx, fi.Skip, c.wantSkip)
|
||||
}
|
||||
if !c.wantSkip && fi.Name != c.wantName {
|
||||
t.Errorf("field %d: name=%q, want %q", c.fieldIdx, fi.Name, c.wantName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWalkFieldsFlattensEmbedded(t *testing.T) {
|
||||
type Inner struct {
|
||||
A string `json:"a"`
|
||||
B int `json:"b"`
|
||||
}
|
||||
type Outer struct {
|
||||
Inner
|
||||
C bool `json:"c"`
|
||||
}
|
||||
var names []string
|
||||
err := walkFields(reflect.TypeFor[Outer](), func(fi fieldInfo) error {
|
||||
names = append(names, fi.Name)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("walkFields: %v", err)
|
||||
}
|
||||
want := []string{"a", "b", "c"}
|
||||
if !reflect.DeepEqual(names, want) {
|
||||
t.Fatalf("got %v, want %v", names, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStructType(t *testing.T) {
|
||||
type S struct{ X int }
|
||||
cases := []struct {
|
||||
name string
|
||||
in reflect.Type
|
||||
wantNil bool
|
||||
}{
|
||||
{"struct", reflect.TypeFor[S](), false},
|
||||
{"*struct", reflect.TypeFor[*S](), false},
|
||||
{"[]struct", reflect.TypeFor[[]S](), false},
|
||||
{"time.Time", reflect.TypeFor[time.Time](), true},
|
||||
{"string", reflect.TypeFor[string](), true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := structType(c.in)
|
||||
if (got == nil) != c.wantNil {
|
||||
t.Errorf("%s: got %v, wantNil %v", c.name, got, c.wantNil)
|
||||
}
|
||||
}
|
||||
}
|
||||
112
services/search/pkg/mapping/opensearch.go
Normal file
112
services/search/pkg/mapping/opensearch.go
Normal file
@@ -0,0 +1,112 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
)
|
||||
|
||||
// OpenSearchBuildMapping builds the OpenSearch "properties" map (the value
|
||||
// of mappings.properties) for type t by walking the struct via reflection.
|
||||
// Field names come from json tags; overrides are keyed by those names.
|
||||
//
|
||||
// The returned map contains plain JSON-friendly values (strings, bools,
|
||||
// nested maps) and can be marshalled directly.
|
||||
func OpenSearchBuildMapping(t reflect.Type, overrides map[string]FieldOpts) (map[string]any, error) {
|
||||
return buildOpenSearchProperties(t, overrides, "")
|
||||
}
|
||||
|
||||
func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, prefix string) (map[string]any, error) {
|
||||
props := map[string]any{}
|
||||
err := walkFields(t, func(fi fieldInfo) error {
|
||||
key := fi.Name
|
||||
if prefix != "" {
|
||||
key = prefix + "." + fi.Name
|
||||
}
|
||||
opts := overrides[key]
|
||||
fieldType := opts.Type
|
||||
if fieldType == "" {
|
||||
fieldType = inferType(fi.GoField.Type)
|
||||
}
|
||||
|
||||
if fieldType == TypeObject {
|
||||
sub := structType(fi.GoField.Type)
|
||||
if sub == nil {
|
||||
return fmt.Errorf("mapping: object type on non-struct field %q", key)
|
||||
}
|
||||
subProps, err := buildOpenSearchProperties(sub, overrides, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
props[fi.Name] = map[string]any{"properties": subProps}
|
||||
return nil
|
||||
}
|
||||
|
||||
fm, err := openSearchFieldMapping(fieldType, opts, fi.GoField.Type)
|
||||
if err != nil {
|
||||
return fmt.Errorf("mapping: field %q: %w", key, err)
|
||||
}
|
||||
props[fi.Name] = fm
|
||||
return nil
|
||||
})
|
||||
return props, err
|
||||
}
|
||||
|
||||
func openSearchFieldMapping(fieldType string, opts FieldOpts, goType reflect.Type) (map[string]any, error) {
|
||||
switch fieldType {
|
||||
case TypeKeyword:
|
||||
m := map[string]any{"type": "keyword"}
|
||||
if opts.Analyzer != "" {
|
||||
m["type"] = "text"
|
||||
m["analyzer"] = opts.Analyzer
|
||||
}
|
||||
return m, nil
|
||||
case TypeFulltext:
|
||||
m := map[string]any{
|
||||
"type": "text",
|
||||
"term_vector": "with_positions_offsets",
|
||||
}
|
||||
if opts.Analyzer != "" {
|
||||
m["analyzer"] = opts.Analyzer
|
||||
}
|
||||
return m, nil
|
||||
case TypePath:
|
||||
m := map[string]any{"type": "text"}
|
||||
if opts.Analyzer != "" {
|
||||
m["analyzer"] = opts.Analyzer
|
||||
} else {
|
||||
m["analyzer"] = "path_hierarchy"
|
||||
}
|
||||
return m, nil
|
||||
case TypeWildcard:
|
||||
// OpenSearch stores wildcard fields with doc_values=false by
|
||||
// default, so emit it explicitly to keep local and remote
|
||||
// mappings in sync for the Apply comparison.
|
||||
return map[string]any{"type": "wildcard", "doc_values": false}, nil
|
||||
case TypeNumeric:
|
||||
return map[string]any{"type": openSearchNumericType(goType)}, nil
|
||||
case TypeBool:
|
||||
return map[string]any{"type": "boolean"}, nil
|
||||
case TypeDatetime:
|
||||
return map[string]any{"type": "date"}, nil
|
||||
case "":
|
||||
return nil, fmt.Errorf("no type inferred and no override")
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported type %q", fieldType)
|
||||
}
|
||||
|
||||
// openSearchNumericType maps a Go numeric type to an OpenSearch numeric
|
||||
// field type.
|
||||
func openSearchNumericType(t reflect.Type) string {
|
||||
t = deref(t)
|
||||
switch t.Kind() {
|
||||
case reflect.Float32:
|
||||
return "float"
|
||||
case reflect.Float64:
|
||||
return "double"
|
||||
case reflect.Int8, reflect.Uint8, reflect.Int16, reflect.Uint16:
|
||||
return "short"
|
||||
case reflect.Int32, reflect.Uint32:
|
||||
return "integer"
|
||||
}
|
||||
return "long"
|
||||
}
|
||||
132
services/search/pkg/mapping/opensearch_test.go
Normal file
132
services/search/pkg/mapping/opensearch_test.go
Normal file
@@ -0,0 +1,132 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type osDoc struct {
|
||||
ID string `json:"ID"`
|
||||
Size uint64 `json:"Size"`
|
||||
Deleted bool `json:"Deleted"`
|
||||
CreatedAt time.Time `json:"CreatedAt"`
|
||||
Rating float64 `json:"Rating"`
|
||||
Nested *struct {
|
||||
Artist string `json:"artist"`
|
||||
Year int32 `json:"year"`
|
||||
} `json:"nested,omitempty"`
|
||||
}
|
||||
|
||||
func TestOpenSearchNumericTypes(t *testing.T) {
|
||||
type doc struct {
|
||||
A int8 `json:"a"`
|
||||
B int16 `json:"b"`
|
||||
C int32 `json:"c"`
|
||||
D int64 `json:"d"`
|
||||
E uint8 `json:"e"`
|
||||
F uint64 `json:"f"`
|
||||
G float32 `json:"g"`
|
||||
H float64 `json:"h"`
|
||||
}
|
||||
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSearchBuildMapping: %v", err)
|
||||
}
|
||||
want := map[string]string{"a": "short", "b": "short", "c": "integer", "d": "long", "e": "short", "f": "long", "g": "float", "h": "double"}
|
||||
for k, wt := range want {
|
||||
if got := props[k].(map[string]any)["type"]; got != wt {
|
||||
t.Errorf("%s: type = %v, want %v", k, got, wt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenSearchBuildMappingInferred(t *testing.T) {
|
||||
props, err := OpenSearchBuildMapping(reflect.TypeFor[osDoc](), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSearchBuildMapping: %v", err)
|
||||
}
|
||||
want := map[string]string{
|
||||
"ID": "keyword",
|
||||
"Size": "long",
|
||||
"Deleted": "boolean",
|
||||
"CreatedAt": "date",
|
||||
"Rating": "double",
|
||||
}
|
||||
for k, v := range want {
|
||||
m, ok := props[k].(map[string]any)
|
||||
if !ok {
|
||||
t.Errorf("%s: missing or not a map: %#v", k, props[k])
|
||||
continue
|
||||
}
|
||||
if got := m["type"]; got != v {
|
||||
t.Errorf("%s: type %v, want %v", k, got, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenSearchBuildMappingNested(t *testing.T) {
|
||||
props, err := OpenSearchBuildMapping(reflect.TypeFor[osDoc](), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSearchBuildMapping: %v", err)
|
||||
}
|
||||
nested, ok := props["nested"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("nested: not a map: %#v", props["nested"])
|
||||
}
|
||||
sub, ok := nested["properties"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("nested.properties: missing: %#v", nested)
|
||||
}
|
||||
artist, ok := sub["artist"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("nested.artist: %#v", sub)
|
||||
}
|
||||
if artist["type"] != "keyword" {
|
||||
t.Errorf("nested.artist.type: %v", artist["type"])
|
||||
}
|
||||
year, ok := sub["year"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("nested.year: %#v", sub)
|
||||
}
|
||||
if year["type"] != "integer" {
|
||||
t.Errorf("nested.year.type: %v (int32 → integer expected)", year["type"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenSearchBuildMappingOverrides(t *testing.T) {
|
||||
type doc struct {
|
||||
Name string `json:"Name"`
|
||||
Content string `json:"Content"`
|
||||
Path string `json:"Path"`
|
||||
MimeType string `json:"MimeType"`
|
||||
}
|
||||
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{
|
||||
"Name": {Analyzer: "lowercaseKeyword"},
|
||||
"Content": {Type: TypeFulltext},
|
||||
"Path": {Type: TypePath},
|
||||
"MimeType": {Type: TypeWildcard},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("OpenSearchBuildMapping: %v", err)
|
||||
}
|
||||
name := props["Name"].(map[string]any)
|
||||
if name["type"] != "text" || name["analyzer"] != "lowercaseKeyword" {
|
||||
t.Errorf("Name: %#v", name)
|
||||
}
|
||||
content := props["Content"].(map[string]any)
|
||||
if content["type"] != "text" || content["term_vector"] != "with_positions_offsets" {
|
||||
t.Errorf("Content: %#v", content)
|
||||
}
|
||||
if _, ok := content["analyzer"]; ok {
|
||||
t.Errorf("Content should leave analyzer unset (use OpenSearch default), got %#v", content["analyzer"])
|
||||
}
|
||||
path := props["Path"].(map[string]any)
|
||||
if path["type"] != "text" || path["analyzer"] != "path_hierarchy" {
|
||||
t.Errorf("Path: %#v", path)
|
||||
}
|
||||
mime := props["MimeType"].(map[string]any)
|
||||
if mime["type"] != "wildcard" {
|
||||
t.Errorf("MimeType: %#v", mime)
|
||||
}
|
||||
}
|
||||
34
services/search/pkg/mapping/opts.go
Normal file
34
services/search/pkg/mapping/opts.go
Normal file
@@ -0,0 +1,34 @@
|
||||
// Package mapping builds search index mappings for bleve and OpenSearch from
|
||||
// a Go struct via reflection. Field names come from json tags; the caller
|
||||
// provides overrides for fields that need a specific type or analyzer.
|
||||
package mapping
|
||||
|
||||
// Field type constants used in FieldOpts.Type. An empty Type means the type
|
||||
// is inferred from the Go field via reflection.
|
||||
const (
|
||||
TypeKeyword = "keyword"
|
||||
TypeFulltext = "fulltext"
|
||||
TypePath = "path"
|
||||
TypeWildcard = "wildcard"
|
||||
TypeNumeric = "numeric"
|
||||
TypeDatetime = "datetime"
|
||||
TypeBool = "bool"
|
||||
TypeObject = "object"
|
||||
)
|
||||
|
||||
// FieldOpts overrides the default type inference for a struct field. Keys in
|
||||
// the override map are json-tag names (e.g. "Name", "location", "audio.artist"),
|
||||
// not Go field names.
|
||||
type FieldOpts struct {
|
||||
// Type is one of the Type* constants. Empty means "infer from Go type".
|
||||
Type string
|
||||
|
||||
// Analyzer is the name of a custom analyzer registered on the bleve
|
||||
// IndexMapping (e.g. "lowercaseKeyword", "fulltext"). For OpenSearch it
|
||||
// becomes the analyzer attribute on the field.
|
||||
Analyzer string
|
||||
|
||||
// IncludeInAll controls bleve's _all field inclusion. Nil means "use the
|
||||
// bleve default for this field type". Has no effect on OpenSearch.
|
||||
IncludeInAll *bool
|
||||
}
|
||||
18
services/search/pkg/mapping/serialize.go
Normal file
18
services/search/pkg/mapping/serialize.go
Normal file
@@ -0,0 +1,18 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/conversions"
|
||||
)
|
||||
|
||||
// PrepareForIndex converts v to the flat map[string]any the backend index
|
||||
// clients expect, via a json round-trip (conversions.To). overrides is
|
||||
// reserved for type-specific adaptations wired in by follow-up features.
|
||||
func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, error) {
|
||||
out, err := conversions.To[map[string]any](v)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("mapping: prepare %T: %w", v, err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
83
services/search/pkg/mapping/serialize_test.go
Normal file
83
services/search/pkg/mapping/serialize_test.go
Normal file
@@ -0,0 +1,83 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPrepareForIndexError(t *testing.T) {
|
||||
// a func field can't be json-marshalled -> conversions.To errors
|
||||
type bad struct {
|
||||
F func() `json:"f"`
|
||||
}
|
||||
if _, err := PrepareForIndex(bad{}, nil); err == nil {
|
||||
t.Error("expected error for non-marshallable value")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareForIndexNil(t *testing.T) {
|
||||
// a typed nil pointer marshals to null -> nil map, no error, no panic
|
||||
out, err := PrepareForIndex((*struct{})(nil), nil)
|
||||
if err != nil || out != nil {
|
||||
t.Errorf("got (%v, %v), want (nil, nil)", out, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareForIndexFlattensEmbedded(t *testing.T) {
|
||||
type inner struct {
|
||||
Name string `json:"Name"`
|
||||
Size uint64 `json:"Size"`
|
||||
}
|
||||
type outer struct {
|
||||
inner
|
||||
ID string `json:"ID"`
|
||||
}
|
||||
m, err := PrepareForIndex(outer{inner: inner{Name: "a", Size: 7}, ID: "x"}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("PrepareForIndex: %v", err)
|
||||
}
|
||||
want := map[string]any{"Name": "a", "Size": float64(7), "ID": "x"}
|
||||
if !reflect.DeepEqual(m, want) {
|
||||
t.Fatalf("got %#v, want %#v", m, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareForIndexOmitsNilWithOmitempty(t *testing.T) {
|
||||
type facet struct {
|
||||
Artist string `json:"artist"`
|
||||
}
|
||||
type doc struct {
|
||||
Name string `json:"Name"`
|
||||
Audio *facet `json:"audio,omitempty"`
|
||||
}
|
||||
m, err := PrepareForIndex(doc{Name: "n"}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("PrepareForIndex: %v", err)
|
||||
}
|
||||
if _, ok := m["audio"]; ok {
|
||||
t.Errorf("audio should be omitted when nil: %#v", m)
|
||||
}
|
||||
if m["Name"] != "n" {
|
||||
t.Errorf("Name: %v", m["Name"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareForIndexIncludesNestedWhenSet(t *testing.T) {
|
||||
type facet struct {
|
||||
Artist string `json:"artist"`
|
||||
}
|
||||
type doc struct {
|
||||
Audio *facet `json:"audio,omitempty"`
|
||||
}
|
||||
m, err := PrepareForIndex(doc{Audio: &facet{Artist: "A"}}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("PrepareForIndex: %v", err)
|
||||
}
|
||||
nested, ok := m["audio"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("audio should be a nested map: %#v", m["audio"])
|
||||
}
|
||||
if nested["artist"] != "A" {
|
||||
t.Errorf("audio.artist: %v", nested["artist"])
|
||||
}
|
||||
}
|
||||
49
services/search/pkg/mapping/validate.go
Normal file
49
services/search/pkg/mapping/validate.go
Normal file
@@ -0,0 +1,49 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Validate returns an error if any override key does not match a known field
|
||||
// name in t. Top-level fields are identified by their json-tag name; nested
|
||||
// named struct fields are reachable as "parent.child". Embedded (anonymous)
|
||||
// structs are flattened, so their fields sit at the parent level (as with
|
||||
// encoding/json).
|
||||
func Validate(t reflect.Type, overrides map[string]FieldOpts) error {
|
||||
if len(overrides) == 0 {
|
||||
return nil
|
||||
}
|
||||
names := collectNames(t, "")
|
||||
var unknown []string
|
||||
for k := range overrides {
|
||||
if _, ok := names[k]; !ok {
|
||||
unknown = append(unknown, k)
|
||||
}
|
||||
}
|
||||
if len(unknown) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Strings(unknown)
|
||||
return fmt.Errorf("mapping: unknown override keys: %s", strings.Join(unknown, ", "))
|
||||
}
|
||||
|
||||
func collectNames(t reflect.Type, prefix string) map[string]struct{} {
|
||||
out := map[string]struct{}{}
|
||||
_ = walkFields(t, func(fi fieldInfo) error {
|
||||
key := fi.Name
|
||||
if prefix != "" {
|
||||
key = prefix + "." + fi.Name
|
||||
}
|
||||
out[key] = struct{}{}
|
||||
if sub := structType(fi.GoField.Type); sub != nil {
|
||||
for k := range collectNames(sub, key) {
|
||||
out[k] = struct{}{}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return out
|
||||
}
|
||||
50
services/search/pkg/mapping/validate_test.go
Normal file
50
services/search/pkg/mapping/validate_test.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type inner struct {
|
||||
Artist string `json:"artist"`
|
||||
}
|
||||
|
||||
type sample struct {
|
||||
Name string `json:"Name"`
|
||||
Audio *inner `json:"audio,omitempty"`
|
||||
Location *struct { //nolint:unused
|
||||
Lon float64 `json:"longitude"`
|
||||
Lat float64 `json:"latitude"`
|
||||
} `json:"location,omitempty"`
|
||||
}
|
||||
|
||||
func TestValidateAccepts(t *testing.T) {
|
||||
err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{
|
||||
"Name": {Analyzer: "lowercaseKeyword"},
|
||||
"audio": {Type: TypeObject},
|
||||
"audio.artist": {Analyzer: "lowercaseKeyword"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateRejectsUnknown(t *testing.T) {
|
||||
err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{
|
||||
"nope": {},
|
||||
"audio.zzz": {},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("expected error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "nope") || !strings.Contains(err.Error(), "audio.zzz") {
|
||||
t.Fatalf("error missing keys: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateEmpty(t *testing.T) {
|
||||
if err := Validate(reflect.TypeFor[sample](), nil); err != nil {
|
||||
t.Fatalf("empty overrides should pass: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/conversions"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
@@ -43,7 +44,7 @@ func NewBatch(client *opensearchgoAPI.Client, index string, size int) (*Batch, e
|
||||
|
||||
func (b *Batch) Upsert(id string, r search.Resource) error {
|
||||
return b.withSizeLimit(func() error {
|
||||
body, err := conversions.To[map[string]any](r)
|
||||
body, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal resource: %w", err)
|
||||
}
|
||||
|
||||
@@ -3,29 +3,32 @@ package opensearch
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"maps"
|
||||
"reflect"
|
||||
|
||||
"github.com/go-jose/go-jose/v3/json"
|
||||
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
|
||||
"github.com/tidwall/gjson"
|
||||
|
||||
searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrManualActionRequired = errors.New("manual action required")
|
||||
IndexManagerLatest = IndexIndexManagerResourceV2
|
||||
IndexIndexManagerResourceV1 IndexManager = "resource_v1.json"
|
||||
IndexIndexManagerResourceV2 IndexManager = "resource_v2.json"
|
||||
IndexIndexManagerResourceV2 IndexManager = "resource_v2"
|
||||
)
|
||||
|
||||
//go:embed internal/indexes/*.json
|
||||
var indexes embed.FS
|
||||
|
||||
type IndexManager string
|
||||
|
||||
// indexGenerators dispatches each IndexManager variant to its builder.
|
||||
var indexGenerators = map[IndexManager]func() ([]byte, error){
|
||||
IndexIndexManagerResourceV2: buildResourceV2Mapping,
|
||||
}
|
||||
|
||||
func (m IndexManager) String() string {
|
||||
b, err := m.MarshalJSON()
|
||||
if err != nil {
|
||||
@@ -36,16 +39,56 @@ func (m IndexManager) String() string {
|
||||
}
|
||||
|
||||
func (m IndexManager) MarshalJSON() ([]byte, error) {
|
||||
filePath := string(m)
|
||||
body, err := indexes.ReadFile(path.Join("./internal/indexes", filePath))
|
||||
switch {
|
||||
case err != nil:
|
||||
return nil, fmt.Errorf("failed to read index file %s: %w", filePath, err)
|
||||
case len(body) <= 0:
|
||||
return nil, fmt.Errorf("index file %s is empty", filePath)
|
||||
gen, ok := indexGenerators[m]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown index manager %q", string(m))
|
||||
}
|
||||
return gen()
|
||||
}
|
||||
|
||||
// buildResourceV2Mapping renders the OpenSearch index template for a
|
||||
// search.Resource from the shared SearchFieldOverrides. OpenSearch-specific
|
||||
// tweaks (wildcard MimeType, path_hierarchy Path) are applied on top.
|
||||
func buildResourceV2Mapping() ([]byte, error) {
|
||||
resourceType := reflect.TypeFor[search.Resource]()
|
||||
overrides := maps.Clone(search.Resource{}.SearchFieldOverrides())
|
||||
overrides["MimeType"] = searchmapping.FieldOpts{Type: searchmapping.TypeWildcard}
|
||||
overrides["Path"] = searchmapping.FieldOpts{Type: searchmapping.TypePath}
|
||||
if err := searchmapping.Validate(resourceType, overrides); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
props, err := searchmapping.OpenSearchBuildMapping(resourceType, overrides)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return body, nil
|
||||
index := map[string]any{
|
||||
"settings": map[string]any{
|
||||
"number_of_shards": "1",
|
||||
"number_of_replicas": "1",
|
||||
"analysis": map[string]any{
|
||||
"analyzer": map[string]any{
|
||||
"path_hierarchy": map[string]any{
|
||||
"type": "custom",
|
||||
"tokenizer": "path_hierarchy",
|
||||
"filter": []string{"lowercase"},
|
||||
},
|
||||
"lowercaseKeyword": map[string]any{
|
||||
"type": "custom",
|
||||
"tokenizer": "keyword",
|
||||
"filter": []string{"lowercase"},
|
||||
},
|
||||
},
|
||||
"tokenizer": map[string]any{
|
||||
"path_hierarchy": map[string]any{"type": "path_hierarchy"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"mappings": map[string]any{
|
||||
"properties": props,
|
||||
},
|
||||
}
|
||||
return json.Marshal(index)
|
||||
}
|
||||
|
||||
func (m IndexManager) Apply(ctx context.Context, name string, client *opensearchgoAPI.Client) error {
|
||||
@@ -118,8 +161,11 @@ func (m IndexManager) Apply(ctx context.Context, name string, client *opensearch
|
||||
|
||||
if errs != nil {
|
||||
return fmt.Errorf(
|
||||
"index %s already exists and is different from the requested version, %w: %w",
|
||||
name,
|
||||
"index %s already exists with a different mapping than the requested version. "+
|
||||
"There is no in-place migration today: drop the index in OpenSearch (DELETE /%s) "+
|
||||
"and restart the search service. The index will be recreated with the new mapping. "+
|
||||
"%w: %w",
|
||||
name, name,
|
||||
ErrManualActionRequired,
|
||||
errors.Join(errs...),
|
||||
)
|
||||
|
||||
@@ -15,6 +15,17 @@ import (
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
// copyFacet converts a typed pointer from the indexed shape (libregraph) to
|
||||
// the protobuf shape via conversions.To. Returns nil when src is nil so the
|
||||
// enclosing Match.Entity field stays nil.
|
||||
func copyFacet[Dst, Src any](src *Src) *Dst {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
dst, _ := conversions.To[*Dst](src)
|
||||
return dst
|
||||
}
|
||||
|
||||
func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match, error) {
|
||||
resource, err := conversions.To[search.Resource](hit.Source)
|
||||
if err != nil {
|
||||
@@ -68,26 +79,10 @@ func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match,
|
||||
|
||||
return strings.Join(contentHighlights[:], "; ")
|
||||
}(),
|
||||
Audio: func() *searchMessage.Audio {
|
||||
if !strings.HasPrefix(resource.MimeType, "audio/") {
|
||||
return nil
|
||||
}
|
||||
|
||||
audio, _ := conversions.To[*searchMessage.Audio](resource.Audio)
|
||||
return audio
|
||||
}(),
|
||||
Image: func() *searchMessage.Image {
|
||||
image, _ := conversions.To[*searchMessage.Image](resource.Image)
|
||||
return image
|
||||
}(),
|
||||
Location: func() *searchMessage.GeoCoordinates {
|
||||
geoCoordinates, _ := conversions.To[*searchMessage.GeoCoordinates](resource.Location)
|
||||
return geoCoordinates
|
||||
}(),
|
||||
Photo: func() *searchMessage.Photo {
|
||||
photo, _ := conversions.To[*searchMessage.Photo](resource.Photo)
|
||||
return photo
|
||||
}(),
|
||||
Audio: copyFacet[searchMessage.Audio](resource.Audio),
|
||||
Image: copyFacet[searchMessage.Image](resource.Image),
|
||||
Location: copyFacet[searchMessage.GeoCoordinates](resource.Location),
|
||||
Photo: copyFacet[searchMessage.Photo](resource.Photo),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
{
|
||||
"settings": {
|
||||
"number_of_shards": "1",
|
||||
"number_of_replicas": "1",
|
||||
"analysis": {
|
||||
"analyzer": {
|
||||
"path_hierarchy": {
|
||||
"filter": [
|
||||
"lowercase"
|
||||
],
|
||||
"tokenizer": "path_hierarchy",
|
||||
"type": "custom"
|
||||
}
|
||||
},
|
||||
"tokenizer": {
|
||||
"path_hierarchy": {
|
||||
"type": "path_hierarchy"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"ID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"ParentID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"RootID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"MimeType": {
|
||||
"type": "wildcard",
|
||||
"doc_values": false
|
||||
},
|
||||
"Path": {
|
||||
"type": "text",
|
||||
"analyzer": "path_hierarchy"
|
||||
},
|
||||
"Deleted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"Hidden": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
{
|
||||
"settings": {
|
||||
"number_of_shards": "1",
|
||||
"number_of_replicas": "1",
|
||||
"analysis": {
|
||||
"analyzer": {
|
||||
"path_hierarchy": {
|
||||
"filter": [
|
||||
"lowercase"
|
||||
],
|
||||
"tokenizer": "path_hierarchy",
|
||||
"type": "custom"
|
||||
}
|
||||
},
|
||||
"tokenizer": {
|
||||
"path_hierarchy": {
|
||||
"type": "path_hierarchy"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"mappings": {
|
||||
"properties": {
|
||||
"Content": {
|
||||
"type": "text",
|
||||
"term_vector": "with_positions_offsets"
|
||||
},
|
||||
"ID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"ParentID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"RootID": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"MimeType": {
|
||||
"type": "wildcard",
|
||||
"doc_values": false
|
||||
},
|
||||
"Path": {
|
||||
"type": "text",
|
||||
"analyzer": "path_hierarchy"
|
||||
},
|
||||
"Deleted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"Hidden": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"Favorites": {
|
||||
"type": "keyword"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,17 +8,25 @@ import (
|
||||
bleveQuery "github.com/blevesearch/bleve/v2/search/query"
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/pkg/kql"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
// lowercaseFields lists the bleve fields whose index mapping uses a
|
||||
// lowercasing analyzer. Values bound to these fields are pre-lowercased
|
||||
// so query-side matching stays consistent with the index.
|
||||
// Keep in sync with services/search/pkg/bleve/index.go NewMapping.
|
||||
var lowercaseFields = map[string]struct{}{
|
||||
"Name": {},
|
||||
"Tags": {},
|
||||
"Favorites": {},
|
||||
"Content": {},
|
||||
// lowercaseFields is derived from Resource.SearchFieldOverrides(): any
|
||||
// field whose override picks a lowercasing analyzer (`lowercaseKeyword`)
|
||||
// or the fulltext type (which uses a lowercasing analyzer under the hood)
|
||||
// gets its query-side value pre-lowercased so compile-time matches the
|
||||
// index-time tokenization. Anything else keeps its original casing.
|
||||
var lowercaseFields = buildLowercaseFields()
|
||||
|
||||
func buildLowercaseFields() map[string]struct{} {
|
||||
out := map[string]struct{}{}
|
||||
for key, opts := range (search.Resource{}).SearchFieldOverrides() {
|
||||
if opts.Analyzer == "lowercaseKeyword" || opts.Type == mapping.TypeFulltext {
|
||||
out[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var _fields = map[string]string{
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
|
||||
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
|
||||
@@ -19,6 +20,7 @@ import (
|
||||
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
|
||||
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
)
|
||||
|
||||
var scopeRegex = regexp.MustCompile(`scope:\s*([^" "\n\r]*)`)
|
||||
@@ -51,13 +53,36 @@ type BatchOperator interface {
|
||||
type Resource struct {
|
||||
content.Document
|
||||
|
||||
ID string
|
||||
RootID string
|
||||
Path string
|
||||
ParentID string
|
||||
Type uint64
|
||||
Deleted bool
|
||||
Hidden bool
|
||||
ID string `json:"ID"`
|
||||
RootID string `json:"RootID"`
|
||||
Path string `json:"Path"`
|
||||
ParentID string `json:"ParentID"`
|
||||
Type uint64 `json:"Type"`
|
||||
Deleted bool `json:"Deleted"`
|
||||
Hidden bool `json:"Hidden"`
|
||||
}
|
||||
|
||||
// resourceFieldOverrides is built once (it never changes) and reused on hot
|
||||
// paths instead of reallocating per call.
|
||||
var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts {
|
||||
excludeFromAll := false
|
||||
return map[string]mapping.FieldOpts{
|
||||
"Name": {Analyzer: "lowercaseKeyword"},
|
||||
"Content": {Type: mapping.TypeFulltext},
|
||||
"Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll},
|
||||
"Favorites": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll},
|
||||
// Mtime is stored as an RFC3339 string; type it as a date so mtime:>...
|
||||
// range queries are chronological on both backends (bleve DateRangeQuery
|
||||
// / OpenSearch date range), not a lexicographic keyword compare.
|
||||
"Mtime": {Type: mapping.TypeDatetime},
|
||||
}
|
||||
})
|
||||
|
||||
// SearchFieldOverrides returns the field options the mapping package needs to
|
||||
// build per-backend index mappings for a Resource (keys are json-tag names).
|
||||
// The map is shared and read-only; clone it before mutating.
|
||||
func (Resource) SearchFieldOverrides() map[string]mapping.FieldOpts {
|
||||
return resourceFieldOverrides()
|
||||
}
|
||||
|
||||
// ResolveReference makes sure the path is relative to the space root
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -628,10 +629,10 @@ func (s *Service) doUpsertItem(ref *provider.Reference, batch BatchOperator) {
|
||||
|
||||
// determine if metadata needs to be stored in storage as well
|
||||
metadata := map[string]string{}
|
||||
addAudioMetadata(metadata, doc.Audio)
|
||||
addImageMetadata(metadata, doc.Image)
|
||||
addLocationMetadata(metadata, doc.Location)
|
||||
addPhotoMetadata(metadata, doc.Photo)
|
||||
facetToMetadata(metadata, doc.Audio, "libre.graph.audio.")
|
||||
facetToMetadata(metadata, doc.Image, "libre.graph.image.")
|
||||
facetToMetadata(metadata, doc.Location, "libre.graph.location.")
|
||||
facetToMetadata(metadata, doc.Photo, "libre.graph.photo.")
|
||||
if len(metadata) == 0 {
|
||||
return
|
||||
}
|
||||
@@ -656,43 +657,25 @@ func (s *Service) doUpsertItem(ref *provider.Reference, batch BatchOperator) {
|
||||
}
|
||||
}
|
||||
|
||||
func addAudioMetadata(metadata map[string]string, audio *libregraph.Audio) {
|
||||
if audio == nil {
|
||||
return
|
||||
// facetToMetadata flattens a libregraph facet (Audio / Image / Location / Photo
|
||||
// pointer) into the metadata map under the given prefix via the model's ToMap.
|
||||
// No-op when the facet is nil.
|
||||
func facetToMetadata[T libregraph.MappedNullable](metadata map[string]string, facet T, prefix string) {
|
||||
// Only nilable kinds can be nil; IsNil panics on a value type (some
|
||||
// libregraph models satisfy MappedNullable with a value receiver).
|
||||
switch v := reflect.ValueOf(facet); v.Kind() {
|
||||
case reflect.Ptr, reflect.Map, reflect.Slice, reflect.Interface, reflect.Chan, reflect.Func:
|
||||
if v.IsNil() {
|
||||
return
|
||||
}
|
||||
}
|
||||
marshalToStringMap(audio, metadata, "libre.graph.audio.")
|
||||
}
|
||||
|
||||
func addImageMetadata(metadata map[string]string, image *libregraph.Image) {
|
||||
if image == nil {
|
||||
return
|
||||
}
|
||||
marshalToStringMap(image, metadata, "libre.graph.image.")
|
||||
}
|
||||
|
||||
func addLocationMetadata(metadata map[string]string, location *libregraph.GeoCoordinates) {
|
||||
if location == nil {
|
||||
return
|
||||
}
|
||||
marshalToStringMap(location, metadata, "libre.graph.location.")
|
||||
}
|
||||
|
||||
func addPhotoMetadata(metadata map[string]string, photo *libregraph.Photo) {
|
||||
if photo == nil {
|
||||
return
|
||||
}
|
||||
marshalToStringMap(photo, metadata, "libre.graph.photo.")
|
||||
}
|
||||
|
||||
func marshalToStringMap[T libregraph.MappedNullable](source T, target map[string]string, prefix string) {
|
||||
// ToMap never returns a non-nil error ...
|
||||
m, _ := source.ToMap()
|
||||
|
||||
// ToMap never returns a non-nil error.
|
||||
m, _ := facet.ToMap()
|
||||
for k, v := range m {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
target[prefix+k] = valueToString(v)
|
||||
metadata[prefix+k] = valueToString(v)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user