refactor: reflection-based search mapping

Build the bleve and OpenSearch index mappings from the Go struct via
reflection (json tags + per-field overrides) instead of hand-rolled
mappings and hit deserializers. New mapping package: BleveBuildMapping,
OpenSearchBuildMapping, Deserialize[T], PrepareForIndex; field decoding is
fail-soft. Mtime is typed as a date so mtime ranges are chronological on
both backends. Route CS3 facet parsing through mapping.DeserializeStringMap.

The any-valued (bleve hit) and string-valued (CS3 metadata) deserializers
share one generic fillStruct walker with a per-value setLeaf callback.
This commit is contained in:
Dominik Schmidt
2026-07-02 14:25:04 +02:00
parent 11449b5943
commit ac86d9b907
31 changed files with 1842 additions and 477 deletions

View File

@@ -9,9 +9,7 @@ import (
"net/http"
"net/url"
"path"
"reflect"
"strconv"
"strings"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
@@ -28,6 +26,7 @@ import (
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
)
// CreateUploadSession create an upload session to allow your app to upload files up to the maximum file size.
@@ -452,130 +451,20 @@ func cs3ResourceToDriveItem(logger *log.Logger, publicBaseURL *url.URL, res *sto
}
}
if res.GetArbitraryMetadata() != nil {
driveItem.Audio = cs3ResourceToDriveItemAudioFacet(logger, res)
driveItem.Image = cs3ResourceToDriveItemImageFacet(logger, res)
driveItem.Location = cs3ResourceToDriveItemLocationFacet(logger, res)
driveItem.Photo = cs3ResourceToDriveItemPhotoFacet(logger, res)
if metadata := res.GetArbitraryMetadata().GetMetadata(); metadata != nil {
driveItem.Audio = metadataToFacet[libregraph.Audio](metadata, "audio")
driveItem.Image = metadataToFacet[libregraph.Image](metadata, "image")
driveItem.Location = metadataToFacet[libregraph.GeoCoordinates](metadata, "location")
driveItem.Photo = metadataToFacet[libregraph.Photo](metadata, "photo")
}
return driveItem, nil
}
func cs3ResourceToDriveItemAudioFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Audio {
if !strings.HasPrefix(res.GetMimeType(), "audio/") {
return nil
}
k := res.GetArbitraryMetadata().GetMetadata()
if k == nil {
return nil
}
var audio = &libregraph.Audio{}
if ok := unmarshalStringMap(logger, audio, k, "libre.graph.audio."); ok {
return audio
}
return nil
}
func cs3ResourceToDriveItemImageFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Image {
k := res.GetArbitraryMetadata().GetMetadata()
if k == nil {
return nil
}
var image = &libregraph.Image{}
if ok := unmarshalStringMap(logger, image, k, "libre.graph.image."); ok {
return image
}
return nil
}
func cs3ResourceToDriveItemLocationFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.GeoCoordinates {
k := res.GetArbitraryMetadata().GetMetadata()
if k == nil {
return nil
}
var location = &libregraph.GeoCoordinates{}
if ok := unmarshalStringMap(logger, location, k, "libre.graph.location."); ok {
return location
}
return nil
}
func cs3ResourceToDriveItemPhotoFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Photo {
k := res.GetArbitraryMetadata().GetMetadata()
if k == nil {
return nil
}
var photo = &libregraph.Photo{}
if ok := unmarshalStringMap(logger, photo, k, "libre.graph.photo."); ok {
return photo
}
return nil
}
func getFieldName(structField reflect.StructField) string {
tag := structField.Tag.Get("json")
if tag == "" {
return structField.Name
}
return strings.Split(tag, ",")[0]
}
func unmarshalStringMap(logger *log.Logger, out any, flatMap map[string]string, prefix string) bool {
nonEmpty := false
obj := reflect.ValueOf(out).Elem()
timeKind := reflect.TypeOf(&time.Time{}).Elem().Kind()
for i := 0; i < obj.NumField(); i++ {
field := obj.Field(i)
structField := obj.Type().Field(i)
mapKey := prefix + getFieldName(structField)
if value, ok := flatMap[mapKey]; ok {
if field.Kind() == reflect.Ptr {
newValue := reflect.New(field.Type().Elem())
var tmp any
var err error
switch t := newValue.Type().Elem().Kind(); t {
case reflect.String:
tmp = value
case reflect.Int32:
tmp, err = strconv.ParseInt(value, 10, 32)
case reflect.Int64:
tmp, err = strconv.ParseInt(value, 10, 64)
case reflect.Float32:
tmp, err = strconv.ParseFloat(value, 32)
case reflect.Float64:
tmp, err = strconv.ParseFloat(value, 64)
case reflect.Bool:
tmp, err = strconv.ParseBool(value)
case timeKind:
tmp, err = time.Parse(time.RFC3339, value)
default:
err = errors.New("unsupported type")
logger.Error().Err(err).Str("type", t.String()).Str("mapKey", mapKey).Msg("target field type for value of mapKey is not supported")
}
if err != nil {
logger.Error().Err(err).Str("mapKey", mapKey).Msg("unmarshalling failed")
continue
}
newValue.Elem().Set(reflect.ValueOf(tmp).Convert(field.Type().Elem()))
field.Set(newValue)
nonEmpty = true
}
}
}
return nonEmpty
// metadataToFacet builds a DriveItem facet *T from CS3 arbitrary metadata under
// the "libre.graph.<facet>." key prefix. Nil when no such keys are present.
func metadataToFacet[T any](metadata map[string]string, facet string) *T {
return mapping.DeserializeStringsAt[T](metadata, "libre.graph."+facet+".")
}
func cs3ResourceToRemoteItem(res *storageprovider.ResourceInfo) (*libregraph.RemoteItem, error) {

View File

@@ -136,10 +136,10 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
Tags: getFieldSliceValue[string](hit.Fields, "Tags"),
Favorites: getFieldSliceValue[string](hit.Fields, "Favorites"),
Highlights: getFragmentValue(hit.Fragments, "Content", 0),
Audio: getAudioValue[searchMessage.Audio](hit.Fields),
Image: getImageValue[searchMessage.Image](hit.Fields),
Location: getLocationValue[searchMessage.GeoCoordinates](hit.Fields),
Photo: getPhotoValue[searchMessage.Photo](hit.Fields),
Audio: hitToFacet[searchMessage.Audio](hit.Fields, "audio"),
Image: hitToFacet[searchMessage.Image](hit.Fields, "image"),
Location: hitToFacet[searchMessage.GeoCoordinates](hit.Fields, "location"),
Photo: hitToFacet[searchMessage.Photo](hit.Fields, "photo"),
},
}

View File

@@ -10,6 +10,7 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/utils"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
@@ -36,10 +37,21 @@ func NewBatch(index bleve.Index, size int) (*Batch, error) {
func (b *Batch) Upsert(id string, r search.Resource) error {
return b.withSizeLimit(func() error {
return b.batch.Index(id, r)
return b.indexResource(id, r)
})
}
// indexResource prepares r for bleve (resolving json tags and splicing in
// type-specific adaptations via the mapping package) and appends it to the
// batch under id.
func (b *Batch) indexResource(id string, r search.Resource) error {
doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
if err != nil {
return err
}
return b.batch.Index(id, doc)
}
func (b *Batch) Move(id, parentID, location string) error {
return b.withSizeLimit(func() error {
rootResource, err := searchResourceByID(id, b.index)
@@ -68,7 +80,7 @@ func (b *Batch) Move(id, parentID, location string) error {
}
for _, resource := range resources {
if err := b.batch.Index(resource.ID, resource); err != nil {
if err := b.indexResource(resource.ID, *resource); err != nil {
return err
}
if b.batch.Size() >= b.size {
@@ -90,7 +102,7 @@ func (b *Batch) Delete(id string) error {
}
for _, resource := range affectedResources {
if err := b.batch.Index(resource.ID, resource); err != nil {
if err := b.indexResource(resource.ID, *resource); err != nil {
return err
}
if b.batch.Size() >= b.size {
@@ -112,7 +124,7 @@ func (b *Batch) Restore(id string) error {
}
for _, resource := range affectedResources {
if err := b.batch.Index(resource.ID, resource); err != nil {
if err := b.indexResource(resource.ID, *resource); err != nil {
return err
}
if b.batch.Size() >= b.size {

View File

@@ -1,18 +1,13 @@
package bleve
import (
"reflect"
"regexp"
"strings"
"time"
bleveSearch "github.com/blevesearch/bleve/v2/search"
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"google.golang.org/protobuf/types/known/timestamppb"
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
@@ -75,131 +70,19 @@ func getFragmentValue(m bleveSearch.FieldFragmentMap, key string, idx int) strin
return val[idx]
}
func getAudioValue[T any](fields map[string]any) *T {
if !strings.HasPrefix(getFieldValue[string](fields, "MimeType"), "audio/") {
return nil
}
var audio = newPointerOfType[T]()
if ok := unmarshalInterfaceMap(audio, fields, "audio."); ok {
return audio
}
return nil
}
func getImageValue[T any](fields map[string]any) *T {
var image = newPointerOfType[T]()
if ok := unmarshalInterfaceMap(image, fields, "image."); ok {
return image
}
return nil
}
func getLocationValue[T any](fields map[string]any) *T {
var location = newPointerOfType[T]()
if ok := unmarshalInterfaceMap(location, fields, "location."); ok {
return location
}
return nil
}
func getPhotoValue[T any](fields map[string]any) *T {
var photo = newPointerOfType[T]()
if ok := unmarshalInterfaceMap(photo, fields, "photo."); ok {
return photo
}
return nil
}
func newPointerOfType[T any]() *T {
t := reflect.TypeOf((*T)(nil)).Elem()
ptr := reflect.New(t).Interface()
return ptr.(*T)
}
func unmarshalInterfaceMap(out any, flatMap map[string]any, prefix string) bool {
nonEmpty := false
obj := reflect.ValueOf(out).Elem()
for i := 0; i < obj.NumField(); i++ {
field := obj.Field(i)
structField := obj.Type().Field(i)
mapKey := prefix + getFieldName(structField)
if value, ok := flatMap[mapKey]; ok {
if field.Kind() == reflect.Ptr {
alloc := reflect.New(field.Type().Elem())
elemType := field.Type().Elem()
// convert time strings from index for search requests
if elemType == reflect.TypeOf(timestamppb.Timestamp{}) {
if strValue, ok := value.(string); ok {
if parsedTime, err := time.Parse(time.RFC3339, strValue); err == nil {
alloc.Elem().Set(reflect.ValueOf(*timestamppb.New(parsedTime)))
field.Set(alloc)
nonEmpty = true
}
}
continue
}
// convert time strings from index for libregraph structs when updating resources
if elemType == reflect.TypeOf(time.Time{}) {
if strValue, ok := value.(string); ok {
if parsedTime, err := time.Parse(time.RFC3339, strValue); err == nil {
alloc.Elem().Set(reflect.ValueOf(parsedTime))
field.Set(alloc)
nonEmpty = true
}
}
continue
}
alloc.Elem().Set(reflect.ValueOf(value).Convert(elemType))
field.Set(alloc)
nonEmpty = true
}
}
}
return nonEmpty
}
func getFieldName(structField reflect.StructField) string {
tag := structField.Tag.Get("json")
if tag == "" {
return structField.Name
}
return strings.Split(tag, ",")[0]
// hitToFacet builds a search Entity facet *T from a bleve hit's fields under the
// given key prefix. Nil when the hit has no such fields.
func hitToFacet[T any](fields map[string]any, prefix string) *T {
return mapping.DeserializeAt[T](fields, prefix)
}
// matchToResource reconstructs a search.Resource from a bleve hit. Used by
// the Move / Delete / Restore / Purge paths that round-trip a record through
// the index. Always returns a non-nil *Resource: Deserialize is fail-soft
// for per-field parse errors, so corrupted hit values surface as zero
// values on individual fields instead of dropping the whole record.
func matchToResource(match *bleveSearch.DocumentMatch) *search.Resource {
return &search.Resource{
ID: getFieldValue[string](match.Fields, "ID"),
RootID: getFieldValue[string](match.Fields, "RootID"),
Path: getFieldValue[string](match.Fields, "Path"),
ParentID: getFieldValue[string](match.Fields, "ParentID"),
Type: uint64(getFieldValue[float64](match.Fields, "Type")),
Deleted: getFieldValue[bool](match.Fields, "Deleted"),
Document: content.Document{
Name: getFieldValue[string](match.Fields, "Name"),
Title: getFieldValue[string](match.Fields, "Title"),
Size: uint64(getFieldValue[float64](match.Fields, "Size")),
Mtime: getFieldValue[string](match.Fields, "Mtime"),
MimeType: getFieldValue[string](match.Fields, "MimeType"),
Content: getFieldValue[string](match.Fields, "Content"),
Tags: getFieldSliceValue[string](match.Fields, "Tags"),
Favorites: getFieldSliceValue[string](match.Fields, "Favorites"),
Audio: getAudioValue[libregraph.Audio](match.Fields),
Image: getImageValue[libregraph.Image](match.Fields),
Location: getLocationValue[libregraph.GeoCoordinates](match.Fields),
Photo: getPhotoValue[libregraph.Photo](match.Fields),
},
}
return mapping.Deserialize[search.Resource](match.Fields)
}
func escapeQuery(s string) string {

View File

@@ -4,6 +4,7 @@ import (
"errors"
"math"
"path/filepath"
"reflect"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
@@ -15,6 +16,7 @@ import (
"github.com/blevesearch/bleve/v2/mapping"
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
@@ -38,27 +40,20 @@ func NewIndex(root string) (bleve.Index, error) {
}
func NewMapping() (mapping.IndexMapping, error) {
nameMapping := bleve.NewTextFieldMapping()
nameMapping.Analyzer = "lowercaseKeyword"
lowercaseMapping := bleve.NewTextFieldMapping()
lowercaseMapping.IncludeInAll = false
lowercaseMapping.Analyzer = "lowercaseKeyword"
fulltextFieldMapping := bleve.NewTextFieldMapping()
fulltextFieldMapping.Analyzer = "fulltext"
fulltextFieldMapping.IncludeInAll = false
docMapping := bleve.NewDocumentMapping()
docMapping.AddFieldMappingsAt("Name", nameMapping)
docMapping.AddFieldMappingsAt("Tags", lowercaseMapping)
docMapping.AddFieldMappingsAt("Favorites", lowercaseMapping)
docMapping.AddFieldMappingsAt("Content", fulltextFieldMapping)
resourceType := reflect.TypeFor[search.Resource]()
overrides := search.Resource{}.SearchFieldOverrides()
if err := searchmapping.Validate(resourceType, overrides); err != nil {
return nil, err
}
docMapping, err := searchmapping.BleveBuildMapping(resourceType, overrides)
if err != nil {
return nil, err
}
indexMapping := bleve.NewIndexMapping()
indexMapping.DefaultAnalyzer = keyword.Name
indexMapping.DefaultMapping = docMapping
err := indexMapping.AddCustomAnalyzer("lowercaseKeyword",
err = indexMapping.AddCustomAnalyzer("lowercaseKeyword",
map[string]any{
"type": custom.Name,
"tokenizer": single.Name,

View File

@@ -0,0 +1,48 @@
package bleve_test
import (
"testing"
bleveSearch "github.com/blevesearch/bleve/v2"
bquery "github.com/blevesearch/bleve/v2/search/query"
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
// Mtime is typed as a date, so range queries are chronological, not a
// lexicographic keyword compare.
func TestMtimeDateRange(t *testing.T) {
m, err := bleve.NewMapping()
if err != nil {
t.Fatal(err)
}
idx, err := bleveSearch.NewMemOnly(m)
if err != nil {
t.Fatal(err)
}
r := search.Resource{ID: "x", Document: content.Document{Name: "f", Mtime: "2026-03-15T12:00:00.123456789Z"}}
doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
if err != nil {
t.Fatal(err)
}
if err := idx.Index(r.ID, doc); err != nil {
t.Fatal(err)
}
hits := func(qs string) uint64 {
res, err := idx.Search(bleveSearch.NewSearchRequest(bquery.NewQueryStringQuery(qs)))
if err != nil {
t.Fatalf("%s: %v", qs, err)
}
return res.Total
}
if got := hits(`Mtime:>"2026-01-01T00:00:00Z"`); got != 1 {
t.Errorf("in-range: got %d hits, want 1", got)
}
if got := hits(`Mtime:>"2026-06-01T00:00:00Z"`); got != 0 {
t.Errorf("out-of-range: got %d hits, want 0", got)
}
}

View File

@@ -14,14 +14,14 @@ func init() {
// Document wraps all resource meta fields,
// it is used as a content extraction result.
type Document struct {
Title string
Name string
Content string
Size uint64
Mtime string
MimeType string
Tags []string
Favorites []string
Title string `json:"Title"`
Name string `json:"Name"`
Content string `json:"Content"`
Size uint64 `json:"Size"`
Mtime string `json:"Mtime"`
MimeType string `json:"MimeType"`
Tags []string `json:"Tags"`
Favorites []string `json:"Favorites"`
Audio *libregraph.Audio `json:"audio,omitempty"`
Image *libregraph.Image `json:"image,omitempty"`
Location *libregraph.GeoCoordinates `json:"location,omitempty"`

View File

@@ -0,0 +1,92 @@
package mapping
import (
"fmt"
"reflect"
"github.com/blevesearch/bleve/v2"
bleveMapping "github.com/blevesearch/bleve/v2/mapping"
)
// BleveBuildMapping builds a bleve DocumentMapping for t by walking the
// struct via reflection. Field names come from json tags; overrides are
// keyed by those names (or dotted paths for nested fields).
//
// The returned mapping references analyzer names (Analyzer field on the
// FieldOpts, plus "fulltext" / "path_hierarchy" for the corresponding Types);
// the caller is responsible for registering those analyzers on the enclosing
// IndexMapping.
func BleveBuildMapping(t reflect.Type, overrides map[string]FieldOpts) (*bleveMapping.DocumentMapping, error) {
return buildBleveDocMapping(t, overrides, "")
}
func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix string) (*bleveMapping.DocumentMapping, error) {
doc := bleve.NewDocumentMapping()
err := walkFields(t, func(fi fieldInfo) error {
key := fi.Name
if prefix != "" {
key = prefix + "." + fi.Name
}
opts := overrides[key]
fieldType := opts.Type
if fieldType == "" {
fieldType = inferType(fi.GoField.Type)
}
if fieldType == TypeObject {
sub := structType(fi.GoField.Type)
if sub == nil {
return fmt.Errorf("mapping: object type on non-struct field %q", key)
}
subDoc, err := buildBleveDocMapping(sub, overrides, key)
if err != nil {
return err
}
doc.AddSubDocumentMapping(fi.Name, subDoc)
return nil
}
fm, err := bleveFieldMapping(fieldType, opts)
if err != nil {
return fmt.Errorf("mapping: field %q: %w", key, err)
}
doc.AddFieldMappingsAt(fi.Name, fm)
return nil
})
return doc, err
}
func bleveFieldMapping(fieldType string, opts FieldOpts) (*bleveMapping.FieldMapping, error) {
switch fieldType {
case TypeWildcard:
// bleve has no wildcard type; fall back to keyword-ish text.
fieldType = TypeKeyword
fallthrough
case TypeKeyword, TypeFulltext, TypePath:
fm := bleve.NewTextFieldMapping()
switch {
case opts.Analyzer != "":
fm.Analyzer = opts.Analyzer
case fieldType == TypeFulltext:
fm.Analyzer = "fulltext"
case fieldType == TypePath:
fm.Analyzer = "path_hierarchy"
}
switch {
case opts.IncludeInAll != nil:
fm.IncludeInAll = *opts.IncludeInAll
case fieldType == TypeFulltext, fieldType == TypePath:
fm.IncludeInAll = false
}
return fm, nil
case TypeNumeric:
return bleve.NewNumericFieldMapping(), nil
case TypeBool:
return bleve.NewBooleanFieldMapping(), nil
case TypeDatetime:
return bleve.NewDateTimeFieldMapping(), nil
case "":
return nil, fmt.Errorf("no type inferred and no override")
}
return nil, fmt.Errorf("unsupported type %q", fieldType)
}

View File

@@ -0,0 +1,116 @@
package mapping
import (
"reflect"
"testing"
"time"
)
type bleveDoc struct {
Name string `json:"Name"`
Content string `json:"Content"`
Tags []string `json:"Tags"`
Size uint64 `json:"Size"`
Deleted bool `json:"Deleted"`
CreatedAt time.Time `json:"CreatedAt"`
Nested *nested `json:"nested,omitempty"`
}
type nested struct {
Artist string `json:"artist"`
Year int `json:"year"`
}
// bleve wildcard falls back to keyword-ish text (bleve has no wildcard type).
func TestBleveWildcardFallback(t *testing.T) {
type doc struct {
Mime string `json:"mime"`
}
dm, err := BleveBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"mime": {Type: TypeWildcard}})
if err != nil {
t.Fatalf("BleveBuildMapping: %v", err)
}
fms := dm.Properties["mime"].Fields
if len(fms) != 1 || fms[0].Type != "text" {
t.Fatalf("wildcard should map to text, got %+v", fms)
}
}
func TestBleveBuildMappingInferredTypes(t *testing.T) {
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil)
if err != nil {
t.Fatalf("BleveBuildMapping: %v", err)
}
cases := map[string]string{
"Name": "text",
"Content": "text",
"Tags": "text",
"Size": "number",
"Deleted": "boolean",
"CreatedAt": "datetime",
}
for field, wantType := range cases {
prop := dm.Properties[field]
if prop == nil {
t.Errorf("missing property %q", field)
continue
}
if len(prop.Fields) == 0 {
t.Errorf("%q: no field mappings", field)
continue
}
if got := prop.Fields[0].Type; got != wantType {
t.Errorf("%q: got type %q, want %q", field, got, wantType)
}
}
}
func TestBleveBuildMappingNestedIsSubDocument(t *testing.T) {
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil)
if err != nil {
t.Fatalf("BleveBuildMapping: %v", err)
}
sub := dm.Properties["nested"]
if sub == nil {
t.Fatal("missing nested sub-document")
}
if sub.Properties["artist"] == nil || sub.Properties["year"] == nil {
t.Fatalf("nested fields missing: %#v", sub.Properties)
}
if got := sub.Properties["artist"].Fields[0].Type; got != "text" {
t.Errorf("nested.artist: type %q, want text", got)
}
if got := sub.Properties["year"].Fields[0].Type; got != "number" {
t.Errorf("nested.year: type %q, want number", got)
}
}
func TestBleveBuildMappingOverrides(t *testing.T) {
includeInAllFalse := false
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{
"Name": {Analyzer: "lowercaseKeyword"},
"Content": {Type: TypeFulltext},
"Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &includeInAllFalse},
})
if err != nil {
t.Fatalf("BleveBuildMapping: %v", err)
}
nameField := dm.Properties["Name"].Fields[0]
if nameField.Analyzer != "lowercaseKeyword" {
t.Errorf("Name analyzer: %q, want lowercaseKeyword", nameField.Analyzer)
}
if !nameField.IncludeInAll {
t.Errorf("Name IncludeInAll should stay default-true when not overridden")
}
contentField := dm.Properties["Content"].Fields[0]
if contentField.Analyzer != "fulltext" {
t.Errorf("Content analyzer: %q, want fulltext", contentField.Analyzer)
}
if contentField.IncludeInAll {
t.Errorf("Content IncludeInAll should default to false for fulltext type")
}
tagsField := dm.Properties["Tags"].Fields[0]
if tagsField.IncludeInAll {
t.Errorf("Tags IncludeInAll should honor the explicit false override")
}
}

View File

@@ -0,0 +1,172 @@
package mapping
import (
"fmt"
"reflect"
"time"
"google.golang.org/protobuf/types/known/timestamppb"
)
// Deserialize builds a *T from bleve's flat hit.Fields map (json-tag keys,
// "parent.child" for nested pointers). Used to rebuild a search Resource from a
// hit. Fail-soft: an unparseable field stays at its zero value.
func Deserialize[T any](fields map[string]any) *T {
t := reflect.TypeFor[T]()
if t.Kind() != reflect.Struct {
panic(fmt.Sprintf("mapping: Deserialize requires a struct type, got %v", t))
}
out := reflect.New(t)
fillStruct(out.Elem(), fields, "", setValue)
return out.Interface().(*T)
}
// DeserializeAt is Deserialize scoped to a dotted prefix, used to rebuild one
// search-result facet (e.g. "audio"). Returns nil when nothing matched, so the
// caller can leave the enclosing pointer nil.
func DeserializeAt[T any](fields map[string]any, prefix string) *T {
t := reflect.TypeFor[T]()
if t.Kind() != reflect.Struct {
panic(fmt.Sprintf("mapping: DeserializeAt requires a struct type, got %v", t))
}
out := reflect.New(t)
if !fillStruct(out.Elem(), fields, prefix, setValue) {
return nil
}
return out.Interface().(*T)
}
// fillStruct walks v's exported fields, reading values from the flat fields map
// (json-tag keys, "parent.child" for nested pointers). setLeaf converts each raw
// value into a leaf, which is what lets the any-valued and string-valued
// deserializers share one walker. Returns true if any leaf was populated.
func fillStruct[V any](v reflect.Value, fields map[string]V, prefix string, setLeaf func(reflect.Value, V) error) bool {
t := v.Type()
touched := false
for i := 0; i < t.NumField(); i++ {
fi := resolveField(t.Field(i))
if fi.Skip {
continue
}
fv := v.Field(i)
if fi.Embedded {
// Embedded *struct: allocate, recurse, keep the pointer only if set.
if fv.Kind() == reflect.Ptr {
if fv.Type().Elem().Kind() != reflect.Struct || !fv.CanSet() {
continue
}
alloc := reflect.New(fv.Type().Elem())
if fillStruct(alloc.Elem(), fields, prefix, setLeaf) {
fv.Set(alloc)
touched = true
}
continue
}
if fv.Kind() == reflect.Struct && fillStruct(fv, fields, prefix, setLeaf) {
touched = true
}
continue
}
key := fi.Name
if prefix != "" {
key = prefix + "." + fi.Name
}
// Pointer to a nested (non-time) struct: recurse, keep the pointer only
// if a field was populated.
if fv.Kind() == reflect.Ptr {
if elem := fv.Type().Elem(); elem.Kind() == reflect.Struct && elem != timeType && elem != timestampType {
alloc := reflect.New(elem)
if fillStruct(alloc.Elem(), fields, key, setLeaf) {
fv.Set(alloc)
touched = true
}
continue
}
}
if raw, ok := fields[key]; ok && setLeaf(fv, raw) == nil {
touched = true
}
}
return touched
}
// setValue writes raw (an any value from a bleve hit) into v, converting to the
// field's type. Returns an error on nil or a type mismatch.
func setValue(v reflect.Value, raw any) error {
if v.Kind() == reflect.Ptr {
alloc := reflect.New(v.Type().Elem())
if err := setValue(alloc.Elem(), raw); err != nil {
return err
}
v.Set(alloc)
return nil
}
if v.Type() == timeType || v.Type() == timestampType {
t, ok := parseTime(raw)
if !ok {
return fmt.Errorf("not an RFC3339 time: %v", raw)
}
setParsedTime(v, t)
return nil
}
if v.Kind() == reflect.Slice {
return setSlice(v, raw)
}
rv := reflect.ValueOf(raw)
if !rv.IsValid() {
return fmt.Errorf("nil value for %s", v.Type())
}
if !rv.Type().ConvertibleTo(v.Type()) {
return fmt.Errorf("cannot convert %s to %s", rv.Type(), v.Type())
}
v.Set(rv.Convert(v.Type()))
return nil
}
func setSlice(v reflect.Value, raw any) error {
items, ok := raw.([]any)
if !ok {
// bleve unwraps single-element slices; re-wrap here.
items = []any{raw}
}
// Compact in place with a single MakeSlice: unparseable elements are
// dropped, Slice(0, j) trims the tail.
out := reflect.MakeSlice(v.Type(), len(items), len(items))
j := 0
for _, item := range items {
if setValue(out.Index(j), item) == nil {
j++
}
}
if j == 0 {
return fmt.Errorf("no slice elements set from %T", raw)
}
v.Set(out.Slice(0, j))
return nil
}
func parseTime(raw any) (time.Time, bool) {
s, ok := raw.(string)
if !ok {
return time.Time{}, false
}
t, err := time.Parse(time.RFC3339, s)
if err != nil {
return time.Time{}, false
}
return t, true
}
// setParsedTime writes t into v, which must be a time.Time or
// timestamppb.Timestamp field.
func setParsedTime(v reflect.Value, t time.Time) {
if v.Type() == timestampType {
v.Set(reflect.ValueOf(*timestamppb.New(t)))
return
}
v.Set(reflect.ValueOf(t))
}

View File

@@ -0,0 +1,82 @@
package mapping
import (
"fmt"
"reflect"
"strconv"
"strings"
"time"
)
// DeserializeStringsAt is DeserializeAt for a string-valued map (e.g. CS3
// ArbitraryMetadata), parsing each string into the field's Go type via strconv/
// time.Parse. Used to build a graph DriveItem facet. Returns nil when nothing
// under the prefix matched.
func DeserializeStringsAt[T any](fields map[string]string, prefix string) *T {
t := reflect.TypeFor[T]()
if t.Kind() != reflect.Struct {
panic(fmt.Sprintf("mapping: DeserializeStringsAt requires a struct type, got %v", t))
}
out := reflect.New(t)
// Callers pass a flat-key prefix with a trailing dot (e.g.
// "libre.graph.audio."); fillStruct joins segments with ".", so drop it.
if !fillStruct(out.Elem(), fields, strings.TrimSuffix(prefix, "."), setValueFromString) {
return nil
}
return out.Interface().(*T)
}
// setValueFromString parses the string raw into v's Go type via strconv/
// time.Parse, returning a descriptive error on failure.
func setValueFromString(v reflect.Value, raw string) error {
if v.Kind() == reflect.Ptr {
alloc := reflect.New(v.Type().Elem())
if err := setValueFromString(alloc.Elem(), raw); err != nil {
return err
}
v.Set(alloc)
return nil
}
if v.Type() == timeType || v.Type() == timestampType {
t, err := time.Parse(time.RFC3339, raw)
if err != nil {
return fmt.Errorf("parse time %q: %w", raw, err)
}
setParsedTime(v, t)
return nil
}
switch v.Kind() {
case reflect.String:
v.SetString(raw)
return nil
case reflect.Bool:
b, err := strconv.ParseBool(raw)
if err != nil {
return fmt.Errorf("parse bool %q: %w", raw, err)
}
v.SetBool(b)
return nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
n, err := strconv.ParseInt(raw, 10, v.Type().Bits())
if err != nil {
return fmt.Errorf("parse int %q: %w", raw, err)
}
v.SetInt(n)
return nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
n, err := strconv.ParseUint(raw, 10, v.Type().Bits())
if err != nil {
return fmt.Errorf("parse uint %q: %w", raw, err)
}
v.SetUint(n)
return nil
case reflect.Float32, reflect.Float64:
f, err := strconv.ParseFloat(raw, v.Type().Bits())
if err != nil {
return fmt.Errorf("parse float %q: %w", raw, err)
}
v.SetFloat(f)
return nil
}
return fmt.Errorf("unsupported target kind %s", v.Kind())
}

View File

@@ -0,0 +1,122 @@
package mapping
import (
"reflect"
"testing"
"time"
"google.golang.org/protobuf/types/known/timestamppb"
)
type stringFacet struct {
Artist *string `json:"artist,omitempty"`
Year *int32 `json:"year,omitempty"`
Duration *int64 `json:"duration,omitempty"`
Rating *float64 `json:"rating,omitempty"`
Explicit *bool `json:"explicit,omitempty"`
Taken *time.Time `json:"takenDateTime,omitempty"`
}
func TestSetValueFromStringUnsupportedKind(t *testing.T) {
v := reflect.New(reflect.TypeFor[[]int]()).Elem() // settable slice
if err := setValueFromString(v, "x"); err == nil {
t.Error("expected error for unsupported target kind (slice)")
}
}
func TestDeserializeStringsAtBasicTypes(t *testing.T) {
r := DeserializeStringsAt[stringFacet](map[string]string{
"libre.graph.audio.artist": "Queen",
"libre.graph.audio.year": "1975",
"libre.graph.audio.duration": "354000",
"libre.graph.audio.rating": "4.9",
"libre.graph.audio.explicit": "true",
"libre.graph.audio.takenDateTime": "2024-01-02T03:04:05Z",
}, "libre.graph.audio.")
if r == nil {
t.Fatal("expected non-nil *stringFacet")
}
if r.Artist == nil || *r.Artist != "Queen" {
t.Errorf("Artist: %#v", r.Artist)
}
if r.Year == nil || *r.Year != 1975 {
t.Errorf("Year: %#v", r.Year)
}
if r.Duration == nil || *r.Duration != 354000 {
t.Errorf("Duration: %#v", r.Duration)
}
if r.Rating == nil || *r.Rating != 4.9 {
t.Errorf("Rating: %#v", r.Rating)
}
if r.Explicit == nil || !*r.Explicit {
t.Errorf("Explicit: %#v", r.Explicit)
}
if r.Taken == nil || !r.Taken.Equal(time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC)) {
t.Errorf("Taken: %#v", r.Taken)
}
}
func TestDeserializeStringsAtReturnsNilWhenEmpty(t *testing.T) {
r := DeserializeStringsAt[stringFacet](map[string]string{
"libre.graph.image.width": "1200",
}, "libre.graph.audio.")
if r != nil {
t.Fatalf("expected nil, got %#v", r)
}
}
func TestDeserializeStringsAtTimestamppb(t *testing.T) {
type photoFacet struct {
Taken *timestamppb.Timestamp `json:"takenDateTime,omitempty"`
}
r := DeserializeStringsAt[photoFacet](map[string]string{
"libre.graph.photo.takenDateTime": "2024-05-06T07:08:09Z",
}, "libre.graph.photo.")
if r == nil || r.Taken == nil {
t.Fatalf("Taken missing: %#v", r)
}
want := time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC)
if !r.Taken.AsTime().Equal(want) {
t.Errorf("Taken: got %v, want %v", r.Taken.AsTime(), want)
}
}
func TestDeserializeStringsAtIsFailSoft(t *testing.T) {
// A single malformed field (year is unparseable as int) must not drop
// the whole facet. The bad field stays at zero value, the rest of the
// facet still populates. Mirrors the bleve-hit Deserialize behavior.
r := DeserializeStringsAt[stringFacet](map[string]string{
"libre.graph.audio.artist": "Iron Maiden",
"libre.graph.audio.year": "not-a-number",
"libre.graph.audio.duration": "354000",
"libre.graph.audio.explicit": "not-a-bool",
"libre.graph.audio.rating": "4.9",
}, "libre.graph.audio.")
if r == nil {
t.Fatal("expected non-nil *stringFacet despite bad fields")
}
if r.Artist == nil || *r.Artist != "Iron Maiden" {
t.Errorf("Artist should still be populated, got %#v", r.Artist)
}
if r.Duration == nil || *r.Duration != 354000 {
t.Errorf("Duration should still be populated, got %#v", r.Duration)
}
if r.Rating == nil || *r.Rating != 4.9 {
t.Errorf("Rating should still be populated, got %#v", r.Rating)
}
if r.Year != nil {
t.Errorf("Year should stay nil for bad int, got %#v", r.Year)
}
if r.Explicit != nil {
t.Errorf("Explicit should stay nil for bad bool, got %#v", r.Explicit)
}
}
func TestDeserializeStringsAtPanicsOnNonStruct(t *testing.T) {
defer func() {
if r := recover(); r == nil {
t.Fatal("expected panic for non-struct T")
}
}()
DeserializeStringsAt[int](nil, "")
}

View File

@@ -0,0 +1,155 @@
package mapping
import (
"testing"
"time"
"google.golang.org/protobuf/types/known/timestamppb"
)
type Leaf struct {
Name string `json:"Name"`
Size uint64 `json:"Size"`
Deleted bool `json:"Deleted"`
Tags []string `json:"Tags"`
Favorites []string `json:"Favorites"`
}
type audio struct {
Artist *string `json:"artist,omitempty"`
Year *int32 `json:"year,omitempty"`
}
type photo struct {
Taken *timestamppb.Timestamp `json:"takenDateTime,omitempty"`
Mtime *time.Time `json:"mtime,omitempty"`
}
type embedded struct {
Leaf
Audio *audio `json:"audio,omitempty"`
Photo *photo `json:"photo,omitempty"`
}
func TestDeserializeAtNonStructPanics(t *testing.T) {
defer func() {
if recover() == nil {
t.Error("expected panic for non-struct type")
}
}()
_ = DeserializeAt[int](map[string]any{}, "")
}
func TestDeserializeLeafFields(t *testing.T) {
r := Deserialize[Leaf](map[string]any{
"Name": "n",
"Size": float64(42),
"Deleted": true,
})
if r.Name != "n" || r.Size != 42 || !r.Deleted {
t.Fatalf("got %#v", r)
}
}
func TestDeserializeScalarToSlice(t *testing.T) {
r := Deserialize[Leaf](map[string]any{
"Tags": "single",
"Favorites": []any{"a", "b"},
})
if len(r.Tags) != 1 || r.Tags[0] != "single" {
t.Errorf("Tags: %#v", r.Tags)
}
if len(r.Favorites) != 2 || r.Favorites[0] != "a" || r.Favorites[1] != "b" {
t.Errorf("Favorites: %#v", r.Favorites)
}
}
func TestDeserializeTimestamp(t *testing.T) {
r := Deserialize[embedded](map[string]any{
"photo.takenDateTime": "2024-01-02T03:04:05Z",
"photo.mtime": "2024-05-06T07:08:09Z",
})
if r.Photo == nil {
t.Fatal("Photo is nil")
}
if r.Photo.Taken == nil {
t.Fatal("Taken is nil")
}
expected := time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC)
if !r.Photo.Taken.AsTime().Equal(expected) {
t.Errorf("Taken: got %v, want %v", r.Photo.Taken.AsTime(), expected)
}
if r.Photo.Mtime == nil {
t.Fatal("Mtime is nil")
}
if !r.Photo.Mtime.Equal(time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC)) {
t.Errorf("Mtime: %v", r.Photo.Mtime)
}
}
func TestDeserializeIsFailSoft(t *testing.T) {
// Malformed values (type mismatch, unparseable time) leave the
// affected field at its zero value instead of dropping the whole
// record. Matches the pre-refactor getFieldValue behavior so
// matchToResource never returns nil on a corrupted hit.
r := Deserialize[embedded](map[string]any{
"Name": "n",
"Size": "not-a-number", // wrong type
"Deleted": true,
"photo.takenDateTime": "not-an-rfc3339-time",
"photo.mtime": "2024-05-06T07:08:09Z",
})
if r == nil {
t.Fatal("expected non-nil *embedded even with partial corruption")
}
if r.Name != "n" {
t.Errorf("Name: %q", r.Name)
}
if r.Size != 0 {
t.Errorf("Size should stay zero on mismatch, got %d", r.Size)
}
if !r.Deleted {
t.Errorf("Deleted should still be true")
}
if r.Photo == nil {
t.Fatal("Photo should be populated because Mtime parsed ok")
}
if r.Photo.Taken != nil {
t.Errorf("Taken should stay nil for unparseable time, got %v", r.Photo.Taken)
}
if r.Photo.Mtime == nil {
t.Error("Mtime should be parsed")
}
}
func TestDeserializePanicsOnNonStruct(t *testing.T) {
defer func() {
if r := recover(); r == nil {
t.Fatal("expected panic for non-struct T")
}
}()
Deserialize[int](nil)
}
func TestDeserializeAtReturnsNilWhenNothingMatches(t *testing.T) {
r := DeserializeAt[audio](map[string]any{"Name": "n"}, "audio")
if r != nil {
t.Fatalf("expected nil, got %#v", r)
}
}
func TestDeserializeAtReturnsValueWhenPrefixMatches(t *testing.T) {
r := DeserializeAt[audio](map[string]any{
"audio.artist": "A",
"audio.year": float64(2024), // setValue: pointer + numeric convert
}, "audio")
if r == nil {
t.Fatal("expected non-nil *audio")
}
if r.Artist == nil || *r.Artist != "A" {
t.Errorf("Artist: %#v", r.Artist)
}
if r.Year == nil || *r.Year != 2024 {
t.Errorf("Year: %#v", r.Year)
}
}

View File

@@ -0,0 +1,127 @@
package mapping
import (
"errors"
"reflect"
"testing"
)
// The walker is shared by both deserializers, so its structural behavior
// (flattening embedded structs, recursing into nested pointers, joining the
// prefix, keeping a pointer only when a field was set, fail-soft skipping) is
// tested here once against a trivial string setter instead of twice through
// Deserialize and DeserializeStringsAt.
// FsEmbVal/FsEmbPtr are exported so their embedded field is exported; an
// unexported embedded type would be skipped by resolveField.
type FsEmbVal struct {
EV string `json:"ev"`
}
type FsEmbPtr struct {
EP string `json:"ep"`
}
type fsNested struct {
N string `json:"n"`
}
type fsRoot struct {
FsEmbVal // embedded value struct: fields promoted
*FsEmbPtr // embedded pointer struct: allocated on demand
Leaf string `json:"leaf"`
Nested *fsNested `json:"nested"` // nested pointer: recursed under "nested."
}
var errBadLeaf = errors.New("bad leaf")
// fsSet writes raw into a string field; the "BAD" sentinel simulates a parse
// failure so the fail-soft path can be exercised.
func fsSet(v reflect.Value, raw string) error {
if raw == "BAD" {
return errBadLeaf
}
v.SetString(raw)
return nil
}
func TestFillStruct(t *testing.T) {
fill := func(fields map[string]string, prefix string) (fsRoot, bool) {
var root fsRoot
touched := fillStruct(reflect.ValueOf(&root).Elem(), fields, prefix, fsSet)
return root, touched
}
t.Run("flattens embedded, recurses nested", func(t *testing.T) {
root, touched := fill(map[string]string{
"leaf": "L",
"ev": "EV",
"ep": "EP",
"nested.n": "N",
}, "")
if !touched {
t.Fatal("expected touched")
}
if root.Leaf != "L" {
t.Errorf("Leaf: %q", root.Leaf)
}
if root.EV != "EV" {
t.Errorf("embedded value not promoted: %q", root.EV)
}
if root.FsEmbPtr == nil || root.EP != "EP" {
t.Errorf("embedded pointer not allocated: %+v", root.FsEmbPtr)
}
if root.Nested == nil || root.Nested.N != "N" {
t.Errorf("nested pointer not populated: %+v", root.Nested)
}
})
t.Run("nothing matches: touched false, pointers stay nil", func(t *testing.T) {
root, touched := fill(map[string]string{"other": "x"}, "")
if touched {
t.Fatal("expected untouched")
}
if root.Nested != nil {
t.Errorf("Nested should stay nil: %+v", root.Nested)
}
if root.FsEmbPtr != nil {
t.Errorf("embedded pointer should stay nil: %+v", root.FsEmbPtr)
}
})
t.Run("prefix arg is joined with the field name", func(t *testing.T) {
root, touched := fill(map[string]string{
"pre.leaf": "L",
"pre.nested.n": "N",
}, "pre")
if !touched || root.Leaf != "L" || root.Nested == nil || root.Nested.N != "N" {
t.Fatalf("prefix not joined: leaf=%q nested=%+v", root.Leaf, root.Nested)
}
})
t.Run("fail-soft: errored leaf stays zero, walk continues", func(t *testing.T) {
root, touched := fill(map[string]string{
"leaf": "BAD",
"ev": "EV",
}, "")
if !touched {
t.Fatal("expected touched because ev was set")
}
if root.Leaf != "" {
t.Errorf("errored leaf should stay zero, got %q", root.Leaf)
}
if root.EV != "EV" {
t.Errorf("walk should continue past the error: %q", root.EV)
}
})
t.Run("embedded pointer dropped when its only field errors", func(t *testing.T) {
root, touched := fill(map[string]string{"ep": "BAD"}, "")
if touched {
t.Fatal("expected untouched")
}
if root.FsEmbPtr != nil {
t.Errorf("embedded pointer should stay nil on error, got %+v", root.FsEmbPtr)
}
})
}

View File

@@ -0,0 +1,116 @@
package mapping
import (
"reflect"
"strings"
"time"
"google.golang.org/protobuf/types/known/timestamppb"
)
var (
timeType = reflect.TypeFor[time.Time]()
timestampType = reflect.TypeFor[timestamppb.Timestamp]()
)
// deref unwraps pointer and slice types to their element type.
func deref(t reflect.Type) reflect.Type {
for t.Kind() == reflect.Ptr || t.Kind() == reflect.Slice {
t = t.Elem()
}
return t
}
// inferType returns the mapping type for a Go type. Pointers and slices are
// unwrapped to their element type. time.Time and timestamppb.Timestamp become
// datetime; other structs become object.
func inferType(t reflect.Type) string {
t = deref(t)
switch t.Kind() {
case reflect.String:
return TypeKeyword
case reflect.Bool:
return TypeBool
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
reflect.Float32, reflect.Float64:
return TypeNumeric
case reflect.Struct:
if t == timeType || t == timestampType {
return TypeDatetime
}
return TypeObject
}
return ""
}
// fieldInfo is the resolved metadata for one struct field.
type fieldInfo struct {
Name string
GoField reflect.StructField
Skip bool
Embedded bool
}
// resolveField resolves a struct field's json-tag name and skip/embed state.
func resolveField(sf reflect.StructField) fieldInfo {
if !sf.IsExported() {
return fieldInfo{Skip: true}
}
name := sf.Name
tag := sf.Tag.Get("json")
if tag != "" {
first, _, _ := strings.Cut(tag, ",")
if first == "-" {
return fieldInfo{Skip: true}
}
if first != "" {
name = first
}
}
return fieldInfo{
Name: name,
GoField: sf,
Embedded: sf.Anonymous,
}
}
// walkFields visits exported leaf fields of t, flattening embedded structs
// onto the enclosing level. It returns the first error returned by fn.
func walkFields(t reflect.Type, fn func(fi fieldInfo) error) error {
for t.Kind() == reflect.Ptr {
t = t.Elem()
}
if t.Kind() != reflect.Struct {
return nil
}
for i := 0; i < t.NumField(); i++ {
fi := resolveField(t.Field(i))
if fi.Skip {
continue
}
if fi.Embedded {
if err := walkFields(fi.GoField.Type, fn); err != nil {
return err
}
continue
}
if err := fn(fi); err != nil {
return err
}
}
return nil
}
// structType returns the underlying struct type, unwrapping pointers and
// slices. Returns nil when t is not a walkable struct (e.g. time.Time).
func structType(t reflect.Type) reflect.Type {
t = deref(t)
if t.Kind() != reflect.Struct {
return nil
}
if t == timeType || t == timestampType {
return nil
}
return t
}

View File

@@ -0,0 +1,125 @@
package mapping
import (
"reflect"
"testing"
"time"
"google.golang.org/protobuf/types/known/timestamppb"
)
func TestInferTypeUnsupported(t *testing.T) {
if got := inferType(reflect.TypeFor[map[string]int]()); got != "" {
t.Errorf("map: got %q, want empty", got)
}
if got := inferType(reflect.TypeFor[chan int]()); got != "" {
t.Errorf("chan: got %q, want empty", got)
}
}
func TestInferType(t *testing.T) {
cases := []struct {
name string
in any
want string
}{
{"string", "", TypeKeyword},
{"*string", (*string)(nil), TypeKeyword},
{"[]string", []string(nil), TypeKeyword},
{"bool", false, TypeBool},
{"int", int(0), TypeNumeric},
{"int64", int64(0), TypeNumeric},
{"uint64", uint64(0), TypeNumeric},
{"float64", float64(0), TypeNumeric},
{"time.Time", time.Time{}, TypeDatetime},
{"*time.Time", (*time.Time)(nil), TypeDatetime},
{"*timestamppb.Timestamp", (*timestamppb.Timestamp)(nil), TypeDatetime},
{"struct", struct{ X int }{}, TypeObject},
{"*struct", (*struct{ X int })(nil), TypeObject},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got := inferType(reflect.TypeOf(c.in))
if got != c.want {
t.Fatalf("inferType(%s): got %q, want %q", c.name, got, c.want)
}
})
}
}
func TestResolveField(t *testing.T) {
type S struct {
Exported string `json:"exp"`
Renamed string `json:"renamed,omitempty"`
NoTag string
OmitOnly string `json:",omitempty"`
Skipped string `json:"-"`
unexported string //nolint:unused
}
st := reflect.TypeFor[S]()
cases := []struct {
fieldIdx int
wantName string
wantSkip bool
}{
{0, "exp", false},
{1, "renamed", false},
{2, "NoTag", false},
{3, "OmitOnly", false},
{4, "", true},
{5, "", true},
}
for _, c := range cases {
fi := resolveField(st.Field(c.fieldIdx))
if fi.Skip != c.wantSkip {
t.Errorf("field %d: skip=%v, want %v", c.fieldIdx, fi.Skip, c.wantSkip)
}
if !c.wantSkip && fi.Name != c.wantName {
t.Errorf("field %d: name=%q, want %q", c.fieldIdx, fi.Name, c.wantName)
}
}
}
func TestWalkFieldsFlattensEmbedded(t *testing.T) {
type Inner struct {
A string `json:"a"`
B int `json:"b"`
}
type Outer struct {
Inner
C bool `json:"c"`
}
var names []string
err := walkFields(reflect.TypeFor[Outer](), func(fi fieldInfo) error {
names = append(names, fi.Name)
return nil
})
if err != nil {
t.Fatalf("walkFields: %v", err)
}
want := []string{"a", "b", "c"}
if !reflect.DeepEqual(names, want) {
t.Fatalf("got %v, want %v", names, want)
}
}
func TestStructType(t *testing.T) {
type S struct{ X int }
cases := []struct {
name string
in reflect.Type
wantNil bool
}{
{"struct", reflect.TypeFor[S](), false},
{"*struct", reflect.TypeFor[*S](), false},
{"[]struct", reflect.TypeFor[[]S](), false},
{"time.Time", reflect.TypeFor[time.Time](), true},
{"string", reflect.TypeFor[string](), true},
}
for _, c := range cases {
got := structType(c.in)
if (got == nil) != c.wantNil {
t.Errorf("%s: got %v, wantNil %v", c.name, got, c.wantNil)
}
}
}

View File

@@ -0,0 +1,112 @@
package mapping
import (
"fmt"
"reflect"
)
// OpenSearchBuildMapping builds the OpenSearch "properties" map (the value
// of mappings.properties) for type t by walking the struct via reflection.
// Field names come from json tags; overrides are keyed by those names.
//
// The returned map contains plain JSON-friendly values (strings, bools,
// nested maps) and can be marshalled directly.
func OpenSearchBuildMapping(t reflect.Type, overrides map[string]FieldOpts) (map[string]any, error) {
return buildOpenSearchProperties(t, overrides, "")
}
func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, prefix string) (map[string]any, error) {
props := map[string]any{}
err := walkFields(t, func(fi fieldInfo) error {
key := fi.Name
if prefix != "" {
key = prefix + "." + fi.Name
}
opts := overrides[key]
fieldType := opts.Type
if fieldType == "" {
fieldType = inferType(fi.GoField.Type)
}
if fieldType == TypeObject {
sub := structType(fi.GoField.Type)
if sub == nil {
return fmt.Errorf("mapping: object type on non-struct field %q", key)
}
subProps, err := buildOpenSearchProperties(sub, overrides, key)
if err != nil {
return err
}
props[fi.Name] = map[string]any{"properties": subProps}
return nil
}
fm, err := openSearchFieldMapping(fieldType, opts, fi.GoField.Type)
if err != nil {
return fmt.Errorf("mapping: field %q: %w", key, err)
}
props[fi.Name] = fm
return nil
})
return props, err
}
func openSearchFieldMapping(fieldType string, opts FieldOpts, goType reflect.Type) (map[string]any, error) {
switch fieldType {
case TypeKeyword:
m := map[string]any{"type": "keyword"}
if opts.Analyzer != "" {
m["type"] = "text"
m["analyzer"] = opts.Analyzer
}
return m, nil
case TypeFulltext:
m := map[string]any{
"type": "text",
"term_vector": "with_positions_offsets",
}
if opts.Analyzer != "" {
m["analyzer"] = opts.Analyzer
}
return m, nil
case TypePath:
m := map[string]any{"type": "text"}
if opts.Analyzer != "" {
m["analyzer"] = opts.Analyzer
} else {
m["analyzer"] = "path_hierarchy"
}
return m, nil
case TypeWildcard:
// OpenSearch stores wildcard fields with doc_values=false by
// default, so emit it explicitly to keep local and remote
// mappings in sync for the Apply comparison.
return map[string]any{"type": "wildcard", "doc_values": false}, nil
case TypeNumeric:
return map[string]any{"type": openSearchNumericType(goType)}, nil
case TypeBool:
return map[string]any{"type": "boolean"}, nil
case TypeDatetime:
return map[string]any{"type": "date"}, nil
case "":
return nil, fmt.Errorf("no type inferred and no override")
}
return nil, fmt.Errorf("unsupported type %q", fieldType)
}
// openSearchNumericType maps a Go numeric type to an OpenSearch numeric
// field type.
func openSearchNumericType(t reflect.Type) string {
t = deref(t)
switch t.Kind() {
case reflect.Float32:
return "float"
case reflect.Float64:
return "double"
case reflect.Int8, reflect.Uint8, reflect.Int16, reflect.Uint16:
return "short"
case reflect.Int32, reflect.Uint32:
return "integer"
}
return "long"
}

View File

@@ -0,0 +1,132 @@
package mapping
import (
"reflect"
"testing"
"time"
)
type osDoc struct {
ID string `json:"ID"`
Size uint64 `json:"Size"`
Deleted bool `json:"Deleted"`
CreatedAt time.Time `json:"CreatedAt"`
Rating float64 `json:"Rating"`
Nested *struct {
Artist string `json:"artist"`
Year int32 `json:"year"`
} `json:"nested,omitempty"`
}
func TestOpenSearchNumericTypes(t *testing.T) {
type doc struct {
A int8 `json:"a"`
B int16 `json:"b"`
C int32 `json:"c"`
D int64 `json:"d"`
E uint8 `json:"e"`
F uint64 `json:"f"`
G float32 `json:"g"`
H float64 `json:"h"`
}
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), nil)
if err != nil {
t.Fatalf("OpenSearchBuildMapping: %v", err)
}
want := map[string]string{"a": "short", "b": "short", "c": "integer", "d": "long", "e": "short", "f": "long", "g": "float", "h": "double"}
for k, wt := range want {
if got := props[k].(map[string]any)["type"]; got != wt {
t.Errorf("%s: type = %v, want %v", k, got, wt)
}
}
}
func TestOpenSearchBuildMappingInferred(t *testing.T) {
props, err := OpenSearchBuildMapping(reflect.TypeFor[osDoc](), nil)
if err != nil {
t.Fatalf("OpenSearchBuildMapping: %v", err)
}
want := map[string]string{
"ID": "keyword",
"Size": "long",
"Deleted": "boolean",
"CreatedAt": "date",
"Rating": "double",
}
for k, v := range want {
m, ok := props[k].(map[string]any)
if !ok {
t.Errorf("%s: missing or not a map: %#v", k, props[k])
continue
}
if got := m["type"]; got != v {
t.Errorf("%s: type %v, want %v", k, got, v)
}
}
}
func TestOpenSearchBuildMappingNested(t *testing.T) {
props, err := OpenSearchBuildMapping(reflect.TypeFor[osDoc](), nil)
if err != nil {
t.Fatalf("OpenSearchBuildMapping: %v", err)
}
nested, ok := props["nested"].(map[string]any)
if !ok {
t.Fatalf("nested: not a map: %#v", props["nested"])
}
sub, ok := nested["properties"].(map[string]any)
if !ok {
t.Fatalf("nested.properties: missing: %#v", nested)
}
artist, ok := sub["artist"].(map[string]any)
if !ok {
t.Fatalf("nested.artist: %#v", sub)
}
if artist["type"] != "keyword" {
t.Errorf("nested.artist.type: %v", artist["type"])
}
year, ok := sub["year"].(map[string]any)
if !ok {
t.Fatalf("nested.year: %#v", sub)
}
if year["type"] != "integer" {
t.Errorf("nested.year.type: %v (int32 → integer expected)", year["type"])
}
}
func TestOpenSearchBuildMappingOverrides(t *testing.T) {
type doc struct {
Name string `json:"Name"`
Content string `json:"Content"`
Path string `json:"Path"`
MimeType string `json:"MimeType"`
}
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{
"Name": {Analyzer: "lowercaseKeyword"},
"Content": {Type: TypeFulltext},
"Path": {Type: TypePath},
"MimeType": {Type: TypeWildcard},
})
if err != nil {
t.Fatalf("OpenSearchBuildMapping: %v", err)
}
name := props["Name"].(map[string]any)
if name["type"] != "text" || name["analyzer"] != "lowercaseKeyword" {
t.Errorf("Name: %#v", name)
}
content := props["Content"].(map[string]any)
if content["type"] != "text" || content["term_vector"] != "with_positions_offsets" {
t.Errorf("Content: %#v", content)
}
if _, ok := content["analyzer"]; ok {
t.Errorf("Content should leave analyzer unset (use OpenSearch default), got %#v", content["analyzer"])
}
path := props["Path"].(map[string]any)
if path["type"] != "text" || path["analyzer"] != "path_hierarchy" {
t.Errorf("Path: %#v", path)
}
mime := props["MimeType"].(map[string]any)
if mime["type"] != "wildcard" {
t.Errorf("MimeType: %#v", mime)
}
}

View File

@@ -0,0 +1,34 @@
// Package mapping builds search index mappings for bleve and OpenSearch from
// a Go struct via reflection. Field names come from json tags; the caller
// provides overrides for fields that need a specific type or analyzer.
package mapping
// Field type constants used in FieldOpts.Type. An empty Type means the type
// is inferred from the Go field via reflection.
const (
TypeKeyword = "keyword"
TypeFulltext = "fulltext"
TypePath = "path"
TypeWildcard = "wildcard"
TypeNumeric = "numeric"
TypeDatetime = "datetime"
TypeBool = "bool"
TypeObject = "object"
)
// FieldOpts overrides the default type inference for a struct field. Keys in
// the override map are json-tag names (e.g. "Name", "location", "audio.artist"),
// not Go field names.
type FieldOpts struct {
// Type is one of the Type* constants. Empty means "infer from Go type".
Type string
// Analyzer is the name of a custom analyzer registered on the bleve
// IndexMapping (e.g. "lowercaseKeyword", "fulltext"). For OpenSearch it
// becomes the analyzer attribute on the field.
Analyzer string
// IncludeInAll controls bleve's _all field inclusion. Nil means "use the
// bleve default for this field type". Has no effect on OpenSearch.
IncludeInAll *bool
}

View File

@@ -0,0 +1,18 @@
package mapping
import (
"fmt"
"github.com/opencloud-eu/opencloud/pkg/conversions"
)
// PrepareForIndex converts v to the flat map[string]any the backend index
// clients expect, via a json round-trip (conversions.To). overrides is
// reserved for type-specific adaptations wired in by follow-up features.
func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, error) {
out, err := conversions.To[map[string]any](v)
if err != nil {
return nil, fmt.Errorf("mapping: prepare %T: %w", v, err)
}
return out, nil
}

View File

@@ -0,0 +1,83 @@
package mapping
import (
"reflect"
"testing"
)
func TestPrepareForIndexError(t *testing.T) {
// a func field can't be json-marshalled -> conversions.To errors
type bad struct {
F func() `json:"f"`
}
if _, err := PrepareForIndex(bad{}, nil); err == nil {
t.Error("expected error for non-marshallable value")
}
}
func TestPrepareForIndexNil(t *testing.T) {
// a typed nil pointer marshals to null -> nil map, no error, no panic
out, err := PrepareForIndex((*struct{})(nil), nil)
if err != nil || out != nil {
t.Errorf("got (%v, %v), want (nil, nil)", out, err)
}
}
func TestPrepareForIndexFlattensEmbedded(t *testing.T) {
type inner struct {
Name string `json:"Name"`
Size uint64 `json:"Size"`
}
type outer struct {
inner
ID string `json:"ID"`
}
m, err := PrepareForIndex(outer{inner: inner{Name: "a", Size: 7}, ID: "x"}, nil)
if err != nil {
t.Fatalf("PrepareForIndex: %v", err)
}
want := map[string]any{"Name": "a", "Size": float64(7), "ID": "x"}
if !reflect.DeepEqual(m, want) {
t.Fatalf("got %#v, want %#v", m, want)
}
}
func TestPrepareForIndexOmitsNilWithOmitempty(t *testing.T) {
type facet struct {
Artist string `json:"artist"`
}
type doc struct {
Name string `json:"Name"`
Audio *facet `json:"audio,omitempty"`
}
m, err := PrepareForIndex(doc{Name: "n"}, nil)
if err != nil {
t.Fatalf("PrepareForIndex: %v", err)
}
if _, ok := m["audio"]; ok {
t.Errorf("audio should be omitted when nil: %#v", m)
}
if m["Name"] != "n" {
t.Errorf("Name: %v", m["Name"])
}
}
func TestPrepareForIndexIncludesNestedWhenSet(t *testing.T) {
type facet struct {
Artist string `json:"artist"`
}
type doc struct {
Audio *facet `json:"audio,omitempty"`
}
m, err := PrepareForIndex(doc{Audio: &facet{Artist: "A"}}, nil)
if err != nil {
t.Fatalf("PrepareForIndex: %v", err)
}
nested, ok := m["audio"].(map[string]any)
if !ok {
t.Fatalf("audio should be a nested map: %#v", m["audio"])
}
if nested["artist"] != "A" {
t.Errorf("audio.artist: %v", nested["artist"])
}
}

View File

@@ -0,0 +1,49 @@
package mapping
import (
"fmt"
"reflect"
"sort"
"strings"
)
// Validate returns an error if any override key does not match a known field
// name in t. Top-level fields are identified by their json-tag name; nested
// named struct fields are reachable as "parent.child". Embedded (anonymous)
// structs are flattened, so their fields sit at the parent level (as with
// encoding/json).
func Validate(t reflect.Type, overrides map[string]FieldOpts) error {
if len(overrides) == 0 {
return nil
}
names := collectNames(t, "")
var unknown []string
for k := range overrides {
if _, ok := names[k]; !ok {
unknown = append(unknown, k)
}
}
if len(unknown) == 0 {
return nil
}
sort.Strings(unknown)
return fmt.Errorf("mapping: unknown override keys: %s", strings.Join(unknown, ", "))
}
func collectNames(t reflect.Type, prefix string) map[string]struct{} {
out := map[string]struct{}{}
_ = walkFields(t, func(fi fieldInfo) error {
key := fi.Name
if prefix != "" {
key = prefix + "." + fi.Name
}
out[key] = struct{}{}
if sub := structType(fi.GoField.Type); sub != nil {
for k := range collectNames(sub, key) {
out[k] = struct{}{}
}
}
return nil
})
return out
}

View File

@@ -0,0 +1,50 @@
package mapping
import (
"reflect"
"strings"
"testing"
)
type inner struct {
Artist string `json:"artist"`
}
type sample struct {
Name string `json:"Name"`
Audio *inner `json:"audio,omitempty"`
Location *struct { //nolint:unused
Lon float64 `json:"longitude"`
Lat float64 `json:"latitude"`
} `json:"location,omitempty"`
}
func TestValidateAccepts(t *testing.T) {
err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{
"Name": {Analyzer: "lowercaseKeyword"},
"audio": {Type: TypeObject},
"audio.artist": {Analyzer: "lowercaseKeyword"},
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
func TestValidateRejectsUnknown(t *testing.T) {
err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{
"nope": {},
"audio.zzz": {},
})
if err == nil {
t.Fatalf("expected error")
}
if !strings.Contains(err.Error(), "nope") || !strings.Contains(err.Error(), "audio.zzz") {
t.Fatalf("error missing keys: %v", err)
}
}
func TestValidateEmpty(t *testing.T) {
if err := Validate(reflect.TypeFor[sample](), nil); err != nil {
t.Fatalf("empty overrides should pass: %v", err)
}
}

View File

@@ -14,6 +14,7 @@ import (
"github.com/opencloud-eu/opencloud/pkg/conversions"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
@@ -43,7 +44,7 @@ func NewBatch(client *opensearchgoAPI.Client, index string, size int) (*Batch, e
func (b *Batch) Upsert(id string, r search.Resource) error {
return b.withSizeLimit(func() error {
body, err := conversions.To[map[string]any](r)
body, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
if err != nil {
return fmt.Errorf("failed to marshal resource: %w", err)
}

View File

@@ -3,29 +3,32 @@ package opensearch
import (
"bytes"
"context"
"embed"
"errors"
"fmt"
"path"
"maps"
"reflect"
"github.com/go-jose/go-jose/v3/json"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/tidwall/gjson"
searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
var (
ErrManualActionRequired = errors.New("manual action required")
IndexManagerLatest = IndexIndexManagerResourceV2
IndexIndexManagerResourceV1 IndexManager = "resource_v1.json"
IndexIndexManagerResourceV2 IndexManager = "resource_v2.json"
IndexIndexManagerResourceV2 IndexManager = "resource_v2"
)
//go:embed internal/indexes/*.json
var indexes embed.FS
type IndexManager string
// indexGenerators dispatches each IndexManager variant to its builder.
var indexGenerators = map[IndexManager]func() ([]byte, error){
IndexIndexManagerResourceV2: buildResourceV2Mapping,
}
func (m IndexManager) String() string {
b, err := m.MarshalJSON()
if err != nil {
@@ -36,16 +39,56 @@ func (m IndexManager) String() string {
}
func (m IndexManager) MarshalJSON() ([]byte, error) {
filePath := string(m)
body, err := indexes.ReadFile(path.Join("./internal/indexes", filePath))
switch {
case err != nil:
return nil, fmt.Errorf("failed to read index file %s: %w", filePath, err)
case len(body) <= 0:
return nil, fmt.Errorf("index file %s is empty", filePath)
gen, ok := indexGenerators[m]
if !ok {
return nil, fmt.Errorf("unknown index manager %q", string(m))
}
return gen()
}
// buildResourceV2Mapping renders the OpenSearch index template for a
// search.Resource from the shared SearchFieldOverrides. OpenSearch-specific
// tweaks (wildcard MimeType, path_hierarchy Path) are applied on top.
func buildResourceV2Mapping() ([]byte, error) {
resourceType := reflect.TypeFor[search.Resource]()
overrides := maps.Clone(search.Resource{}.SearchFieldOverrides())
overrides["MimeType"] = searchmapping.FieldOpts{Type: searchmapping.TypeWildcard}
overrides["Path"] = searchmapping.FieldOpts{Type: searchmapping.TypePath}
if err := searchmapping.Validate(resourceType, overrides); err != nil {
return nil, err
}
props, err := searchmapping.OpenSearchBuildMapping(resourceType, overrides)
if err != nil {
return nil, err
}
return body, nil
index := map[string]any{
"settings": map[string]any{
"number_of_shards": "1",
"number_of_replicas": "1",
"analysis": map[string]any{
"analyzer": map[string]any{
"path_hierarchy": map[string]any{
"type": "custom",
"tokenizer": "path_hierarchy",
"filter": []string{"lowercase"},
},
"lowercaseKeyword": map[string]any{
"type": "custom",
"tokenizer": "keyword",
"filter": []string{"lowercase"},
},
},
"tokenizer": map[string]any{
"path_hierarchy": map[string]any{"type": "path_hierarchy"},
},
},
},
"mappings": map[string]any{
"properties": props,
},
}
return json.Marshal(index)
}
func (m IndexManager) Apply(ctx context.Context, name string, client *opensearchgoAPI.Client) error {
@@ -118,8 +161,11 @@ func (m IndexManager) Apply(ctx context.Context, name string, client *opensearch
if errs != nil {
return fmt.Errorf(
"index %s already exists and is different from the requested version, %w: %w",
name,
"index %s already exists with a different mapping than the requested version. "+
"There is no in-place migration today: drop the index in OpenSearch (DELETE /%s) "+
"and restart the search service. The index will be recreated with the new mapping. "+
"%w: %w",
name, name,
ErrManualActionRequired,
errors.Join(errs...),
)

View File

@@ -15,6 +15,17 @@ import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
// copyFacet converts a typed pointer from the indexed shape (libregraph) to
// the protobuf shape via conversions.To. Returns nil when src is nil so the
// enclosing Match.Entity field stays nil.
func copyFacet[Dst, Src any](src *Src) *Dst {
if src == nil {
return nil
}
dst, _ := conversions.To[*Dst](src)
return dst
}
func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match, error) {
resource, err := conversions.To[search.Resource](hit.Source)
if err != nil {
@@ -68,26 +79,10 @@ func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match,
return strings.Join(contentHighlights[:], "; ")
}(),
Audio: func() *searchMessage.Audio {
if !strings.HasPrefix(resource.MimeType, "audio/") {
return nil
}
audio, _ := conversions.To[*searchMessage.Audio](resource.Audio)
return audio
}(),
Image: func() *searchMessage.Image {
image, _ := conversions.To[*searchMessage.Image](resource.Image)
return image
}(),
Location: func() *searchMessage.GeoCoordinates {
geoCoordinates, _ := conversions.To[*searchMessage.GeoCoordinates](resource.Location)
return geoCoordinates
}(),
Photo: func() *searchMessage.Photo {
photo, _ := conversions.To[*searchMessage.Photo](resource.Photo)
return photo
}(),
Audio: copyFacet[searchMessage.Audio](resource.Audio),
Image: copyFacet[searchMessage.Image](resource.Image),
Location: copyFacet[searchMessage.GeoCoordinates](resource.Location),
Photo: copyFacet[searchMessage.Photo](resource.Photo),
},
}

View File

@@ -1,49 +0,0 @@
{
"settings": {
"number_of_shards": "1",
"number_of_replicas": "1",
"analysis": {
"analyzer": {
"path_hierarchy": {
"filter": [
"lowercase"
],
"tokenizer": "path_hierarchy",
"type": "custom"
}
},
"tokenizer": {
"path_hierarchy": {
"type": "path_hierarchy"
}
}
}
},
"mappings": {
"properties": {
"ID": {
"type": "keyword"
},
"ParentID": {
"type": "keyword"
},
"RootID": {
"type": "keyword"
},
"MimeType": {
"type": "wildcard",
"doc_values": false
},
"Path": {
"type": "text",
"analyzer": "path_hierarchy"
},
"Deleted": {
"type": "boolean"
},
"Hidden": {
"type": "boolean"
}
}
}
}

View File

@@ -1,56 +0,0 @@
{
"settings": {
"number_of_shards": "1",
"number_of_replicas": "1",
"analysis": {
"analyzer": {
"path_hierarchy": {
"filter": [
"lowercase"
],
"tokenizer": "path_hierarchy",
"type": "custom"
}
},
"tokenizer": {
"path_hierarchy": {
"type": "path_hierarchy"
}
}
}
},
"mappings": {
"properties": {
"Content": {
"type": "text",
"term_vector": "with_positions_offsets"
},
"ID": {
"type": "keyword"
},
"ParentID": {
"type": "keyword"
},
"RootID": {
"type": "keyword"
},
"MimeType": {
"type": "wildcard",
"doc_values": false
},
"Path": {
"type": "text",
"analyzer": "path_hierarchy"
},
"Deleted": {
"type": "boolean"
},
"Hidden": {
"type": "boolean"
},
"Favorites": {
"type": "keyword"
}
}
}
}

View File

@@ -8,17 +8,25 @@ import (
bleveQuery "github.com/blevesearch/bleve/v2/search/query"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/pkg/kql"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
// lowercaseFields lists the bleve fields whose index mapping uses a
// lowercasing analyzer. Values bound to these fields are pre-lowercased
// so query-side matching stays consistent with the index.
// Keep in sync with services/search/pkg/bleve/index.go NewMapping.
var lowercaseFields = map[string]struct{}{
"Name": {},
"Tags": {},
"Favorites": {},
"Content": {},
// lowercaseFields is derived from Resource.SearchFieldOverrides(): any
// field whose override picks a lowercasing analyzer (`lowercaseKeyword`)
// or the fulltext type (which uses a lowercasing analyzer under the hood)
// gets its query-side value pre-lowercased so compile-time matches the
// index-time tokenization. Anything else keeps its original casing.
var lowercaseFields = buildLowercaseFields()
func buildLowercaseFields() map[string]struct{} {
out := map[string]struct{}{}
for key, opts := range (search.Resource{}).SearchFieldOverrides() {
if opts.Analyzer == "lowercaseKeyword" || opts.Type == mapping.TypeFulltext {
out[key] = struct{}{}
}
}
return out
}
var _fields = map[string]string{

View File

@@ -6,6 +6,7 @@ import (
"fmt"
"regexp"
"strings"
"sync"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
@@ -19,6 +20,7 @@ import (
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
)
var scopeRegex = regexp.MustCompile(`scope:\s*([^" "\n\r]*)`)
@@ -51,13 +53,36 @@ type BatchOperator interface {
type Resource struct {
content.Document
ID string
RootID string
Path string
ParentID string
Type uint64
Deleted bool
Hidden bool
ID string `json:"ID"`
RootID string `json:"RootID"`
Path string `json:"Path"`
ParentID string `json:"ParentID"`
Type uint64 `json:"Type"`
Deleted bool `json:"Deleted"`
Hidden bool `json:"Hidden"`
}
// resourceFieldOverrides is built once (it never changes) and reused on hot
// paths instead of reallocating per call.
var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts {
excludeFromAll := false
return map[string]mapping.FieldOpts{
"Name": {Analyzer: "lowercaseKeyword"},
"Content": {Type: mapping.TypeFulltext},
"Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll},
"Favorites": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll},
// Mtime is stored as an RFC3339 string; type it as a date so mtime:>...
// range queries are chronological on both backends (bleve DateRangeQuery
// / OpenSearch date range), not a lexicographic keyword compare.
"Mtime": {Type: mapping.TypeDatetime},
}
})
// SearchFieldOverrides returns the field options the mapping package needs to
// build per-backend index mappings for a Resource (keys are json-tag names).
// The map is shared and read-only; clone it before mutating.
func (Resource) SearchFieldOverrides() map[string]mapping.FieldOpts {
return resourceFieldOverrides()
}
// ResolveReference makes sure the path is relative to the space root

View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
@@ -628,10 +629,10 @@ func (s *Service) doUpsertItem(ref *provider.Reference, batch BatchOperator) {
// determine if metadata needs to be stored in storage as well
metadata := map[string]string{}
addAudioMetadata(metadata, doc.Audio)
addImageMetadata(metadata, doc.Image)
addLocationMetadata(metadata, doc.Location)
addPhotoMetadata(metadata, doc.Photo)
facetToMetadata(metadata, doc.Audio, "libre.graph.audio.")
facetToMetadata(metadata, doc.Image, "libre.graph.image.")
facetToMetadata(metadata, doc.Location, "libre.graph.location.")
facetToMetadata(metadata, doc.Photo, "libre.graph.photo.")
if len(metadata) == 0 {
return
}
@@ -656,43 +657,25 @@ func (s *Service) doUpsertItem(ref *provider.Reference, batch BatchOperator) {
}
}
func addAudioMetadata(metadata map[string]string, audio *libregraph.Audio) {
if audio == nil {
return
// facetToMetadata flattens a libregraph facet (Audio / Image / Location / Photo
// pointer) into the metadata map under the given prefix via the model's ToMap.
// No-op when the facet is nil.
func facetToMetadata[T libregraph.MappedNullable](metadata map[string]string, facet T, prefix string) {
// Only nilable kinds can be nil; IsNil panics on a value type (some
// libregraph models satisfy MappedNullable with a value receiver).
switch v := reflect.ValueOf(facet); v.Kind() {
case reflect.Ptr, reflect.Map, reflect.Slice, reflect.Interface, reflect.Chan, reflect.Func:
if v.IsNil() {
return
}
}
marshalToStringMap(audio, metadata, "libre.graph.audio.")
}
func addImageMetadata(metadata map[string]string, image *libregraph.Image) {
if image == nil {
return
}
marshalToStringMap(image, metadata, "libre.graph.image.")
}
func addLocationMetadata(metadata map[string]string, location *libregraph.GeoCoordinates) {
if location == nil {
return
}
marshalToStringMap(location, metadata, "libre.graph.location.")
}
func addPhotoMetadata(metadata map[string]string, photo *libregraph.Photo) {
if photo == nil {
return
}
marshalToStringMap(photo, metadata, "libre.graph.photo.")
}
func marshalToStringMap[T libregraph.MappedNullable](source T, target map[string]string, prefix string) {
// ToMap never returns a non-nil error ...
m, _ := source.ToMap()
// ToMap never returns a non-nil error.
m, _ := facet.ToMap()
for k, v := range m {
if v == nil {
continue
}
target[prefix+k] = valueToString(v)
metadata[prefix+k] = valueToString(v)
}
}