From e78cad1cbe7aa7ee00ebb6719d35bed418ef8dfe Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Thu, 2 Jul 2026 14:25:04 +0200 Subject: [PATCH 01/54] refactor: reflection-based search mapping Build the bleve and OpenSearch index mappings from the Go struct via reflection (json tags + per-field overrides) instead of hand-rolled mappings and hit deserializers. New mapping package: BleveBuildMapping, OpenSearchBuildMapping, Deserialize[T], PrepareForIndex; field decoding is fail-soft. Mtime is typed as a date so mtime ranges are chronological on both backends. Route CS3 facet parsing through mapping.DeserializeStringMap. The any-valued (bleve hit) and string-valued (CS3 metadata) deserializers share one generic fillStruct walker with a per-value setLeaf callback. --- services/graph/pkg/service/v0/driveitems.go | 131 +------------ services/search/pkg/bleve/backend.go | 8 +- services/search/pkg/bleve/batch.go | 24 ++- services/search/pkg/bleve/bleve.go | 140 ++------------ services/search/pkg/bleve/index.go | 59 +++--- services/search/pkg/bleve/mtime_test.go | 48 +++++ services/search/pkg/content/content.go | 16 +- services/search/pkg/mapping/bleve.go | 92 ++++++++++ services/search/pkg/mapping/bleve_test.go | 116 ++++++++++++ services/search/pkg/mapping/deserialize.go | 172 ++++++++++++++++++ .../search/pkg/mapping/deserialize_string.go | 82 +++++++++ .../pkg/mapping/deserialize_string_test.go | 122 +++++++++++++ .../search/pkg/mapping/deserialize_test.go | 155 ++++++++++++++++ .../search/pkg/mapping/fillstruct_test.go | 127 +++++++++++++ services/search/pkg/mapping/infer.go | 116 ++++++++++++ services/search/pkg/mapping/infer_test.go | 125 +++++++++++++ services/search/pkg/mapping/opensearch.go | 112 ++++++++++++ .../search/pkg/mapping/opensearch_test.go | 132 ++++++++++++++ services/search/pkg/mapping/opts.go | 34 ++++ services/search/pkg/mapping/serialize.go | 18 ++ services/search/pkg/mapping/serialize_test.go | 83 +++++++++ services/search/pkg/mapping/validate.go | 49 +++++ services/search/pkg/mapping/validate_test.go | 50 +++++ services/search/pkg/opensearch/batch.go | 3 +- services/search/pkg/opensearch/index.go | 106 +++++++---- .../opensearch/internal/convert/opensearch.go | 35 ++-- .../internal/indexes/resource_v1.json | 49 ----- .../internal/indexes/resource_v2.json | 56 ------ services/search/pkg/query/bleve/compiler.go | 25 ++- services/search/pkg/search/search.go | 39 +++- services/search/pkg/search/service.go | 55 ++---- 31 files changed, 1862 insertions(+), 517 deletions(-) create mode 100644 services/search/pkg/bleve/mtime_test.go create mode 100644 services/search/pkg/mapping/bleve.go create mode 100644 services/search/pkg/mapping/bleve_test.go create mode 100644 services/search/pkg/mapping/deserialize.go create mode 100644 services/search/pkg/mapping/deserialize_string.go create mode 100644 services/search/pkg/mapping/deserialize_string_test.go create mode 100644 services/search/pkg/mapping/deserialize_test.go create mode 100644 services/search/pkg/mapping/fillstruct_test.go create mode 100644 services/search/pkg/mapping/infer.go create mode 100644 services/search/pkg/mapping/infer_test.go create mode 100644 services/search/pkg/mapping/opensearch.go create mode 100644 services/search/pkg/mapping/opensearch_test.go create mode 100644 services/search/pkg/mapping/opts.go create mode 100644 services/search/pkg/mapping/serialize.go create mode 100644 services/search/pkg/mapping/serialize_test.go create mode 100644 services/search/pkg/mapping/validate.go create mode 100644 services/search/pkg/mapping/validate_test.go delete mode 100644 services/search/pkg/opensearch/internal/indexes/resource_v1.json delete mode 100644 services/search/pkg/opensearch/internal/indexes/resource_v2.json diff --git a/services/graph/pkg/service/v0/driveitems.go b/services/graph/pkg/service/v0/driveitems.go index f9ae229da8..1acd7b9215 100644 --- a/services/graph/pkg/service/v0/driveitems.go +++ b/services/graph/pkg/service/v0/driveitems.go @@ -9,9 +9,7 @@ import ( "net/http" "net/url" "path" - "reflect" "strconv" - "strings" "time" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" @@ -28,6 +26,7 @@ import ( "github.com/opencloud-eu/opencloud/pkg/log" "github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" ) // CreateUploadSession create an upload session to allow your app to upload files up to the maximum file size. @@ -452,130 +451,20 @@ func cs3ResourceToDriveItem(logger *log.Logger, publicBaseURL *url.URL, res *sto } } - if res.GetArbitraryMetadata() != nil { - driveItem.Audio = cs3ResourceToDriveItemAudioFacet(logger, res) - driveItem.Image = cs3ResourceToDriveItemImageFacet(logger, res) - driveItem.Location = cs3ResourceToDriveItemLocationFacet(logger, res) - driveItem.Photo = cs3ResourceToDriveItemPhotoFacet(logger, res) + if metadata := res.GetArbitraryMetadata().GetMetadata(); metadata != nil { + driveItem.Audio = metadataToFacet[libregraph.Audio](metadata, "audio") + driveItem.Image = metadataToFacet[libregraph.Image](metadata, "image") + driveItem.Location = metadataToFacet[libregraph.GeoCoordinates](metadata, "location") + driveItem.Photo = metadataToFacet[libregraph.Photo](metadata, "photo") } return driveItem, nil } -func cs3ResourceToDriveItemAudioFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Audio { - if !strings.HasPrefix(res.GetMimeType(), "audio/") { - return nil - } - - k := res.GetArbitraryMetadata().GetMetadata() - if k == nil { - return nil - } - - var audio = &libregraph.Audio{} - if ok := unmarshalStringMap(logger, audio, k, "libre.graph.audio."); ok { - return audio - } - - return nil -} - -func cs3ResourceToDriveItemImageFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Image { - k := res.GetArbitraryMetadata().GetMetadata() - if k == nil { - return nil - } - - var image = &libregraph.Image{} - if ok := unmarshalStringMap(logger, image, k, "libre.graph.image."); ok { - return image - } - - return nil -} - -func cs3ResourceToDriveItemLocationFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.GeoCoordinates { - k := res.GetArbitraryMetadata().GetMetadata() - if k == nil { - return nil - } - - var location = &libregraph.GeoCoordinates{} - if ok := unmarshalStringMap(logger, location, k, "libre.graph.location."); ok { - return location - } - - return nil -} - -func cs3ResourceToDriveItemPhotoFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Photo { - k := res.GetArbitraryMetadata().GetMetadata() - if k == nil { - return nil - } - - var photo = &libregraph.Photo{} - if ok := unmarshalStringMap(logger, photo, k, "libre.graph.photo."); ok { - return photo - } - - return nil -} - -func getFieldName(structField reflect.StructField) string { - tag := structField.Tag.Get("json") - if tag == "" { - return structField.Name - } - - return strings.Split(tag, ",")[0] -} - -func unmarshalStringMap(logger *log.Logger, out any, flatMap map[string]string, prefix string) bool { - nonEmpty := false - obj := reflect.ValueOf(out).Elem() - timeKind := reflect.TypeOf(&time.Time{}).Elem().Kind() - for i := 0; i < obj.NumField(); i++ { - field := obj.Field(i) - structField := obj.Type().Field(i) - mapKey := prefix + getFieldName(structField) - - if value, ok := flatMap[mapKey]; ok { - if field.Kind() == reflect.Ptr { - newValue := reflect.New(field.Type().Elem()) - var tmp any - var err error - switch t := newValue.Type().Elem().Kind(); t { - case reflect.String: - tmp = value - case reflect.Int32: - tmp, err = strconv.ParseInt(value, 10, 32) - case reflect.Int64: - tmp, err = strconv.ParseInt(value, 10, 64) - case reflect.Float32: - tmp, err = strconv.ParseFloat(value, 32) - case reflect.Float64: - tmp, err = strconv.ParseFloat(value, 64) - case reflect.Bool: - tmp, err = strconv.ParseBool(value) - case timeKind: - tmp, err = time.Parse(time.RFC3339, value) - default: - err = errors.New("unsupported type") - logger.Error().Err(err).Str("type", t.String()).Str("mapKey", mapKey).Msg("target field type for value of mapKey is not supported") - } - if err != nil { - logger.Error().Err(err).Str("mapKey", mapKey).Msg("unmarshalling failed") - continue - } - newValue.Elem().Set(reflect.ValueOf(tmp).Convert(field.Type().Elem())) - field.Set(newValue) - nonEmpty = true - } - } - } - - return nonEmpty +// metadataToFacet builds a DriveItem facet *T from CS3 arbitrary metadata under +// the "libre.graph.." key prefix. Nil when no such keys are present. +func metadataToFacet[T any](metadata map[string]string, facet string) *T { + return mapping.DeserializeStringsAt[T](metadata, "libre.graph."+facet+".") } func cs3ResourceToRemoteItem(res *storageprovider.ResourceInfo) (*libregraph.RemoteItem, error) { diff --git a/services/search/pkg/bleve/backend.go b/services/search/pkg/bleve/backend.go index a9b7ffd1ac..eda67ead85 100644 --- a/services/search/pkg/bleve/backend.go +++ b/services/search/pkg/bleve/backend.go @@ -136,10 +136,10 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques Tags: getFieldSliceValue[string](hit.Fields, "Tags"), Favorites: getFieldSliceValue[string](hit.Fields, "Favorites"), Highlights: getFragmentValue(hit.Fragments, "Content", 0), - Audio: getAudioValue[searchMessage.Audio](hit.Fields), - Image: getImageValue[searchMessage.Image](hit.Fields), - Location: getLocationValue[searchMessage.GeoCoordinates](hit.Fields), - Photo: getPhotoValue[searchMessage.Photo](hit.Fields), + Audio: hitToFacet[searchMessage.Audio](hit.Fields, "audio"), + Image: hitToFacet[searchMessage.Image](hit.Fields, "image"), + Location: hitToFacet[searchMessage.GeoCoordinates](hit.Fields, "location"), + Photo: hitToFacet[searchMessage.Photo](hit.Fields, "photo"), }, } diff --git a/services/search/pkg/bleve/batch.go b/services/search/pkg/bleve/batch.go index 13010831ca..a6966a8a02 100644 --- a/services/search/pkg/bleve/batch.go +++ b/services/search/pkg/bleve/batch.go @@ -10,6 +10,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/utils" "github.com/opencloud-eu/opencloud/pkg/log" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) @@ -36,18 +37,29 @@ func NewBatch(index bleve.Index, size int) (*Batch, error) { func (b *Batch) Upsert(id string, r search.Resource) error { return b.withSizeLimit(func() error { - return b.batch.Index(id, r) + return b.indexResource(id, r) }) } -func (b *Batch) Move(id string, parentID string, targetPath string) error { +// indexResource prepares r for bleve (resolving json tags and splicing in +// type-specific adaptations via the mapping package) and appends it to the +// batch under id. +func (b *Batch) indexResource(id string, r search.Resource) error { + doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides()) + if err != nil { + return err + } + return b.batch.Index(id, doc) +} + +func (b *Batch) Move(id, parentID, location string) error { return b.withSizeLimit(func() error { rootResource, err := searchResourceByID(id, b.index) if err != nil { return err } currentPath := rootResource.Path - nextPath := utils.MakeRelativePath(targetPath) + nextPath := utils.MakeRelativePath(location) rootResource.Path = nextPath rootResource.Name = path.Base(nextPath) @@ -70,7 +82,7 @@ func (b *Batch) Move(id string, parentID string, targetPath string) error { for _, resource := range resources { resource.Hidden = search.IsHidden(resource.Path) - if err := b.batch.Index(resource.ID, resource); err != nil { + if err := b.indexResource(resource.ID, *resource); err != nil { return err } if b.batch.Size() >= b.size { @@ -92,7 +104,7 @@ func (b *Batch) Delete(id string) error { } for _, resource := range affectedResources { - if err := b.batch.Index(resource.ID, resource); err != nil { + if err := b.indexResource(resource.ID, *resource); err != nil { return err } if b.batch.Size() >= b.size { @@ -114,7 +126,7 @@ func (b *Batch) Restore(id string) error { } for _, resource := range affectedResources { - if err := b.batch.Index(resource.ID, resource); err != nil { + if err := b.indexResource(resource.ID, *resource); err != nil { return err } if b.batch.Size() >= b.size { diff --git a/services/search/pkg/bleve/bleve.go b/services/search/pkg/bleve/bleve.go index d396957eca..1bbf53857d 100644 --- a/services/search/pkg/bleve/bleve.go +++ b/services/search/pkg/bleve/bleve.go @@ -1,18 +1,13 @@ package bleve import ( - "reflect" "regexp" - "strings" - "time" bleveSearch "github.com/blevesearch/bleve/v2/search" storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - libregraph "github.com/opencloud-eu/libre-graph-api-go" - "google.golang.org/protobuf/types/known/timestamppb" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" - "github.com/opencloud-eu/opencloud/services/search/pkg/content" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) @@ -75,132 +70,19 @@ func getFragmentValue(m bleveSearch.FieldFragmentMap, key string, idx int) strin return val[idx] } -func getAudioValue[T any](fields map[string]any) *T { - if !strings.HasPrefix(getFieldValue[string](fields, "MimeType"), "audio/") { - return nil - } - - var audio = newPointerOfType[T]() - if ok := unmarshalInterfaceMap(audio, fields, "audio."); ok { - return audio - } - - return nil -} - -func getImageValue[T any](fields map[string]any) *T { - var image = newPointerOfType[T]() - if ok := unmarshalInterfaceMap(image, fields, "image."); ok { - return image - } - - return nil -} - -func getLocationValue[T any](fields map[string]any) *T { - var location = newPointerOfType[T]() - if ok := unmarshalInterfaceMap(location, fields, "location."); ok { - return location - } - - return nil -} - -func getPhotoValue[T any](fields map[string]any) *T { - var photo = newPointerOfType[T]() - if ok := unmarshalInterfaceMap(photo, fields, "photo."); ok { - return photo - } - - return nil -} - -func newPointerOfType[T any]() *T { - t := reflect.TypeOf((*T)(nil)).Elem() - ptr := reflect.New(t).Interface() - return ptr.(*T) -} - -func unmarshalInterfaceMap(out any, flatMap map[string]any, prefix string) bool { - nonEmpty := false - obj := reflect.ValueOf(out).Elem() - for i := 0; i < obj.NumField(); i++ { - field := obj.Field(i) - structField := obj.Type().Field(i) - mapKey := prefix + getFieldName(structField) - - if value, ok := flatMap[mapKey]; ok { - if field.Kind() == reflect.Ptr { - alloc := reflect.New(field.Type().Elem()) - elemType := field.Type().Elem() - - // convert time strings from index for search requests - if elemType == reflect.TypeOf(timestamppb.Timestamp{}) { - if strValue, ok := value.(string); ok { - if parsedTime, err := time.Parse(time.RFC3339, strValue); err == nil { - alloc.Elem().Set(reflect.ValueOf(*timestamppb.New(parsedTime))) - field.Set(alloc) - nonEmpty = true - } - } - continue - } - - // convert time strings from index for libregraph structs when updating resources - if elemType == reflect.TypeOf(time.Time{}) { - if strValue, ok := value.(string); ok { - if parsedTime, err := time.Parse(time.RFC3339, strValue); err == nil { - alloc.Elem().Set(reflect.ValueOf(parsedTime)) - field.Set(alloc) - nonEmpty = true - } - } - continue - } - - alloc.Elem().Set(reflect.ValueOf(value).Convert(elemType)) - field.Set(alloc) - nonEmpty = true - } - } - } - - return nonEmpty -} - -func getFieldName(structField reflect.StructField) string { - tag := structField.Tag.Get("json") - if tag == "" { - return structField.Name - } - - return strings.Split(tag, ",")[0] +// hitToFacet builds a search Entity facet *T from a bleve hit's fields under the +// given key prefix. Nil when the hit has no such fields. +func hitToFacet[T any](fields map[string]any, prefix string) *T { + return mapping.DeserializeAt[T](fields, prefix) } +// matchToResource reconstructs a search.Resource from a bleve hit. Used by +// the Move / Delete / Restore / Purge paths that round-trip a record through +// the index. Always returns a non-nil *Resource: Deserialize is fail-soft +// for per-field parse errors, so corrupted hit values surface as zero +// values on individual fields instead of dropping the whole record. func matchToResource(match *bleveSearch.DocumentMatch) *search.Resource { - return &search.Resource{ - ID: getFieldValue[string](match.Fields, "ID"), - RootID: getFieldValue[string](match.Fields, "RootID"), - Path: getFieldValue[string](match.Fields, "Path"), - ParentID: getFieldValue[string](match.Fields, "ParentID"), - Type: uint64(getFieldValue[float64](match.Fields, "Type")), - Deleted: getFieldValue[bool](match.Fields, "Deleted"), - Hidden: getFieldValue[bool](match.Fields, "Hidden"), - Document: content.Document{ - Name: getFieldValue[string](match.Fields, "Name"), - Title: getFieldValue[string](match.Fields, "Title"), - Size: uint64(getFieldValue[float64](match.Fields, "Size")), - Mtime: getFieldValue[string](match.Fields, "Mtime"), - MimeType: getFieldValue[string](match.Fields, "MimeType"), - Content: getFieldValue[string](match.Fields, "Content"), - Tags: getFieldSliceValue[string](match.Fields, "Tags"), - Favorites: getFieldSliceValue[string](match.Fields, "Favorites"), - Audio: getAudioValue[libregraph.Audio](match.Fields), - Image: getImageValue[libregraph.Image](match.Fields), - Location: getLocationValue[libregraph.GeoCoordinates](match.Fields), - Photo: getPhotoValue[libregraph.Photo](match.Fields), - }, - } + return mapping.Deserialize[search.Resource](match.Fields) } func escapeQuery(s string) string { diff --git a/services/search/pkg/bleve/index.go b/services/search/pkg/bleve/index.go index 0aa61b4c48..8f52d847cb 100644 --- a/services/search/pkg/bleve/index.go +++ b/services/search/pkg/bleve/index.go @@ -4,17 +4,20 @@ import ( "errors" "math" "path/filepath" + "reflect" "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" regexpCharFilter "github.com/blevesearch/bleve/v2/analysis/char/regexp" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/token/porter" "github.com/blevesearch/bleve/v2/analysis/tokenizer/single" "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" "github.com/blevesearch/bleve/v2/mapping" storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) @@ -47,46 +50,20 @@ func NewIndex(root string) (bleve.Index, error) { } func NewMapping() (mapping.IndexMapping, error) { - words := func() *mapping.FieldMapping { - fm := bleve.NewTextFieldMapping() - fm.Analyzer = "lowercaseWords" - - return fm + resourceType := reflect.TypeFor[search.Resource]() + overrides := search.Resource{}.SearchFieldOverrides() + if err := searchmapping.Validate(resourceType, overrides); err != nil { + return nil, err } - - whole := func(field string) *mapping.FieldMapping { - fm := bleve.NewTextFieldMapping() - fm.Analyzer = "lowercaseKeyword" - fm.IncludeInAll = false - fm.Name = field + wildcardSuffix - - return fm + docMapping, err := searchmapping.BleveBuildMapping(resourceType, overrides) + if err != nil { + return nil, err } - lowercaseMapping := bleve.NewTextFieldMapping() - lowercaseMapping.IncludeInAll = false - lowercaseMapping.Analyzer = "lowercaseKeyword" - - contentMapping := words() - contentMapping.IncludeInAll = false - - docMapping := bleve.NewDocumentMapping() - docMapping.AddFieldMappingsAt("Name", - words(), - whole("Name"), - ) - docMapping.AddFieldMappingsAt("Title", - words(), - whole("Title"), - ) - docMapping.AddFieldMappingsAt("Tags", lowercaseMapping) - docMapping.AddFieldMappingsAt("Favorites", lowercaseMapping) - docMapping.AddFieldMappingsAt("Content", contentMapping) - indexMapping := bleve.NewIndexMapping() indexMapping.DefaultAnalyzer = keyword.Name indexMapping.DefaultMapping = docMapping - err := indexMapping.AddCustomCharFilter("dotToSpace", + err = indexMapping.AddCustomCharFilter("dotToSpace", map[string]any{ "type": regexpCharFilter.Name, "regexp": `\.`, @@ -124,6 +101,20 @@ func NewMapping() (mapping.IndexMapping, error) { return nil, err } + err = indexMapping.AddCustomAnalyzer("fulltext", + map[string]any{ + "type": custom.Name, + "tokenizer": unicode.Name, + "token_filters": []string{ + lowercase.Name, + porter.Name, + }, + }, + ) + if err != nil { + return nil, err + } + return indexMapping, nil } diff --git a/services/search/pkg/bleve/mtime_test.go b/services/search/pkg/bleve/mtime_test.go new file mode 100644 index 0000000000..d7475b0316 --- /dev/null +++ b/services/search/pkg/bleve/mtime_test.go @@ -0,0 +1,48 @@ +package bleve_test + +import ( + "testing" + + bleveSearch "github.com/blevesearch/bleve/v2" + bquery "github.com/blevesearch/bleve/v2/search/query" + + "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" + "github.com/opencloud-eu/opencloud/services/search/pkg/content" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +// Mtime is typed as a date, so range queries are chronological, not a +// lexicographic keyword compare. +func TestMtimeDateRange(t *testing.T) { + m, err := bleve.NewMapping() + if err != nil { + t.Fatal(err) + } + idx, err := bleveSearch.NewMemOnly(m) + if err != nil { + t.Fatal(err) + } + r := search.Resource{ID: "x", Document: content.Document{Name: "f", Mtime: "2026-03-15T12:00:00.123456789Z"}} + doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides()) + if err != nil { + t.Fatal(err) + } + if err := idx.Index(r.ID, doc); err != nil { + t.Fatal(err) + } + + hits := func(qs string) uint64 { + res, err := idx.Search(bleveSearch.NewSearchRequest(bquery.NewQueryStringQuery(qs))) + if err != nil { + t.Fatalf("%s: %v", qs, err) + } + return res.Total + } + if got := hits(`Mtime:>"2026-01-01T00:00:00Z"`); got != 1 { + t.Errorf("in-range: got %d hits, want 1", got) + } + if got := hits(`Mtime:>"2026-06-01T00:00:00Z"`); got != 0 { + t.Errorf("out-of-range: got %d hits, want 0", got) + } +} diff --git a/services/search/pkg/content/content.go b/services/search/pkg/content/content.go index 1c5d914231..5a5d3bc1b2 100644 --- a/services/search/pkg/content/content.go +++ b/services/search/pkg/content/content.go @@ -14,14 +14,14 @@ func init() { // Document wraps all resource meta fields, // it is used as a content extraction result. type Document struct { - Title string - Name string - Content string - Size uint64 - Mtime string `json:"Mtime,omitempty"` - MimeType string - Tags []string - Favorites []string + Title string `json:"Title"` + Name string `json:"Name"` + Content string `json:"Content"` + Size uint64 `json:"Size"` + Mtime string `json:"Mtime,omitempty"` + MimeType string `json:"MimeType"` + Tags []string `json:"Tags"` + Favorites []string `json:"Favorites"` Audio *libregraph.Audio `json:"audio,omitempty"` Image *libregraph.Image `json:"image,omitempty"` Location *libregraph.GeoCoordinates `json:"location,omitempty"` diff --git a/services/search/pkg/mapping/bleve.go b/services/search/pkg/mapping/bleve.go new file mode 100644 index 0000000000..d8329b72f8 --- /dev/null +++ b/services/search/pkg/mapping/bleve.go @@ -0,0 +1,92 @@ +package mapping + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2" + bleveMapping "github.com/blevesearch/bleve/v2/mapping" +) + +// BleveBuildMapping builds a bleve DocumentMapping for t by walking the +// struct via reflection. Field names come from json tags; overrides are +// keyed by those names (or dotted paths for nested fields). +// +// The returned mapping references analyzer names (Analyzer field on the +// FieldOpts, plus "fulltext" / "path_hierarchy" for the corresponding Types); +// the caller is responsible for registering those analyzers on the enclosing +// IndexMapping. +func BleveBuildMapping(t reflect.Type, overrides map[string]FieldOpts) (*bleveMapping.DocumentMapping, error) { + return buildBleveDocMapping(t, overrides, "") +} + +func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix string) (*bleveMapping.DocumentMapping, error) { + doc := bleve.NewDocumentMapping() + err := walkFields(t, func(fi fieldInfo) error { + key := fi.Name + if prefix != "" { + key = prefix + "." + fi.Name + } + opts := overrides[key] + fieldType := opts.Type + if fieldType == "" { + fieldType = inferType(fi.GoField.Type) + } + + if fieldType == TypeObject { + sub := structType(fi.GoField.Type) + if sub == nil { + return fmt.Errorf("mapping: object type on non-struct field %q", key) + } + subDoc, err := buildBleveDocMapping(sub, overrides, key) + if err != nil { + return err + } + doc.AddSubDocumentMapping(fi.Name, subDoc) + return nil + } + + fm, err := bleveFieldMapping(fieldType, opts) + if err != nil { + return fmt.Errorf("mapping: field %q: %w", key, err) + } + doc.AddFieldMappingsAt(fi.Name, fm) + return nil + }) + return doc, err +} + +func bleveFieldMapping(fieldType string, opts FieldOpts) (*bleveMapping.FieldMapping, error) { + switch fieldType { + case TypeWildcard: + // bleve has no wildcard type; fall back to keyword-ish text. + fieldType = TypeKeyword + fallthrough + case TypeKeyword, TypeFulltext, TypePath: + fm := bleve.NewTextFieldMapping() + switch { + case opts.Analyzer != "": + fm.Analyzer = opts.Analyzer + case fieldType == TypeFulltext: + fm.Analyzer = "fulltext" + case fieldType == TypePath: + fm.Analyzer = "path_hierarchy" + } + switch { + case opts.IncludeInAll != nil: + fm.IncludeInAll = *opts.IncludeInAll + case fieldType == TypeFulltext, fieldType == TypePath: + fm.IncludeInAll = false + } + return fm, nil + case TypeNumeric: + return bleve.NewNumericFieldMapping(), nil + case TypeBool: + return bleve.NewBooleanFieldMapping(), nil + case TypeDatetime: + return bleve.NewDateTimeFieldMapping(), nil + case "": + return nil, fmt.Errorf("no type inferred and no override") + } + return nil, fmt.Errorf("unsupported type %q", fieldType) +} diff --git a/services/search/pkg/mapping/bleve_test.go b/services/search/pkg/mapping/bleve_test.go new file mode 100644 index 0000000000..8e2764630e --- /dev/null +++ b/services/search/pkg/mapping/bleve_test.go @@ -0,0 +1,116 @@ +package mapping + +import ( + "reflect" + "testing" + "time" +) + +type bleveDoc struct { + Name string `json:"Name"` + Content string `json:"Content"` + Tags []string `json:"Tags"` + Size uint64 `json:"Size"` + Deleted bool `json:"Deleted"` + CreatedAt time.Time `json:"CreatedAt"` + Nested *nested `json:"nested,omitempty"` +} + +type nested struct { + Artist string `json:"artist"` + Year int `json:"year"` +} + +// bleve wildcard falls back to keyword-ish text (bleve has no wildcard type). +func TestBleveWildcardFallback(t *testing.T) { + type doc struct { + Mime string `json:"mime"` + } + dm, err := BleveBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"mime": {Type: TypeWildcard}}) + if err != nil { + t.Fatalf("BleveBuildMapping: %v", err) + } + fms := dm.Properties["mime"].Fields + if len(fms) != 1 || fms[0].Type != "text" { + t.Fatalf("wildcard should map to text, got %+v", fms) + } +} + +func TestBleveBuildMappingInferredTypes(t *testing.T) { + dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil) + if err != nil { + t.Fatalf("BleveBuildMapping: %v", err) + } + cases := map[string]string{ + "Name": "text", + "Content": "text", + "Tags": "text", + "Size": "number", + "Deleted": "boolean", + "CreatedAt": "datetime", + } + for field, wantType := range cases { + prop := dm.Properties[field] + if prop == nil { + t.Errorf("missing property %q", field) + continue + } + if len(prop.Fields) == 0 { + t.Errorf("%q: no field mappings", field) + continue + } + if got := prop.Fields[0].Type; got != wantType { + t.Errorf("%q: got type %q, want %q", field, got, wantType) + } + } +} + +func TestBleveBuildMappingNestedIsSubDocument(t *testing.T) { + dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil) + if err != nil { + t.Fatalf("BleveBuildMapping: %v", err) + } + sub := dm.Properties["nested"] + if sub == nil { + t.Fatal("missing nested sub-document") + } + if sub.Properties["artist"] == nil || sub.Properties["year"] == nil { + t.Fatalf("nested fields missing: %#v", sub.Properties) + } + if got := sub.Properties["artist"].Fields[0].Type; got != "text" { + t.Errorf("nested.artist: type %q, want text", got) + } + if got := sub.Properties["year"].Fields[0].Type; got != "number" { + t.Errorf("nested.year: type %q, want number", got) + } +} + +func TestBleveBuildMappingOverrides(t *testing.T) { + includeInAllFalse := false + dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{ + "Name": {Analyzer: "lowercaseKeyword"}, + "Content": {Type: TypeFulltext}, + "Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &includeInAllFalse}, + }) + if err != nil { + t.Fatalf("BleveBuildMapping: %v", err) + } + nameField := dm.Properties["Name"].Fields[0] + if nameField.Analyzer != "lowercaseKeyword" { + t.Errorf("Name analyzer: %q, want lowercaseKeyword", nameField.Analyzer) + } + if !nameField.IncludeInAll { + t.Errorf("Name IncludeInAll should stay default-true when not overridden") + } + contentField := dm.Properties["Content"].Fields[0] + if contentField.Analyzer != "fulltext" { + t.Errorf("Content analyzer: %q, want fulltext", contentField.Analyzer) + } + if contentField.IncludeInAll { + t.Errorf("Content IncludeInAll should default to false for fulltext type") + } + tagsField := dm.Properties["Tags"].Fields[0] + if tagsField.IncludeInAll { + t.Errorf("Tags IncludeInAll should honor the explicit false override") + } +} diff --git a/services/search/pkg/mapping/deserialize.go b/services/search/pkg/mapping/deserialize.go new file mode 100644 index 0000000000..f8c2f844a1 --- /dev/null +++ b/services/search/pkg/mapping/deserialize.go @@ -0,0 +1,172 @@ +package mapping + +import ( + "fmt" + "reflect" + "time" + + "google.golang.org/protobuf/types/known/timestamppb" +) + +// Deserialize builds a *T from bleve's flat hit.Fields map (json-tag keys, +// "parent.child" for nested pointers). Used to rebuild a search Resource from a +// hit. Fail-soft: an unparseable field stays at its zero value. +func Deserialize[T any](fields map[string]any) *T { + t := reflect.TypeFor[T]() + if t.Kind() != reflect.Struct { + panic(fmt.Sprintf("mapping: Deserialize requires a struct type, got %v", t)) + } + out := reflect.New(t) + fillStruct(out.Elem(), fields, "", setValue) + return out.Interface().(*T) +} + +// DeserializeAt is Deserialize scoped to a dotted prefix, used to rebuild one +// search-result facet (e.g. "audio"). Returns nil when nothing matched, so the +// caller can leave the enclosing pointer nil. +func DeserializeAt[T any](fields map[string]any, prefix string) *T { + t := reflect.TypeFor[T]() + if t.Kind() != reflect.Struct { + panic(fmt.Sprintf("mapping: DeserializeAt requires a struct type, got %v", t)) + } + out := reflect.New(t) + if !fillStruct(out.Elem(), fields, prefix, setValue) { + return nil + } + return out.Interface().(*T) +} + +// fillStruct walks v's exported fields, reading values from the flat fields map +// (json-tag keys, "parent.child" for nested pointers). setLeaf converts each raw +// value into a leaf, which is what lets the any-valued and string-valued +// deserializers share one walker. Returns true if any leaf was populated. +func fillStruct[V any](v reflect.Value, fields map[string]V, prefix string, setLeaf func(reflect.Value, V) error) bool { + t := v.Type() + touched := false + for i := 0; i < t.NumField(); i++ { + fi := resolveField(t.Field(i)) + if fi.Skip { + continue + } + fv := v.Field(i) + + if fi.Embedded { + // Embedded *struct: allocate, recurse, keep the pointer only if set. + if fv.Kind() == reflect.Ptr { + if fv.Type().Elem().Kind() != reflect.Struct || !fv.CanSet() { + continue + } + alloc := reflect.New(fv.Type().Elem()) + if fillStruct(alloc.Elem(), fields, prefix, setLeaf) { + fv.Set(alloc) + touched = true + } + continue + } + if fv.Kind() == reflect.Struct && fillStruct(fv, fields, prefix, setLeaf) { + touched = true + } + continue + } + + key := fi.Name + if prefix != "" { + key = prefix + "." + fi.Name + } + + // Pointer to a nested (non-time) struct: recurse, keep the pointer only + // if a field was populated. + if fv.Kind() == reflect.Ptr { + if elem := fv.Type().Elem(); elem.Kind() == reflect.Struct && elem != timeType && elem != timestampType { + alloc := reflect.New(elem) + if fillStruct(alloc.Elem(), fields, key, setLeaf) { + fv.Set(alloc) + touched = true + } + continue + } + } + + if raw, ok := fields[key]; ok && setLeaf(fv, raw) == nil { + touched = true + } + } + return touched +} + +// setValue writes raw (an any value from a bleve hit) into v, converting to the +// field's type. Returns an error on nil or a type mismatch. +func setValue(v reflect.Value, raw any) error { + if v.Kind() == reflect.Ptr { + alloc := reflect.New(v.Type().Elem()) + if err := setValue(alloc.Elem(), raw); err != nil { + return err + } + v.Set(alloc) + return nil + } + if v.Type() == timeType || v.Type() == timestampType { + t, ok := parseTime(raw) + if !ok { + return fmt.Errorf("not an RFC3339 time: %v", raw) + } + setParsedTime(v, t) + return nil + } + if v.Kind() == reflect.Slice { + return setSlice(v, raw) + } + rv := reflect.ValueOf(raw) + if !rv.IsValid() { + return fmt.Errorf("nil value for %s", v.Type()) + } + if !rv.Type().ConvertibleTo(v.Type()) { + return fmt.Errorf("cannot convert %s to %s", rv.Type(), v.Type()) + } + v.Set(rv.Convert(v.Type())) + return nil +} + +func setSlice(v reflect.Value, raw any) error { + items, ok := raw.([]any) + if !ok { + // bleve unwraps single-element slices; re-wrap here. + items = []any{raw} + } + // Compact in place with a single MakeSlice: unparseable elements are + // dropped, Slice(0, j) trims the tail. + out := reflect.MakeSlice(v.Type(), len(items), len(items)) + j := 0 + for _, item := range items { + if setValue(out.Index(j), item) == nil { + j++ + } + } + if j == 0 { + return fmt.Errorf("no slice elements set from %T", raw) + } + v.Set(out.Slice(0, j)) + return nil +} + +func parseTime(raw any) (time.Time, bool) { + s, ok := raw.(string) + if !ok { + return time.Time{}, false + } + t, err := time.Parse(time.RFC3339, s) + if err != nil { + return time.Time{}, false + } + return t, true +} + +// setParsedTime writes t into v, which must be a time.Time or +// timestamppb.Timestamp field. +func setParsedTime(v reflect.Value, t time.Time) { + if v.Type() == timestampType { + v.Set(reflect.ValueOf(*timestamppb.New(t))) + return + } + v.Set(reflect.ValueOf(t)) +} diff --git a/services/search/pkg/mapping/deserialize_string.go b/services/search/pkg/mapping/deserialize_string.go new file mode 100644 index 0000000000..16eff0a08d --- /dev/null +++ b/services/search/pkg/mapping/deserialize_string.go @@ -0,0 +1,82 @@ +package mapping + +import ( + "fmt" + "reflect" + "strconv" + "strings" + "time" +) + +// DeserializeStringsAt is DeserializeAt for a string-valued map (e.g. CS3 +// ArbitraryMetadata), parsing each string into the field's Go type via strconv/ +// time.Parse. Used to build a graph DriveItem facet. Returns nil when nothing +// under the prefix matched. +func DeserializeStringsAt[T any](fields map[string]string, prefix string) *T { + t := reflect.TypeFor[T]() + if t.Kind() != reflect.Struct { + panic(fmt.Sprintf("mapping: DeserializeStringsAt requires a struct type, got %v", t)) + } + out := reflect.New(t) + // Callers pass a flat-key prefix with a trailing dot (e.g. + // "libre.graph.audio."); fillStruct joins segments with ".", so drop it. + if !fillStruct(out.Elem(), fields, strings.TrimSuffix(prefix, "."), setValueFromString) { + return nil + } + return out.Interface().(*T) +} + +// setValueFromString parses the string raw into v's Go type via strconv/ +// time.Parse, returning a descriptive error on failure. +func setValueFromString(v reflect.Value, raw string) error { + if v.Kind() == reflect.Ptr { + alloc := reflect.New(v.Type().Elem()) + if err := setValueFromString(alloc.Elem(), raw); err != nil { + return err + } + v.Set(alloc) + return nil + } + if v.Type() == timeType || v.Type() == timestampType { + t, err := time.Parse(time.RFC3339, raw) + if err != nil { + return fmt.Errorf("parse time %q: %w", raw, err) + } + setParsedTime(v, t) + return nil + } + switch v.Kind() { + case reflect.String: + v.SetString(raw) + return nil + case reflect.Bool: + b, err := strconv.ParseBool(raw) + if err != nil { + return fmt.Errorf("parse bool %q: %w", raw, err) + } + v.SetBool(b) + return nil + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + n, err := strconv.ParseInt(raw, 10, v.Type().Bits()) + if err != nil { + return fmt.Errorf("parse int %q: %w", raw, err) + } + v.SetInt(n) + return nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + n, err := strconv.ParseUint(raw, 10, v.Type().Bits()) + if err != nil { + return fmt.Errorf("parse uint %q: %w", raw, err) + } + v.SetUint(n) + return nil + case reflect.Float32, reflect.Float64: + f, err := strconv.ParseFloat(raw, v.Type().Bits()) + if err != nil { + return fmt.Errorf("parse float %q: %w", raw, err) + } + v.SetFloat(f) + return nil + } + return fmt.Errorf("unsupported target kind %s", v.Kind()) +} diff --git a/services/search/pkg/mapping/deserialize_string_test.go b/services/search/pkg/mapping/deserialize_string_test.go new file mode 100644 index 0000000000..981f5e50a5 --- /dev/null +++ b/services/search/pkg/mapping/deserialize_string_test.go @@ -0,0 +1,122 @@ +package mapping + +import ( + "reflect" + "testing" + "time" + + "google.golang.org/protobuf/types/known/timestamppb" +) + +type stringFacet struct { + Artist *string `json:"artist,omitempty"` + Year *int32 `json:"year,omitempty"` + Duration *int64 `json:"duration,omitempty"` + Rating *float64 `json:"rating,omitempty"` + Explicit *bool `json:"explicit,omitempty"` + Taken *time.Time `json:"takenDateTime,omitempty"` +} + +func TestSetValueFromStringUnsupportedKind(t *testing.T) { + v := reflect.New(reflect.TypeFor[[]int]()).Elem() // settable slice + if err := setValueFromString(v, "x"); err == nil { + t.Error("expected error for unsupported target kind (slice)") + } +} + +func TestDeserializeStringsAtBasicTypes(t *testing.T) { + r := DeserializeStringsAt[stringFacet](map[string]string{ + "libre.graph.audio.artist": "Queen", + "libre.graph.audio.year": "1975", + "libre.graph.audio.duration": "354000", + "libre.graph.audio.rating": "4.9", + "libre.graph.audio.explicit": "true", + "libre.graph.audio.takenDateTime": "2024-01-02T03:04:05Z", + }, "libre.graph.audio.") + if r == nil { + t.Fatal("expected non-nil *stringFacet") + } + if r.Artist == nil || *r.Artist != "Queen" { + t.Errorf("Artist: %#v", r.Artist) + } + if r.Year == nil || *r.Year != 1975 { + t.Errorf("Year: %#v", r.Year) + } + if r.Duration == nil || *r.Duration != 354000 { + t.Errorf("Duration: %#v", r.Duration) + } + if r.Rating == nil || *r.Rating != 4.9 { + t.Errorf("Rating: %#v", r.Rating) + } + if r.Explicit == nil || !*r.Explicit { + t.Errorf("Explicit: %#v", r.Explicit) + } + if r.Taken == nil || !r.Taken.Equal(time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC)) { + t.Errorf("Taken: %#v", r.Taken) + } +} + +func TestDeserializeStringsAtReturnsNilWhenEmpty(t *testing.T) { + r := DeserializeStringsAt[stringFacet](map[string]string{ + "libre.graph.image.width": "1200", + }, "libre.graph.audio.") + if r != nil { + t.Fatalf("expected nil, got %#v", r) + } +} + +func TestDeserializeStringsAtTimestamppb(t *testing.T) { + type photoFacet struct { + Taken *timestamppb.Timestamp `json:"takenDateTime,omitempty"` + } + r := DeserializeStringsAt[photoFacet](map[string]string{ + "libre.graph.photo.takenDateTime": "2024-05-06T07:08:09Z", + }, "libre.graph.photo.") + if r == nil || r.Taken == nil { + t.Fatalf("Taken missing: %#v", r) + } + want := time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC) + if !r.Taken.AsTime().Equal(want) { + t.Errorf("Taken: got %v, want %v", r.Taken.AsTime(), want) + } +} + +func TestDeserializeStringsAtIsFailSoft(t *testing.T) { + // A single malformed field (year is unparseable as int) must not drop + // the whole facet. The bad field stays at zero value, the rest of the + // facet still populates. Mirrors the bleve-hit Deserialize behavior. + r := DeserializeStringsAt[stringFacet](map[string]string{ + "libre.graph.audio.artist": "Iron Maiden", + "libre.graph.audio.year": "not-a-number", + "libre.graph.audio.duration": "354000", + "libre.graph.audio.explicit": "not-a-bool", + "libre.graph.audio.rating": "4.9", + }, "libre.graph.audio.") + if r == nil { + t.Fatal("expected non-nil *stringFacet despite bad fields") + } + if r.Artist == nil || *r.Artist != "Iron Maiden" { + t.Errorf("Artist should still be populated, got %#v", r.Artist) + } + if r.Duration == nil || *r.Duration != 354000 { + t.Errorf("Duration should still be populated, got %#v", r.Duration) + } + if r.Rating == nil || *r.Rating != 4.9 { + t.Errorf("Rating should still be populated, got %#v", r.Rating) + } + if r.Year != nil { + t.Errorf("Year should stay nil for bad int, got %#v", r.Year) + } + if r.Explicit != nil { + t.Errorf("Explicit should stay nil for bad bool, got %#v", r.Explicit) + } +} + +func TestDeserializeStringsAtPanicsOnNonStruct(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic for non-struct T") + } + }() + DeserializeStringsAt[int](nil, "") +} diff --git a/services/search/pkg/mapping/deserialize_test.go b/services/search/pkg/mapping/deserialize_test.go new file mode 100644 index 0000000000..ec92191ce3 --- /dev/null +++ b/services/search/pkg/mapping/deserialize_test.go @@ -0,0 +1,155 @@ +package mapping + +import ( + "testing" + "time" + + "google.golang.org/protobuf/types/known/timestamppb" +) + +type Leaf struct { + Name string `json:"Name"` + Size uint64 `json:"Size"` + Deleted bool `json:"Deleted"` + Tags []string `json:"Tags"` + Favorites []string `json:"Favorites"` +} + +type audio struct { + Artist *string `json:"artist,omitempty"` + Year *int32 `json:"year,omitempty"` +} + +type photo struct { + Taken *timestamppb.Timestamp `json:"takenDateTime,omitempty"` + Mtime *time.Time `json:"mtime,omitempty"` +} + +type embedded struct { + Leaf + Audio *audio `json:"audio,omitempty"` + Photo *photo `json:"photo,omitempty"` +} + +func TestDeserializeAtNonStructPanics(t *testing.T) { + defer func() { + if recover() == nil { + t.Error("expected panic for non-struct type") + } + }() + _ = DeserializeAt[int](map[string]any{}, "") +} + +func TestDeserializeLeafFields(t *testing.T) { + r := Deserialize[Leaf](map[string]any{ + "Name": "n", + "Size": float64(42), + "Deleted": true, + }) + if r.Name != "n" || r.Size != 42 || !r.Deleted { + t.Fatalf("got %#v", r) + } +} + +func TestDeserializeScalarToSlice(t *testing.T) { + r := Deserialize[Leaf](map[string]any{ + "Tags": "single", + "Favorites": []any{"a", "b"}, + }) + if len(r.Tags) != 1 || r.Tags[0] != "single" { + t.Errorf("Tags: %#v", r.Tags) + } + if len(r.Favorites) != 2 || r.Favorites[0] != "a" || r.Favorites[1] != "b" { + t.Errorf("Favorites: %#v", r.Favorites) + } +} + +func TestDeserializeTimestamp(t *testing.T) { + r := Deserialize[embedded](map[string]any{ + "photo.takenDateTime": "2024-01-02T03:04:05Z", + "photo.mtime": "2024-05-06T07:08:09Z", + }) + if r.Photo == nil { + t.Fatal("Photo is nil") + } + if r.Photo.Taken == nil { + t.Fatal("Taken is nil") + } + expected := time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC) + if !r.Photo.Taken.AsTime().Equal(expected) { + t.Errorf("Taken: got %v, want %v", r.Photo.Taken.AsTime(), expected) + } + if r.Photo.Mtime == nil { + t.Fatal("Mtime is nil") + } + if !r.Photo.Mtime.Equal(time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC)) { + t.Errorf("Mtime: %v", r.Photo.Mtime) + } +} + +func TestDeserializeIsFailSoft(t *testing.T) { + // Malformed values (type mismatch, unparseable time) leave the + // affected field at its zero value instead of dropping the whole + // record. Matches the pre-refactor getFieldValue behavior so + // matchToResource never returns nil on a corrupted hit. + r := Deserialize[embedded](map[string]any{ + "Name": "n", + "Size": "not-a-number", // wrong type + "Deleted": true, + "photo.takenDateTime": "not-an-rfc3339-time", + "photo.mtime": "2024-05-06T07:08:09Z", + }) + if r == nil { + t.Fatal("expected non-nil *embedded even with partial corruption") + } + if r.Name != "n" { + t.Errorf("Name: %q", r.Name) + } + if r.Size != 0 { + t.Errorf("Size should stay zero on mismatch, got %d", r.Size) + } + if !r.Deleted { + t.Errorf("Deleted should still be true") + } + if r.Photo == nil { + t.Fatal("Photo should be populated because Mtime parsed ok") + } + if r.Photo.Taken != nil { + t.Errorf("Taken should stay nil for unparseable time, got %v", r.Photo.Taken) + } + if r.Photo.Mtime == nil { + t.Error("Mtime should be parsed") + } +} + +func TestDeserializePanicsOnNonStruct(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic for non-struct T") + } + }() + Deserialize[int](nil) +} + +func TestDeserializeAtReturnsNilWhenNothingMatches(t *testing.T) { + r := DeserializeAt[audio](map[string]any{"Name": "n"}, "audio") + if r != nil { + t.Fatalf("expected nil, got %#v", r) + } +} + +func TestDeserializeAtReturnsValueWhenPrefixMatches(t *testing.T) { + r := DeserializeAt[audio](map[string]any{ + "audio.artist": "A", + "audio.year": float64(2024), // setValue: pointer + numeric convert + }, "audio") + if r == nil { + t.Fatal("expected non-nil *audio") + } + if r.Artist == nil || *r.Artist != "A" { + t.Errorf("Artist: %#v", r.Artist) + } + if r.Year == nil || *r.Year != 2024 { + t.Errorf("Year: %#v", r.Year) + } +} diff --git a/services/search/pkg/mapping/fillstruct_test.go b/services/search/pkg/mapping/fillstruct_test.go new file mode 100644 index 0000000000..4e5856fe80 --- /dev/null +++ b/services/search/pkg/mapping/fillstruct_test.go @@ -0,0 +1,127 @@ +package mapping + +import ( + "errors" + "reflect" + "testing" +) + +// The walker is shared by both deserializers, so its structural behavior +// (flattening embedded structs, recursing into nested pointers, joining the +// prefix, keeping a pointer only when a field was set, fail-soft skipping) is +// tested here once against a trivial string setter instead of twice through +// Deserialize and DeserializeStringsAt. + +// FsEmbVal/FsEmbPtr are exported so their embedded field is exported; an +// unexported embedded type would be skipped by resolveField. +type FsEmbVal struct { + EV string `json:"ev"` +} + +type FsEmbPtr struct { + EP string `json:"ep"` +} + +type fsNested struct { + N string `json:"n"` +} + +type fsRoot struct { + FsEmbVal // embedded value struct: fields promoted + *FsEmbPtr // embedded pointer struct: allocated on demand + Leaf string `json:"leaf"` + Nested *fsNested `json:"nested"` // nested pointer: recursed under "nested." +} + +var errBadLeaf = errors.New("bad leaf") + +// fsSet writes raw into a string field; the "BAD" sentinel simulates a parse +// failure so the fail-soft path can be exercised. +func fsSet(v reflect.Value, raw string) error { + if raw == "BAD" { + return errBadLeaf + } + v.SetString(raw) + return nil +} + +func TestFillStruct(t *testing.T) { + fill := func(fields map[string]string, prefix string) (fsRoot, bool) { + var root fsRoot + touched := fillStruct(reflect.ValueOf(&root).Elem(), fields, prefix, fsSet) + return root, touched + } + + t.Run("flattens embedded, recurses nested", func(t *testing.T) { + root, touched := fill(map[string]string{ + "leaf": "L", + "ev": "EV", + "ep": "EP", + "nested.n": "N", + }, "") + if !touched { + t.Fatal("expected touched") + } + if root.Leaf != "L" { + t.Errorf("Leaf: %q", root.Leaf) + } + if root.EV != "EV" { + t.Errorf("embedded value not promoted: %q", root.EV) + } + if root.FsEmbPtr == nil || root.EP != "EP" { + t.Errorf("embedded pointer not allocated: %+v", root.FsEmbPtr) + } + if root.Nested == nil || root.Nested.N != "N" { + t.Errorf("nested pointer not populated: %+v", root.Nested) + } + }) + + t.Run("nothing matches: touched false, pointers stay nil", func(t *testing.T) { + root, touched := fill(map[string]string{"other": "x"}, "") + if touched { + t.Fatal("expected untouched") + } + if root.Nested != nil { + t.Errorf("Nested should stay nil: %+v", root.Nested) + } + if root.FsEmbPtr != nil { + t.Errorf("embedded pointer should stay nil: %+v", root.FsEmbPtr) + } + }) + + t.Run("prefix arg is joined with the field name", func(t *testing.T) { + root, touched := fill(map[string]string{ + "pre.leaf": "L", + "pre.nested.n": "N", + }, "pre") + if !touched || root.Leaf != "L" || root.Nested == nil || root.Nested.N != "N" { + t.Fatalf("prefix not joined: leaf=%q nested=%+v", root.Leaf, root.Nested) + } + }) + + t.Run("fail-soft: errored leaf stays zero, walk continues", func(t *testing.T) { + root, touched := fill(map[string]string{ + "leaf": "BAD", + "ev": "EV", + }, "") + if !touched { + t.Fatal("expected touched because ev was set") + } + if root.Leaf != "" { + t.Errorf("errored leaf should stay zero, got %q", root.Leaf) + } + if root.EV != "EV" { + t.Errorf("walk should continue past the error: %q", root.EV) + } + }) + + t.Run("embedded pointer dropped when its only field errors", func(t *testing.T) { + root, touched := fill(map[string]string{"ep": "BAD"}, "") + if touched { + t.Fatal("expected untouched") + } + if root.FsEmbPtr != nil { + t.Errorf("embedded pointer should stay nil on error, got %+v", root.FsEmbPtr) + } + }) +} diff --git a/services/search/pkg/mapping/infer.go b/services/search/pkg/mapping/infer.go new file mode 100644 index 0000000000..76a49bd669 --- /dev/null +++ b/services/search/pkg/mapping/infer.go @@ -0,0 +1,116 @@ +package mapping + +import ( + "reflect" + "strings" + "time" + + "google.golang.org/protobuf/types/known/timestamppb" +) + +var ( + timeType = reflect.TypeFor[time.Time]() + timestampType = reflect.TypeFor[timestamppb.Timestamp]() +) + +// deref unwraps pointer and slice types to their element type. +func deref(t reflect.Type) reflect.Type { + for t.Kind() == reflect.Ptr || t.Kind() == reflect.Slice { + t = t.Elem() + } + return t +} + +// inferType returns the mapping type for a Go type. Pointers and slices are +// unwrapped to their element type. time.Time and timestamppb.Timestamp become +// datetime; other structs become object. +func inferType(t reflect.Type) string { + t = deref(t) + switch t.Kind() { + case reflect.String: + return TypeKeyword + case reflect.Bool: + return TypeBool + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64: + return TypeNumeric + case reflect.Struct: + if t == timeType || t == timestampType { + return TypeDatetime + } + return TypeObject + } + return "" +} + +// fieldInfo is the resolved metadata for one struct field. +type fieldInfo struct { + Name string + GoField reflect.StructField + Skip bool + Embedded bool +} + +// resolveField resolves a struct field's json-tag name and skip/embed state. +func resolveField(sf reflect.StructField) fieldInfo { + if !sf.IsExported() { + return fieldInfo{Skip: true} + } + name := sf.Name + tag := sf.Tag.Get("json") + if tag != "" { + first, _, _ := strings.Cut(tag, ",") + if first == "-" { + return fieldInfo{Skip: true} + } + if first != "" { + name = first + } + } + return fieldInfo{ + Name: name, + GoField: sf, + Embedded: sf.Anonymous, + } +} + +// walkFields visits exported leaf fields of t, flattening embedded structs +// onto the enclosing level. It returns the first error returned by fn. +func walkFields(t reflect.Type, fn func(fi fieldInfo) error) error { + for t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Struct { + return nil + } + for i := 0; i < t.NumField(); i++ { + fi := resolveField(t.Field(i)) + if fi.Skip { + continue + } + if fi.Embedded { + if err := walkFields(fi.GoField.Type, fn); err != nil { + return err + } + continue + } + if err := fn(fi); err != nil { + return err + } + } + return nil +} + +// structType returns the underlying struct type, unwrapping pointers and +// slices. Returns nil when t is not a walkable struct (e.g. time.Time). +func structType(t reflect.Type) reflect.Type { + t = deref(t) + if t.Kind() != reflect.Struct { + return nil + } + if t == timeType || t == timestampType { + return nil + } + return t +} diff --git a/services/search/pkg/mapping/infer_test.go b/services/search/pkg/mapping/infer_test.go new file mode 100644 index 0000000000..9ee57b656a --- /dev/null +++ b/services/search/pkg/mapping/infer_test.go @@ -0,0 +1,125 @@ +package mapping + +import ( + "reflect" + "testing" + "time" + + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestInferTypeUnsupported(t *testing.T) { + if got := inferType(reflect.TypeFor[map[string]int]()); got != "" { + t.Errorf("map: got %q, want empty", got) + } + if got := inferType(reflect.TypeFor[chan int]()); got != "" { + t.Errorf("chan: got %q, want empty", got) + } +} + +func TestInferType(t *testing.T) { + cases := []struct { + name string + in any + want string + }{ + {"string", "", TypeKeyword}, + {"*string", (*string)(nil), TypeKeyword}, + {"[]string", []string(nil), TypeKeyword}, + {"bool", false, TypeBool}, + {"int", int(0), TypeNumeric}, + {"int64", int64(0), TypeNumeric}, + {"uint64", uint64(0), TypeNumeric}, + {"float64", float64(0), TypeNumeric}, + {"time.Time", time.Time{}, TypeDatetime}, + {"*time.Time", (*time.Time)(nil), TypeDatetime}, + {"*timestamppb.Timestamp", (*timestamppb.Timestamp)(nil), TypeDatetime}, + {"struct", struct{ X int }{}, TypeObject}, + {"*struct", (*struct{ X int })(nil), TypeObject}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := inferType(reflect.TypeOf(c.in)) + if got != c.want { + t.Fatalf("inferType(%s): got %q, want %q", c.name, got, c.want) + } + }) + } +} + +func TestResolveField(t *testing.T) { + type S struct { + Exported string `json:"exp"` + Renamed string `json:"renamed,omitempty"` + NoTag string + OmitOnly string `json:",omitempty"` + Skipped string `json:"-"` + unexported string //nolint:unused + } + st := reflect.TypeFor[S]() + cases := []struct { + fieldIdx int + wantName string + wantSkip bool + }{ + {0, "exp", false}, + {1, "renamed", false}, + {2, "NoTag", false}, + {3, "OmitOnly", false}, + {4, "", true}, + {5, "", true}, + } + for _, c := range cases { + fi := resolveField(st.Field(c.fieldIdx)) + if fi.Skip != c.wantSkip { + t.Errorf("field %d: skip=%v, want %v", c.fieldIdx, fi.Skip, c.wantSkip) + } + if !c.wantSkip && fi.Name != c.wantName { + t.Errorf("field %d: name=%q, want %q", c.fieldIdx, fi.Name, c.wantName) + } + } +} + +func TestWalkFieldsFlattensEmbedded(t *testing.T) { + type Inner struct { + A string `json:"a"` + B int `json:"b"` + } + type Outer struct { + Inner + C bool `json:"c"` + } + var names []string + err := walkFields(reflect.TypeFor[Outer](), func(fi fieldInfo) error { + names = append(names, fi.Name) + return nil + }) + if err != nil { + t.Fatalf("walkFields: %v", err) + } + want := []string{"a", "b", "c"} + if !reflect.DeepEqual(names, want) { + t.Fatalf("got %v, want %v", names, want) + } +} + +func TestStructType(t *testing.T) { + type S struct{ X int } + cases := []struct { + name string + in reflect.Type + wantNil bool + }{ + {"struct", reflect.TypeFor[S](), false}, + {"*struct", reflect.TypeFor[*S](), false}, + {"[]struct", reflect.TypeFor[[]S](), false}, + {"time.Time", reflect.TypeFor[time.Time](), true}, + {"string", reflect.TypeFor[string](), true}, + } + for _, c := range cases { + got := structType(c.in) + if (got == nil) != c.wantNil { + t.Errorf("%s: got %v, wantNil %v", c.name, got, c.wantNil) + } + } +} diff --git a/services/search/pkg/mapping/opensearch.go b/services/search/pkg/mapping/opensearch.go new file mode 100644 index 0000000000..fa6152feb7 --- /dev/null +++ b/services/search/pkg/mapping/opensearch.go @@ -0,0 +1,112 @@ +package mapping + +import ( + "fmt" + "reflect" +) + +// OpenSearchBuildMapping builds the OpenSearch "properties" map (the value +// of mappings.properties) for type t by walking the struct via reflection. +// Field names come from json tags; overrides are keyed by those names. +// +// The returned map contains plain JSON-friendly values (strings, bools, +// nested maps) and can be marshalled directly. +func OpenSearchBuildMapping(t reflect.Type, overrides map[string]FieldOpts) (map[string]any, error) { + return buildOpenSearchProperties(t, overrides, "") +} + +func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, prefix string) (map[string]any, error) { + props := map[string]any{} + err := walkFields(t, func(fi fieldInfo) error { + key := fi.Name + if prefix != "" { + key = prefix + "." + fi.Name + } + opts := overrides[key] + fieldType := opts.Type + if fieldType == "" { + fieldType = inferType(fi.GoField.Type) + } + + if fieldType == TypeObject { + sub := structType(fi.GoField.Type) + if sub == nil { + return fmt.Errorf("mapping: object type on non-struct field %q", key) + } + subProps, err := buildOpenSearchProperties(sub, overrides, key) + if err != nil { + return err + } + props[fi.Name] = map[string]any{"properties": subProps} + return nil + } + + fm, err := openSearchFieldMapping(fieldType, opts, fi.GoField.Type) + if err != nil { + return fmt.Errorf("mapping: field %q: %w", key, err) + } + props[fi.Name] = fm + return nil + }) + return props, err +} + +func openSearchFieldMapping(fieldType string, opts FieldOpts, goType reflect.Type) (map[string]any, error) { + switch fieldType { + case TypeKeyword: + m := map[string]any{"type": "keyword"} + if opts.Analyzer != "" { + m["type"] = "text" + m["analyzer"] = opts.Analyzer + } + return m, nil + case TypeFulltext: + m := map[string]any{ + "type": "text", + "term_vector": "with_positions_offsets", + } + if opts.Analyzer != "" { + m["analyzer"] = opts.Analyzer + } + return m, nil + case TypePath: + m := map[string]any{"type": "text"} + if opts.Analyzer != "" { + m["analyzer"] = opts.Analyzer + } else { + m["analyzer"] = "path_hierarchy" + } + return m, nil + case TypeWildcard: + // OpenSearch stores wildcard fields with doc_values=false by + // default, so emit it explicitly to keep local and remote + // mappings in sync for the Apply comparison. + return map[string]any{"type": "wildcard", "doc_values": false}, nil + case TypeNumeric: + return map[string]any{"type": openSearchNumericType(goType)}, nil + case TypeBool: + return map[string]any{"type": "boolean"}, nil + case TypeDatetime: + return map[string]any{"type": "date"}, nil + case "": + return nil, fmt.Errorf("no type inferred and no override") + } + return nil, fmt.Errorf("unsupported type %q", fieldType) +} + +// openSearchNumericType maps a Go numeric type to an OpenSearch numeric +// field type. +func openSearchNumericType(t reflect.Type) string { + t = deref(t) + switch t.Kind() { + case reflect.Float32: + return "float" + case reflect.Float64: + return "double" + case reflect.Int8, reflect.Uint8, reflect.Int16, reflect.Uint16: + return "short" + case reflect.Int32, reflect.Uint32: + return "integer" + } + return "long" +} diff --git a/services/search/pkg/mapping/opensearch_test.go b/services/search/pkg/mapping/opensearch_test.go new file mode 100644 index 0000000000..1ad6985546 --- /dev/null +++ b/services/search/pkg/mapping/opensearch_test.go @@ -0,0 +1,132 @@ +package mapping + +import ( + "reflect" + "testing" + "time" +) + +type osDoc struct { + ID string `json:"ID"` + Size uint64 `json:"Size"` + Deleted bool `json:"Deleted"` + CreatedAt time.Time `json:"CreatedAt"` + Rating float64 `json:"Rating"` + Nested *struct { + Artist string `json:"artist"` + Year int32 `json:"year"` + } `json:"nested,omitempty"` +} + +func TestOpenSearchNumericTypes(t *testing.T) { + type doc struct { + A int8 `json:"a"` + B int16 `json:"b"` + C int32 `json:"c"` + D int64 `json:"d"` + E uint8 `json:"e"` + F uint64 `json:"f"` + G float32 `json:"g"` + H float64 `json:"h"` + } + props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), nil) + if err != nil { + t.Fatalf("OpenSearchBuildMapping: %v", err) + } + want := map[string]string{"a": "short", "b": "short", "c": "integer", "d": "long", "e": "short", "f": "long", "g": "float", "h": "double"} + for k, wt := range want { + if got := props[k].(map[string]any)["type"]; got != wt { + t.Errorf("%s: type = %v, want %v", k, got, wt) + } + } +} + +func TestOpenSearchBuildMappingInferred(t *testing.T) { + props, err := OpenSearchBuildMapping(reflect.TypeFor[osDoc](), nil) + if err != nil { + t.Fatalf("OpenSearchBuildMapping: %v", err) + } + want := map[string]string{ + "ID": "keyword", + "Size": "long", + "Deleted": "boolean", + "CreatedAt": "date", + "Rating": "double", + } + for k, v := range want { + m, ok := props[k].(map[string]any) + if !ok { + t.Errorf("%s: missing or not a map: %#v", k, props[k]) + continue + } + if got := m["type"]; got != v { + t.Errorf("%s: type %v, want %v", k, got, v) + } + } +} + +func TestOpenSearchBuildMappingNested(t *testing.T) { + props, err := OpenSearchBuildMapping(reflect.TypeFor[osDoc](), nil) + if err != nil { + t.Fatalf("OpenSearchBuildMapping: %v", err) + } + nested, ok := props["nested"].(map[string]any) + if !ok { + t.Fatalf("nested: not a map: %#v", props["nested"]) + } + sub, ok := nested["properties"].(map[string]any) + if !ok { + t.Fatalf("nested.properties: missing: %#v", nested) + } + artist, ok := sub["artist"].(map[string]any) + if !ok { + t.Fatalf("nested.artist: %#v", sub) + } + if artist["type"] != "keyword" { + t.Errorf("nested.artist.type: %v", artist["type"]) + } + year, ok := sub["year"].(map[string]any) + if !ok { + t.Fatalf("nested.year: %#v", sub) + } + if year["type"] != "integer" { + t.Errorf("nested.year.type: %v (int32 → integer expected)", year["type"]) + } +} + +func TestOpenSearchBuildMappingOverrides(t *testing.T) { + type doc struct { + Name string `json:"Name"` + Content string `json:"Content"` + Path string `json:"Path"` + MimeType string `json:"MimeType"` + } + props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{ + "Name": {Analyzer: "lowercaseKeyword"}, + "Content": {Type: TypeFulltext}, + "Path": {Type: TypePath}, + "MimeType": {Type: TypeWildcard}, + }) + if err != nil { + t.Fatalf("OpenSearchBuildMapping: %v", err) + } + name := props["Name"].(map[string]any) + if name["type"] != "text" || name["analyzer"] != "lowercaseKeyword" { + t.Errorf("Name: %#v", name) + } + content := props["Content"].(map[string]any) + if content["type"] != "text" || content["term_vector"] != "with_positions_offsets" { + t.Errorf("Content: %#v", content) + } + if _, ok := content["analyzer"]; ok { + t.Errorf("Content should leave analyzer unset (use OpenSearch default), got %#v", content["analyzer"]) + } + path := props["Path"].(map[string]any) + if path["type"] != "text" || path["analyzer"] != "path_hierarchy" { + t.Errorf("Path: %#v", path) + } + mime := props["MimeType"].(map[string]any) + if mime["type"] != "wildcard" { + t.Errorf("MimeType: %#v", mime) + } +} diff --git a/services/search/pkg/mapping/opts.go b/services/search/pkg/mapping/opts.go new file mode 100644 index 0000000000..39129961db --- /dev/null +++ b/services/search/pkg/mapping/opts.go @@ -0,0 +1,34 @@ +// Package mapping builds search index mappings for bleve and OpenSearch from +// a Go struct via reflection. Field names come from json tags; the caller +// provides overrides for fields that need a specific type or analyzer. +package mapping + +// Field type constants used in FieldOpts.Type. An empty Type means the type +// is inferred from the Go field via reflection. +const ( + TypeKeyword = "keyword" + TypeFulltext = "fulltext" + TypePath = "path" + TypeWildcard = "wildcard" + TypeNumeric = "numeric" + TypeDatetime = "datetime" + TypeBool = "bool" + TypeObject = "object" +) + +// FieldOpts overrides the default type inference for a struct field. Keys in +// the override map are json-tag names (e.g. "Name", "location", "audio.artist"), +// not Go field names. +type FieldOpts struct { + // Type is one of the Type* constants. Empty means "infer from Go type". + Type string + + // Analyzer is the name of a custom analyzer registered on the bleve + // IndexMapping (e.g. "lowercaseKeyword", "fulltext"). For OpenSearch it + // becomes the analyzer attribute on the field. + Analyzer string + + // IncludeInAll controls bleve's _all field inclusion. Nil means "use the + // bleve default for this field type". Has no effect on OpenSearch. + IncludeInAll *bool +} diff --git a/services/search/pkg/mapping/serialize.go b/services/search/pkg/mapping/serialize.go new file mode 100644 index 0000000000..bf72344662 --- /dev/null +++ b/services/search/pkg/mapping/serialize.go @@ -0,0 +1,18 @@ +package mapping + +import ( + "fmt" + + "github.com/opencloud-eu/opencloud/pkg/conversions" +) + +// PrepareForIndex converts v to the flat map[string]any the backend index +// clients expect, via a json round-trip (conversions.To). overrides is +// reserved for type-specific adaptations wired in by follow-up features. +func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, error) { + out, err := conversions.To[map[string]any](v) + if err != nil { + return nil, fmt.Errorf("mapping: prepare %T: %w", v, err) + } + return out, nil +} diff --git a/services/search/pkg/mapping/serialize_test.go b/services/search/pkg/mapping/serialize_test.go new file mode 100644 index 0000000000..b352708999 --- /dev/null +++ b/services/search/pkg/mapping/serialize_test.go @@ -0,0 +1,83 @@ +package mapping + +import ( + "reflect" + "testing" +) + +func TestPrepareForIndexError(t *testing.T) { + // a func field can't be json-marshalled -> conversions.To errors + type bad struct { + F func() `json:"f"` + } + if _, err := PrepareForIndex(bad{}, nil); err == nil { + t.Error("expected error for non-marshallable value") + } +} + +func TestPrepareForIndexNil(t *testing.T) { + // a typed nil pointer marshals to null -> nil map, no error, no panic + out, err := PrepareForIndex((*struct{})(nil), nil) + if err != nil || out != nil { + t.Errorf("got (%v, %v), want (nil, nil)", out, err) + } +} + +func TestPrepareForIndexFlattensEmbedded(t *testing.T) { + type inner struct { + Name string `json:"Name"` + Size uint64 `json:"Size"` + } + type outer struct { + inner + ID string `json:"ID"` + } + m, err := PrepareForIndex(outer{inner: inner{Name: "a", Size: 7}, ID: "x"}, nil) + if err != nil { + t.Fatalf("PrepareForIndex: %v", err) + } + want := map[string]any{"Name": "a", "Size": float64(7), "ID": "x"} + if !reflect.DeepEqual(m, want) { + t.Fatalf("got %#v, want %#v", m, want) + } +} + +func TestPrepareForIndexOmitsNilWithOmitempty(t *testing.T) { + type facet struct { + Artist string `json:"artist"` + } + type doc struct { + Name string `json:"Name"` + Audio *facet `json:"audio,omitempty"` + } + m, err := PrepareForIndex(doc{Name: "n"}, nil) + if err != nil { + t.Fatalf("PrepareForIndex: %v", err) + } + if _, ok := m["audio"]; ok { + t.Errorf("audio should be omitted when nil: %#v", m) + } + if m["Name"] != "n" { + t.Errorf("Name: %v", m["Name"]) + } +} + +func TestPrepareForIndexIncludesNestedWhenSet(t *testing.T) { + type facet struct { + Artist string `json:"artist"` + } + type doc struct { + Audio *facet `json:"audio,omitempty"` + } + m, err := PrepareForIndex(doc{Audio: &facet{Artist: "A"}}, nil) + if err != nil { + t.Fatalf("PrepareForIndex: %v", err) + } + nested, ok := m["audio"].(map[string]any) + if !ok { + t.Fatalf("audio should be a nested map: %#v", m["audio"]) + } + if nested["artist"] != "A" { + t.Errorf("audio.artist: %v", nested["artist"]) + } +} diff --git a/services/search/pkg/mapping/validate.go b/services/search/pkg/mapping/validate.go new file mode 100644 index 0000000000..761309992a --- /dev/null +++ b/services/search/pkg/mapping/validate.go @@ -0,0 +1,49 @@ +package mapping + +import ( + "fmt" + "reflect" + "sort" + "strings" +) + +// Validate returns an error if any override key does not match a known field +// name in t. Top-level fields are identified by their json-tag name; nested +// named struct fields are reachable as "parent.child". Embedded (anonymous) +// structs are flattened, so their fields sit at the parent level (as with +// encoding/json). +func Validate(t reflect.Type, overrides map[string]FieldOpts) error { + if len(overrides) == 0 { + return nil + } + names := collectNames(t, "") + var unknown []string + for k := range overrides { + if _, ok := names[k]; !ok { + unknown = append(unknown, k) + } + } + if len(unknown) == 0 { + return nil + } + sort.Strings(unknown) + return fmt.Errorf("mapping: unknown override keys: %s", strings.Join(unknown, ", ")) +} + +func collectNames(t reflect.Type, prefix string) map[string]struct{} { + out := map[string]struct{}{} + _ = walkFields(t, func(fi fieldInfo) error { + key := fi.Name + if prefix != "" { + key = prefix + "." + fi.Name + } + out[key] = struct{}{} + if sub := structType(fi.GoField.Type); sub != nil { + for k := range collectNames(sub, key) { + out[k] = struct{}{} + } + } + return nil + }) + return out +} diff --git a/services/search/pkg/mapping/validate_test.go b/services/search/pkg/mapping/validate_test.go new file mode 100644 index 0000000000..fdf514c29b --- /dev/null +++ b/services/search/pkg/mapping/validate_test.go @@ -0,0 +1,50 @@ +package mapping + +import ( + "reflect" + "strings" + "testing" +) + +type inner struct { + Artist string `json:"artist"` +} + +type sample struct { + Name string `json:"Name"` + Audio *inner `json:"audio,omitempty"` + Location *struct { //nolint:unused + Lon float64 `json:"longitude"` + Lat float64 `json:"latitude"` + } `json:"location,omitempty"` +} + +func TestValidateAccepts(t *testing.T) { + err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{ + "Name": {Analyzer: "lowercaseKeyword"}, + "audio": {Type: TypeObject}, + "audio.artist": {Analyzer: "lowercaseKeyword"}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestValidateRejectsUnknown(t *testing.T) { + err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{ + "nope": {}, + "audio.zzz": {}, + }) + if err == nil { + t.Fatalf("expected error") + } + if !strings.Contains(err.Error(), "nope") || !strings.Contains(err.Error(), "audio.zzz") { + t.Fatalf("error missing keys: %v", err) + } +} + +func TestValidateEmpty(t *testing.T) { + if err := Validate(reflect.TypeFor[sample](), nil); err != nil { + t.Fatalf("empty overrides should pass: %v", err) + } +} diff --git a/services/search/pkg/opensearch/batch.go b/services/search/pkg/opensearch/batch.go index ff76b06b16..cd1a89e386 100644 --- a/services/search/pkg/opensearch/batch.go +++ b/services/search/pkg/opensearch/batch.go @@ -14,6 +14,7 @@ import ( "github.com/opencloud-eu/opencloud/pkg/conversions" "github.com/opencloud-eu/opencloud/pkg/log" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) @@ -43,7 +44,7 @@ func NewBatch(client *opensearchgoAPI.Client, index string, size int) (*Batch, e func (b *Batch) Upsert(id string, r search.Resource) error { return b.withSizeLimit(func() error { - body, err := conversions.To[map[string]any](r) + body, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides()) if err != nil { return fmt.Errorf("failed to marshal resource: %w", err) } diff --git a/services/search/pkg/opensearch/index.go b/services/search/pkg/opensearch/index.go index 277c9f5c5c..318358fde9 100644 --- a/services/search/pkg/opensearch/index.go +++ b/services/search/pkg/opensearch/index.go @@ -3,51 +3,38 @@ package opensearch import ( "bytes" "context" - "embed" "errors" "fmt" - "path" + "maps" "reflect" - "strings" "github.com/go-jose/go-jose/v3/json" opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "github.com/tidwall/gjson" + + searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) var ( ErrManualActionRequired = errors.New("manual action required") - IndexManagerLatest = IndexIndexManagerResourceV3 - IndexIndexManagerResourceV3 IndexManager = "resource_v3.json" + IndexManagerLatest = IndexIndexManagerResourceV2 + IndexIndexManagerResourceV2 IndexManager = "resource_v2" ) -//go:embed internal/indexes/*.json -var indexes embed.FS - type IndexManager string -// Version is the part of the definition file name that says which generation it -// is, resource_v3.json carries v3. -func (m IndexManager) Version() string { - name := strings.TrimSuffix(string(m), path.Ext(string(m))) - _, version, found := strings.Cut(name, "_") - if !found { - return "" - } - - return version +// IndexName puts the schema generation behind the configured name, so a new +// generation starts on an index of its own instead of refusing to work with +// the one that is there. Interim: derived from a constant here, from the +// shared SchemaVersion later in this series. +func IndexName(name string) string { + return name + "-v2" } -// IndexName puts the generation of the definition behind the configured name, -// so a new one starts on an index of its own instead of refusing to work with -// the one that is there. -func IndexName(name string) string { - version := IndexManagerLatest.Version() - if version == "" { - return name - } - - return name + "-" + version +// indexGenerators dispatches each IndexManager variant to its builder. +var indexGenerators = map[IndexManager]func() ([]byte, error){ + IndexIndexManagerResourceV2: buildResourceV2Mapping, } func (m IndexManager) String() string { @@ -60,16 +47,56 @@ func (m IndexManager) String() string { } func (m IndexManager) MarshalJSON() ([]byte, error) { - filePath := string(m) - body, err := indexes.ReadFile(path.Join("./internal/indexes", filePath)) - switch { - case err != nil: - return nil, fmt.Errorf("failed to read index file %s: %w", filePath, err) - case len(body) <= 0: - return nil, fmt.Errorf("index file %s is empty", filePath) + gen, ok := indexGenerators[m] + if !ok { + return nil, fmt.Errorf("unknown index manager %q", string(m)) + } + return gen() +} + +// buildResourceV2Mapping renders the OpenSearch index template for a +// search.Resource from the shared SearchFieldOverrides. OpenSearch-specific +// tweaks (wildcard MimeType, path_hierarchy Path) are applied on top. +func buildResourceV2Mapping() ([]byte, error) { + resourceType := reflect.TypeFor[search.Resource]() + overrides := maps.Clone(search.Resource{}.SearchFieldOverrides()) + overrides["MimeType"] = searchmapping.FieldOpts{Type: searchmapping.TypeWildcard} + overrides["Path"] = searchmapping.FieldOpts{Type: searchmapping.TypePath} + if err := searchmapping.Validate(resourceType, overrides); err != nil { + return nil, err + } + props, err := searchmapping.OpenSearchBuildMapping(resourceType, overrides) + if err != nil { + return nil, err } - return body, nil + index := map[string]any{ + "settings": map[string]any{ + "number_of_shards": "1", + "number_of_replicas": "1", + "analysis": map[string]any{ + "analyzer": map[string]any{ + "path_hierarchy": map[string]any{ + "type": "custom", + "tokenizer": "path_hierarchy", + "filter": []string{"lowercase"}, + }, + "lowercaseKeyword": map[string]any{ + "type": "custom", + "tokenizer": "keyword", + "filter": []string{"lowercase"}, + }, + }, + "tokenizer": map[string]any{ + "path_hierarchy": map[string]any{"type": "path_hierarchy"}, + }, + }, + }, + "mappings": map[string]any{ + "properties": props, + }, + } + return json.Marshal(index) } func coveredAt(declared, index gjson.Result, declaredPath, indexPath string) (string, string, bool) { @@ -163,8 +190,11 @@ func (m IndexManager) Apply(ctx context.Context, name string, client *opensearch if errs != nil { return fmt.Errorf( - "index %s already exists and is different from the requested version, %w: %w", - name, + "index %s already exists with a different mapping than the requested version. "+ + "There is no in-place migration today: drop the index in OpenSearch (DELETE /%s) "+ + "and restart the search service. The index will be recreated with the new mapping. "+ + "%w: %w", + name, name, ErrManualActionRequired, errors.Join(errs...), ) diff --git a/services/search/pkg/opensearch/internal/convert/opensearch.go b/services/search/pkg/opensearch/internal/convert/opensearch.go index 16c156314f..aef6db363b 100644 --- a/services/search/pkg/opensearch/internal/convert/opensearch.go +++ b/services/search/pkg/opensearch/internal/convert/opensearch.go @@ -15,6 +15,17 @@ import ( "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) +// copyFacet converts a typed pointer from the indexed shape (libregraph) to +// the protobuf shape via conversions.To. Returns nil when src is nil so the +// enclosing Match.Entity field stays nil. +func copyFacet[Dst, Src any](src *Src) *Dst { + if src == nil { + return nil + } + dst, _ := conversions.To[*Dst](src) + return dst +} + func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match, error) { resource, err := conversions.To[search.Resource](hit.Source) if err != nil { @@ -69,26 +80,10 @@ func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match, return strings.Join(contentHighlights[:], "; ") }(), - Audio: func() *searchMessage.Audio { - if !strings.HasPrefix(resource.MimeType, "audio/") { - return nil - } - - audio, _ := conversions.To[*searchMessage.Audio](resource.Audio) - return audio - }(), - Image: func() *searchMessage.Image { - image, _ := conversions.To[*searchMessage.Image](resource.Image) - return image - }(), - Location: func() *searchMessage.GeoCoordinates { - geoCoordinates, _ := conversions.To[*searchMessage.GeoCoordinates](resource.Location) - return geoCoordinates - }(), - Photo: func() *searchMessage.Photo { - photo, _ := conversions.To[*searchMessage.Photo](resource.Photo) - return photo - }(), + Audio: copyFacet[searchMessage.Audio](resource.Audio), + Image: copyFacet[searchMessage.Image](resource.Image), + Location: copyFacet[searchMessage.GeoCoordinates](resource.Location), + Photo: copyFacet[searchMessage.Photo](resource.Photo), }, } diff --git a/services/search/pkg/opensearch/internal/indexes/resource_v1.json b/services/search/pkg/opensearch/internal/indexes/resource_v1.json deleted file mode 100644 index f0f719c4c5..0000000000 --- a/services/search/pkg/opensearch/internal/indexes/resource_v1.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "settings": { - "number_of_shards": "1", - "number_of_replicas": "1", - "analysis": { - "analyzer": { - "path_hierarchy": { - "filter": [ - "lowercase" - ], - "tokenizer": "path_hierarchy", - "type": "custom" - } - }, - "tokenizer": { - "path_hierarchy": { - "type": "path_hierarchy" - } - } - } - }, - "mappings": { - "properties": { - "ID": { - "type": "keyword" - }, - "ParentID": { - "type": "keyword" - }, - "RootID": { - "type": "keyword" - }, - "MimeType": { - "type": "wildcard", - "doc_values": false - }, - "Path": { - "type": "text", - "analyzer": "path_hierarchy" - }, - "Deleted": { - "type": "boolean" - }, - "Hidden": { - "type": "boolean" - } - } - } -} diff --git a/services/search/pkg/opensearch/internal/indexes/resource_v2.json b/services/search/pkg/opensearch/internal/indexes/resource_v2.json deleted file mode 100644 index 64b450ef51..0000000000 --- a/services/search/pkg/opensearch/internal/indexes/resource_v2.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "settings": { - "number_of_shards": "1", - "number_of_replicas": "1", - "analysis": { - "analyzer": { - "path_hierarchy": { - "filter": [ - "lowercase" - ], - "tokenizer": "path_hierarchy", - "type": "custom" - } - }, - "tokenizer": { - "path_hierarchy": { - "type": "path_hierarchy" - } - } - } - }, - "mappings": { - "properties": { - "Content": { - "type": "text", - "term_vector": "with_positions_offsets" - }, - "ID": { - "type": "keyword" - }, - "ParentID": { - "type": "keyword" - }, - "RootID": { - "type": "keyword" - }, - "MimeType": { - "type": "wildcard", - "doc_values": false - }, - "Path": { - "type": "text", - "analyzer": "path_hierarchy" - }, - "Deleted": { - "type": "boolean" - }, - "Hidden": { - "type": "boolean" - }, - "Favorites": { - "type": "keyword" - } - } - } -} \ No newline at end of file diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index 0a84fabfc4..218dc43104 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -12,16 +12,25 @@ import ( bleveQuery "github.com/blevesearch/bleve/v2/search/query" "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/pkg/kql" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) -var lowercaseFields = map[string]struct{}{ - "Name": {}, - "Title": {}, - "Tags": {}, - "Favorites": {}, - "Content": {}, - "MimeType": {}, - "Hidden": {}, +// lowercaseFields is derived from Resource.SearchFieldOverrides(): any +// field whose override picks a lowercasing analyzer (`lowercaseKeyword`) +// or the fulltext type (which uses a lowercasing analyzer under the hood) +// gets its query-side value pre-lowercased so compile-time matches the +// index-time tokenization. Anything else keeps its original casing. +var lowercaseFields = buildLowercaseFields() + +func buildLowercaseFields() map[string]struct{} { + out := map[string]struct{}{} + for key, opts := range (search.Resource{}).SearchFieldOverrides() { + if opts.Analyzer == "lowercaseKeyword" || opts.Type == mapping.TypeFulltext { + out[key] = struct{}{} + } + } + return out } var _fields = map[string]string{ diff --git a/services/search/pkg/search/search.go b/services/search/pkg/search/search.go index ad3792a8d4..035b18a71a 100644 --- a/services/search/pkg/search/search.go +++ b/services/search/pkg/search/search.go @@ -6,6 +6,7 @@ import ( "fmt" "regexp" "strings" + "sync" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" @@ -19,6 +20,7 @@ import ( searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" "github.com/opencloud-eu/opencloud/services/search/pkg/content" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" ) var scopeRegex = regexp.MustCompile(`scope:\s*([^" "\n\r]*)`) @@ -52,13 +54,36 @@ type BatchOperator interface { type Resource struct { content.Document - ID string - RootID string - Path string - ParentID string - Type uint64 - Deleted bool - Hidden bool + ID string `json:"ID"` + RootID string `json:"RootID"` + Path string `json:"Path"` + ParentID string `json:"ParentID"` + Type uint64 `json:"Type"` + Deleted bool `json:"Deleted"` + Hidden bool `json:"Hidden"` +} + +// resourceFieldOverrides is built once (it never changes) and reused on hot +// paths instead of reallocating per call. +var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts { + excludeFromAll := false + return map[string]mapping.FieldOpts{ + "Name": {Analyzer: "lowercaseKeyword"}, + "Content": {Type: mapping.TypeFulltext}, + "Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll}, + "Favorites": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll}, + // Mtime is stored as an RFC3339 string; type it as a date so mtime:>... + // range queries are chronological on both backends (bleve DateRangeQuery + // / OpenSearch date range), not a lexicographic keyword compare. + "Mtime": {Type: mapping.TypeDatetime}, + } +}) + +// SearchFieldOverrides returns the field options the mapping package needs to +// build per-backend index mappings for a Resource (keys are json-tag names). +// The map is shared and read-only; clone it before mutating. +func (Resource) SearchFieldOverrides() map[string]mapping.FieldOpts { + return resourceFieldOverrides() } // ResolveReference makes sure the path is relative to the space root diff --git a/services/search/pkg/search/service.go b/services/search/pkg/search/service.go index 8f647158cf..0c6611be10 100644 --- a/services/search/pkg/search/service.go +++ b/services/search/pkg/search/service.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path/filepath" + "reflect" "sort" "strconv" "strings" @@ -666,10 +667,10 @@ func (s *Service) doUpsertItem(ref *provider.Reference, batch BatchOperator) { // determine if metadata needs to be stored in storage as well metadata := map[string]string{} - addAudioMetadata(metadata, doc.Audio) - addImageMetadata(metadata, doc.Image) - addLocationMetadata(metadata, doc.Location) - addPhotoMetadata(metadata, doc.Photo) + facetToMetadata(metadata, doc.Audio, "libre.graph.audio.") + facetToMetadata(metadata, doc.Image, "libre.graph.image.") + facetToMetadata(metadata, doc.Location, "libre.graph.location.") + facetToMetadata(metadata, doc.Photo, "libre.graph.photo.") if len(metadata) == 0 { return } @@ -705,43 +706,25 @@ func IsHidden(path string) bool { return false } -func addAudioMetadata(metadata map[string]string, audio *libregraph.Audio) { - if audio == nil { - return +// facetToMetadata flattens a libregraph facet (Audio / Image / Location / Photo +// pointer) into the metadata map under the given prefix via the model's ToMap. +// No-op when the facet is nil. +func facetToMetadata[T libregraph.MappedNullable](metadata map[string]string, facet T, prefix string) { + // Only nilable kinds can be nil; IsNil panics on a value type (some + // libregraph models satisfy MappedNullable with a value receiver). + switch v := reflect.ValueOf(facet); v.Kind() { + case reflect.Ptr, reflect.Map, reflect.Slice, reflect.Interface, reflect.Chan, reflect.Func: + if v.IsNil() { + return + } } - marshalToStringMap(audio, metadata, "libre.graph.audio.") -} - -func addImageMetadata(metadata map[string]string, image *libregraph.Image) { - if image == nil { - return - } - marshalToStringMap(image, metadata, "libre.graph.image.") -} - -func addLocationMetadata(metadata map[string]string, location *libregraph.GeoCoordinates) { - if location == nil { - return - } - marshalToStringMap(location, metadata, "libre.graph.location.") -} - -func addPhotoMetadata(metadata map[string]string, photo *libregraph.Photo) { - if photo == nil { - return - } - marshalToStringMap(photo, metadata, "libre.graph.photo.") -} - -func marshalToStringMap[T libregraph.MappedNullable](source T, target map[string]string, prefix string) { - // ToMap never returns a non-nil error ... - m, _ := source.ToMap() - + // ToMap never returns a non-nil error. + m, _ := facet.ToMap() for k, v := range m { if v == nil { continue } - target[prefix+k] = valueToString(v) + metadata[prefix+k] = valueToString(v) } } From 29cfdb01a072871aee3a92ebb89c691ab4366e31 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Thu, 2 Jul 2026 14:25:35 +0200 Subject: [PATCH 02/54] feat(search): index Location as a geopoint on both backends Add a TypeGeopoint field type. The libregraph Location facet is kept as an object (retrieval / numeric queries) and a sibling _geopoint field carries the {lat,lon} form for geo-distance / bbox / polygon queries, uniform across bleve and OpenSearch via the shared mapping. PrepareForIndex splices the sibling in at write time. --- services/search/pkg/bleve/geo_verify_test.go | 178 ++++++++++++++++++ services/search/pkg/mapping/bleve.go | 17 ++ services/search/pkg/mapping/bleve_test.go | 44 +++++ services/search/pkg/mapping/geo.go | 51 +++++ services/search/pkg/mapping/geo_test.go | 160 ++++++++++++++++ services/search/pkg/mapping/opensearch.go | 17 ++ .../search/pkg/mapping/opensearch_test.go | 39 ++++ services/search/pkg/mapping/opts.go | 1 + services/search/pkg/mapping/serialize.go | 9 +- services/search/pkg/mapping/validate_test.go | 1 + services/search/pkg/search/search.go | 1 + 11 files changed, 516 insertions(+), 2 deletions(-) create mode 100644 services/search/pkg/bleve/geo_verify_test.go create mode 100644 services/search/pkg/mapping/geo.go create mode 100644 services/search/pkg/mapping/geo_test.go diff --git a/services/search/pkg/bleve/geo_verify_test.go b/services/search/pkg/bleve/geo_verify_test.go new file mode 100644 index 0000000000..27cf0e7a01 --- /dev/null +++ b/services/search/pkg/bleve/geo_verify_test.go @@ -0,0 +1,178 @@ +package bleve_test + +import ( + "sort" + "testing" + + bleveSearch "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" + libregraph "github.com/opencloud-eu/libre-graph-api-go" + + "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" + "github.com/opencloud-eu/opencloud/services/search/pkg/content" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +// geoFixture builds an in-memory bleve index with a single resource that +// carries the given lon/lat/alt. Used by the search tests below. +func geoFixture(t *testing.T, lon, lat, alt float64) bleveSearch.Index { + t.Helper() + idxMapping, err := bleve.NewMapping() + if err != nil { + t.Fatalf("NewMapping: %v", err) + } + idx, err := bleveSearch.NewMemOnly(idxMapping) + if err != nil { + t.Fatalf("NewMemOnly: %v", err) + } + r := search.Resource{ + ID: "x", + Document: content.Document{ + Name: "team.jpg", + Location: &libregraph.GeoCoordinates{ + Longitude: &lon, + Latitude: &lat, + Altitude: &alt, + }, + }, + } + doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides()) + if err != nil { + t.Fatalf("PrepareForIndex: %v", err) + } + if err := idx.Index(r.ID, doc); err != nil { + t.Fatalf("Index: %v", err) + } + return idx +} + +// TestLocationAltitudeRoundTrip proves that every subfield of Location +// (including altitude) ends up in hit.Fields when a Resource is indexed +// through the full bleve pipeline. This is the invariant the Move / +// Delete / Restore round-trip depends on. +func TestLocationAltitudeRoundTrip(t *testing.T) { + idx := geoFixture(t, 11.103870357204285, 49.48675890884328, 1047.7) + + req := bleveSearch.NewSearchRequest(bleveSearch.NewMatchAllQuery()) + req.Fields = []string{"*"} + res, err := idx.Search(req) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(res.Hits) == 0 { + t.Fatal("no hits") + } + keys := make([]string, 0, len(res.Hits[0].Fields)) + for k := range res.Hits[0].Fields { + keys = append(keys, k) + } + sort.Strings(keys) + t.Logf("hit.Fields keys: %v", keys) + + for _, k := range []string{"location.longitude", "location.latitude", "location.altitude"} { + if _, ok := res.Hits[0].Fields[k]; !ok { + t.Errorf("missing %q in hit.Fields (got %v)", k, keys) + } + } +} + +func TestLocationLatitudeRangeQueryMatches(t *testing.T) { + idx := geoFixture(t, 11.1, 49.48, 1000) + + // numeric range on the sub-field + min, max := 49.0, 50.0 + incl := true + q := query.NewNumericRangeInclusiveQuery(&min, &max, &incl, &incl) + q.SetField("location.latitude") + res, err := idx.Search(bleveSearch.NewSearchRequest(q)) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(res.Hits) != 1 { + t.Fatalf("latitude range: got %d hits, want 1", len(res.Hits)) + } + + // same range excluding the indexed latitude => no match + lowMin, lowMax := 0.0, 10.0 + q2 := query.NewNumericRangeInclusiveQuery(&lowMin, &lowMax, &incl, &incl) + q2.SetField("location.latitude") + res, err = idx.Search(bleveSearch.NewSearchRequest(q2)) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(res.Hits) != 0 { + t.Errorf("latitude range outside value: got %d hits, want 0", len(res.Hits)) + } +} + +func TestLocationLongitudeRangeQueryMatches(t *testing.T) { + idx := geoFixture(t, 11.1, 49.48, 1000) + + min, max := 11.0, 12.0 + incl := true + q := query.NewNumericRangeInclusiveQuery(&min, &max, &incl, &incl) + q.SetField("location.longitude") + res, err := idx.Search(bleveSearch.NewSearchRequest(q)) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(res.Hits) != 1 { + t.Fatalf("longitude range: got %d hits, want 1", len(res.Hits)) + } +} + +func TestLocationAltitudeRangeQueryMatches(t *testing.T) { + idx := geoFixture(t, 11.1, 49.48, 1047.7) + + min := 1000.0 + incl := true + q := query.NewNumericRangeInclusiveQuery(&min, nil, &incl, nil) + q.SetField("location.altitude") + res, err := idx.Search(bleveSearch.NewSearchRequest(q)) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(res.Hits) != 1 { + t.Fatalf("altitude >= 1000: got %d hits, want 1", len(res.Hits)) + } + + // altitude floor above the indexed value => no hits + highMin := 2000.0 + q2 := query.NewNumericRangeInclusiveQuery(&highMin, nil, &incl, nil) + q2.SetField("location.altitude") + res, err = idx.Search(bleveSearch.NewSearchRequest(q2)) + if err != nil { + t.Fatalf("Search: %v", err) + } + if len(res.Hits) != 0 { + t.Errorf("altitude >= 2000 against 1047.7: got %d hits, want 0", len(res.Hits)) + } +} + +func TestLocationGeoDistanceQueryMatches(t *testing.T) { + // Nuremberg-ish coordinates. + idx := geoFixture(t, 11.103870357204285, 49.48675890884328, 1047.7) + + // 10 km radius around the indexed point should match. + near := query.NewGeoDistanceQuery(11.103870357204285, 49.48675890884328, "10km") + near.SetField("location" + mapping.GeopointSuffix) + res, err := idx.Search(bleveSearch.NewSearchRequest(near)) + if err != nil { + t.Fatalf("Search (near): %v", err) + } + if len(res.Hits) != 1 { + t.Fatalf("geo distance near: got %d hits, want 1", len(res.Hits)) + } + + // Far away (Berlin, ~400 km) with a 10 km radius should miss. + far := query.NewGeoDistanceQuery(13.404954, 52.520008, "10km") + far.SetField("location" + mapping.GeopointSuffix) + res, err = idx.Search(bleveSearch.NewSearchRequest(far)) + if err != nil { + t.Fatalf("Search (far): %v", err) + } + if len(res.Hits) != 0 { + t.Errorf("geo distance far: got %d hits, want 0", len(res.Hits)) + } +} diff --git a/services/search/pkg/mapping/bleve.go b/services/search/pkg/mapping/bleve.go index d8329b72f8..f357e4f305 100644 --- a/services/search/pkg/mapping/bleve.go +++ b/services/search/pkg/mapping/bleve.go @@ -46,6 +46,21 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix return nil } + if fieldType == TypeGeopoint { + // Keep the facet object, add a sibling _geopoint field (see GeopointSuffix). + sub := structType(fi.GoField.Type) + if sub == nil { + return fmt.Errorf("mapping: geopoint type on non-struct field %q", key) + } + subDoc, err := buildBleveDocMapping(sub, overrides, key) + if err != nil { + return err + } + doc.AddSubDocumentMapping(fi.Name, subDoc) + doc.AddFieldMappingsAt(fi.Name+GeopointSuffix, bleve.NewGeoPointFieldMapping()) + return nil + } + fm, err := bleveFieldMapping(fieldType, opts) if err != nil { return fmt.Errorf("mapping: field %q: %w", key, err) @@ -85,6 +100,8 @@ func bleveFieldMapping(fieldType string, opts FieldOpts) (*bleveMapping.FieldMap return bleve.NewBooleanFieldMapping(), nil case TypeDatetime: return bleve.NewDateTimeFieldMapping(), nil + case TypeGeopoint: + return bleve.NewGeoPointFieldMapping(), nil case "": return nil, fmt.Errorf("no type inferred and no override") } diff --git a/services/search/pkg/mapping/bleve_test.go b/services/search/pkg/mapping/bleve_test.go index 8e2764630e..59d58fd6b9 100644 --- a/services/search/pkg/mapping/bleve_test.go +++ b/services/search/pkg/mapping/bleve_test.go @@ -114,3 +114,47 @@ func TestBleveBuildMappingOverrides(t *testing.T) { t.Errorf("Tags IncludeInAll should honor the explicit false override") } } + +func TestBleveBuildMappingGeopoint(t *testing.T) { + type geoDoc struct { + Location *struct { + Lon *float64 `json:"longitude,omitempty"` + Lat *float64 `json:"latitude,omitempty"` + Alt *float64 `json:"altitude,omitempty"` + } `json:"location,omitempty"` + } + dm, err := BleveBuildMapping(reflect.TypeFor[geoDoc](), map[string]FieldOpts{ + "location": {Type: TypeGeopoint}, + }) + if err != nil { + t.Fatalf("BleveBuildMapping: %v", err) + } + // Original facet stays as an object sub-document with numeric + // sub-properties - for data retrieval via hit.Fields and ordinary + // numeric queries. + loc := dm.Properties["location"] + if loc == nil { + t.Fatalf("location sub-document missing: %#v", dm.Properties) + } + if len(loc.Fields) != 0 { + t.Errorf("location should not carry field mappings directly, got %#v", loc.Fields) + } + for _, sub := range []string{"longitude", "latitude", "altitude"} { + prop, ok := loc.Properties[sub] + if !ok { + t.Errorf("missing sub-field %q under location (properties: %v)", sub, loc.Properties) + continue + } + if len(prop.Fields) == 0 || prop.Fields[0].Type != "number" { + t.Errorf("location.%s Fields: %#v, want [number]", sub, prop.Fields) + } + } + // Sibling geopoint at "_geopoint" for geo-distance queries. + sibling := dm.Properties["location"+GeopointSuffix] + if sibling == nil { + t.Fatalf("location%s missing: %#v", GeopointSuffix, dm.Properties) + } + if len(sibling.Fields) == 0 || sibling.Fields[0].Type != "geopoint" { + t.Errorf("location%s Fields: %#v, want [geopoint]", GeopointSuffix, sibling.Fields) + } +} diff --git a/services/search/pkg/mapping/geo.go b/services/search/pkg/mapping/geo.go new file mode 100644 index 0000000000..3d0adf61a7 --- /dev/null +++ b/services/search/pkg/mapping/geo.go @@ -0,0 +1,51 @@ +package mapping + +import "strings" + +// GeopointSuffix is appended to a field's name to produce the sibling key +// that carries the geo_point / bleve-geopoint representation of the +// original facet. For example, a libregraph "location" object with +// longitude / latitude / altitude is preserved as-is under "location" (for +// data retrieval and numeric queries) while "location_geopoint" carries +// the {lat, lon} form the geo indices understand. +const GeopointSuffix = "_geopoint" + +// addGeopointSiblings walks the overrides; for each TypeGeopoint entry at +// a dotted path (e.g. "location" or "journey.start") it writes a sibling +// under the suffixed key with the {lat, lon} form both bleve's +// ExtractGeoPoint and OpenSearch's geo_point parser accept. The original +// facet object stays untouched so downstream code still sees the full +// libregraph shape (including altitude). +func addGeopointSiblings(m map[string]any, overrides map[string]FieldOpts) { + for key, opts := range overrides { + if opts.Type == TypeGeopoint { + addGeopointSibling(m, key) + } + } +} + +// addGeopointSibling resolves dottedPath within m and, if the target is a +// libregraph-shaped geo object (with numeric "longitude" and "latitude"), +// writes the `{lat, lon}` sibling at the same level under the suffixed key. +func addGeopointSibling(m map[string]any, dottedPath string) { + parts := strings.Split(dottedPath, ".") + parent := m + for _, p := range parts[:len(parts)-1] { + next, ok := parent[p].(map[string]any) + if !ok { + return + } + parent = next + } + leaf := parts[len(parts)-1] + obj, ok := parent[leaf].(map[string]any) + if !ok { + return + } + lon, hasLon := obj["longitude"].(float64) + lat, hasLat := obj["latitude"].(float64) + if !hasLon || !hasLat { + return + } + parent[leaf+GeopointSuffix] = map[string]any{"lat": lat, "lon": lon} +} diff --git a/services/search/pkg/mapping/geo_test.go b/services/search/pkg/mapping/geo_test.go new file mode 100644 index 0000000000..78cd3b5af7 --- /dev/null +++ b/services/search/pkg/mapping/geo_test.go @@ -0,0 +1,160 @@ +package mapping + +import ( + "reflect" + "testing" +) + +// build-mapping error paths: an override that doesn't fit the Go field. +func TestBuildMappingErrors(t *testing.T) { + type doc struct { + Name string `json:"name"` + } + // Geopoint on a non-struct field must error on both backends. + if _, err := BleveBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"name": {Type: TypeGeopoint}}); err == nil { + t.Error("bleve: expected error for geopoint on string field") + } + if _, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"name": {Type: TypeGeopoint}}); err == nil { + t.Error("opensearch: expected error for geopoint on string field") + } +} + +func TestAddGeopointSiblingMissingIntermediate(t *testing.T) { + m := map[string]any{"journey": "not-a-map"} + addGeopointSibling(m, "journey.start") // must bail, not panic + if _, ok := m["journey.start"+GeopointSuffix]; ok { + t.Error("no sibling should be written when the path can't be resolved") + } +} + +func TestPrepareForIndexAddsGeopointSibling(t *testing.T) { + type geoDoc struct { + Location *struct { + Longitude *float64 `json:"longitude,omitempty"` + Latitude *float64 `json:"latitude,omitempty"` + Altitude *float64 `json:"altitude,omitempty"` + } `json:"location,omitempty"` + } + lon, lat, alt := 11.1, 49.4, 1047.7 + doc := geoDoc{Location: &struct { + Longitude *float64 `json:"longitude,omitempty"` + Latitude *float64 `json:"latitude,omitempty"` + Altitude *float64 `json:"altitude,omitempty"` + }{Longitude: &lon, Latitude: &lat, Altitude: &alt}} + + m, err := PrepareForIndex(doc, map[string]FieldOpts{ + "location": {Type: TypeGeopoint}, + }) + if err != nil { + t.Fatalf("PrepareForIndex: %v", err) + } + + // Original location object stays untouched (full libregraph shape). + orig, ok := m["location"].(map[string]any) + if !ok { + t.Fatalf("expected location object preserved, got %T", m["location"]) + } + if orig["longitude"] != lon || orig["latitude"] != lat || orig["altitude"] != alt { + t.Errorf("location object: %#v", orig) + } + + // Sibling location_geopoint has {lat, lon} for the geo indices. + gp, ok := m["location"+GeopointSuffix].(map[string]any) + if !ok { + t.Fatalf("expected location_geopoint sibling, got %T", m["location"+GeopointSuffix]) + } + if gp["lat"] != lat || gp["lon"] != lon { + t.Errorf("sibling: %#v", gp) + } +} + +func TestPrepareForIndexSkipsIncompleteGeopoint(t *testing.T) { + type geoDoc struct { + Location *struct { + Altitude *float64 `json:"altitude,omitempty"` + } `json:"location,omitempty"` + } + alt := 100.0 + doc := geoDoc{Location: &struct { + Altitude *float64 `json:"altitude,omitempty"` + }{Altitude: &alt}} + + m, err := PrepareForIndex(doc, map[string]FieldOpts{ + "location": {Type: TypeGeopoint}, + }) + if err != nil { + t.Fatalf("PrepareForIndex: %v", err) + } + // Original stays (altitude alone is still useful metadata). + if _, ok := m["location"]; !ok { + t.Error("location should still be present when only altitude is set") + } + // No sibling without both lon and lat. + if _, ok := m["location"+GeopointSuffix]; ok { + t.Errorf("no sibling expected, got %#v", m["location"+GeopointSuffix]) + } +} + +func TestPrepareForIndexWithoutOverrideNoSibling(t *testing.T) { + type geoDoc struct { + Location *struct { + Longitude *float64 `json:"longitude,omitempty"` + Latitude *float64 `json:"latitude,omitempty"` + } `json:"location,omitempty"` + } + lon, lat := 11.1, 49.4 + doc := geoDoc{Location: &struct { + Longitude *float64 `json:"longitude,omitempty"` + Latitude *float64 `json:"latitude,omitempty"` + }{Longitude: &lon, Latitude: &lat}} + + m, err := PrepareForIndex(doc, nil) + if err != nil { + t.Fatalf("PrepareForIndex: %v", err) + } + if _, ok := m["location"+GeopointSuffix]; ok { + t.Errorf("no sibling expected without override, got %#v", m["location"+GeopointSuffix]) + } +} + +func TestPrepareForIndexHandlesNestedGeopoint(t *testing.T) { + // journey.start and journey.end - two geopoints in the same facet, + // demonstrating the dotted-path walker. + type geo struct { + Longitude *float64 `json:"longitude,omitempty"` + Latitude *float64 `json:"latitude,omitempty"` + } + type journey struct { + Start *geo `json:"start,omitempty"` + End *geo `json:"end,omitempty"` + } + type doc struct { + Journey *journey `json:"journey,omitempty"` + } + slon, slat := 11.0, 49.0 + elon, elat := 13.4, 52.5 + d := doc{Journey: &journey{ + Start: &geo{Longitude: &slon, Latitude: &slat}, + End: &geo{Longitude: &elon, Latitude: &elat}, + }} + + m, err := PrepareForIndex(d, map[string]FieldOpts{ + "journey.start": {Type: TypeGeopoint}, + "journey.end": {Type: TypeGeopoint}, + }) + if err != nil { + t.Fatalf("PrepareForIndex: %v", err) + } + j, ok := m["journey"].(map[string]any) + if !ok { + t.Fatalf("journey not an object: %T", m["journey"]) + } + startGp, ok := j["start"+GeopointSuffix].(map[string]any) + if !ok || startGp["lat"] != slat || startGp["lon"] != slon { + t.Errorf("journey.start sibling: %#v", j["start"+GeopointSuffix]) + } + endGp, ok := j["end"+GeopointSuffix].(map[string]any) + if !ok || endGp["lat"] != elat || endGp["lon"] != elon { + t.Errorf("journey.end sibling: %#v", j["end"+GeopointSuffix]) + } +} diff --git a/services/search/pkg/mapping/opensearch.go b/services/search/pkg/mapping/opensearch.go index fa6152feb7..abeeb96d65 100644 --- a/services/search/pkg/mapping/opensearch.go +++ b/services/search/pkg/mapping/opensearch.go @@ -41,6 +41,21 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p return nil } + if fieldType == TypeGeopoint { + // Keep the facet object, add a sibling _geopoint field (see GeopointSuffix). + sub := structType(fi.GoField.Type) + if sub == nil { + return fmt.Errorf("mapping: geopoint type on non-struct field %q", key) + } + subProps, err := buildOpenSearchProperties(sub, overrides, key) + if err != nil { + return err + } + props[fi.Name] = map[string]any{"properties": subProps} + props[fi.Name+GeopointSuffix] = map[string]any{"type": "geo_point"} + return nil + } + fm, err := openSearchFieldMapping(fieldType, opts, fi.GoField.Type) if err != nil { return fmt.Errorf("mapping: field %q: %w", key, err) @@ -88,6 +103,8 @@ func openSearchFieldMapping(fieldType string, opts FieldOpts, goType reflect.Typ return map[string]any{"type": "boolean"}, nil case TypeDatetime: return map[string]any{"type": "date"}, nil + case TypeGeopoint: + return map[string]any{"type": "geo_point"}, nil case "": return nil, fmt.Errorf("no type inferred and no override") } diff --git a/services/search/pkg/mapping/opensearch_test.go b/services/search/pkg/mapping/opensearch_test.go index 1ad6985546..de93e8d100 100644 --- a/services/search/pkg/mapping/opensearch_test.go +++ b/services/search/pkg/mapping/opensearch_test.go @@ -130,3 +130,42 @@ func TestOpenSearchBuildMappingOverrides(t *testing.T) { t.Errorf("MimeType: %#v", mime) } } + +func TestOpenSearchBuildMappingGeopoint(t *testing.T) { + type doc struct { + Location *struct { + Lon float64 `json:"longitude"` + Lat float64 `json:"latitude"` + Alt float64 `json:"altitude"` + } `json:"location,omitempty"` + } + props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{ + "location": {Type: TypeGeopoint}, + }) + if err != nil { + t.Fatalf("OpenSearchBuildMapping: %v", err) + } + // Object for libregraph-shape data retrieval. + loc, ok := props["location"].(map[string]any) + if !ok { + t.Fatalf("location: %#v", props["location"]) + } + sub, ok := loc["properties"].(map[string]any) + if !ok { + t.Fatalf("location should have numeric sub-properties, got %#v", loc) + } + for _, k := range []string{"longitude", "latitude", "altitude"} { + prop, ok := sub[k].(map[string]any) + if !ok || prop["type"] != "double" { + t.Errorf("location.%s: %#v", k, sub[k]) + } + } + // Sibling geo_point for spatial queries. + gp, ok := props["location"+GeopointSuffix].(map[string]any) + if !ok { + t.Fatalf("location%s: %#v", GeopointSuffix, props["location"+GeopointSuffix]) + } + if gp["type"] != "geo_point" { + t.Errorf("location%s.type: %v", GeopointSuffix, gp["type"]) + } +} diff --git a/services/search/pkg/mapping/opts.go b/services/search/pkg/mapping/opts.go index 39129961db..3daa258a62 100644 --- a/services/search/pkg/mapping/opts.go +++ b/services/search/pkg/mapping/opts.go @@ -14,6 +14,7 @@ const ( TypeDatetime = "datetime" TypeBool = "bool" TypeObject = "object" + TypeGeopoint = "geopoint" ) // FieldOpts overrides the default type inference for a struct field. Keys in diff --git a/services/search/pkg/mapping/serialize.go b/services/search/pkg/mapping/serialize.go index bf72344662..027d10da71 100644 --- a/services/search/pkg/mapping/serialize.go +++ b/services/search/pkg/mapping/serialize.go @@ -7,12 +7,17 @@ import ( ) // PrepareForIndex converts v to the flat map[string]any the backend index -// clients expect, via a json round-trip (conversions.To). overrides is -// reserved for type-specific adaptations wired in by follow-up features. +// clients expect: a json round-trip (conversions.To) plus type-specific +// adaptations (currently geopoint siblings). Pass the same overrides as the +// *BuildMapping calls so the document and the mapping stay in sync. func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, error) { out, err := conversions.To[map[string]any](v) if err != nil { return nil, fmt.Errorf("mapping: prepare %T: %w", v, err) } + if out == nil { + return out, nil + } + addGeopointSiblings(out, overrides) return out, nil } diff --git a/services/search/pkg/mapping/validate_test.go b/services/search/pkg/mapping/validate_test.go index fdf514c29b..2d8500c2b8 100644 --- a/services/search/pkg/mapping/validate_test.go +++ b/services/search/pkg/mapping/validate_test.go @@ -24,6 +24,7 @@ func TestValidateAccepts(t *testing.T) { "Name": {Analyzer: "lowercaseKeyword"}, "audio": {Type: TypeObject}, "audio.artist": {Analyzer: "lowercaseKeyword"}, + "location": {Type: TypeGeopoint}, }) if err != nil { t.Fatalf("unexpected error: %v", err) diff --git a/services/search/pkg/search/search.go b/services/search/pkg/search/search.go index 035b18a71a..c27123f173 100644 --- a/services/search/pkg/search/search.go +++ b/services/search/pkg/search/search.go @@ -72,6 +72,7 @@ var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts "Content": {Type: mapping.TypeFulltext}, "Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll}, "Favorites": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll}, + "location": {Type: mapping.TypeGeopoint}, // Mtime is stored as an RFC3339 string; type it as a date so mtime:>... // range queries are chronological on both backends (bleve DateRangeQuery // / OpenSearch date range), not a lexicographic keyword compare. From c384322d94bcc003153118d9c4395846af9670bd Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Sun, 5 Jul 2026 16:56:10 +0200 Subject: [PATCH 03/54] test(search): use RFC3339 Mtime in opensearch fixture Mtime is now a date field; the fixture's Go-format string fails OpenSearch date parsing. --- .../search/internal/opensearchtest/testdata/resource_file.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/search/internal/opensearchtest/testdata/resource_file.json b/services/search/internal/opensearchtest/testdata/resource_file.json index 3d63d3e385..cc2a1fda15 100644 --- a/services/search/internal/opensearchtest/testdata/resource_file.json +++ b/services/search/internal/opensearchtest/testdata/resource_file.json @@ -8,7 +8,7 @@ "Name" : "dummy name", "Content" : "dummy content", "Size" : 42, - "Mtime" : "2025-07-24 15:15:01.324093 +0200 CEST m=+0.000056251", + "Mtime" : "2025-07-24T15:15:01.324093+02:00", "MimeType" : "image/jpeg", "Tags" : [ "dummy" ], "Deleted" : false, From c6c36212f55e6737971a4a115d30bf0bdc57dc44 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Sun, 5 Jul 2026 16:56:10 +0200 Subject: [PATCH 04/54] test(search): convert bleve geo/mtime tests to ginkgo The package's engine suite is ginkgo; these new tests were plain. --- services/search/pkg/bleve/geo_verify_test.go | 217 +++++++------------ services/search/pkg/bleve/mtime_test.go | 53 ++--- 2 files changed, 104 insertions(+), 166 deletions(-) diff --git a/services/search/pkg/bleve/geo_verify_test.go b/services/search/pkg/bleve/geo_verify_test.go index 27cf0e7a01..2bfb0d6a69 100644 --- a/services/search/pkg/bleve/geo_verify_test.go +++ b/services/search/pkg/bleve/geo_verify_test.go @@ -1,11 +1,10 @@ package bleve_test import ( - "sort" - "testing" - bleveSearch "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/search/query" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" libregraph "github.com/opencloud-eu/libre-graph-api-go" "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" @@ -14,18 +13,14 @@ import ( "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) -// geoFixture builds an in-memory bleve index with a single resource that -// carries the given lon/lat/alt. Used by the search tests below. -func geoFixture(t *testing.T, lon, lat, alt float64) bleveSearch.Index { - t.Helper() +// geoFixture builds an in-memory bleve index with a single resource carrying +// the given lon/lat/alt, indexed through the full bleve pipeline. +func geoFixture(lon, lat, alt float64) bleveSearch.Index { idxMapping, err := bleve.NewMapping() - if err != nil { - t.Fatalf("NewMapping: %v", err) - } + Expect(err).ToNot(HaveOccurred()) idx, err := bleveSearch.NewMemOnly(idxMapping) - if err != nil { - t.Fatalf("NewMemOnly: %v", err) - } + Expect(err).ToNot(HaveOccurred()) + r := search.Resource{ ID: "x", Document: content.Document{ @@ -38,141 +33,95 @@ func geoFixture(t *testing.T, lon, lat, alt float64) bleveSearch.Index { }, } doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides()) - if err != nil { - t.Fatalf("PrepareForIndex: %v", err) - } - if err := idx.Index(r.ID, doc); err != nil { - t.Fatalf("Index: %v", err) - } + Expect(err).ToNot(HaveOccurred()) + Expect(idx.Index(r.ID, doc)).To(Succeed()) return idx } -// TestLocationAltitudeRoundTrip proves that every subfield of Location -// (including altitude) ends up in hit.Fields when a Resource is indexed -// through the full bleve pipeline. This is the invariant the Move / -// Delete / Restore round-trip depends on. -func TestLocationAltitudeRoundTrip(t *testing.T) { - idx := geoFixture(t, 11.103870357204285, 49.48675890884328, 1047.7) +var _ = Describe("Location geo queries", func() { + // Every Location subfield (including altitude) must end up in hit.Fields + // when a Resource is indexed through the full bleve pipeline. This is the + // invariant the Move / Delete / Restore round-trip depends on. + It("round-trips every Location subfield into hit.Fields", func() { + idx := geoFixture(11.103870357204285, 49.48675890884328, 1047.7) - req := bleveSearch.NewSearchRequest(bleveSearch.NewMatchAllQuery()) - req.Fields = []string{"*"} - res, err := idx.Search(req) - if err != nil { - t.Fatalf("Search: %v", err) - } - if len(res.Hits) == 0 { - t.Fatal("no hits") - } - keys := make([]string, 0, len(res.Hits[0].Fields)) - for k := range res.Hits[0].Fields { - keys = append(keys, k) - } - sort.Strings(keys) - t.Logf("hit.Fields keys: %v", keys) + req := bleveSearch.NewSearchRequest(bleveSearch.NewMatchAllQuery()) + req.Fields = []string{"*"} + res, err := idx.Search(req) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits).ToNot(BeEmpty()) - for _, k := range []string{"location.longitude", "location.latitude", "location.altitude"} { - if _, ok := res.Hits[0].Fields[k]; !ok { - t.Errorf("missing %q in hit.Fields (got %v)", k, keys) + for _, k := range []string{"location.longitude", "location.latitude", "location.altitude"} { + Expect(res.Hits[0].Fields).To(HaveKey(k)) } - } -} + }) -func TestLocationLatitudeRangeQueryMatches(t *testing.T) { - idx := geoFixture(t, 11.1, 49.48, 1000) + It("matches a latitude numeric range and misses outside it", func() { + idx := geoFixture(11.1, 49.48, 1000) - // numeric range on the sub-field - min, max := 49.0, 50.0 - incl := true - q := query.NewNumericRangeInclusiveQuery(&min, &max, &incl, &incl) - q.SetField("location.latitude") - res, err := idx.Search(bleveSearch.NewSearchRequest(q)) - if err != nil { - t.Fatalf("Search: %v", err) - } - if len(res.Hits) != 1 { - t.Fatalf("latitude range: got %d hits, want 1", len(res.Hits)) - } + min, max := 49.0, 50.0 + incl := true + q := query.NewNumericRangeInclusiveQuery(&min, &max, &incl, &incl) + q.SetField("location.latitude") + res, err := idx.Search(bleveSearch.NewSearchRequest(q)) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits).To(HaveLen(1)) - // same range excluding the indexed latitude => no match - lowMin, lowMax := 0.0, 10.0 - q2 := query.NewNumericRangeInclusiveQuery(&lowMin, &lowMax, &incl, &incl) - q2.SetField("location.latitude") - res, err = idx.Search(bleveSearch.NewSearchRequest(q2)) - if err != nil { - t.Fatalf("Search: %v", err) - } - if len(res.Hits) != 0 { - t.Errorf("latitude range outside value: got %d hits, want 0", len(res.Hits)) - } -} + lowMin, lowMax := 0.0, 10.0 + q2 := query.NewNumericRangeInclusiveQuery(&lowMin, &lowMax, &incl, &incl) + q2.SetField("location.latitude") + res, err = idx.Search(bleveSearch.NewSearchRequest(q2)) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits).To(BeEmpty()) + }) -func TestLocationLongitudeRangeQueryMatches(t *testing.T) { - idx := geoFixture(t, 11.1, 49.48, 1000) + It("matches a longitude numeric range", func() { + idx := geoFixture(11.1, 49.48, 1000) - min, max := 11.0, 12.0 - incl := true - q := query.NewNumericRangeInclusiveQuery(&min, &max, &incl, &incl) - q.SetField("location.longitude") - res, err := idx.Search(bleveSearch.NewSearchRequest(q)) - if err != nil { - t.Fatalf("Search: %v", err) - } - if len(res.Hits) != 1 { - t.Fatalf("longitude range: got %d hits, want 1", len(res.Hits)) - } -} + min, max := 11.0, 12.0 + incl := true + q := query.NewNumericRangeInclusiveQuery(&min, &max, &incl, &incl) + q.SetField("location.longitude") + res, err := idx.Search(bleveSearch.NewSearchRequest(q)) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits).To(HaveLen(1)) + }) -func TestLocationAltitudeRangeQueryMatches(t *testing.T) { - idx := geoFixture(t, 11.1, 49.48, 1047.7) + It("matches an altitude lower-bound range and misses above it", func() { + idx := geoFixture(11.1, 49.48, 1047.7) - min := 1000.0 - incl := true - q := query.NewNumericRangeInclusiveQuery(&min, nil, &incl, nil) - q.SetField("location.altitude") - res, err := idx.Search(bleveSearch.NewSearchRequest(q)) - if err != nil { - t.Fatalf("Search: %v", err) - } - if len(res.Hits) != 1 { - t.Fatalf("altitude >= 1000: got %d hits, want 1", len(res.Hits)) - } + min := 1000.0 + incl := true + q := query.NewNumericRangeInclusiveQuery(&min, nil, &incl, nil) + q.SetField("location.altitude") + res, err := idx.Search(bleveSearch.NewSearchRequest(q)) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits).To(HaveLen(1)) - // altitude floor above the indexed value => no hits - highMin := 2000.0 - q2 := query.NewNumericRangeInclusiveQuery(&highMin, nil, &incl, nil) - q2.SetField("location.altitude") - res, err = idx.Search(bleveSearch.NewSearchRequest(q2)) - if err != nil { - t.Fatalf("Search: %v", err) - } - if len(res.Hits) != 0 { - t.Errorf("altitude >= 2000 against 1047.7: got %d hits, want 0", len(res.Hits)) - } -} + highMin := 2000.0 + q2 := query.NewNumericRangeInclusiveQuery(&highMin, nil, &incl, nil) + q2.SetField("location.altitude") + res, err = idx.Search(bleveSearch.NewSearchRequest(q2)) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits).To(BeEmpty()) + }) -func TestLocationGeoDistanceQueryMatches(t *testing.T) { - // Nuremberg-ish coordinates. - idx := geoFixture(t, 11.103870357204285, 49.48675890884328, 1047.7) + It("matches a geo-distance query near the point and misses far away", func() { + // Nuremberg-ish coordinates. + idx := geoFixture(11.103870357204285, 49.48675890884328, 1047.7) - // 10 km radius around the indexed point should match. - near := query.NewGeoDistanceQuery(11.103870357204285, 49.48675890884328, "10km") - near.SetField("location" + mapping.GeopointSuffix) - res, err := idx.Search(bleveSearch.NewSearchRequest(near)) - if err != nil { - t.Fatalf("Search (near): %v", err) - } - if len(res.Hits) != 1 { - t.Fatalf("geo distance near: got %d hits, want 1", len(res.Hits)) - } + // 10 km radius around the indexed point should match. + near := query.NewGeoDistanceQuery(11.103870357204285, 49.48675890884328, "10km") + near.SetField("location" + mapping.GeopointSuffix) + res, err := idx.Search(bleveSearch.NewSearchRequest(near)) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits).To(HaveLen(1)) - // Far away (Berlin, ~400 km) with a 10 km radius should miss. - far := query.NewGeoDistanceQuery(13.404954, 52.520008, "10km") - far.SetField("location" + mapping.GeopointSuffix) - res, err = idx.Search(bleveSearch.NewSearchRequest(far)) - if err != nil { - t.Fatalf("Search (far): %v", err) - } - if len(res.Hits) != 0 { - t.Errorf("geo distance far: got %d hits, want 0", len(res.Hits)) - } -} + // Far away (Berlin, ~400 km) with a 10 km radius should miss. + far := query.NewGeoDistanceQuery(13.404954, 52.520008, "10km") + far.SetField("location" + mapping.GeopointSuffix) + res, err = idx.Search(bleveSearch.NewSearchRequest(far)) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits).To(BeEmpty()) + }) +}) diff --git a/services/search/pkg/bleve/mtime_test.go b/services/search/pkg/bleve/mtime_test.go index d7475b0316..c29c83d9b8 100644 --- a/services/search/pkg/bleve/mtime_test.go +++ b/services/search/pkg/bleve/mtime_test.go @@ -1,10 +1,10 @@ package bleve_test import ( - "testing" - bleveSearch "github.com/blevesearch/bleve/v2" bquery "github.com/blevesearch/bleve/v2/search/query" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" "github.com/opencloud-eu/opencloud/services/search/pkg/content" @@ -14,35 +14,24 @@ import ( // Mtime is typed as a date, so range queries are chronological, not a // lexicographic keyword compare. -func TestMtimeDateRange(t *testing.T) { - m, err := bleve.NewMapping() - if err != nil { - t.Fatal(err) - } - idx, err := bleveSearch.NewMemOnly(m) - if err != nil { - t.Fatal(err) - } - r := search.Resource{ID: "x", Document: content.Document{Name: "f", Mtime: "2026-03-15T12:00:00.123456789Z"}} - doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides()) - if err != nil { - t.Fatal(err) - } - if err := idx.Index(r.ID, doc); err != nil { - t.Fatal(err) - } +var _ = Describe("Mtime date range", func() { + It("compares chronologically, not lexicographically", func() { + m, err := bleve.NewMapping() + Expect(err).ToNot(HaveOccurred()) + idx, err := bleveSearch.NewMemOnly(m) + Expect(err).ToNot(HaveOccurred()) - hits := func(qs string) uint64 { - res, err := idx.Search(bleveSearch.NewSearchRequest(bquery.NewQueryStringQuery(qs))) - if err != nil { - t.Fatalf("%s: %v", qs, err) + r := search.Resource{ID: "x", Document: content.Document{Name: "f", Mtime: "2026-03-15T12:00:00.123456789Z"}} + doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides()) + Expect(err).ToNot(HaveOccurred()) + Expect(idx.Index(r.ID, doc)).To(Succeed()) + + hits := func(qs string) uint64 { + res, err := idx.Search(bleveSearch.NewSearchRequest(bquery.NewQueryStringQuery(qs))) + Expect(err).ToNot(HaveOccurred(), qs) + return res.Total } - return res.Total - } - if got := hits(`Mtime:>"2026-01-01T00:00:00Z"`); got != 1 { - t.Errorf("in-range: got %d hits, want 1", got) - } - if got := hits(`Mtime:>"2026-06-01T00:00:00Z"`); got != 0 { - t.Errorf("out-of-range: got %d hits, want 0", got) - } -} + Expect(hits(`Mtime:>"2026-01-01T00:00:00Z"`)).To(Equal(uint64(1)), "in-range") + Expect(hits(`Mtime:>"2026-06-01T00:00:00Z"`)).To(Equal(uint64(0)), "out-of-range") + }) +}) From 6695f99c90b1deda0092c2dc6b7a8c950893b75a Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Sun, 5 Jul 2026 16:56:10 +0200 Subject: [PATCH 05/54] test(search): convert mapping package tests to ginkgo New package, so use the repo's standard test framework. --- services/search/pkg/mapping/bleve_test.go | 219 ++++++--------- .../pkg/mapping/deserialize_string_test.go | 176 +++++------- .../search/pkg/mapping/deserialize_test.go | 189 +++++-------- .../search/pkg/mapping/fillstruct_test.go | 79 ++---- services/search/pkg/mapping/geo_test.go | 259 ++++++++---------- services/search/pkg/mapping/infer_test.go | 177 +++++------- .../search/pkg/mapping/mapping_suite_test.go | 13 + .../search/pkg/mapping/opensearch_test.go | 259 ++++++++---------- services/search/pkg/mapping/serialize_test.go | 132 ++++----- services/search/pkg/mapping/validate_test.go | 52 ++-- 10 files changed, 672 insertions(+), 883 deletions(-) create mode 100644 services/search/pkg/mapping/mapping_suite_test.go diff --git a/services/search/pkg/mapping/bleve_test.go b/services/search/pkg/mapping/bleve_test.go index 59d58fd6b9..a9cd81954f 100644 --- a/services/search/pkg/mapping/bleve_test.go +++ b/services/search/pkg/mapping/bleve_test.go @@ -2,8 +2,10 @@ package mapping import ( "reflect" - "testing" "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) type bleveDoc struct { @@ -21,140 +23,93 @@ type nested struct { Year int `json:"year"` } -// bleve wildcard falls back to keyword-ish text (bleve has no wildcard type). -func TestBleveWildcardFallback(t *testing.T) { - type doc struct { - Mime string `json:"mime"` - } - dm, err := BleveBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"mime": {Type: TypeWildcard}}) - if err != nil { - t.Fatalf("BleveBuildMapping: %v", err) - } - fms := dm.Properties["mime"].Fields - if len(fms) != 1 || fms[0].Type != "text" { - t.Fatalf("wildcard should map to text, got %+v", fms) - } -} - -func TestBleveBuildMappingInferredTypes(t *testing.T) { - dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil) - if err != nil { - t.Fatalf("BleveBuildMapping: %v", err) - } - cases := map[string]string{ - "Name": "text", - "Content": "text", - "Tags": "text", - "Size": "number", - "Deleted": "boolean", - "CreatedAt": "datetime", - } - for field, wantType := range cases { - prop := dm.Properties[field] - if prop == nil { - t.Errorf("missing property %q", field) - continue +var _ = Describe("BleveBuildMapping", func() { + It("falls back to text for wildcard fields", func() { + // bleve wildcard falls back to keyword-ish text (bleve has no wildcard type). + type doc struct { + Mime string `json:"mime"` } - if len(prop.Fields) == 0 { - t.Errorf("%q: no field mappings", field) - continue - } - if got := prop.Fields[0].Type; got != wantType { - t.Errorf("%q: got type %q, want %q", field, got, wantType) - } - } -} - -func TestBleveBuildMappingNestedIsSubDocument(t *testing.T) { - dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil) - if err != nil { - t.Fatalf("BleveBuildMapping: %v", err) - } - sub := dm.Properties["nested"] - if sub == nil { - t.Fatal("missing nested sub-document") - } - if sub.Properties["artist"] == nil || sub.Properties["year"] == nil { - t.Fatalf("nested fields missing: %#v", sub.Properties) - } - if got := sub.Properties["artist"].Fields[0].Type; got != "text" { - t.Errorf("nested.artist: type %q, want text", got) - } - if got := sub.Properties["year"].Fields[0].Type; got != "number" { - t.Errorf("nested.year: type %q, want number", got) - } -} - -func TestBleveBuildMappingOverrides(t *testing.T) { - includeInAllFalse := false - dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{ - "Name": {Analyzer: "lowercaseKeyword"}, - "Content": {Type: TypeFulltext}, - "Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &includeInAllFalse}, + dm, err := BleveBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"mime": {Type: TypeWildcard}}) + Expect(err).ToNot(HaveOccurred()) + fms := dm.Properties["mime"].Fields + Expect(fms).To(HaveLen(1)) + Expect(fms[0].Type).To(Equal("text"), "wildcard should map to text") }) - if err != nil { - t.Fatalf("BleveBuildMapping: %v", err) - } - nameField := dm.Properties["Name"].Fields[0] - if nameField.Analyzer != "lowercaseKeyword" { - t.Errorf("Name analyzer: %q, want lowercaseKeyword", nameField.Analyzer) - } - if !nameField.IncludeInAll { - t.Errorf("Name IncludeInAll should stay default-true when not overridden") - } - contentField := dm.Properties["Content"].Fields[0] - if contentField.Analyzer != "fulltext" { - t.Errorf("Content analyzer: %q, want fulltext", contentField.Analyzer) - } - if contentField.IncludeInAll { - t.Errorf("Content IncludeInAll should default to false for fulltext type") - } - tagsField := dm.Properties["Tags"].Fields[0] - if tagsField.IncludeInAll { - t.Errorf("Tags IncludeInAll should honor the explicit false override") - } -} -func TestBleveBuildMappingGeopoint(t *testing.T) { - type geoDoc struct { - Location *struct { - Lon *float64 `json:"longitude,omitempty"` - Lat *float64 `json:"latitude,omitempty"` - Alt *float64 `json:"altitude,omitempty"` - } `json:"location,omitempty"` - } - dm, err := BleveBuildMapping(reflect.TypeFor[geoDoc](), map[string]FieldOpts{ - "location": {Type: TypeGeopoint}, + DescribeTable("infers field types", + func(field, wantType string) { + dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil) + Expect(err).ToNot(HaveOccurred()) + prop := dm.Properties[field] + Expect(prop).ToNot(BeNil(), "missing property %q", field) + Expect(prop.Fields).ToNot(BeEmpty(), "%q: no field mappings", field) + Expect(prop.Fields[0].Type).To(Equal(wantType), "%q type", field) + }, + Entry("Name", "Name", "text"), + Entry("Content", "Content", "text"), + Entry("Tags", "Tags", "text"), + Entry("Size", "Size", "number"), + Entry("Deleted", "Deleted", "boolean"), + Entry("CreatedAt", "CreatedAt", "datetime"), + ) + + It("maps a nested struct as a sub-document", func() { + dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil) + Expect(err).ToNot(HaveOccurred()) + sub := dm.Properties["nested"] + Expect(sub).ToNot(BeNil(), "missing nested sub-document") + Expect(sub.Properties["artist"]).ToNot(BeNil()) + Expect(sub.Properties["year"]).ToNot(BeNil()) + Expect(sub.Properties["artist"].Fields[0].Type).To(Equal("text"), "nested.artist") + Expect(sub.Properties["year"].Fields[0].Type).To(Equal("number"), "nested.year") }) - if err != nil { - t.Fatalf("BleveBuildMapping: %v", err) - } - // Original facet stays as an object sub-document with numeric - // sub-properties - for data retrieval via hit.Fields and ordinary - // numeric queries. - loc := dm.Properties["location"] - if loc == nil { - t.Fatalf("location sub-document missing: %#v", dm.Properties) - } - if len(loc.Fields) != 0 { - t.Errorf("location should not carry field mappings directly, got %#v", loc.Fields) - } - for _, sub := range []string{"longitude", "latitude", "altitude"} { - prop, ok := loc.Properties[sub] - if !ok { - t.Errorf("missing sub-field %q under location (properties: %v)", sub, loc.Properties) - continue + + It("applies field overrides", func() { + includeInAllFalse := false + dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{ + "Name": {Analyzer: "lowercaseKeyword"}, + "Content": {Type: TypeFulltext}, + "Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &includeInAllFalse}, + }) + Expect(err).ToNot(HaveOccurred()) + nameField := dm.Properties["Name"].Fields[0] + Expect(nameField.Analyzer).To(Equal("lowercaseKeyword"), "Name analyzer") + Expect(nameField.IncludeInAll).To(BeTrue(), "Name IncludeInAll should stay default-true when not overridden") + contentField := dm.Properties["Content"].Fields[0] + Expect(contentField.Analyzer).To(Equal("fulltext"), "Content analyzer") + Expect(contentField.IncludeInAll).To(BeFalse(), "Content IncludeInAll should default to false for fulltext type") + tagsField := dm.Properties["Tags"].Fields[0] + Expect(tagsField.IncludeInAll).To(BeFalse(), "Tags IncludeInAll should honor the explicit false override") + }) + + It("builds an object sub-document plus a geopoint sibling", func() { + type geoDoc struct { + Location *struct { + Lon *float64 `json:"longitude,omitempty"` + Lat *float64 `json:"latitude,omitempty"` + Alt *float64 `json:"altitude,omitempty"` + } `json:"location,omitempty"` } - if len(prop.Fields) == 0 || prop.Fields[0].Type != "number" { - t.Errorf("location.%s Fields: %#v, want [number]", sub, prop.Fields) + dm, err := BleveBuildMapping(reflect.TypeFor[geoDoc](), map[string]FieldOpts{ + "location": {Type: TypeGeopoint}, + }) + Expect(err).ToNot(HaveOccurred()) + // Original facet stays as an object sub-document with numeric + // sub-properties - for data retrieval via hit.Fields and ordinary + // numeric queries. + loc := dm.Properties["location"] + Expect(loc).ToNot(BeNil(), "location sub-document missing") + Expect(loc.Fields).To(BeEmpty(), "location should not carry field mappings directly") + for _, sub := range []string{"longitude", "latitude", "altitude"} { + prop, ok := loc.Properties[sub] + Expect(ok).To(BeTrue(), "missing sub-field %q under location", sub) + Expect(prop.Fields).ToNot(BeEmpty(), "location.%s Fields", sub) + Expect(prop.Fields[0].Type).To(Equal("number"), "location.%s type", sub) } - } - // Sibling geopoint at "_geopoint" for geo-distance queries. - sibling := dm.Properties["location"+GeopointSuffix] - if sibling == nil { - t.Fatalf("location%s missing: %#v", GeopointSuffix, dm.Properties) - } - if len(sibling.Fields) == 0 || sibling.Fields[0].Type != "geopoint" { - t.Errorf("location%s Fields: %#v, want [geopoint]", GeopointSuffix, sibling.Fields) - } -} + // Sibling geopoint at "_geopoint" for geo-distance queries. + sibling := dm.Properties["location"+GeopointSuffix] + Expect(sibling).ToNot(BeNil(), "location%s missing", GeopointSuffix) + Expect(sibling.Fields).ToNot(BeEmpty(), "location%s Fields", GeopointSuffix) + Expect(sibling.Fields[0].Type).To(Equal("geopoint"), "location%s type", GeopointSuffix) + }) +}) diff --git a/services/search/pkg/mapping/deserialize_string_test.go b/services/search/pkg/mapping/deserialize_string_test.go index 981f5e50a5..9fed25b4d9 100644 --- a/services/search/pkg/mapping/deserialize_string_test.go +++ b/services/search/pkg/mapping/deserialize_string_test.go @@ -2,9 +2,10 @@ package mapping import ( "reflect" - "testing" "time" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -17,106 +18,81 @@ type stringFacet struct { Taken *time.Time `json:"takenDateTime,omitempty"` } -func TestSetValueFromStringUnsupportedKind(t *testing.T) { - v := reflect.New(reflect.TypeFor[[]int]()).Elem() // settable slice - if err := setValueFromString(v, "x"); err == nil { - t.Error("expected error for unsupported target kind (slice)") - } -} +var _ = Describe("DeserializeStringsAt", func() { + It("errors on an unsupported target kind", func() { + v := reflect.New(reflect.TypeFor[[]int]()).Elem() // settable slice + Expect(setValueFromString(v, "x")).To(HaveOccurred(), "expected error for unsupported target kind (slice)") + }) -func TestDeserializeStringsAtBasicTypes(t *testing.T) { - r := DeserializeStringsAt[stringFacet](map[string]string{ - "libre.graph.audio.artist": "Queen", - "libre.graph.audio.year": "1975", - "libre.graph.audio.duration": "354000", - "libre.graph.audio.rating": "4.9", - "libre.graph.audio.explicit": "true", - "libre.graph.audio.takenDateTime": "2024-01-02T03:04:05Z", - }, "libre.graph.audio.") - if r == nil { - t.Fatal("expected non-nil *stringFacet") - } - if r.Artist == nil || *r.Artist != "Queen" { - t.Errorf("Artist: %#v", r.Artist) - } - if r.Year == nil || *r.Year != 1975 { - t.Errorf("Year: %#v", r.Year) - } - if r.Duration == nil || *r.Duration != 354000 { - t.Errorf("Duration: %#v", r.Duration) - } - if r.Rating == nil || *r.Rating != 4.9 { - t.Errorf("Rating: %#v", r.Rating) - } - if r.Explicit == nil || !*r.Explicit { - t.Errorf("Explicit: %#v", r.Explicit) - } - if r.Taken == nil || !r.Taken.Equal(time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC)) { - t.Errorf("Taken: %#v", r.Taken) - } -} + It("parses basic types", func() { + r := DeserializeStringsAt[stringFacet](map[string]string{ + "libre.graph.audio.artist": "Queen", + "libre.graph.audio.year": "1975", + "libre.graph.audio.duration": "354000", + "libre.graph.audio.rating": "4.9", + "libre.graph.audio.explicit": "true", + "libre.graph.audio.takenDateTime": "2024-01-02T03:04:05Z", + }, "libre.graph.audio.") + Expect(r).ToNot(BeNil()) + Expect(r.Artist).ToNot(BeNil()) + Expect(*r.Artist).To(Equal("Queen")) + Expect(r.Year).ToNot(BeNil()) + Expect(*r.Year).To(Equal(int32(1975))) + Expect(r.Duration).ToNot(BeNil()) + Expect(*r.Duration).To(Equal(int64(354000))) + Expect(r.Rating).ToNot(BeNil()) + Expect(*r.Rating).To(Equal(4.9)) + Expect(r.Explicit).ToNot(BeNil()) + Expect(*r.Explicit).To(BeTrue()) + Expect(r.Taken).ToNot(BeNil()) + Expect(r.Taken.Equal(time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC))).To(BeTrue(), "Taken: %#v", r.Taken) + }) -func TestDeserializeStringsAtReturnsNilWhenEmpty(t *testing.T) { - r := DeserializeStringsAt[stringFacet](map[string]string{ - "libre.graph.image.width": "1200", - }, "libre.graph.audio.") - if r != nil { - t.Fatalf("expected nil, got %#v", r) - } -} + It("returns nil when nothing matches the prefix", func() { + r := DeserializeStringsAt[stringFacet](map[string]string{ + "libre.graph.image.width": "1200", + }, "libre.graph.audio.") + Expect(r).To(BeNil()) + }) -func TestDeserializeStringsAtTimestamppb(t *testing.T) { - type photoFacet struct { - Taken *timestamppb.Timestamp `json:"takenDateTime,omitempty"` - } - r := DeserializeStringsAt[photoFacet](map[string]string{ - "libre.graph.photo.takenDateTime": "2024-05-06T07:08:09Z", - }, "libre.graph.photo.") - if r == nil || r.Taken == nil { - t.Fatalf("Taken missing: %#v", r) - } - want := time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC) - if !r.Taken.AsTime().Equal(want) { - t.Errorf("Taken: got %v, want %v", r.Taken.AsTime(), want) - } -} - -func TestDeserializeStringsAtIsFailSoft(t *testing.T) { - // A single malformed field (year is unparseable as int) must not drop - // the whole facet. The bad field stays at zero value, the rest of the - // facet still populates. Mirrors the bleve-hit Deserialize behavior. - r := DeserializeStringsAt[stringFacet](map[string]string{ - "libre.graph.audio.artist": "Iron Maiden", - "libre.graph.audio.year": "not-a-number", - "libre.graph.audio.duration": "354000", - "libre.graph.audio.explicit": "not-a-bool", - "libre.graph.audio.rating": "4.9", - }, "libre.graph.audio.") - if r == nil { - t.Fatal("expected non-nil *stringFacet despite bad fields") - } - if r.Artist == nil || *r.Artist != "Iron Maiden" { - t.Errorf("Artist should still be populated, got %#v", r.Artist) - } - if r.Duration == nil || *r.Duration != 354000 { - t.Errorf("Duration should still be populated, got %#v", r.Duration) - } - if r.Rating == nil || *r.Rating != 4.9 { - t.Errorf("Rating should still be populated, got %#v", r.Rating) - } - if r.Year != nil { - t.Errorf("Year should stay nil for bad int, got %#v", r.Year) - } - if r.Explicit != nil { - t.Errorf("Explicit should stay nil for bad bool, got %#v", r.Explicit) - } -} - -func TestDeserializeStringsAtPanicsOnNonStruct(t *testing.T) { - defer func() { - if r := recover(); r == nil { - t.Fatal("expected panic for non-struct T") + It("parses into a timestamppb.Timestamp", func() { + type photoFacet struct { + Taken *timestamppb.Timestamp `json:"takenDateTime,omitempty"` } - }() - DeserializeStringsAt[int](nil, "") -} + r := DeserializeStringsAt[photoFacet](map[string]string{ + "libre.graph.photo.takenDateTime": "2024-05-06T07:08:09Z", + }, "libre.graph.photo.") + Expect(r).ToNot(BeNil()) + Expect(r.Taken).ToNot(BeNil()) + want := time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC) + Expect(r.Taken.AsTime().Equal(want)).To(BeTrue(), "Taken: got %v, want %v", r.Taken.AsTime(), want) + }) + + It("is fail-soft on malformed fields", func() { + // A single malformed field (year is unparseable as int) must not drop + // the whole facet. The bad field stays at zero value, the rest of the + // facet still populates. Mirrors the bleve-hit Deserialize behavior. + r := DeserializeStringsAt[stringFacet](map[string]string{ + "libre.graph.audio.artist": "Iron Maiden", + "libre.graph.audio.year": "not-a-number", + "libre.graph.audio.duration": "354000", + "libre.graph.audio.explicit": "not-a-bool", + "libre.graph.audio.rating": "4.9", + }, "libre.graph.audio.") + Expect(r).ToNot(BeNil()) + Expect(r.Artist).ToNot(BeNil()) + Expect(*r.Artist).To(Equal("Iron Maiden"), "Artist should still be populated") + Expect(r.Duration).ToNot(BeNil()) + Expect(*r.Duration).To(Equal(int64(354000)), "Duration should still be populated") + Expect(r.Rating).ToNot(BeNil()) + Expect(*r.Rating).To(Equal(4.9), "Rating should still be populated") + Expect(r.Year).To(BeNil(), "Year should stay nil for bad int") + Expect(r.Explicit).To(BeNil(), "Explicit should stay nil for bad bool") + }) + + It("panics for a non-struct T", func() { + Expect(func() { + DeserializeStringsAt[int](nil, "") + }).To(Panic()) + }) +}) diff --git a/services/search/pkg/mapping/deserialize_test.go b/services/search/pkg/mapping/deserialize_test.go index ec92191ce3..55824aeaa5 100644 --- a/services/search/pkg/mapping/deserialize_test.go +++ b/services/search/pkg/mapping/deserialize_test.go @@ -1,9 +1,10 @@ package mapping import ( - "testing" "time" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -31,125 +32,87 @@ type embedded struct { Photo *photo `json:"photo,omitempty"` } -func TestDeserializeAtNonStructPanics(t *testing.T) { - defer func() { - if recover() == nil { - t.Error("expected panic for non-struct type") - } - }() - _ = DeserializeAt[int](map[string]any{}, "") -} - -func TestDeserializeLeafFields(t *testing.T) { - r := Deserialize[Leaf](map[string]any{ - "Name": "n", - "Size": float64(42), - "Deleted": true, +var _ = Describe("Deserialize", func() { + It("panics for a non-struct type at a prefix", func() { + Expect(func() { + _ = DeserializeAt[int](map[string]any{}, "") + }).To(Panic()) }) - if r.Name != "n" || r.Size != 42 || !r.Deleted { - t.Fatalf("got %#v", r) - } -} -func TestDeserializeScalarToSlice(t *testing.T) { - r := Deserialize[Leaf](map[string]any{ - "Tags": "single", - "Favorites": []any{"a", "b"}, + It("panics for a non-struct T", func() { + Expect(func() { + Deserialize[int](nil) + }).To(Panic()) }) - if len(r.Tags) != 1 || r.Tags[0] != "single" { - t.Errorf("Tags: %#v", r.Tags) - } - if len(r.Favorites) != 2 || r.Favorites[0] != "a" || r.Favorites[1] != "b" { - t.Errorf("Favorites: %#v", r.Favorites) - } -} -func TestDeserializeTimestamp(t *testing.T) { - r := Deserialize[embedded](map[string]any{ - "photo.takenDateTime": "2024-01-02T03:04:05Z", - "photo.mtime": "2024-05-06T07:08:09Z", + It("deserializes leaf fields", func() { + r := Deserialize[Leaf](map[string]any{ + "Name": "n", + "Size": float64(42), + "Deleted": true, + }) + Expect(r.Name).To(Equal("n")) + Expect(r.Size).To(Equal(uint64(42))) + Expect(r.Deleted).To(BeTrue()) }) - if r.Photo == nil { - t.Fatal("Photo is nil") - } - if r.Photo.Taken == nil { - t.Fatal("Taken is nil") - } - expected := time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC) - if !r.Photo.Taken.AsTime().Equal(expected) { - t.Errorf("Taken: got %v, want %v", r.Photo.Taken.AsTime(), expected) - } - if r.Photo.Mtime == nil { - t.Fatal("Mtime is nil") - } - if !r.Photo.Mtime.Equal(time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC)) { - t.Errorf("Mtime: %v", r.Photo.Mtime) - } -} -func TestDeserializeIsFailSoft(t *testing.T) { - // Malformed values (type mismatch, unparseable time) leave the - // affected field at its zero value instead of dropping the whole - // record. Matches the pre-refactor getFieldValue behavior so - // matchToResource never returns nil on a corrupted hit. - r := Deserialize[embedded](map[string]any{ - "Name": "n", - "Size": "not-a-number", // wrong type - "Deleted": true, - "photo.takenDateTime": "not-an-rfc3339-time", - "photo.mtime": "2024-05-06T07:08:09Z", + It("coerces a scalar into a slice field", func() { + r := Deserialize[Leaf](map[string]any{ + "Tags": "single", + "Favorites": []any{"a", "b"}, + }) + Expect(r.Tags).To(Equal([]string{"single"})) + Expect(r.Favorites).To(Equal([]string{"a", "b"})) }) - if r == nil { - t.Fatal("expected non-nil *embedded even with partial corruption") - } - if r.Name != "n" { - t.Errorf("Name: %q", r.Name) - } - if r.Size != 0 { - t.Errorf("Size should stay zero on mismatch, got %d", r.Size) - } - if !r.Deleted { - t.Errorf("Deleted should still be true") - } - if r.Photo == nil { - t.Fatal("Photo should be populated because Mtime parsed ok") - } - if r.Photo.Taken != nil { - t.Errorf("Taken should stay nil for unparseable time, got %v", r.Photo.Taken) - } - if r.Photo.Mtime == nil { - t.Error("Mtime should be parsed") - } -} -func TestDeserializePanicsOnNonStruct(t *testing.T) { - defer func() { - if r := recover(); r == nil { - t.Fatal("expected panic for non-struct T") - } - }() - Deserialize[int](nil) -} + It("parses timestamps", func() { + r := Deserialize[embedded](map[string]any{ + "photo.takenDateTime": "2024-01-02T03:04:05Z", + "photo.mtime": "2024-05-06T07:08:09Z", + }) + Expect(r.Photo).ToNot(BeNil()) + Expect(r.Photo.Taken).ToNot(BeNil()) + expected := time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC) + Expect(r.Photo.Taken.AsTime().Equal(expected)).To(BeTrue(), "Taken: got %v, want %v", r.Photo.Taken.AsTime(), expected) + Expect(r.Photo.Mtime).ToNot(BeNil()) + Expect(r.Photo.Mtime.Equal(time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC))).To(BeTrue(), "Mtime: %v", r.Photo.Mtime) + }) -func TestDeserializeAtReturnsNilWhenNothingMatches(t *testing.T) { - r := DeserializeAt[audio](map[string]any{"Name": "n"}, "audio") - if r != nil { - t.Fatalf("expected nil, got %#v", r) - } -} + It("is fail-soft on malformed values", func() { + // Malformed values (type mismatch, unparseable time) leave the + // affected field at its zero value instead of dropping the whole + // record. Matches the pre-refactor getFieldValue behavior so + // matchToResource never returns nil on a corrupted hit. + r := Deserialize[embedded](map[string]any{ + "Name": "n", + "Size": "not-a-number", // wrong type + "Deleted": true, + "photo.takenDateTime": "not-an-rfc3339-time", + "photo.mtime": "2024-05-06T07:08:09Z", + }) + Expect(r).ToNot(BeNil()) + Expect(r.Name).To(Equal("n")) + Expect(r.Size).To(Equal(uint64(0)), "Size should stay zero on mismatch") + Expect(r.Deleted).To(BeTrue()) + Expect(r.Photo).ToNot(BeNil(), "Photo should be populated because Mtime parsed ok") + Expect(r.Photo.Taken).To(BeNil(), "Taken should stay nil for unparseable time") + Expect(r.Photo.Mtime).ToNot(BeNil(), "Mtime should be parsed") + }) -func TestDeserializeAtReturnsValueWhenPrefixMatches(t *testing.T) { - r := DeserializeAt[audio](map[string]any{ - "audio.artist": "A", - "audio.year": float64(2024), // setValue: pointer + numeric convert - }, "audio") - if r == nil { - t.Fatal("expected non-nil *audio") - } - if r.Artist == nil || *r.Artist != "A" { - t.Errorf("Artist: %#v", r.Artist) - } - if r.Year == nil || *r.Year != 2024 { - t.Errorf("Year: %#v", r.Year) - } -} + It("returns nil when nothing matches the prefix", func() { + r := DeserializeAt[audio](map[string]any{"Name": "n"}, "audio") + Expect(r).To(BeNil()) + }) + + It("returns a value when the prefix matches", func() { + r := DeserializeAt[audio](map[string]any{ + "audio.artist": "A", + "audio.year": float64(2024), // setValue: pointer + numeric convert + }, "audio") + Expect(r).ToNot(BeNil()) + Expect(r.Artist).ToNot(BeNil()) + Expect(*r.Artist).To(Equal("A")) + Expect(r.Year).ToNot(BeNil()) + Expect(*r.Year).To(Equal(int32(2024))) + }) +}) diff --git a/services/search/pkg/mapping/fillstruct_test.go b/services/search/pkg/mapping/fillstruct_test.go index 4e5856fe80..70727d8bb7 100644 --- a/services/search/pkg/mapping/fillstruct_test.go +++ b/services/search/pkg/mapping/fillstruct_test.go @@ -3,7 +3,9 @@ package mapping import ( "errors" "reflect" - "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) // The walker is shared by both deserializers, so its structural behavior @@ -45,83 +47,60 @@ func fsSet(v reflect.Value, raw string) error { return nil } -func TestFillStruct(t *testing.T) { +var _ = Describe("fillStruct", func() { fill := func(fields map[string]string, prefix string) (fsRoot, bool) { var root fsRoot touched := fillStruct(reflect.ValueOf(&root).Elem(), fields, prefix, fsSet) return root, touched } - t.Run("flattens embedded, recurses nested", func(t *testing.T) { + It("flattens embedded and recurses nested", func() { root, touched := fill(map[string]string{ "leaf": "L", "ev": "EV", "ep": "EP", "nested.n": "N", }, "") - if !touched { - t.Fatal("expected touched") - } - if root.Leaf != "L" { - t.Errorf("Leaf: %q", root.Leaf) - } - if root.EV != "EV" { - t.Errorf("embedded value not promoted: %q", root.EV) - } - if root.FsEmbPtr == nil || root.EP != "EP" { - t.Errorf("embedded pointer not allocated: %+v", root.FsEmbPtr) - } - if root.Nested == nil || root.Nested.N != "N" { - t.Errorf("nested pointer not populated: %+v", root.Nested) - } + Expect(touched).To(BeTrue()) + Expect(root.Leaf).To(Equal("L")) + Expect(root.EV).To(Equal("EV"), "embedded value not promoted") + Expect(root.FsEmbPtr).ToNot(BeNil(), "embedded pointer not allocated") + Expect(root.EP).To(Equal("EP")) + Expect(root.Nested).ToNot(BeNil(), "nested pointer not populated") + Expect(root.Nested.N).To(Equal("N")) }) - t.Run("nothing matches: touched false, pointers stay nil", func(t *testing.T) { + It("leaves touched false and pointers nil when nothing matches", func() { root, touched := fill(map[string]string{"other": "x"}, "") - if touched { - t.Fatal("expected untouched") - } - if root.Nested != nil { - t.Errorf("Nested should stay nil: %+v", root.Nested) - } - if root.FsEmbPtr != nil { - t.Errorf("embedded pointer should stay nil: %+v", root.FsEmbPtr) - } + Expect(touched).To(BeFalse()) + Expect(root.Nested).To(BeNil(), "Nested should stay nil") + Expect(root.FsEmbPtr).To(BeNil(), "embedded pointer should stay nil") }) - t.Run("prefix arg is joined with the field name", func(t *testing.T) { + It("joins the prefix arg with the field name", func() { root, touched := fill(map[string]string{ "pre.leaf": "L", "pre.nested.n": "N", }, "pre") - if !touched || root.Leaf != "L" || root.Nested == nil || root.Nested.N != "N" { - t.Fatalf("prefix not joined: leaf=%q nested=%+v", root.Leaf, root.Nested) - } + Expect(touched).To(BeTrue()) + Expect(root.Leaf).To(Equal("L")) + Expect(root.Nested).ToNot(BeNil()) + Expect(root.Nested.N).To(Equal("N")) }) - t.Run("fail-soft: errored leaf stays zero, walk continues", func(t *testing.T) { + It("is fail-soft: errored leaf stays zero, walk continues", func() { root, touched := fill(map[string]string{ "leaf": "BAD", "ev": "EV", }, "") - if !touched { - t.Fatal("expected touched because ev was set") - } - if root.Leaf != "" { - t.Errorf("errored leaf should stay zero, got %q", root.Leaf) - } - if root.EV != "EV" { - t.Errorf("walk should continue past the error: %q", root.EV) - } + Expect(touched).To(BeTrue(), "expected touched because ev was set") + Expect(root.Leaf).To(BeEmpty(), "errored leaf should stay zero") + Expect(root.EV).To(Equal("EV"), "walk should continue past the error") }) - t.Run("embedded pointer dropped when its only field errors", func(t *testing.T) { + It("drops the embedded pointer when its only field errors", func() { root, touched := fill(map[string]string{"ep": "BAD"}, "") - if touched { - t.Fatal("expected untouched") - } - if root.FsEmbPtr != nil { - t.Errorf("embedded pointer should stay nil on error, got %+v", root.FsEmbPtr) - } + Expect(touched).To(BeFalse()) + Expect(root.FsEmbPtr).To(BeNil(), "embedded pointer should stay nil on error") }) -} +}) diff --git a/services/search/pkg/mapping/geo_test.go b/services/search/pkg/mapping/geo_test.go index 78cd3b5af7..c0eac54fef 100644 --- a/services/search/pkg/mapping/geo_test.go +++ b/services/search/pkg/mapping/geo_test.go @@ -2,159 +2,142 @@ package mapping import ( "reflect" - "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) -// build-mapping error paths: an override that doesn't fit the Go field. -func TestBuildMappingErrors(t *testing.T) { - type doc struct { - Name string `json:"name"` - } - // Geopoint on a non-struct field must error on both backends. - if _, err := BleveBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"name": {Type: TypeGeopoint}}); err == nil { - t.Error("bleve: expected error for geopoint on string field") - } - if _, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"name": {Type: TypeGeopoint}}); err == nil { - t.Error("opensearch: expected error for geopoint on string field") - } -} +var _ = Describe("BuildMapping geopoint errors", func() { + // build-mapping error paths: an override that doesn't fit the Go field. + It("errors for geopoint on a non-struct field on both backends", func() { + type doc struct { + Name string `json:"name"` + } + // Geopoint on a non-struct field must error on both backends. + _, err := BleveBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"name": {Type: TypeGeopoint}}) + Expect(err).To(HaveOccurred(), "bleve: expected error for geopoint on string field") + _, err = OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"name": {Type: TypeGeopoint}}) + Expect(err).To(HaveOccurred(), "opensearch: expected error for geopoint on string field") + }) +}) -func TestAddGeopointSiblingMissingIntermediate(t *testing.T) { - m := map[string]any{"journey": "not-a-map"} - addGeopointSibling(m, "journey.start") // must bail, not panic - if _, ok := m["journey.start"+GeopointSuffix]; ok { - t.Error("no sibling should be written when the path can't be resolved") - } -} +var _ = Describe("addGeopointSibling", func() { + It("bails without panicking when the intermediate is missing", func() { + m := map[string]any{"journey": "not-a-map"} + addGeopointSibling(m, "journey.start") // must bail, not panic + Expect(m).ToNot(HaveKey("journey.start" + GeopointSuffix)) + }) +}) -func TestPrepareForIndexAddsGeopointSibling(t *testing.T) { - type geoDoc struct { - Location *struct { +var _ = Describe("PrepareForIndex geopoint", func() { + It("adds a geopoint sibling", func() { + type geoDoc struct { + Location *struct { + Longitude *float64 `json:"longitude,omitempty"` + Latitude *float64 `json:"latitude,omitempty"` + Altitude *float64 `json:"altitude,omitempty"` + } `json:"location,omitempty"` + } + lon, lat, alt := 11.1, 49.4, 1047.7 + doc := geoDoc{Location: &struct { Longitude *float64 `json:"longitude,omitempty"` Latitude *float64 `json:"latitude,omitempty"` Altitude *float64 `json:"altitude,omitempty"` - } `json:"location,omitempty"` - } - lon, lat, alt := 11.1, 49.4, 1047.7 - doc := geoDoc{Location: &struct { - Longitude *float64 `json:"longitude,omitempty"` - Latitude *float64 `json:"latitude,omitempty"` - Altitude *float64 `json:"altitude,omitempty"` - }{Longitude: &lon, Latitude: &lat, Altitude: &alt}} + }{Longitude: &lon, Latitude: &lat, Altitude: &alt}} - m, err := PrepareForIndex(doc, map[string]FieldOpts{ - "location": {Type: TypeGeopoint}, + m, err := PrepareForIndex(doc, map[string]FieldOpts{ + "location": {Type: TypeGeopoint}, + }) + Expect(err).ToNot(HaveOccurred()) + + // Original location object stays untouched (full libregraph shape). + orig, ok := m["location"].(map[string]any) + Expect(ok).To(BeTrue(), "expected location object preserved, got %T", m["location"]) + Expect(orig["longitude"]).To(Equal(lon)) + Expect(orig["latitude"]).To(Equal(lat)) + Expect(orig["altitude"]).To(Equal(alt)) + + // Sibling location_geopoint has {lat, lon} for the geo indices. + gp, ok := m["location"+GeopointSuffix].(map[string]any) + Expect(ok).To(BeTrue(), "expected location_geopoint sibling, got %T", m["location"+GeopointSuffix]) + Expect(gp["lat"]).To(Equal(lat)) + Expect(gp["lon"]).To(Equal(lon)) }) - if err != nil { - t.Fatalf("PrepareForIndex: %v", err) - } - // Original location object stays untouched (full libregraph shape). - orig, ok := m["location"].(map[string]any) - if !ok { - t.Fatalf("expected location object preserved, got %T", m["location"]) - } - if orig["longitude"] != lon || orig["latitude"] != lat || orig["altitude"] != alt { - t.Errorf("location object: %#v", orig) - } - - // Sibling location_geopoint has {lat, lon} for the geo indices. - gp, ok := m["location"+GeopointSuffix].(map[string]any) - if !ok { - t.Fatalf("expected location_geopoint sibling, got %T", m["location"+GeopointSuffix]) - } - if gp["lat"] != lat || gp["lon"] != lon { - t.Errorf("sibling: %#v", gp) - } -} - -func TestPrepareForIndexSkipsIncompleteGeopoint(t *testing.T) { - type geoDoc struct { - Location *struct { + It("skips incomplete geopoints", func() { + type geoDoc struct { + Location *struct { + Altitude *float64 `json:"altitude,omitempty"` + } `json:"location,omitempty"` + } + alt := 100.0 + doc := geoDoc{Location: &struct { Altitude *float64 `json:"altitude,omitempty"` - } `json:"location,omitempty"` - } - alt := 100.0 - doc := geoDoc{Location: &struct { - Altitude *float64 `json:"altitude,omitempty"` - }{Altitude: &alt}} + }{Altitude: &alt}} - m, err := PrepareForIndex(doc, map[string]FieldOpts{ - "location": {Type: TypeGeopoint}, + m, err := PrepareForIndex(doc, map[string]FieldOpts{ + "location": {Type: TypeGeopoint}, + }) + Expect(err).ToNot(HaveOccurred()) + // Original stays (altitude alone is still useful metadata). + Expect(m).To(HaveKey("location"), "location should still be present when only altitude is set") + // No sibling without both lon and lat. + Expect(m).ToNot(HaveKey("location"+GeopointSuffix), "no sibling expected") }) - if err != nil { - t.Fatalf("PrepareForIndex: %v", err) - } - // Original stays (altitude alone is still useful metadata). - if _, ok := m["location"]; !ok { - t.Error("location should still be present when only altitude is set") - } - // No sibling without both lon and lat. - if _, ok := m["location"+GeopointSuffix]; ok { - t.Errorf("no sibling expected, got %#v", m["location"+GeopointSuffix]) - } -} -func TestPrepareForIndexWithoutOverrideNoSibling(t *testing.T) { - type geoDoc struct { - Location *struct { + It("writes no sibling without the geopoint override", func() { + type geoDoc struct { + Location *struct { + Longitude *float64 `json:"longitude,omitempty"` + Latitude *float64 `json:"latitude,omitempty"` + } `json:"location,omitempty"` + } + lon, lat := 11.1, 49.4 + doc := geoDoc{Location: &struct { Longitude *float64 `json:"longitude,omitempty"` Latitude *float64 `json:"latitude,omitempty"` - } `json:"location,omitempty"` - } - lon, lat := 11.1, 49.4 - doc := geoDoc{Location: &struct { - Longitude *float64 `json:"longitude,omitempty"` - Latitude *float64 `json:"latitude,omitempty"` - }{Longitude: &lon, Latitude: &lat}} + }{Longitude: &lon, Latitude: &lat}} - m, err := PrepareForIndex(doc, nil) - if err != nil { - t.Fatalf("PrepareForIndex: %v", err) - } - if _, ok := m["location"+GeopointSuffix]; ok { - t.Errorf("no sibling expected without override, got %#v", m["location"+GeopointSuffix]) - } -} - -func TestPrepareForIndexHandlesNestedGeopoint(t *testing.T) { - // journey.start and journey.end - two geopoints in the same facet, - // demonstrating the dotted-path walker. - type geo struct { - Longitude *float64 `json:"longitude,omitempty"` - Latitude *float64 `json:"latitude,omitempty"` - } - type journey struct { - Start *geo `json:"start,omitempty"` - End *geo `json:"end,omitempty"` - } - type doc struct { - Journey *journey `json:"journey,omitempty"` - } - slon, slat := 11.0, 49.0 - elon, elat := 13.4, 52.5 - d := doc{Journey: &journey{ - Start: &geo{Longitude: &slon, Latitude: &slat}, - End: &geo{Longitude: &elon, Latitude: &elat}, - }} - - m, err := PrepareForIndex(d, map[string]FieldOpts{ - "journey.start": {Type: TypeGeopoint}, - "journey.end": {Type: TypeGeopoint}, + m, err := PrepareForIndex(doc, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(m).ToNot(HaveKey("location"+GeopointSuffix), "no sibling expected without override") }) - if err != nil { - t.Fatalf("PrepareForIndex: %v", err) - } - j, ok := m["journey"].(map[string]any) - if !ok { - t.Fatalf("journey not an object: %T", m["journey"]) - } - startGp, ok := j["start"+GeopointSuffix].(map[string]any) - if !ok || startGp["lat"] != slat || startGp["lon"] != slon { - t.Errorf("journey.start sibling: %#v", j["start"+GeopointSuffix]) - } - endGp, ok := j["end"+GeopointSuffix].(map[string]any) - if !ok || endGp["lat"] != elat || endGp["lon"] != elon { - t.Errorf("journey.end sibling: %#v", j["end"+GeopointSuffix]) - } -} + + It("handles nested geopoints via the dotted-path walker", func() { + // journey.start and journey.end - two geopoints in the same facet, + // demonstrating the dotted-path walker. + type geo struct { + Longitude *float64 `json:"longitude,omitempty"` + Latitude *float64 `json:"latitude,omitempty"` + } + type journey struct { + Start *geo `json:"start,omitempty"` + End *geo `json:"end,omitempty"` + } + type doc struct { + Journey *journey `json:"journey,omitempty"` + } + slon, slat := 11.0, 49.0 + elon, elat := 13.4, 52.5 + d := doc{Journey: &journey{ + Start: &geo{Longitude: &slon, Latitude: &slat}, + End: &geo{Longitude: &elon, Latitude: &elat}, + }} + + m, err := PrepareForIndex(d, map[string]FieldOpts{ + "journey.start": {Type: TypeGeopoint}, + "journey.end": {Type: TypeGeopoint}, + }) + Expect(err).ToNot(HaveOccurred()) + j, ok := m["journey"].(map[string]any) + Expect(ok).To(BeTrue(), "journey not an object: %T", m["journey"]) + startGp, ok := j["start"+GeopointSuffix].(map[string]any) + Expect(ok).To(BeTrue(), "journey.start sibling: %#v", j["start"+GeopointSuffix]) + Expect(startGp["lat"]).To(Equal(slat)) + Expect(startGp["lon"]).To(Equal(slon)) + endGp, ok := j["end"+GeopointSuffix].(map[string]any) + Expect(ok).To(BeTrue(), "journey.end sibling: %#v", j["end"+GeopointSuffix]) + Expect(endGp["lat"]).To(Equal(elat)) + Expect(endGp["lon"]).To(Equal(elon)) + }) +}) diff --git a/services/search/pkg/mapping/infer_test.go b/services/search/pkg/mapping/infer_test.go index 9ee57b656a..702fd0dc96 100644 --- a/services/search/pkg/mapping/infer_test.go +++ b/services/search/pkg/mapping/infer_test.go @@ -2,52 +2,40 @@ package mapping import ( "reflect" - "testing" "time" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "google.golang.org/protobuf/types/known/timestamppb" ) -func TestInferTypeUnsupported(t *testing.T) { - if got := inferType(reflect.TypeFor[map[string]int]()); got != "" { - t.Errorf("map: got %q, want empty", got) - } - if got := inferType(reflect.TypeFor[chan int]()); got != "" { - t.Errorf("chan: got %q, want empty", got) - } -} +var _ = Describe("inferType", func() { + It("returns empty for unsupported kinds", func() { + Expect(inferType(reflect.TypeFor[map[string]int]())).To(BeEmpty(), "map") + Expect(inferType(reflect.TypeFor[chan int]())).To(BeEmpty(), "chan") + }) -func TestInferType(t *testing.T) { - cases := []struct { - name string - in any - want string - }{ - {"string", "", TypeKeyword}, - {"*string", (*string)(nil), TypeKeyword}, - {"[]string", []string(nil), TypeKeyword}, - {"bool", false, TypeBool}, - {"int", int(0), TypeNumeric}, - {"int64", int64(0), TypeNumeric}, - {"uint64", uint64(0), TypeNumeric}, - {"float64", float64(0), TypeNumeric}, - {"time.Time", time.Time{}, TypeDatetime}, - {"*time.Time", (*time.Time)(nil), TypeDatetime}, - {"*timestamppb.Timestamp", (*timestamppb.Timestamp)(nil), TypeDatetime}, - {"struct", struct{ X int }{}, TypeObject}, - {"*struct", (*struct{ X int })(nil), TypeObject}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - got := inferType(reflect.TypeOf(c.in)) - if got != c.want { - t.Fatalf("inferType(%s): got %q, want %q", c.name, got, c.want) - } - }) - } -} + DescribeTable("infers the mapping type", + func(in any, want string) { + Expect(inferType(reflect.TypeOf(in))).To(Equal(want)) + }, + Entry("string", "", TypeKeyword), + Entry("*string", (*string)(nil), TypeKeyword), + Entry("[]string", []string(nil), TypeKeyword), + Entry("bool", false, TypeBool), + Entry("int", int(0), TypeNumeric), + Entry("int64", int64(0), TypeNumeric), + Entry("uint64", uint64(0), TypeNumeric), + Entry("float64", float64(0), TypeNumeric), + Entry("time.Time", time.Time{}, TypeDatetime), + Entry("*time.Time", (*time.Time)(nil), TypeDatetime), + Entry("*timestamppb.Timestamp", (*timestamppb.Timestamp)(nil), TypeDatetime), + Entry("struct", struct{ X int }{}, TypeObject), + Entry("*struct", (*struct{ X int })(nil), TypeObject), + ) +}) -func TestResolveField(t *testing.T) { +var _ = Describe("resolveField", func() { type S struct { Exported string `json:"exp"` Renamed string `json:"renamed,omitempty"` @@ -57,69 +45,56 @@ func TestResolveField(t *testing.T) { unexported string //nolint:unused } st := reflect.TypeFor[S]() - cases := []struct { - fieldIdx int - wantName string - wantSkip bool - }{ - {0, "exp", false}, - {1, "renamed", false}, - {2, "NoTag", false}, - {3, "OmitOnly", false}, - {4, "", true}, - {5, "", true}, - } - for _, c := range cases { - fi := resolveField(st.Field(c.fieldIdx)) - if fi.Skip != c.wantSkip { - t.Errorf("field %d: skip=%v, want %v", c.fieldIdx, fi.Skip, c.wantSkip) - } - if !c.wantSkip && fi.Name != c.wantName { - t.Errorf("field %d: name=%q, want %q", c.fieldIdx, fi.Name, c.wantName) - } - } -} -func TestWalkFieldsFlattensEmbedded(t *testing.T) { - type Inner struct { - A string `json:"a"` - B int `json:"b"` - } - type Outer struct { - Inner - C bool `json:"c"` - } - var names []string - err := walkFields(reflect.TypeFor[Outer](), func(fi fieldInfo) error { - names = append(names, fi.Name) - return nil + DescribeTable("resolves the field name and skip flag", + func(fieldIdx int, wantName string, wantSkip bool) { + fi := resolveField(st.Field(fieldIdx)) + Expect(fi.Skip).To(Equal(wantSkip), "field %d skip", fieldIdx) + if !wantSkip { + Expect(fi.Name).To(Equal(wantName), "field %d name", fieldIdx) + } + }, + Entry("exported json tag", 0, "exp", false), + Entry("renamed with omitempty", 1, "renamed", false), + Entry("no tag", 2, "NoTag", false), + Entry("omitempty only", 3, "OmitOnly", false), + Entry("json:- skipped", 4, "", true), + Entry("unexported skipped", 5, "", true), + ) +}) + +var _ = Describe("walkFields", func() { + It("flattens embedded structs", func() { + type Inner struct { + A string `json:"a"` + B int `json:"b"` + } + type Outer struct { + Inner + C bool `json:"c"` + } + var names []string + err := walkFields(reflect.TypeFor[Outer](), func(fi fieldInfo) error { + names = append(names, fi.Name) + return nil + }) + Expect(err).ToNot(HaveOccurred()) + Expect(names).To(Equal([]string{"a", "b", "c"})) }) - if err != nil { - t.Fatalf("walkFields: %v", err) - } - want := []string{"a", "b", "c"} - if !reflect.DeepEqual(names, want) { - t.Fatalf("got %v, want %v", names, want) - } -} +}) -func TestStructType(t *testing.T) { +var _ = Describe("structType", func() { type S struct{ X int } - cases := []struct { - name string - in reflect.Type - wantNil bool - }{ - {"struct", reflect.TypeFor[S](), false}, - {"*struct", reflect.TypeFor[*S](), false}, - {"[]struct", reflect.TypeFor[[]S](), false}, - {"time.Time", reflect.TypeFor[time.Time](), true}, - {"string", reflect.TypeFor[string](), true}, - } - for _, c := range cases { - got := structType(c.in) - if (got == nil) != c.wantNil { - t.Errorf("%s: got %v, wantNil %v", c.name, got, c.wantNil) - } - } -} + + DescribeTable("resolves struct-ish types", + func(in reflect.Type, wantNil bool) { + got := structType(in) + Expect(got == nil).To(Equal(wantNil)) + }, + Entry("struct", reflect.TypeFor[S](), false), + Entry("*struct", reflect.TypeFor[*S](), false), + Entry("[]struct", reflect.TypeFor[[]S](), false), + Entry("time.Time", reflect.TypeFor[time.Time](), true), + Entry("string", reflect.TypeFor[string](), true), + ) +}) diff --git a/services/search/pkg/mapping/mapping_suite_test.go b/services/search/pkg/mapping/mapping_suite_test.go new file mode 100644 index 0000000000..4053109597 --- /dev/null +++ b/services/search/pkg/mapping/mapping_suite_test.go @@ -0,0 +1,13 @@ +package mapping + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMapping(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Mapping Suite") +} diff --git a/services/search/pkg/mapping/opensearch_test.go b/services/search/pkg/mapping/opensearch_test.go index de93e8d100..3ff80f20e3 100644 --- a/services/search/pkg/mapping/opensearch_test.go +++ b/services/search/pkg/mapping/opensearch_test.go @@ -2,8 +2,10 @@ package mapping import ( "reflect" - "testing" "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) type osDoc struct { @@ -18,154 +20,117 @@ type osDoc struct { } `json:"nested,omitempty"` } -func TestOpenSearchNumericTypes(t *testing.T) { - type doc struct { - A int8 `json:"a"` - B int16 `json:"b"` - C int32 `json:"c"` - D int64 `json:"d"` - E uint8 `json:"e"` - F uint64 `json:"f"` - G float32 `json:"g"` - H float64 `json:"h"` - } - props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), nil) - if err != nil { - t.Fatalf("OpenSearchBuildMapping: %v", err) - } - want := map[string]string{"a": "short", "b": "short", "c": "integer", "d": "long", "e": "short", "f": "long", "g": "float", "h": "double"} - for k, wt := range want { - if got := props[k].(map[string]any)["type"]; got != wt { - t.Errorf("%s: type = %v, want %v", k, got, wt) - } - } -} +var _ = Describe("OpenSearchBuildMapping", func() { + DescribeTable("maps numeric Go types to OpenSearch types", + func(field, wantType string) { + type doc struct { + A int8 `json:"a"` + B int16 `json:"b"` + C int32 `json:"c"` + D int64 `json:"d"` + E uint8 `json:"e"` + F uint64 `json:"f"` + G float32 `json:"g"` + H float64 `json:"h"` + } + props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), nil) + Expect(err).ToNot(HaveOccurred()) + Expect(props[field].(map[string]any)["type"]).To(Equal(wantType), "%s type", field) + }, + Entry("int8 -> short", "a", "short"), + Entry("int16 -> short", "b", "short"), + Entry("int32 -> integer", "c", "integer"), + Entry("int64 -> long", "d", "long"), + Entry("uint8 -> short", "e", "short"), + Entry("uint64 -> long", "f", "long"), + Entry("float32 -> float", "g", "float"), + Entry("float64 -> double", "h", "double"), + ) -func TestOpenSearchBuildMappingInferred(t *testing.T) { - props, err := OpenSearchBuildMapping(reflect.TypeFor[osDoc](), nil) - if err != nil { - t.Fatalf("OpenSearchBuildMapping: %v", err) - } - want := map[string]string{ - "ID": "keyword", - "Size": "long", - "Deleted": "boolean", - "CreatedAt": "date", - "Rating": "double", - } - for k, v := range want { - m, ok := props[k].(map[string]any) - if !ok { - t.Errorf("%s: missing or not a map: %#v", k, props[k]) - continue - } - if got := m["type"]; got != v { - t.Errorf("%s: type %v, want %v", k, got, v) - } - } -} + DescribeTable("infers field types", + func(field, wantType string) { + props, err := OpenSearchBuildMapping(reflect.TypeFor[osDoc](), nil) + Expect(err).ToNot(HaveOccurred()) + m, ok := props[field].(map[string]any) + Expect(ok).To(BeTrue(), "%s: missing or not a map: %#v", field, props[field]) + Expect(m["type"]).To(Equal(wantType), "%s type", field) + }, + Entry("ID -> keyword", "ID", "keyword"), + Entry("Size -> long", "Size", "long"), + Entry("Deleted -> boolean", "Deleted", "boolean"), + Entry("CreatedAt -> date", "CreatedAt", "date"), + Entry("Rating -> double", "Rating", "double"), + ) -func TestOpenSearchBuildMappingNested(t *testing.T) { - props, err := OpenSearchBuildMapping(reflect.TypeFor[osDoc](), nil) - if err != nil { - t.Fatalf("OpenSearchBuildMapping: %v", err) - } - nested, ok := props["nested"].(map[string]any) - if !ok { - t.Fatalf("nested: not a map: %#v", props["nested"]) - } - sub, ok := nested["properties"].(map[string]any) - if !ok { - t.Fatalf("nested.properties: missing: %#v", nested) - } - artist, ok := sub["artist"].(map[string]any) - if !ok { - t.Fatalf("nested.artist: %#v", sub) - } - if artist["type"] != "keyword" { - t.Errorf("nested.artist.type: %v", artist["type"]) - } - year, ok := sub["year"].(map[string]any) - if !ok { - t.Fatalf("nested.year: %#v", sub) - } - if year["type"] != "integer" { - t.Errorf("nested.year.type: %v (int32 → integer expected)", year["type"]) - } -} - -func TestOpenSearchBuildMappingOverrides(t *testing.T) { - type doc struct { - Name string `json:"Name"` - Content string `json:"Content"` - Path string `json:"Path"` - MimeType string `json:"MimeType"` - } - props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{ - "Name": {Analyzer: "lowercaseKeyword"}, - "Content": {Type: TypeFulltext}, - "Path": {Type: TypePath}, - "MimeType": {Type: TypeWildcard}, + It("maps nested structs with their sub-properties", func() { + props, err := OpenSearchBuildMapping(reflect.TypeFor[osDoc](), nil) + Expect(err).ToNot(HaveOccurred()) + nested, ok := props["nested"].(map[string]any) + Expect(ok).To(BeTrue(), "nested: not a map: %#v", props["nested"]) + sub, ok := nested["properties"].(map[string]any) + Expect(ok).To(BeTrue(), "nested.properties: missing: %#v", nested) + artist, ok := sub["artist"].(map[string]any) + Expect(ok).To(BeTrue(), "nested.artist: %#v", sub) + Expect(artist["type"]).To(Equal("keyword"), "nested.artist.type") + year, ok := sub["year"].(map[string]any) + Expect(ok).To(BeTrue(), "nested.year: %#v", sub) + Expect(year["type"]).To(Equal("integer"), "nested.year.type (int32 -> integer expected)") }) - if err != nil { - t.Fatalf("OpenSearchBuildMapping: %v", err) - } - name := props["Name"].(map[string]any) - if name["type"] != "text" || name["analyzer"] != "lowercaseKeyword" { - t.Errorf("Name: %#v", name) - } - content := props["Content"].(map[string]any) - if content["type"] != "text" || content["term_vector"] != "with_positions_offsets" { - t.Errorf("Content: %#v", content) - } - if _, ok := content["analyzer"]; ok { - t.Errorf("Content should leave analyzer unset (use OpenSearch default), got %#v", content["analyzer"]) - } - path := props["Path"].(map[string]any) - if path["type"] != "text" || path["analyzer"] != "path_hierarchy" { - t.Errorf("Path: %#v", path) - } - mime := props["MimeType"].(map[string]any) - if mime["type"] != "wildcard" { - t.Errorf("MimeType: %#v", mime) - } -} -func TestOpenSearchBuildMappingGeopoint(t *testing.T) { - type doc struct { - Location *struct { - Lon float64 `json:"longitude"` - Lat float64 `json:"latitude"` - Alt float64 `json:"altitude"` - } `json:"location,omitempty"` - } - props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{ - "location": {Type: TypeGeopoint}, - }) - if err != nil { - t.Fatalf("OpenSearchBuildMapping: %v", err) - } - // Object for libregraph-shape data retrieval. - loc, ok := props["location"].(map[string]any) - if !ok { - t.Fatalf("location: %#v", props["location"]) - } - sub, ok := loc["properties"].(map[string]any) - if !ok { - t.Fatalf("location should have numeric sub-properties, got %#v", loc) - } - for _, k := range []string{"longitude", "latitude", "altitude"} { - prop, ok := sub[k].(map[string]any) - if !ok || prop["type"] != "double" { - t.Errorf("location.%s: %#v", k, sub[k]) + It("applies field overrides", func() { + type doc struct { + Name string `json:"Name"` + Content string `json:"Content"` + Path string `json:"Path"` + MimeType string `json:"MimeType"` } - } - // Sibling geo_point for spatial queries. - gp, ok := props["location"+GeopointSuffix].(map[string]any) - if !ok { - t.Fatalf("location%s: %#v", GeopointSuffix, props["location"+GeopointSuffix]) - } - if gp["type"] != "geo_point" { - t.Errorf("location%s.type: %v", GeopointSuffix, gp["type"]) - } -} + props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{ + "Name": {Analyzer: "lowercaseKeyword"}, + "Content": {Type: TypeFulltext}, + "Path": {Type: TypePath}, + "MimeType": {Type: TypeWildcard}, + }) + Expect(err).ToNot(HaveOccurred()) + name := props["Name"].(map[string]any) + Expect(name["type"]).To(Equal("text"), "Name: %#v", name) + Expect(name["analyzer"]).To(Equal("lowercaseKeyword"), "Name: %#v", name) + content := props["Content"].(map[string]any) + Expect(content["type"]).To(Equal("text"), "Content: %#v", content) + Expect(content["term_vector"]).To(Equal("with_positions_offsets"), "Content: %#v", content) + _, ok := content["analyzer"] + Expect(ok).To(BeFalse(), "Content should leave analyzer unset (use OpenSearch default)") + path := props["Path"].(map[string]any) + Expect(path["type"]).To(Equal("text"), "Path: %#v", path) + Expect(path["analyzer"]).To(Equal("path_hierarchy"), "Path: %#v", path) + mime := props["MimeType"].(map[string]any) + Expect(mime["type"]).To(Equal("wildcard"), "MimeType: %#v", mime) + }) + + It("builds an object plus a geo_point sibling for geopoints", func() { + type doc struct { + Location *struct { + Lon float64 `json:"longitude"` + Lat float64 `json:"latitude"` + Alt float64 `json:"altitude"` + } `json:"location,omitempty"` + } + props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{ + "location": {Type: TypeGeopoint}, + }) + Expect(err).ToNot(HaveOccurred()) + // Object for libregraph-shape data retrieval. + loc, ok := props["location"].(map[string]any) + Expect(ok).To(BeTrue(), "location: %#v", props["location"]) + sub, ok := loc["properties"].(map[string]any) + Expect(ok).To(BeTrue(), "location should have numeric sub-properties, got %#v", loc) + for _, k := range []string{"longitude", "latitude", "altitude"} { + prop, ok := sub[k].(map[string]any) + Expect(ok).To(BeTrue(), "location.%s: %#v", k, sub[k]) + Expect(prop["type"]).To(Equal("double"), "location.%s: %#v", k, sub[k]) + } + // Sibling geo_point for spatial queries. + gp, ok := props["location"+GeopointSuffix].(map[string]any) + Expect(ok).To(BeTrue(), "location%s: %#v", GeopointSuffix, props["location"+GeopointSuffix]) + Expect(gp["type"]).To(Equal("geo_point"), "location%s.type", GeopointSuffix) + }) +}) diff --git a/services/search/pkg/mapping/serialize_test.go b/services/search/pkg/mapping/serialize_test.go index b352708999..18051639fa 100644 --- a/services/search/pkg/mapping/serialize_test.go +++ b/services/search/pkg/mapping/serialize_test.go @@ -1,83 +1,67 @@ package mapping import ( - "reflect" - "testing" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) -func TestPrepareForIndexError(t *testing.T) { - // a func field can't be json-marshalled -> conversions.To errors - type bad struct { - F func() `json:"f"` - } - if _, err := PrepareForIndex(bad{}, nil); err == nil { - t.Error("expected error for non-marshallable value") - } -} +var _ = Describe("PrepareForIndex serialization", func() { + It("errors for a non-marshallable value", func() { + // a func field can't be json-marshalled -> conversions.To errors + type bad struct { + F func() `json:"f"` + } + _, err := PrepareForIndex(bad{}, nil) + Expect(err).To(HaveOccurred()) + }) -func TestPrepareForIndexNil(t *testing.T) { - // a typed nil pointer marshals to null -> nil map, no error, no panic - out, err := PrepareForIndex((*struct{})(nil), nil) - if err != nil || out != nil { - t.Errorf("got (%v, %v), want (nil, nil)", out, err) - } -} + It("returns nil map for a typed nil pointer", func() { + // a typed nil pointer marshals to null -> nil map, no error, no panic + out, err := PrepareForIndex((*struct{})(nil), nil) + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(BeNil()) + }) -func TestPrepareForIndexFlattensEmbedded(t *testing.T) { - type inner struct { - Name string `json:"Name"` - Size uint64 `json:"Size"` - } - type outer struct { - inner - ID string `json:"ID"` - } - m, err := PrepareForIndex(outer{inner: inner{Name: "a", Size: 7}, ID: "x"}, nil) - if err != nil { - t.Fatalf("PrepareForIndex: %v", err) - } - want := map[string]any{"Name": "a", "Size": float64(7), "ID": "x"} - if !reflect.DeepEqual(m, want) { - t.Fatalf("got %#v, want %#v", m, want) - } -} + It("flattens embedded structs", func() { + type inner struct { + Name string `json:"Name"` + Size uint64 `json:"Size"` + } + type outer struct { + inner + ID string `json:"ID"` + } + m, err := PrepareForIndex(outer{inner: inner{Name: "a", Size: 7}, ID: "x"}, nil) + Expect(err).ToNot(HaveOccurred()) + want := map[string]any{"Name": "a", "Size": float64(7), "ID": "x"} + Expect(m).To(Equal(want)) + }) -func TestPrepareForIndexOmitsNilWithOmitempty(t *testing.T) { - type facet struct { - Artist string `json:"artist"` - } - type doc struct { - Name string `json:"Name"` - Audio *facet `json:"audio,omitempty"` - } - m, err := PrepareForIndex(doc{Name: "n"}, nil) - if err != nil { - t.Fatalf("PrepareForIndex: %v", err) - } - if _, ok := m["audio"]; ok { - t.Errorf("audio should be omitted when nil: %#v", m) - } - if m["Name"] != "n" { - t.Errorf("Name: %v", m["Name"]) - } -} + It("omits nil fields tagged omitempty", func() { + type facet struct { + Artist string `json:"artist"` + } + type doc struct { + Name string `json:"Name"` + Audio *facet `json:"audio,omitempty"` + } + m, err := PrepareForIndex(doc{Name: "n"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(m).ToNot(HaveKey("audio")) + Expect(m["Name"]).To(Equal("n")) + }) -func TestPrepareForIndexIncludesNestedWhenSet(t *testing.T) { - type facet struct { - Artist string `json:"artist"` - } - type doc struct { - Audio *facet `json:"audio,omitempty"` - } - m, err := PrepareForIndex(doc{Audio: &facet{Artist: "A"}}, nil) - if err != nil { - t.Fatalf("PrepareForIndex: %v", err) - } - nested, ok := m["audio"].(map[string]any) - if !ok { - t.Fatalf("audio should be a nested map: %#v", m["audio"]) - } - if nested["artist"] != "A" { - t.Errorf("audio.artist: %v", nested["artist"]) - } -} + It("includes nested facets when set", func() { + type facet struct { + Artist string `json:"artist"` + } + type doc struct { + Audio *facet `json:"audio,omitempty"` + } + m, err := PrepareForIndex(doc{Audio: &facet{Artist: "A"}}, nil) + Expect(err).ToNot(HaveOccurred()) + nested, ok := m["audio"].(map[string]any) + Expect(ok).To(BeTrue(), "audio should be a nested map: %#v", m["audio"]) + Expect(nested["artist"]).To(Equal("A")) + }) +}) diff --git a/services/search/pkg/mapping/validate_test.go b/services/search/pkg/mapping/validate_test.go index 2d8500c2b8..278bb3820f 100644 --- a/services/search/pkg/mapping/validate_test.go +++ b/services/search/pkg/mapping/validate_test.go @@ -2,8 +2,9 @@ package mapping import ( "reflect" - "strings" - "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) type inner struct { @@ -19,33 +20,28 @@ type sample struct { } `json:"location,omitempty"` } -func TestValidateAccepts(t *testing.T) { - err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{ - "Name": {Analyzer: "lowercaseKeyword"}, - "audio": {Type: TypeObject}, - "audio.artist": {Analyzer: "lowercaseKeyword"}, - "location": {Type: TypeGeopoint}, +var _ = Describe("Validate", func() { + It("accepts known override keys", func() { + err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{ + "Name": {Analyzer: "lowercaseKeyword"}, + "audio": {Type: TypeObject}, + "audio.artist": {Analyzer: "lowercaseKeyword"}, + "location": {Type: TypeGeopoint}, + }) + Expect(err).ToNot(HaveOccurred()) }) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } -} -func TestValidateRejectsUnknown(t *testing.T) { - err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{ - "nope": {}, - "audio.zzz": {}, + It("rejects unknown override keys", func() { + err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{ + "nope": {}, + "audio.zzz": {}, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("nope")) + Expect(err.Error()).To(ContainSubstring("audio.zzz")) }) - if err == nil { - t.Fatalf("expected error") - } - if !strings.Contains(err.Error(), "nope") || !strings.Contains(err.Error(), "audio.zzz") { - t.Fatalf("error missing keys: %v", err) - } -} -func TestValidateEmpty(t *testing.T) { - if err := Validate(reflect.TypeFor[sample](), nil); err != nil { - t.Fatalf("empty overrides should pass: %v", err) - } -} + It("accepts empty overrides", func() { + Expect(Validate(reflect.TypeFor[sample](), nil)).To(Succeed()) + }) +}) From 7cc144a9c337ec0e40989fa0958b2fa945e9a7ce Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 6 Jul 2026 16:30:24 +0200 Subject: [PATCH 06/54] test(search): set Mtime on opensearch folder and root fixtures The Mtime field is mapped as an OpenSearch `date`, which rejects an empty value with `mapper_parsing_exception: cannot parse empty date`. The folder and root fixtures had no Mtime, so serializing them to `"Mtime": ""` made TestEngine_Purge/purge_resource_trees fail when the document was indexed. Give both a valid RFC3339 Mtime, matching the file fixture. --- .../internal/opensearchtest/testdata/resource_folder.json | 3 ++- .../search/internal/opensearchtest/testdata/resource_root.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/services/search/internal/opensearchtest/testdata/resource_folder.json b/services/search/internal/opensearchtest/testdata/resource_folder.json index aa469daf98..345ef71724 100644 --- a/services/search/internal/opensearchtest/testdata/resource_folder.json +++ b/services/search/internal/opensearchtest/testdata/resource_folder.json @@ -3,5 +3,6 @@ "RootID" : "1$1!1", "ParentID" : "1$1!1", "Path" : "./parent d!r", - "Type" : 2 + "Type" : 2, + "Mtime" : "2025-07-24T15:15:01.324093+02:00" } diff --git a/services/search/internal/opensearchtest/testdata/resource_root.json b/services/search/internal/opensearchtest/testdata/resource_root.json index 18e401dfbd..0ea9e063df 100644 --- a/services/search/internal/opensearchtest/testdata/resource_root.json +++ b/services/search/internal/opensearchtest/testdata/resource_root.json @@ -1,5 +1,6 @@ { "ID" : "1$1!1", "RootID" : "1$1!1", - "Path" : "." + "Path" : ".", + "Mtime" : "2025-07-24T15:15:01.324093+02:00" } From 450f3801980931d97c280275b840275de7bf6162 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Thu, 16 Jul 2026 01:33:03 +0200 Subject: [PATCH 07/54] fix(search): type mtime as a date --- services/search/pkg/bleve/mtime_test.go | 6 +++++- services/search/pkg/content/basic.go | 4 ++-- services/search/pkg/content/basic_test.go | 12 ++++++++---- services/search/pkg/content/content.go | 3 ++- .../pkg/opensearch/internal/convert/opensearch.go | 5 ++--- .../opensearch/internal/convert/opensearch_test.go | 2 +- services/search/pkg/search/search.go | 4 ---- 7 files changed, 20 insertions(+), 16 deletions(-) diff --git a/services/search/pkg/bleve/mtime_test.go b/services/search/pkg/bleve/mtime_test.go index c29c83d9b8..2118662289 100644 --- a/services/search/pkg/bleve/mtime_test.go +++ b/services/search/pkg/bleve/mtime_test.go @@ -1,6 +1,10 @@ package bleve_test import ( + "time" + + "github.com/opencloud-eu/opencloud/pkg/conversions" + bleveSearch "github.com/blevesearch/bleve/v2" bquery "github.com/blevesearch/bleve/v2/search/query" . "github.com/onsi/ginkgo/v2" @@ -21,7 +25,7 @@ var _ = Describe("Mtime date range", func() { idx, err := bleveSearch.NewMemOnly(m) Expect(err).ToNot(HaveOccurred()) - r := search.Resource{ID: "x", Document: content.Document{Name: "f", Mtime: "2026-03-15T12:00:00.123456789Z"}} + r := search.Resource{ID: "x", Document: content.Document{Name: "f", Mtime: conversions.ToPointer(time.Date(2026, 3, 15, 12, 0, 0, 123456789, time.UTC))}} doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides()) Expect(err).ToNot(HaveOccurred()) Expect(idx.Index(r.ID, doc)).To(Succeed()) diff --git a/services/search/pkg/content/basic.go b/services/search/pkg/content/basic.go index 1499258c98..7ba8638a18 100644 --- a/services/search/pkg/content/basic.go +++ b/services/search/pkg/content/basic.go @@ -3,9 +3,9 @@ package content import ( "context" "encoding/json" - "time" storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/opencloud-eu/opencloud/pkg/conversions" "github.com/opencloud-eu/opencloud/pkg/log" "github.com/opencloud-eu/reva/v2/pkg/tags" "github.com/opencloud-eu/reva/v2/pkg/utils" @@ -54,7 +54,7 @@ func (b Basic) Extract(_ context.Context, ri *storageProvider.ResourceInfo) (Doc } if ri.Mtime != nil { - doc.Mtime = utils.TSToTime(ri.Mtime).UTC().Format(time.RFC3339Nano) + doc.Mtime = conversions.ToPointer(utils.TSToTime(ri.Mtime).UTC()) } return doc, nil diff --git a/services/search/pkg/content/basic_test.go b/services/search/pkg/content/basic_test.go index 86832ed418..709202fc33 100644 --- a/services/search/pkg/content/basic_test.go +++ b/services/search/pkg/content/basic_test.go @@ -1,6 +1,10 @@ package content_test import ( + "time" + + "github.com/opencloud-eu/opencloud/pkg/conversions" + "context" "encoding/json" @@ -69,11 +73,11 @@ var _ = Describe("Basic", func() { It("RFC3339 mtime", func() { for _, data := range []struct { second uint64 - expect string + expect *time.Time }{ - {second: 4000, expect: "1970-01-01T01:06:40Z"}, - {second: 3000, expect: "1970-01-01T00:50:00Z"}, - {expect: ""}, + {second: 4000, expect: conversions.ToPointer(time.Unix(4000, 0).UTC())}, + {second: 3000, expect: conversions.ToPointer(time.Unix(3000, 0).UTC())}, + {}, } { ri := &storageProvider.ResourceInfo{} diff --git a/services/search/pkg/content/content.go b/services/search/pkg/content/content.go index 5a5d3bc1b2..d44d5874c2 100644 --- a/services/search/pkg/content/content.go +++ b/services/search/pkg/content/content.go @@ -2,6 +2,7 @@ package content import ( "strings" + "time" "github.com/bbalet/stopwords" libregraph "github.com/opencloud-eu/libre-graph-api-go" @@ -18,7 +19,7 @@ type Document struct { Name string `json:"Name"` Content string `json:"Content"` Size uint64 `json:"Size"` - Mtime string `json:"Mtime,omitempty"` + Mtime *time.Time `json:"Mtime,omitempty"` MimeType string `json:"MimeType"` Tags []string `json:"Tags"` Favorites []string `json:"Favorites"` diff --git a/services/search/pkg/opensearch/internal/convert/opensearch.go b/services/search/pkg/opensearch/internal/convert/opensearch.go index aef6db363b..c972414b1d 100644 --- a/services/search/pkg/opensearch/internal/convert/opensearch.go +++ b/services/search/pkg/opensearch/internal/convert/opensearch.go @@ -3,7 +3,6 @@ package convert import ( "fmt" "strings" - "time" opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "google.golang.org/protobuf/types/known/timestamppb" @@ -87,8 +86,8 @@ func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match, }, } - if mtime, err := time.Parse(time.RFC3339, resource.Mtime); err == nil { - match.Entity.LastModifiedTime = ×tamppb.Timestamp{Seconds: mtime.Unix(), Nanos: int32(mtime.Nanosecond())} + if resource.Mtime != nil { + match.Entity.LastModifiedTime = timestamppb.New(*resource.Mtime) } return match, nil diff --git a/services/search/pkg/opensearch/internal/convert/opensearch_test.go b/services/search/pkg/opensearch/internal/convert/opensearch_test.go index 8f034e327c..af72aac432 100644 --- a/services/search/pkg/opensearch/internal/convert/opensearch_test.go +++ b/services/search/pkg/opensearch/internal/convert/opensearch_test.go @@ -36,7 +36,7 @@ var _ = Describe("OpenSearchHitToMatch", func() { resource = opensearchtest.Testdata.Resources.File resource.MimeType = "audio/mpeg" mtime = time.Date(2025, 7, 24, 15, 15, 1, 0, time.UTC) - resource.Mtime = mtime.Format(time.RFC3339) + resource.Mtime = &mtime resource.Favorites = []string{"cbf24bce-3e6e-4d9e-a2a2-cbf24bce3e6e"} hit = opensearchgoAPI.SearchHit{ diff --git a/services/search/pkg/search/search.go b/services/search/pkg/search/search.go index c27123f173..78fe0ac29c 100644 --- a/services/search/pkg/search/search.go +++ b/services/search/pkg/search/search.go @@ -73,10 +73,6 @@ var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts "Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll}, "Favorites": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll}, "location": {Type: mapping.TypeGeopoint}, - // Mtime is stored as an RFC3339 string; type it as a date so mtime:>... - // range queries are chronological on both backends (bleve DateRangeQuery - // / OpenSearch date range), not a lexicographic keyword compare. - "Mtime": {Type: mapping.TypeDatetime}, } }) From 7d5584990017fc73b2ff4bb785d8225772f83d72 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Tue, 28 Jul 2026 19:46:50 +0200 Subject: [PATCH 08/54] feat(search): version the search index by schema version Both backends carry a shared search.SchemaVersion in the index name (OpenSearch -vN) and data path (bleve-vN). A breaking schema change bumps the version so the service builds a fresh index instead of colliding with the incompatible previous one; the old index is left in place. --- services/search/README.md | 2 +- services/search/pkg/bleve/index.go | 7 ++--- services/search/pkg/command/server.go | 3 ++- services/search/pkg/config/engine.go | 2 +- services/search/pkg/opensearch/backend.go | 2 +- services/search/pkg/opensearch/index.go | 28 ++++++++++---------- services/search/pkg/opensearch/index_test.go | 16 +++++++++++ services/search/pkg/search/search.go | 6 +++++ 8 files changed, 43 insertions(+), 23 deletions(-) diff --git a/services/search/README.md b/services/search/README.md index c0e9036aac..f0938d14e9 100644 --- a/services/search/README.md +++ b/services/search/README.md @@ -39,7 +39,7 @@ To enable OpenSearch as a backend, the following settings must be set: Additionally, the following optional settings can be set: -* `SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME=val` (default: `opencloud-resource`): Name of the OpenSearch index +* `SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME=val` (default: `opencloud-resource`): Base name of the OpenSearch index. The running index is suffixed with the schema version (e.g. `opencloud-resource-v3`); a breaking schema change targets a fresh index and leaves the old one in place. * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_USERNAME=val`: Username for HTTP Basic Authentication. * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_PASSWORD=val`: Password for HTTP Basic Authentication. * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_HEADER=val`: HTTP headers to include in requests. diff --git a/services/search/pkg/bleve/index.go b/services/search/pkg/bleve/index.go index 8f52d847cb..264ad9893b 100644 --- a/services/search/pkg/bleve/index.go +++ b/services/search/pkg/bleve/index.go @@ -2,6 +2,7 @@ package bleve import ( "errors" + "fmt" "math" "path/filepath" "reflect" @@ -26,12 +27,8 @@ const ( indexVersion = "v2" ) -func indexPath(root string) string { - return filepath.Join(root, "bleve-"+indexVersion) -} - func NewIndex(root string) (bleve.Index, error) { - destination := indexPath(root) + destination := filepath.Join(root, fmt.Sprintf("bleve-v%d", search.SchemaVersion)) index, err := bleve.Open(destination) if errors.Is(err, bleve.ErrorIndexPathDoesNotExist) { indexMapping, err := NewMapping() diff --git a/services/search/pkg/command/server.go b/services/search/pkg/command/server.go index dc90903929..a93338f79c 100644 --- a/services/search/pkg/command/server.go +++ b/services/search/pkg/command/server.go @@ -119,7 +119,8 @@ func Server(cfg *config.Config) *cobra.Command { return fmt.Errorf("failed to create OpenSearch client: %w", err) } - openSearchBackend, err := opensearch.NewBackend(cfg.Engine.OpenSearch.ResourceIndex.Name, client) + indexName := opensearch.VersionedIndexName(cfg.Engine.OpenSearch.ResourceIndex.Name) + openSearchBackend, err := opensearch.NewBackend(indexName, client) if err != nil { return fmt.Errorf("failed to create OpenSearch backend: %w", err) } diff --git a/services/search/pkg/config/engine.go b/services/search/pkg/config/engine.go index a22d2a92d1..9c9b47997d 100644 --- a/services/search/pkg/config/engine.go +++ b/services/search/pkg/config/engine.go @@ -25,7 +25,7 @@ type EngineOpenSearch struct { // EngineOpenSearchResourceIndex defines the OpenSearch index for resources type EngineOpenSearchResourceIndex struct { - Name string `yaml:"name" env:"SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME" desc:"The name of the OpenSearch index for resources." introductionVersion:"4.0.0"` + Name string `yaml:"name" env:"SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME" desc:"The base name of the OpenSearch index for resources. The running index is suffixed with the schema version, e.g. opencloud-resource-v3." introductionVersion:"4.0.0"` } // EngineOpenSearchClient configures the OpenSearch client diff --git a/services/search/pkg/opensearch/backend.go b/services/search/pkg/opensearch/backend.go index 6082643a93..ef6d25c031 100644 --- a/services/search/pkg/opensearch/backend.go +++ b/services/search/pkg/opensearch/backend.go @@ -35,7 +35,7 @@ type Backend struct { // NewBackend creates a backend on the versioned generation of the named index. func NewBackend(name string, client *opensearchgoAPI.Client) (*Backend, error) { - index := IndexName(name) + index := VersionedIndexName(name) pingResp, err := client.Ping(context.TODO(), &opensearchgoAPI.PingReq{}) switch { diff --git a/services/search/pkg/opensearch/index.go b/services/search/pkg/opensearch/index.go index 318358fde9..120ed6cea6 100644 --- a/services/search/pkg/opensearch/index.go +++ b/services/search/pkg/opensearch/index.go @@ -17,24 +17,24 @@ import ( ) var ( - ErrManualActionRequired = errors.New("manual action required") - IndexManagerLatest = IndexIndexManagerResourceV2 - IndexIndexManagerResourceV2 IndexManager = "resource_v2" + ErrManualActionRequired = errors.New("manual action required") + + // IndexManagerLatest identifies the current resource mapping; its version is + // derived from search.SchemaVersion so it never drifts from the index name. + IndexManagerLatest = IndexManager(fmt.Sprintf("resource_v%d", search.SchemaVersion)) ) +// VersionedIndexName suffixes the base index name with the schema version, e.g. +// "opencloud-resource" -> "opencloud-resource-v3". +func VersionedIndexName(base string) string { + return fmt.Sprintf("%s-v%d", base, search.SchemaVersion) +} + type IndexManager string -// IndexName puts the schema generation behind the configured name, so a new -// generation starts on an index of its own instead of refusing to work with -// the one that is there. Interim: derived from a constant here, from the -// shared SchemaVersion later in this series. -func IndexName(name string) string { - return name + "-v2" -} - // indexGenerators dispatches each IndexManager variant to its builder. var indexGenerators = map[IndexManager]func() ([]byte, error){ - IndexIndexManagerResourceV2: buildResourceV2Mapping, + IndexManagerLatest: buildResourceMapping, } func (m IndexManager) String() string { @@ -54,10 +54,10 @@ func (m IndexManager) MarshalJSON() ([]byte, error) { return gen() } -// buildResourceV2Mapping renders the OpenSearch index template for a +// buildResourceMapping renders the OpenSearch index template for a // search.Resource from the shared SearchFieldOverrides. OpenSearch-specific // tweaks (wildcard MimeType, path_hierarchy Path) are applied on top. -func buildResourceV2Mapping() ([]byte, error) { +func buildResourceMapping() ([]byte, error) { resourceType := reflect.TypeFor[search.Resource]() overrides := maps.Clone(search.Resource{}.SearchFieldOverrides()) overrides["MimeType"] = searchmapping.FieldOpts{Type: searchmapping.TypeWildcard} diff --git a/services/search/pkg/opensearch/index_test.go b/services/search/pkg/opensearch/index_test.go index faba72f674..3ba5a38cec 100644 --- a/services/search/pkg/opensearch/index_test.go +++ b/services/search/pkg/opensearch/index_test.go @@ -1,6 +1,7 @@ package opensearch_test import ( + "fmt" "strings" "testing" @@ -9,8 +10,23 @@ import ( "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) +// TestVersionedIndexName guards that the index name and the generator identity +// carry the same schema version. +func TestVersionedIndexName(t *testing.T) { + require.Equal(t, + fmt.Sprintf("opencloud-resource-v%d", search.SchemaVersion), + opensearch.VersionedIndexName("opencloud-resource"), + ) + require.Equal(t, + fmt.Sprintf("resource_v%d", search.SchemaVersion), + string(opensearch.IndexManagerLatest), + ) +} + func TestIndexManager(t *testing.T) { t.Run("index plausibility", func(t *testing.T) { tests := []opensearchtest.TableTest[opensearch.IndexManager, struct{}]{ diff --git a/services/search/pkg/search/search.go b/services/search/pkg/search/search.go index 78fe0ac29c..bf81837d17 100644 --- a/services/search/pkg/search/search.go +++ b/services/search/pkg/search/search.go @@ -23,6 +23,12 @@ import ( "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" ) +// SchemaVersion is the shared schema version for both search backends. Bump it +// on a breaking mapping change: each version gets its own index (OpenSearch name +// suffix, bleve path suffix), so the service builds a fresh index instead of +// colliding with the old one. No migration; reindex to populate. +const SchemaVersion = 3 + var scopeRegex = regexp.MustCompile(`scope:\s*([^" "\n\r]*)`) // Engine is the interface to the search engine From 40a2f1460074b8f91e37c5eb12aba46779574e2a Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Wed, 29 Jul 2026 16:12:07 +0200 Subject: [PATCH 09/54] fix(search): preserve query value case for case-sensitive fields on OpenSearch OpenSearch lowercased every KQL query value, so exact-match queries on case-preserved keyword fields (facet values, ids) never matched their stored token. Fold the value only for fields with a lowercasing analyzer, mirroring the bleve backend. The field set is derived once in search.LowercaseValueFields and shared by both backends (bleve's local buildLowercaseFields is dropped). --- .../opensearch/internal/convert/kql_expand.go | 12 +++++---- services/search/pkg/query/bleve/compiler.go | 21 +++------------ services/search/pkg/search/search.go | 26 +++++++++++++++++++ 3 files changed, 37 insertions(+), 22 deletions(-) diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand.go b/services/search/pkg/opensearch/internal/convert/kql_expand.go index 17258512c9..5c63a3072f 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand.go @@ -3,13 +3,13 @@ package convert import ( "fmt" "reflect" - "slices" "strconv" "strings" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) func ExpandKQL(nodes []ast.Node) ([]ast.Node, error) { @@ -101,11 +101,13 @@ func (_ kqlExpander) remapKey(current string, defaultKey string) string { } func (_ kqlExpander) lowerValue(key, value string) string { - if slices.Contains([]string{"Name", "Title", "Tags", "Content", "MimeType", "Type", "Hidden"}, key) { - return strings.ToLower(value) + // only fold the value for fields whose index analyzer lowercases too; + // case-preserved (keyword) fields must keep their casing or they never match + // their stored token. Shared with bleve via search.LowercaseValueFields. + if _, ok := search.LowercaseValueFields()[key]; !ok { + return value } - - return value + return strings.ToLower(value) } func (_ kqlExpander) unfoldValue(key, value string) []ast.Node { diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index 218dc43104..9722f45512 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -12,26 +12,13 @@ import ( bleveQuery "github.com/blevesearch/bleve/v2/search/query" "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/pkg/kql" - "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) -// lowercaseFields is derived from Resource.SearchFieldOverrides(): any -// field whose override picks a lowercasing analyzer (`lowercaseKeyword`) -// or the fulltext type (which uses a lowercasing analyzer under the hood) -// gets its query-side value pre-lowercased so compile-time matches the -// index-time tokenization. Anything else keeps its original casing. -var lowercaseFields = buildLowercaseFields() - -func buildLowercaseFields() map[string]struct{} { - out := map[string]struct{}{} - for key, opts := range (search.Resource{}).SearchFieldOverrides() { - if opts.Analyzer == "lowercaseKeyword" || opts.Type == mapping.TypeFulltext { - out[key] = struct{}{} - } - } - return out -} +// lowercaseFields holds the fields whose query-side value is pre-lowercased so +// it matches the index-time lowercasing analyzer. Shared with the OpenSearch +// backend via search.LowercaseValueFields; every other field keeps its casing. +var lowercaseFields = search.LowercaseValueFields() var _fields = map[string]string{ "rootid": "RootID", diff --git a/services/search/pkg/search/search.go b/services/search/pkg/search/search.go index bf81837d17..fb4ebef31e 100644 --- a/services/search/pkg/search/search.go +++ b/services/search/pkg/search/search.go @@ -89,6 +89,32 @@ func (Resource) SearchFieldOverrides() map[string]mapping.FieldOpts { return resourceFieldOverrides() } +// lowercaseValueFields is the set of index field names whose query values must +// be lowercased to match their index-time lowercasing analyzer (lowercaseKeyword +// or the fulltext type). Built once from the field overrides. +var lowercaseValueFields = sync.OnceValue(func() map[string]struct{} { + out := map[string]struct{}{} + for key, opts := range resourceFieldOverrides() { + if opts.Analyzer == "lowercaseKeyword" || opts.Type == mapping.TypeFulltext { + out[key] = struct{}{} + } + } + // stored values are normalized lowercase, so query values must fold too + // even though the index fields preserve case + for _, key := range []string{"MimeType", "Type", "Hidden"} { + out[key] = struct{}{} + } + return out +}) + +// LowercaseValueFields returns the set of index field names whose query values +// must be lowercased so query-side matching lines up with the index-time +// analyzer. Both search backends use it, so value casing stays consistent; every +// other (case-preserved) field keeps its original case. Read-only, do not mutate. +func LowercaseValueFields() map[string]struct{} { + return lowercaseValueFields() +} + // ResolveReference makes sure the path is relative to the space root func ResolveReference(ctx context.Context, ref *provider.Reference, ri *provider.ResourceInfo, gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) (*provider.Reference, error) { if ref.GetResourceId().GetOpaqueId() == ref.GetResourceId().GetSpaceId() { From 65f615d92739975703077dfdd31ee97b07983240 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Wed, 29 Jul 2026 17:39:23 +0200 Subject: [PATCH 10/54] refactor(kql): move parse/validation errors into pkg/kql The KQL parser produced its own validation errors but imported them from the search service's query package. Move them into pkg/kql and let the search backend consume kql.IsValidationError, so the parser stops depending on a service package. --- pkg/kql/cast.go | 5 ++--- pkg/kql/dictionary_test.go | 21 +++++++++---------- .../search/pkg/query => pkg/kql}/error.go | 7 ++++--- pkg/kql/kql_test.go | 3 +-- pkg/kql/validate.go | 7 +++---- services/search/pkg/bleve/backend.go | 3 ++- services/search/pkg/opensearch/backend.go | 4 ++-- 7 files changed, 24 insertions(+), 26 deletions(-) rename {services/search/pkg/query => pkg/kql}/error.go (86%) diff --git a/pkg/kql/cast.go b/pkg/kql/cast.go index ecd0cac27f..fd2eeadeac 100644 --- a/pkg/kql/cast.go +++ b/pkg/kql/cast.go @@ -7,7 +7,6 @@ import ( "github.com/jinzhu/now" "github.com/opencloud-eu/opencloud/pkg/ast" - "github.com/opencloud-eu/opencloud/services/search/pkg/query" ) func toNode[T ast.Node](in any) (T, error) { @@ -85,7 +84,7 @@ func toTimeRange(in any) (*time.Time, *time.Time, error) { value, err := toString(in) if err != nil { - return &from, &to, &query.UnsupportedTimeRangeError{} + return &from, &to, &UnsupportedTimeRangeError{} } c := &now.Config{ @@ -132,7 +131,7 @@ func toTimeRange(in any) (*time.Time, *time.Time, error) { } if from.IsZero() || to.IsZero() { - return nil, nil, &query.UnsupportedTimeRangeError{} + return nil, nil, &UnsupportedTimeRangeError{} } return &from, &to, nil diff --git a/pkg/kql/dictionary_test.go b/pkg/kql/dictionary_test.go index 25f6e2f902..a49d16dc18 100644 --- a/pkg/kql/dictionary_test.go +++ b/pkg/kql/dictionary_test.go @@ -9,7 +9,6 @@ import ( "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/pkg/ast/test" "github.com/opencloud-eu/opencloud/pkg/kql" - "github.com/opencloud-eu/opencloud/services/search/pkg/query" tAssert "github.com/stretchr/testify/assert" ) @@ -34,13 +33,13 @@ func TestParse_Spec(t *testing.T) { }, { name: `AND`, - error: query.StartsWithBinaryOperatorError{ + error: kql.StartsWithBinaryOperatorError{ Node: &ast.OperatorNode{Value: kql.BoolAND}, }, }, { name: `AND cat AND dog`, - error: query.StartsWithBinaryOperatorError{ + error: kql.StartsWithBinaryOperatorError{ Node: &ast.OperatorNode{Value: kql.BoolAND}, }, }, @@ -80,13 +79,13 @@ func TestParse_Spec(t *testing.T) { }, { name: `OR`, - error: query.StartsWithBinaryOperatorError{ + error: kql.StartsWithBinaryOperatorError{ Node: &ast.OperatorNode{Value: kql.BoolOR}, }, }, { name: `OR cat AND dog`, - error: query.StartsWithBinaryOperatorError{ + error: kql.StartsWithBinaryOperatorError{ Node: &ast.OperatorNode{Value: kql.BoolOR}, }, }, @@ -930,37 +929,37 @@ func TestParse_Errors(t *testing.T) { tests := []testCase{ { query: "animal:(mammal:cat mammal:dog reptile:turtle)", - error: query.NamedGroupInvalidNodesError{ + error: kql.NamedGroupInvalidNodesError{ Node: &ast.StringNode{Key: "mammal", Value: "cat"}, }, }, { query: "animal:(cat mammal:dog turtle)", - error: query.NamedGroupInvalidNodesError{ + error: kql.NamedGroupInvalidNodesError{ Node: &ast.StringNode{Key: "mammal", Value: "dog"}, }, }, { query: "animal:(AND cat)", - error: query.StartsWithBinaryOperatorError{ + error: kql.StartsWithBinaryOperatorError{ Node: &ast.OperatorNode{Value: kql.BoolAND}, }, }, { query: "animal:(OR cat)", - error: query.StartsWithBinaryOperatorError{ + error: kql.StartsWithBinaryOperatorError{ Node: &ast.OperatorNode{Value: kql.BoolOR}, }, }, { query: "(AND cat)", - error: query.StartsWithBinaryOperatorError{ + error: kql.StartsWithBinaryOperatorError{ Node: &ast.OperatorNode{Value: kql.BoolAND}, }, }, { query: "(OR cat)", - error: query.StartsWithBinaryOperatorError{ + error: kql.StartsWithBinaryOperatorError{ Node: &ast.OperatorNode{Value: kql.BoolOR}, }, }, diff --git a/services/search/pkg/query/error.go b/pkg/kql/error.go similarity index 86% rename from services/search/pkg/query/error.go rename to pkg/kql/error.go index 27fde9541e..47009001b3 100644 --- a/services/search/pkg/query/error.go +++ b/pkg/kql/error.go @@ -1,4 +1,4 @@ -package query +package kql import ( "errors" @@ -39,8 +39,9 @@ func (e UnsupportedTimeRangeError) Error() string { return fmt.Sprintf("unable to convert '%v' to a time range", e.Value) } -// IsValidationError says whether the query itself is at fault, which makes it a -// bad request and not an error of ours. +// IsValidationError reports whether err is one of the KQL parse/validation +// errors produced by this package, i.e. the query itself is at fault and the +// caller should treat it as a bad request. func IsValidationError(err error) bool { var ( startsWithBinaryOperator *StartsWithBinaryOperatorError diff --git a/pkg/kql/kql_test.go b/pkg/kql/kql_test.go index d3b745ca3f..d90245ccf3 100644 --- a/pkg/kql/kql_test.go +++ b/pkg/kql/kql_test.go @@ -5,7 +5,6 @@ import ( "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/pkg/kql" - "github.com/opencloud-eu/opencloud/services/search/pkg/query" tAssert "github.com/stretchr/testify/assert" ) @@ -22,7 +21,7 @@ func TestNewAST(t *testing.T) { { name: "error", givenQuery: kql.BoolAND, - expectedError: query.StartsWithBinaryOperatorError{ + expectedError: kql.StartsWithBinaryOperatorError{ Node: &ast.OperatorNode{Value: kql.BoolAND}, }, }, diff --git a/pkg/kql/validate.go b/pkg/kql/validate.go index 56bf681887..1f7e38ccba 100644 --- a/pkg/kql/validate.go +++ b/pkg/kql/validate.go @@ -2,7 +2,6 @@ package kql import ( "github.com/opencloud-eu/opencloud/pkg/ast" - "github.com/opencloud-eu/opencloud/services/search/pkg/query" ) func validateAst(a *ast.Ast) error { @@ -10,7 +9,7 @@ func validateAst(a *ast.Ast) error { case *ast.OperatorNode: switch node.Value { case BoolAND, BoolOR: - return &query.StartsWithBinaryOperatorError{Node: node} + return &StartsWithBinaryOperatorError{Node: node} } } return nil @@ -21,14 +20,14 @@ func validateGroupNode(n *ast.GroupNode) error { case *ast.OperatorNode: switch node.Value { case BoolAND, BoolOR: - return &query.StartsWithBinaryOperatorError{Node: node} + return &StartsWithBinaryOperatorError{Node: node} } } if n.Key != "" { for _, node := range n.Nodes { if ast.NodeKey(node) != "" { - return &query.NamedGroupInvalidNodesError{Node: node} + return &NamedGroupInvalidNodesError{Node: node} } } } diff --git a/services/search/pkg/bleve/backend.go b/services/search/pkg/bleve/backend.go index eda67ead85..e325d81261 100644 --- a/services/search/pkg/bleve/backend.go +++ b/services/search/pkg/bleve/backend.go @@ -14,6 +14,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/utils" "google.golang.org/protobuf/types/known/timestamppb" + "github.com/opencloud-eu/opencloud/pkg/kql" "github.com/opencloud-eu/opencloud/pkg/log" "github.com/opencloud-eu/opencloud/services/search/pkg/search" @@ -45,7 +46,7 @@ func NewBackend(index bleve.Index, queryCreator searchQuery.Creator[query.Query] func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) { createdQuery, err := b.queryCreator.Create(sir.Query) if err != nil { - if searchQuery.IsValidationError(err) { + if kql.IsValidationError(err) { return nil, errtypes.BadRequest(err.Error()) } return nil, err diff --git a/services/search/pkg/opensearch/backend.go b/services/search/pkg/opensearch/backend.go index ef6d25c031..93433f502a 100644 --- a/services/search/pkg/opensearch/backend.go +++ b/services/search/pkg/opensearch/backend.go @@ -14,11 +14,11 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/utils" "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/pkg/kql" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - searchQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) @@ -74,7 +74,7 @@ func NewBackend(name string, client *opensearchgoAPI.Client) (*Backend, error) { func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) { boolQuery, err := convert.KQLToOpenSearchBoolQuery(sir.Query) switch { - case searchQuery.IsValidationError(err): + case kql.IsValidationError(err): return nil, errtypes.BadRequest(err.Error()) case err != nil: return nil, fmt.Errorf("failed to convert KQL query to OpenSearch bool query: %w", err) From 45a80966179e48cdfbf628d509cc8a424a0a5578 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Wed, 29 Jul 2026 17:39:23 +0200 Subject: [PATCH 11/54] feat(search): derive a case-insensitive field-name index from the resource struct mapping.FieldNameIndex walks the struct and maps a lowercased field path to the real field name, including nested facet sub-fields. Backend-neutral. --- services/search/pkg/mapping/fieldindex.go | 36 +++++++++ .../search/pkg/mapping/fieldindex_test.go | 79 +++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 services/search/pkg/mapping/fieldindex.go create mode 100644 services/search/pkg/mapping/fieldindex_test.go diff --git a/services/search/pkg/mapping/fieldindex.go b/services/search/pkg/mapping/fieldindex.go new file mode 100644 index 0000000000..5bc5d21b66 --- /dev/null +++ b/services/search/pkg/mapping/fieldindex.go @@ -0,0 +1,36 @@ +package mapping + +import ( + "reflect" + "strings" +) + +// FieldNameIndex maps a lowercased field path to the real field name for every +// field of t, recursing into nested facets (photo.cameraMake, ...). Names come +// from json tags, so it is backend-neutral; the query layer resolves KQL keys +// case-insensitively against it. +func FieldNameIndex(t reflect.Type, overrides map[string]FieldOpts) map[string]string { + out := map[string]string{} + var walk func(t reflect.Type, prefix string) + walk = func(t reflect.Type, prefix string) { + _ = walkFields(t, func(fi fieldInfo) error { + name := fi.Name + if prefix != "" { + name = prefix + "." + fi.Name + } + out[strings.ToLower(name)] = name + + fieldType := overrides[name].Type + if fieldType == "" { + fieldType = inferType(fi.GoField.Type) + } + // recurse into nested facets; time.Time is a struct too but a leaf. + if sub := structType(fi.GoField.Type); sub != nil && fieldType != TypeDatetime { + walk(sub, name) + } + return nil + }) + } + walk(t, "") + return out +} diff --git a/services/search/pkg/mapping/fieldindex_test.go b/services/search/pkg/mapping/fieldindex_test.go new file mode 100644 index 0000000000..d0e47dcb2e --- /dev/null +++ b/services/search/pkg/mapping/fieldindex_test.go @@ -0,0 +1,79 @@ +package mapping_test + +import ( + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func resourceFieldIndex(testing.TB) map[string]string { + return mapping.FieldNameIndex( + reflect.TypeFor[search.Resource](), + search.Resource{}.SearchFieldOverrides(), + ) +} + +// resolve looks up the lowercased key in the index, falling back to the key. +func resolve(idx map[string]string, key string) string { + if v, ok := idx[strings.ToLower(key)]; ok { + return v + } + return key +} + +func TestFieldNameIndex_TopLevelCaseInsensitive(t *testing.T) { + idx := resourceFieldIndex(t) + for in, want := range map[string]string{ + "rootid": "RootID", "ROOTID": "RootID", "RootID": "RootID", + "name": "Name", "NAME": "Name", + "mimetype": "MimeType", "MimeType": "MimeType", + "tags": "Tags", "favorites": "Favorites", + "mtime": "Mtime", "parentid": "ParentID", "id": "ID", + } { + require.Equalf(t, want, resolve(idx, in), "resolve(%q)", in) + } +} + +// Facet sub-fields are lowerCamelCase in the index (from libregraph json tags); +// the derived index resolves them case-insensitively. +func TestFieldNameIndex_FacetsCaseInsensitive(t *testing.T) { + idx := resourceFieldIndex(t) + for in, want := range map[string]string{ + // case-insensitive: same field, different casings + "photo.cameramake": "photo.cameraMake", + "photo.CAMERAMAKE": "photo.cameraMake", + // a representative sub-field across each facet + "photo.takendatetime": "photo.takenDateTime", + "audio.artist": "audio.artist", + "audio.albumartist": "audio.albumArtist", + "image.width": "image.width", + "location.latitude": "location.latitude", + } { + require.Equalf(t, want, resolve(idx, in), "resolve(%q)", in) + } +} + +func TestFieldNameIndex_UnknownPassesThrough(t *testing.T) { + idx := resourceFieldIndex(t) + require.Equal(t, "nope.field", resolve(idx, "nope.field")) + require.Equal(t, "custom", resolve(idx, "custom")) +} + +// All top-level fields are covered from one derived source, so both backends +// resolve them the same way. +func TestFieldNameIndex_CoversAllTopLevelFields(t *testing.T) { + idx := resourceFieldIndex(t) + for in, want := range map[string]string{ + "rootid": "RootID", "path": "Path", "id": "ID", "name": "Name", + "size": "Size", "mtime": "Mtime", "type": "Type", + "content": "Content", "hidden": "Hidden", "tags": "Tags", + "favorites": "Favorites", + } { + require.Equalf(t, want, resolve(idx, in), "derived should cover %q", in) + } +} From bf7d04f7a1b4a9e2b285128c261dbd79d97bee67 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Wed, 29 Jul 2026 17:39:23 +0200 Subject: [PATCH 12/54] feat(search): add shared KQL lowering pass query.Normalize resolves field names (query.ResolveField, from the derived index + a small alias overlay) and expands media-type restrictions (mimetype.Expand) once, between parse and backend compilation. --- .../search/pkg/query/mimetype/mimetype.go | 96 ++++++++++++++++++ .../pkg/query/mimetype/mimetype_test.go | 97 +++++++++++++++++++ services/search/pkg/query/normalize.go | 73 ++++++++++++++ services/search/pkg/query/normalize_test.go | 90 +++++++++++++++++ services/search/pkg/query/resolver.go | 41 ++++++++ 5 files changed, 397 insertions(+) create mode 100644 services/search/pkg/query/mimetype/mimetype.go create mode 100644 services/search/pkg/query/mimetype/mimetype_test.go create mode 100644 services/search/pkg/query/normalize.go create mode 100644 services/search/pkg/query/normalize_test.go create mode 100644 services/search/pkg/query/resolver.go diff --git a/services/search/pkg/query/mimetype/mimetype.go b/services/search/pkg/query/mimetype/mimetype.go new file mode 100644 index 0000000000..98692d9524 --- /dev/null +++ b/services/search/pkg/query/mimetype/mimetype.go @@ -0,0 +1,96 @@ +// Package mimetype maps the "mediatype" KQL restriction (field name and value) +// to a concrete MimeType query. +package mimetype + +import ( + "strings" + + "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/pkg/kql" +) + +// field is the real index field a mediatype restriction targets. +const field = "MimeType" + +// Expand turns mediatype: into the MimeType query it stands for: category +// values (file/document/image/...) expand to their MIME set, anything else is a +// literal MimeType:. Returns nil for non-mediatype keys. +func Expand(key, value string) []ast.Node { + if strings.ToLower(key) != "mediatype" { + return nil + } + switch value { + case "file": + return []ast.Node{ + &ast.OperatorNode{Value: kql.BoolNOT}, + &ast.StringNode{Key: field, Value: "httpd/unix-directory"}, + } + case "folder": + return term("httpd/unix-directory") + case "document": + return group( + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.wordprocessingml.form", + "application/vnd.oasis.opendocument.text", + "text/plain", + "text/markdown", + "application/rtf", + "application/vnd.apple.pages", + ) + case "spreadsheet": + return group( + "application/vnd.ms-excel", + "application/vnd.oasis.opendocument.spreadsheet", + "text/csv", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.apple.numbers", + ) + case "presentation": + return group( + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.ms-powerpoint", + "application/vnd.apple.keynote", + ) + case "pdf": + return term("application/pdf") + case "image": + return term("image/*") + case "video": + return term("video/*") + case "audio": + return term("audio/*") + case "archive": + return group( + "application/zip", + "application/gzip", + "application/x-gzip", + "application/x-7z-compressed", + "application/x-rar-compressed", + "application/x-tar", + "application/x-bzip2", + "application/x-bzip", + "application/x-tgz", + ) + } + // not a category: treat the value as a literal MIME type. + return term(value) +} + +// term is a single MimeType:value restriction. +func term(value string) []ast.Node { + return []ast.Node{&ast.StringNode{Key: field, Value: value}} +} + +// group is a single OR group of MimeType:value restrictions. +func group(values ...string) []ast.Node { + nodes := make([]ast.Node, 0, len(values)*2-1) + for i, v := range values { + if i > 0 { + nodes = append(nodes, &ast.OperatorNode{Value: kql.BoolOR}) + } + nodes = append(nodes, &ast.StringNode{Key: field, Value: v}) + } + return []ast.Node{&ast.GroupNode{Nodes: nodes}} +} diff --git a/services/search/pkg/query/mimetype/mimetype_test.go b/services/search/pkg/query/mimetype/mimetype_test.go new file mode 100644 index 0000000000..0b25844e28 --- /dev/null +++ b/services/search/pkg/query/mimetype/mimetype_test.go @@ -0,0 +1,97 @@ +package mimetype_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/services/search/pkg/query/mimetype" +) + +// This is the single place the mediatype -> MimeType mapping is tested. The +// query pipeline consumes Expand via query.Normalize and must NOT re-test it. + +func TestExpand_onlyTriggersOnMediatype(t *testing.T) { + require.Nil(t, mimetype.Expand("Name", "document")) + require.Nil(t, mimetype.Expand("MimeType", "file")) // the real field name is not the trigger + require.Nil(t, mimetype.Expand("Tags", "file")) +} + +func TestExpand_keyIsCaseInsensitive(t *testing.T) { + require.NotNil(t, mimetype.Expand("MediaType", "file")) +} + +// A non-category value is a literal MIME type and targets the MimeType field. +func TestExpand_literalValuePassesThroughToMimeType(t *testing.T) { + require.Equal(t, []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "application/pdf"}, + }, mimetype.Expand("mediatype", "application/pdf")) + + require.Equal(t, []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "image/jpeg"}, + }, mimetype.Expand("mediatype", "image/jpeg")) +} + +func TestExpand_fileIsNotAFolder(t *testing.T) { + require.Equal(t, []ast.Node{ + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + }, mimetype.Expand("mediatype", "file")) +} + +func TestExpand_folderIsASingleTerm(t *testing.T) { + require.Equal(t, []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + }, mimetype.Expand("mediatype", "folder")) +} + +func TestExpand_wildcardCategories(t *testing.T) { + for value, mime := range map[string]string{ + "image": "image/*", "video": "video/*", "audio": "audio/*", "pdf": "application/pdf", + } { + require.Equal(t, []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: mime}, + }, mimetype.Expand("mediatype", value), value) + } +} + +func TestExpand_documentGroup(t *testing.T) { + got := mimetype.Expand("mediatype", "document") + require.Len(t, got, 1) + group, ok := got[0].(*ast.GroupNode) + require.True(t, ok) + require.Equal(t, mimeValues(group), []string{ + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.wordprocessingml.form", + "application/vnd.oasis.opendocument.text", + "text/plain", + "text/markdown", + "application/rtf", + "application/vnd.apple.pages", + }) +} + +// spreadsheet asserts the exact MIME set, in order, with no duplicate entry. +func TestExpand_spreadsheet(t *testing.T) { + group := mimetype.Expand("mediatype", "spreadsheet")[0].(*ast.GroupNode) + require.Equal(t, mimeValues(group), []string{ + "application/vnd.ms-excel", + "application/vnd.oasis.opendocument.spreadsheet", + "text/csv", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.apple.numbers", + }) +} + +// mimeValues extracts the StringNode values from an OR group, dropping operators. +func mimeValues(group *ast.GroupNode) []string { + var out []string + for _, n := range group.Nodes { + if s, ok := n.(*ast.StringNode); ok { + out = append(out, s.Value) + } + } + return out +} diff --git a/services/search/pkg/query/normalize.go b/services/search/pkg/query/normalize.go new file mode 100644 index 0000000000..8c1269604b --- /dev/null +++ b/services/search/pkg/query/normalize.go @@ -0,0 +1,73 @@ +package query + +import ( + "reflect" + + "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/services/search/pkg/query/mimetype" +) + +// Normalize is the shared KQL lowering pass between parse and compile: it +// resolves keys to real field names (via resolve) and expands media-type +// restrictions, so the backends compile a plain field:value AST. +func Normalize(a *ast.Ast, resolve func(string) string) *ast.Ast { + a.Nodes = normalizeNodes(a.Nodes, resolve, "") + return a +} + +// normalizeNodes rewrites nodes in place. defaultKey is what a bare restriction +// inherits: its enclosing group's key, or "" at the top level. +func normalizeNodes(nodes []ast.Node, resolve func(string) string, defaultKey string) []ast.Node { + resolveKey := func(key string) string { + if key == "" && defaultKey != "" { + return defaultKey // bare child inherits the group key + } + return resolve(key) + } + + out := make([]ast.Node, 0, len(nodes)) + for _, n := range nodes { + n = toPointer(n) // ensure a pointer so in-place key rewrites persist + switch node := n.(type) { + case *ast.StringNode: + node.Key = resolveKey(node.Key) + if exp := mimetype.Expand(node.Key, node.Value); exp != nil { + out = append(out, normalizeNodes(exp, resolve, defaultKey)...) + continue + } + out = append(out, node) + case *ast.DateTimeNode: + node.Key = resolveKey(node.Key) + out = append(out, node) + case *ast.BooleanNode: + node.Key = resolveKey(node.Key) + out = append(out, node) + case *ast.GroupNode: + groupKey := defaultKey + if node.Key != "" { + node.Key = resolve(node.Key) + groupKey = node.Key + } + node.Nodes = normalizeNodes(node.Nodes, resolve, groupKey) + out = append(out, node) + default: + out = append(out, n) + } + } + return out +} + +// toPointer returns n as a pointer; the parser emits some nodes by value and the +// in-place key rewrites would be lost on those. +func toPointer(n ast.Node) ast.Node { + rv := reflect.ValueOf(n) + if rv.Kind() == reflect.Ptr { + return n + } + ptr := reflect.New(rv.Type()) + ptr.Elem().Set(rv) + if pn, ok := ptr.Interface().(ast.Node); ok { + return pn + } + return n +} diff --git a/services/search/pkg/query/normalize_test.go b/services/search/pkg/query/normalize_test.go new file mode 100644 index 0000000000..49cea2c153 --- /dev/null +++ b/services/search/pkg/query/normalize_test.go @@ -0,0 +1,90 @@ +package query_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/services/search/pkg/query" +) + +// This is the single place the shared KQL lowering pass is tested (field +// resolution, media-type expansion, group-key defaulting, pointer conversion). +// The backend query compilers consume its canonical output and must not re-test +// it. + +func norm(nodes ...ast.Node) []ast.Node { + return query.Normalize(&ast.Ast{Nodes: nodes}, query.ResolveField).Nodes +} + +func TestResolveField(t *testing.T) { + require.Equal(t, "Name", query.ResolveField("")) // empty -> free-text default + require.Equal(t, "Name", query.ResolveField("NAME")) // case-insensitive + require.Equal(t, "Tags", query.ResolveField("tag")) // singular alias + require.Equal(t, "MimeType", query.ResolveField("mimetype")) // real field, case-insensitive + require.Equal(t, "photo.cameraMake", query.ResolveField("photo.CAMERAMAKE")) // facet, case-insensitive + require.Equal(t, "unknown.field", query.ResolveField("unknown.field")) // unknown key: unchanged, becomes a dead query +} + +func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) { + got := norm( + &ast.StringNode{Key: "", Value: "free"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "TAG", Value: "x"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "photo.cameramake", Value: "canon"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "mediatype", Value: "file"}, + ) + require.Equal(t, []ast.Node{ + &ast.StringNode{Key: "Name", Value: "free"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Tags", Value: "x"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "photo.cameraMake", Value: "canon"}, + &ast.OperatorNode{Value: "AND"}, + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + }, got) +} + +// A bare restriction inside a named group inherits the group key; a keyed child +// keeps its own key; a bare restriction in an unnamed group falls back to Name. +func TestNormalize_GroupKeyDefaulting(t *testing.T) { + got := norm( + &ast.GroupNode{Key: "author", Nodes: []ast.Node{ + &ast.StringNode{Value: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "name", Value: "d"}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Value: "e"}, + }}, + ) + require.Equal(t, []ast.Node{ + &ast.GroupNode{Key: "author", Nodes: []ast.Node{ + &ast.StringNode{Key: "author", Value: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "Name", Value: "d"}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "e"}, + }}, + }, got) +} + +func TestNormalize_ConvertsValueNodesToPointers(t *testing.T) { + got := norm( + ast.StringNode{Key: "name", Value: "x"}, + ast.OperatorNode{Value: "AND"}, + ast.DateTimeNode{Key: "mtime"}, + ) + require.Equal(t, []ast.Node{ + &ast.StringNode{Key: "Name", Value: "x"}, + &ast.OperatorNode{Value: "AND"}, + &ast.DateTimeNode{Key: "Mtime"}, + }, got) +} diff --git a/services/search/pkg/query/resolver.go b/services/search/pkg/query/resolver.go new file mode 100644 index 0000000000..5a4127a817 --- /dev/null +++ b/services/search/pkg/query/resolver.go @@ -0,0 +1,41 @@ +package query + +import ( + "reflect" + "strings" + "sync" + + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +// aliases are KQL spellings the derived index can't produce (fields are plural). +var aliases = map[string]string{ + "tag": "Tags", + "favorite": "Favorites", +} + +// fieldIndex maps a lowercased KQL key to the real field name: derived once from +// the resource struct, overlaid with the explicit aliases. +var fieldIndex = sync.OnceValue(func() map[string]string { + idx := mapping.FieldNameIndex( + reflect.TypeFor[search.Resource](), + search.Resource{}.SearchFieldOverrides(), + ) + for k, v := range aliases { + idx[k] = v + } + return idx +}) + +// ResolveField maps a KQL key to the index field name: empty -> Name, a known +// key (case-insensitive) -> its field, anything else unchanged. +func ResolveField(name string) string { + if name == "" { + return "Name" + } + if v, ok := fieldIndex()[strings.ToLower(name)]; ok { + return v + } + return name +} From 1cba974c1db2cd557e4c1970cee95e13ebd6af78 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Wed, 29 Jul 2026 17:39:23 +0200 Subject: [PATCH 13/54] refactor(search): run the bleve backend on the shared lowering pass The bleve Creator runs query.Normalize before compiling; the compiler consumes a canonical AST with no field resolution or media-type special-casing. --- services/search/pkg/query/bleve/bleve.go | 4 ++ services/search/pkg/query/bleve/compiler.go | 62 +++++-------------- .../search/pkg/query/bleve/compiler_test.go | 3 +- 3 files changed, 21 insertions(+), 48 deletions(-) diff --git a/services/search/pkg/query/bleve/bleve.go b/services/search/pkg/query/bleve/bleve.go index 0acbc80095..41e260b81c 100644 --- a/services/search/pkg/query/bleve/bleve.go +++ b/services/search/pkg/query/bleve/bleve.go @@ -22,6 +22,10 @@ func (c Creator[T]) Create(qs string) (T, error) { return t, err } + // shared KQL lowering pass: resolve field names + expand media-type aliases + // once, so the compiler below sees only canonical field:value nodes. + builderAst = query.Normalize(builderAst, query.ResolveField) + t, err = c.compiler.Compile(builderAst) if err != nil { return t, err diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index 9722f45512..c5b32835b0 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -95,9 +95,11 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { for i := offset; i < len(nodes); i++ { switch n := nodes[i].(type) { case *ast.StringNode: - k := getField(n.Key) + // keys are resolved and media-type expanded by normalize; MimeType + // values are literal MIME types, so they skip the escaper. + k := n.Key v := n.Value - if k != "ID" && k != "Size" { + if k != "ID" && k != "Size" && k != "MimeType" { v = bleveEscaper.Replace(n.Value) } @@ -105,46 +107,7 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { v = strings.ToLower(v) } - if k == "Type" { - v = resourceType(v) - } - - var q bleveQuery.Query - var group bool - switch { - case k == "Hidden": - value, err := strconv.ParseBool(v) - if err != nil { - q = bleveQuery.NewMatchNoneQuery() - break - } - - bq := bleveQuery.NewBoolFieldQuery(value) - bq.SetField(k) - q = bq - case k == "MimeType": - q, group = mimeType(k, v) - if prev == nil { - isGroup = group - } - case slices.Contains([]string{"Name", "Title"}, k) && strings.ContainsAny(n.Value, "*?"): - patterns := []bleveQuery.Query{bleveQuery.NewQueryStringQuery(k + ".wildcard:" + v)} - if !strings.HasSuffix(v, "*") { - patterns = append(patterns, bleveQuery.NewQueryStringQuery(k+".wildcard:"+v+".*")) - } - - q = closed(bleveQuery.NewDisjunctionQuery(patterns)) - case n.Exact && !strings.ContainsAny(n.Value, "*?") && slices.Contains([]string{"Name", "Title"}, k): - q = bleveQuery.NewQueryStringQuery(k + ".wildcard:" + v) - case k == "Path" && !strings.ContainsAny(n.Value, "*?"): - q = pathAndBelow(k, n.Value) - case slices.Contains([]string{"Name", "Title", "Content"}, k) && !strings.ContainsAny(n.Value, "*?"): - q = phrase(k, n.Value) - case strings.Contains(n.Value, " ") && !strings.ContainsAny(n.Value, "*?"): - q = phrase(k, n.Value) - default: - q = bleveQuery.NewQueryStringQuery(k + ":" + v) - } + q := bleveQuery.NewQueryStringQuery(k + ":" + v) if prev == nil { prev = q @@ -157,7 +120,7 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { End: bleveQuery.BleveQueryTime{}, InclusiveStart: nil, InclusiveEnd: nil, - FieldVal: getField(n.Key), + FieldVal: n.Key, } if n.Operator == nil { @@ -187,7 +150,14 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { next = q } case *ast.NumberNode: - q := numberRange(getField(n.Key), n.Operator, n.Value) + var q bleveQuery.Query + if field := getField(n.Key); slices.Contains([]string{"Size", "Type"}, field) { + q = numberRange(field, n.Operator, n.Value) + } else { + // same answer as the OpenSearch backend: unknown numeric keys + // match nothing instead of querying an arbitrary field + q = bleveQuery.NewMatchNoneQuery() + } if q == nil { continue } @@ -206,9 +176,7 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { next = q } case *ast.GroupNode: - if n.Key != "" { - n = normalizeGroupingProperty(n) - } + // keys resolved and grouping property propagated in normalize q, _, err := walk(0, n.Nodes) if err != nil { return nil, 0, err diff --git a/services/search/pkg/query/bleve/compiler_test.go b/services/search/pkg/query/bleve/compiler_test.go index 30c283f694..f3f1f3ab53 100644 --- a/services/search/pkg/query/bleve/compiler_test.go +++ b/services/search/pkg/query/bleve/compiler_test.go @@ -7,6 +7,7 @@ import ( "github.com/blevesearch/bleve/v2/search/query" "github.com/opencloud-eu/opencloud/pkg/ast" + searchquery "github.com/opencloud-eu/opencloud/services/search/pkg/query" tAssert "github.com/stretchr/testify/assert" ) @@ -590,7 +591,7 @@ func Test_compile(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := compile(tt.args) + got, err := compile(searchquery.Normalize(tt.args, searchquery.ResolveField)) if (err != nil) != tt.wantErr { t.Errorf("compile() error = %v, wantErr %v", err, tt.wantErr) From fb22dd81a4bc955bfc059198a698ac52bf39507b Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Wed, 29 Jul 2026 17:39:23 +0200 Subject: [PATCH 14/54] refactor(search): run the OpenSearch backend on the shared lowering pass KQLToOpenSearchBoolQuery runs query.Normalize, then only value lowercasing stays backend-specific; remapKey and unfoldValue are gone. --- .../opensearch/internal/convert/kql_expand.go | 209 +----- .../internal/convert/kql_expand_test.go | 664 +----------------- .../opensearch/internal/convert/kql_query.go | 8 +- services/search/pkg/query/normalize.go | 3 + services/search/pkg/query/normalize_test.go | 10 +- 5 files changed, 65 insertions(+), 829 deletions(-) diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand.go b/services/search/pkg/opensearch/internal/convert/kql_expand.go index 5c63a3072f..8d382c8745 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand.go @@ -1,210 +1,25 @@ package convert import ( - "fmt" - "reflect" - "strconv" "strings" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) -func ExpandKQL(nodes []ast.Node) ([]ast.Node, error) { - return kqlExpander{}.expand(nodes, "") -} - -type kqlExpander struct{} - -func (e kqlExpander) expand(nodes []ast.Node, defaultKey string) ([]ast.Node, error) { - for i, node := range nodes { - rnode := reflect.ValueOf(node) - - // we need to ensure that the node is a pointer to an ast.Node in every case - if rnode.Kind() != reflect.Ptr { - ptr := reflect.New(rnode.Type()) - ptr.Elem().Set(rnode) - rnode = ptr - cnode, ok := rnode.Interface().(ast.Node) - if !ok { - return nil, fmt.Errorf("expected node to be of type ast.Node, got %T", rnode.Interface()) - } - - node = cnode // Update the original node to the pointer - nodes[i] = node // Update the original slice with the pointer - } - - var unfoldedNodes []ast.Node - switch cnode := node.(type) { - case *ast.GroupNode: - if cnode.Key != "" { // group nodes should not get a default key - cnode.Key = e.remapKey(cnode.Key, defaultKey) - } - - groupNodes, err := e.expand(cnode.Nodes, cnode.Key) - if err != nil { - return nil, err - } - cnode.Nodes = groupNodes +// LowerValues folds restriction values for fields whose index analyzer +// lowercases (search.LowercaseValueFields, shared with bleve); case-preserved +// fields keep their casing. Runs after query.Normalize, so keys are resolved. +func LowerValues(nodes []ast.Node) []ast.Node { + for _, n := range nodes { + switch node := n.(type) { case *ast.StringNode: - cnode.Key = e.remapKey(cnode.Key, defaultKey) - cnode.Value = e.lowerValue(cnode.Key, cnode.Value) - unfoldedNodes = e.unfoldValue(cnode.Key, cnode.Value) - case *ast.DateTimeNode: - cnode.Key = e.remapKey(cnode.Key, defaultKey) - case *ast.BooleanNode: - cnode.Key = e.remapKey(cnode.Key, defaultKey) - case *ast.NumberNode: - cnode.Key = e.remapKey(cnode.Key, defaultKey) - } - - if unfoldedNodes != nil { - // Insert unfolded nodes at the current index - nodes = append(nodes[:i], append(unfoldedNodes, nodes[i+1:]...)...) - // Adjust index to account for new nodes - i += len(unfoldedNodes) - 1 + if _, ok := search.LowercaseValueFields()[node.Key]; ok { + node.Value = strings.ToLower(node.Value) + } + case *ast.GroupNode: + LowerValues(node.Nodes) } } - - return nodes, nil -} - -func (_ kqlExpander) remapKey(current string, defaultKey string) string { - if defaultKey == "" { - defaultKey = "Name" // Set a default key if none is provided - } - - key, ok := map[string]string{ - "": defaultKey, // Default case if current is empty - "title": "Title", - "rootid": "RootID", - "path": "Path", - "id": "ID", - "name": "Name", - "size": "Size", - "mtime": "Mtime", - "mediatype": "MimeType", - "type": "Type", - "tag": "Tags", - "tags": "Tags", - "content": "Content", - "hidden": "Hidden", - "favorite": "Favorites", - }[strings.ToLower(current)] - if !ok { - return current // Return the original key if not found - } - - return key -} - -func (_ kqlExpander) lowerValue(key, value string) string { - // only fold the value for fields whose index analyzer lowercases too; - // case-preserved (keyword) fields must keep their casing or they never match - // their stored token. Shared with bleve via search.LowercaseValueFields. - if _, ok := search.LowercaseValueFields()[key]; !ok { - return value - } - return strings.ToLower(value) -} - -func (_ kqlExpander) unfoldValue(key, value string) []ast.Node { - result, ok := map[string][]ast.Node{ - "Type:file": { - &ast.StringNode{Key: key, Value: strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10)}, - }, - "Type:folder": { - &ast.StringNode{Key: key, Value: strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10)}, - }, - "MimeType:file": { - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, - }, - "MimeType:folder": { - &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, - }, - "MimeType:document": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/msword"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.text"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "text/plain"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "text/markdown"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/rtf"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.apple.pages"}, - }}, - }, - "MimeType:spreadsheet": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/vnd.ms-excel"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadsheet"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "text/csv"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"}, - }}, - }, - "MimeType:presentation": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.presentation"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.ms-powerpoint"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.apple.keynote"}, - }}, - }, - "MimeType:pdf": { - &ast.StringNode{Key: key, Value: "application/pdf"}, - }, - "MimeType:image": { - &ast.StringNode{Key: key, Value: "image/*"}, - }, - "MimeType:video": { - &ast.StringNode{Key: key, Value: "video/*"}, - }, - "MimeType:audio": { - &ast.StringNode{Key: key, Value: "audio/*"}, - }, - "MimeType:archive": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/zip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/gzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-gzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-7z-compressed"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-rar-compressed"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-tar"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-bzip2"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-bzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-tgz"}, - }}, - }, - }[fmt.Sprintf("%s:%s", key, value)] - if !ok { - return nil - } - - return result + return nodes } diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go index 44534d7bcd..ef1927b48d 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go @@ -1,643 +1,57 @@ package convert_test import ( - "fmt" "testing" "github.com/stretchr/testify/require" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" - "github.com/opencloud-eu/opencloud/pkg/ast" - "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" ) -func TestExpandKQLAST(t *testing.T) { - t.Run("always converts a value node to a pointer node", func(t *testing.T) { - tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - { - Name: "ast.node.V -> ast.node.PTR", - Got: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - ast.OperatorNode{Value: "AND"}, - &ast.DateTimeNode{Key: "c"}, - &ast.OperatorNode{Value: "OR"}, - ast.DateTimeNode{Key: "d"}, - ast.OperatorNode{Value: "OR"}, - &ast.BooleanNode{Key: "f"}, - &ast.OperatorNode{Value: "NOT"}, - ast.BooleanNode{Key: "g"}, - ast.OperatorNode{Value: "NOT"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - }}, - }}, - }}, - ast.GroupNode{Key: "i", Nodes: []ast.Node{ - ast.StringNode{Key: "a"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - ast.OperatorNode{Value: "OR"}, - ast.GroupNode{Key: "h", Nodes: []ast.Node{ - ast.StringNode{Key: "a"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - ast.OperatorNode{Value: "OR"}, - ast.GroupNode{Key: "h", Nodes: []ast.Node{ - ast.StringNode{Key: "a"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "b"}, - }}, - }}, - }}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "AND"}, - &ast.DateTimeNode{Key: "c"}, - &ast.OperatorNode{Value: "OR"}, - &ast.DateTimeNode{Key: "d"}, - &ast.OperatorNode{Value: "OR"}, - &ast.BooleanNode{Key: "f"}, - &ast.OperatorNode{Value: "NOT"}, - &ast.BooleanNode{Key: "g"}, - &ast.OperatorNode{Value: "NOT"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - }}, - }}, - }}, - &ast.GroupNode{Key: "i", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: "h", Nodes: []ast.Node{ - &ast.StringNode{Key: "a"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "b"}, - }}, - }}, - }}, - }, +// LowerValues runs after the shared query.Normalize pass, so it operates on +// already-resolved pointer nodes. Field resolution, media-type expansion and +// group-key defaulting are tested once at the query.Normalize level (see +// pkg/query normalize_test), not here. Only lowercase-analyzed fields get their +// value folded; case-preserved fields keep their casing. +func TestLowerValues(t *testing.T) { + tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ + { + Name: "lowercase-analyzed field: value is folded, recursing into groups", + Got: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "StringNode"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "StringNode"}, + }}, }, - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - result, err := convert.ExpandKQL(test.Got) - require.NoError(t, err) - require.Equal(t, test.Want, result) - }) - } - }) - - t.Run("remaps some keys", func(t *testing.T) { - var tests []opensearchtest.TableTest[[]ast.Node, []ast.Node] - - for k, v := range map[string]string{ - "": "Name", // Default to "Name" if no key is provided - "rootid": "RootID", - "path": "Path", - "id": "ID", - "name": "Name", - "size": "Size", - "mtime": "Mtime", - "mediatype": "MimeType", - "type": "Type", - "tag": "Tags", - "tags": "Tags", - "content": "Content", - "hidden": "Hidden", - "favorite": "Favorites", - "any": "any", // Example of an unknown key that should remain unchanged - } { - tests = append(tests, opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - Name: fmt.Sprintf("%s -> %s", k, v), - Got: []ast.Node{ - &ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: k}, - ast.OperatorNode{Value: "AND"}, - &ast.DateTimeNode{Key: k}, - &ast.OperatorNode{Value: "OR"}, - ast.DateTimeNode{Key: k}, - ast.OperatorNode{Value: "OR"}, - &ast.BooleanNode{Key: k}, - &ast.OperatorNode{Value: "NOT"}, - ast.BooleanNode{Key: k}, - ast.OperatorNode{Value: "NOT"}, - &ast.GroupNode{Key: k, Nodes: []ast.Node{ - &ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: k, Nodes: []ast.Node{ - &ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: k, Nodes: []ast.Node{ - &ast.StringNode{Key: k}, - &ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: k}, - }}, - }}, - }}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "AND"}, - &ast.DateTimeNode{Key: v}, - &ast.OperatorNode{Value: "OR"}, - &ast.DateTimeNode{Key: v}, - &ast.OperatorNode{Value: "OR"}, - &ast.BooleanNode{Key: v}, - &ast.OperatorNode{Value: "NOT"}, - &ast.BooleanNode{Key: v}, - &ast.OperatorNode{Value: "NOT"}, - &ast.GroupNode{Key: func() string { - switch { - case k == "": - return k - default: - return v - } - }(), Nodes: []ast.Node{ - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: func() string { - switch { - case k == "": - return k - default: - return v - } - }(), Nodes: []ast.Node{ - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "OR"}, - &ast.GroupNode{Key: func() string { - switch { - case k == "": - return k - default: - return v - } - }(), Nodes: []ast.Node{ - &ast.StringNode{Key: v}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: v}, - }}, - }}, - }}, - }, - }) - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - result, err := convert.ExpandKQL(test.Got) - require.NoError(t, err) - require.Equal(t, test.Want, result) - }) - } - }) - - t.Run("lowercases some values", func(t *testing.T) { - tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - { - Name: "Name: StringNode -> stringnode", - Got: []ast.Node{ - ast.StringNode{Key: "Name", Value: "StringNode"}, - ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "Name", Value: "StringNode"}, - }}, - }, - Want: []ast.Node{ + Want: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "stringnode"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ &ast.StringNode{Key: "Name", Value: "stringnode"}, - &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "stringnode"}, - }}, - }, + }}, }, - { - Name: "aBc: StringNode -> StringNode", - Got: []ast.Node{ - ast.StringNode{Key: "aBc", Value: "StringNode"}, - }, - Want: []ast.Node{ + }, + { + Name: "case-preserved field: value keeps its casing", + Got: []ast.Node{ + &ast.StringNode{Key: "aBc", Value: "StringNode"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ &ast.StringNode{Key: "aBc", Value: "StringNode"}, - }, + }}, }, - { - Name: "Path: ./Documents -> ./Documents", - Got: []ast.Node{ - ast.StringNode{Key: "Path", Value: "./Documents"}, - ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "Path", Value: "./Documents"}, - }}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: "Path", Value: "./Documents"}, - &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "Path", Value: "./Documents"}, - }}, - }, + Want: []ast.Node{ + &ast.StringNode{Key: "aBc", Value: "StringNode"}, + &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ + &ast.StringNode{Key: "aBc", Value: "StringNode"}, + }}, }, - { - Name: "Hidden: TRUE -> true", - Got: []ast.Node{ - ast.StringNode{Key: "Hidden", Value: "TRUE"}, - ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - ast.StringNode{Key: "Hidden", Value: "TRUE"}, - }}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: "Hidden", Value: "true"}, - &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "Hidden", Value: "true"}, - }}, - }, - }, - { - Name: "ID: 1$1!AB23 -> 1$1!AB23", - Got: []ast.Node{ - ast.StringNode{Key: "ID", Value: "1$1!AB23"}, - ast.StringNode{Key: "RootID", Value: "1$1!AB23"}, - ast.StringNode{Key: "ParentID", Value: "1$1!AB23"}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: "ID", Value: "1$1!AB23"}, - &ast.StringNode{Key: "RootID", Value: "1$1!AB23"}, - &ast.StringNode{Key: "ParentID", Value: "1$1!AB23"}, - }, - }, - } + }, + } - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - result, err := convert.ExpandKQL(test.Got) - require.NoError(t, err) - require.Equal(t, test.Want, result) - }) - } - }) - - t.Run("unfolds some values", func(t *testing.T) { - tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - { - Name: "MimeType:unknown", - Got: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "unknown"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "unknown"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:file", - Got: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "file"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:folder", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "folder"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:document", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "document"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "application/msword"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.text"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "text/plain"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "text/markdown"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/rtf"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.pages"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:spreadsheet", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "spreadsheet"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-excel"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadsheet"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "text/csv"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.numbers"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:presentation", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "presentation"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.presentation"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-powerpoint"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.keynote"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:pdf", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "pdf"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "MimeType", Value: "application/pdf"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:image", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "image"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "MimeType", Value: "image/*"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:video", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "video"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "MimeType", Value: "video/*"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:audio", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "audio"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "MimeType", Value: "audio/*"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - { - Name: "MimeType:archive", - Got: []ast.Node{ - ast.BooleanNode{Key: "Deleted", Value: false}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Key: "MimeType", Value: "archive"}, - ast.OperatorNode{Value: "AND"}, - ast.StringNode{Value: "some-name"}, - }, - Want: []ast.Node{ - &ast.BooleanNode{Key: "Deleted", Value: false}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "application/zip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/gzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-gzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-7z-compressed"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-rar-compressed"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-tar"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-bzip2"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-bzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "MimeType", Value: "application/x-tgz"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Name", Value: `some-name`}, - }, - }, - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - if test.Skip { - t.Skip("Skipping test due to known issue") - } - result, err := convert.ExpandKQL(test.Got) - require.NoError(t, err) - require.EqualValues(t, test.Want, result) - }) - } - }) - - t.Run("different cases", func(t *testing.T) { - tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - { - Name: "use the group node key as default key", - Got: []ast.Node{ - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Value: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Key: "a", Nodes: []ast.Node{ - &ast.StringNode{Value: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Key: "mediatype", Nodes: []ast.Node{ - &ast.StringNode{Value: "file"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "mediatype", Value: "file"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - }, - Want: []ast.Node{ - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Key: "a", Nodes: []ast.Node{ - &ast.StringNode{Key: "a", Value: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Key: "MimeType", Nodes: []ast.Node{ - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "c", Value: "d"}, - }}, - }, - }, - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - if test.Skip { - t.Skip("Skipping test due to known issue") - } - result, err := convert.ExpandKQL(test.Got) - require.NoError(t, err) - require.EqualValues(t, test.Want, result) - }) - } - }) + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + require.Equal(t, test.Want, convert.LowerValues(test.Got)) + }) + } } diff --git a/services/search/pkg/opensearch/internal/convert/kql_query.go b/services/search/pkg/opensearch/internal/convert/kql_query.go index f824990e6d..18687dda74 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_query.go +++ b/services/search/pkg/opensearch/internal/convert/kql_query.go @@ -5,6 +5,7 @@ import ( "github.com/opencloud-eu/opencloud/pkg/kql" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" + "github.com/opencloud-eu/opencloud/services/search/pkg/query" ) var ( @@ -17,10 +18,9 @@ func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) { return nil, err } - kqlNodes, err := ExpandKQL(kqlAst.Nodes) - if err != nil { - return nil, fmt.Errorf("failed to expand KQL AST nodes: %w", err) - } + // shared lowering (field resolution + media-type), then value lowercasing. + kqlAst = query.Normalize(kqlAst, query.ResolveField) + kqlNodes := LowerValues(kqlAst.Nodes) builder, err := TranspileKQLToOpenSearch(kqlNodes) if err != nil { diff --git a/services/search/pkg/query/normalize.go b/services/search/pkg/query/normalize.go index 8c1269604b..78a4a55fe9 100644 --- a/services/search/pkg/query/normalize.go +++ b/services/search/pkg/query/normalize.go @@ -42,6 +42,9 @@ func normalizeNodes(nodes []ast.Node, resolve func(string) string, defaultKey st case *ast.BooleanNode: node.Key = resolveKey(node.Key) out = append(out, node) + case *ast.NumberNode: + node.Key = resolveKey(node.Key) + out = append(out, node) case *ast.GroupNode: groupKey := defaultKey if node.Key != "" { diff --git a/services/search/pkg/query/normalize_test.go b/services/search/pkg/query/normalize_test.go index 49cea2c153..66ba3243f0 100644 --- a/services/search/pkg/query/normalize_test.go +++ b/services/search/pkg/query/normalize_test.go @@ -19,9 +19,9 @@ func norm(nodes ...ast.Node) []ast.Node { } func TestResolveField(t *testing.T) { - require.Equal(t, "Name", query.ResolveField("")) // empty -> free-text default - require.Equal(t, "Name", query.ResolveField("NAME")) // case-insensitive - require.Equal(t, "Tags", query.ResolveField("tag")) // singular alias + require.Equal(t, "Name", query.ResolveField("")) // empty -> free-text default + require.Equal(t, "Name", query.ResolveField("NAME")) // case-insensitive + require.Equal(t, "Tags", query.ResolveField("tag")) // singular alias require.Equal(t, "MimeType", query.ResolveField("mimetype")) // real field, case-insensitive require.Equal(t, "photo.cameraMake", query.ResolveField("photo.CAMERAMAKE")) // facet, case-insensitive require.Equal(t, "unknown.field", query.ResolveField("unknown.field")) // unknown key: unchanged, becomes a dead query @@ -36,6 +36,8 @@ func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) { &ast.StringNode{Key: "photo.cameramake", Value: "canon"}, &ast.OperatorNode{Value: "AND"}, &ast.StringNode{Key: "mediatype", Value: "file"}, + &ast.OperatorNode{Value: "AND"}, + ast.NumberNode{Key: "size", Value: 100}, ) require.Equal(t, []ast.Node{ &ast.StringNode{Key: "Name", Value: "free"}, @@ -46,6 +48,8 @@ func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) { &ast.OperatorNode{Value: "AND"}, &ast.OperatorNode{Value: "NOT"}, &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + &ast.OperatorNode{Value: "AND"}, + &ast.NumberNode{Key: "Size", Value: 100}, }, got) } From ec58861e4ea2701209c96d0f50cd6a001c8f46ce Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 10 Aug 2026 14:50:39 +0200 Subject: [PATCH 15/54] feat(search): per-field case-insensitive search via _lowercase siblings Keyword and path fields always index their case-preserved base and, when CaseInsensitive is set, an additional _lowercase sibling used only for matching. The KQL lowering marks a restriction case-insensitive; each backend searches the sibling and lowercases the query value the same way the sibling is precomputed at index time (Go strings.ToLower on both sides, so non-ASCII stays consistent). Search always returns the case-preserved base, so the sibling never has to be read back. In bleve it is indexed but not stored, kept out of _all, and without doc values. In OpenSearch it deliberately stays in _source: excluding it would make every update-by-query script rebuild all siblings from the document via painless toLowerCase, which lowercases differently than Go and would drift from the query side. Keeping it in _source avoids that, and a lowercased copy of a name or path is negligible disk in a cluster. The OpenSearch move script keeps the base and its sibling in sync by swapping the moved prefix in Path_lowercase and setting Name_lowercase from Go-lowercased params, so case-insensitive search still finds a file after it moves (previously the sibling went stale). bleve re-indexes the whole document on move/delete/restore, so its siblings stay fresh for free. This also repairs OpenSearch path search (the query value was no longer folded to lowercase, so path: returned nothing) and makes bleve path queries match a folder and its descendants like OpenSearch's path_hierarchy. The Path base stays case-preserved so the move/delete descendant update (an exact TermQuery on Path) matches mixed-case folders. --- pkg/ast/ast.go | 3 + services/search/pkg/bleve/backend_test.go | 842 ++++++++++++++++++ services/search/pkg/bleve/index.go | 40 - services/search/pkg/mapping/bleve.go | 52 +- services/search/pkg/mapping/bleve_test.go | 26 +- services/search/pkg/mapping/casing.go | 57 ++ services/search/pkg/mapping/casing_test.go | 41 + services/search/pkg/mapping/opensearch.go | 42 +- .../search/pkg/mapping/opensearch_test.go | 17 +- services/search/pkg/mapping/opts.go | 13 +- services/search/pkg/mapping/serialize.go | 1 + services/search/pkg/mapping/validate_test.go | 4 +- services/search/pkg/opensearch/batch.go | 48 +- services/search/pkg/opensearch/index.go | 8 +- .../opensearch/internal/convert/kql_expand.go | 25 - .../opensearch/internal/convert/kql_query.go | 5 +- .../internal/convert/kql_transpile.go | 132 +-- .../internal/convert/kql_transpile_test.go | 50 +- services/search/pkg/query/bleve/compiler.go | 54 +- .../search/pkg/query/bleve/compiler_test.go | 155 ++-- services/search/pkg/query/normalize.go | 5 + services/search/pkg/query/normalize_test.go | 27 +- services/search/pkg/query/resolver.go | 67 +- services/search/pkg/search/search.go | 35 +- 24 files changed, 1309 insertions(+), 440 deletions(-) create mode 100644 services/search/pkg/bleve/backend_test.go create mode 100644 services/search/pkg/mapping/casing.go create mode 100644 services/search/pkg/mapping/casing_test.go delete mode 100644 services/search/pkg/opensearch/internal/convert/kql_expand.go diff --git a/pkg/ast/ast.go b/pkg/ast/ast.go index f1a7e3263e..f63b989ff0 100644 --- a/pkg/ast/ast.go +++ b/pkg/ast/ast.go @@ -44,6 +44,9 @@ type StringNode struct { Key string Value string Exact bool + // CaseInsensitive marks a case-insensitive restriction; set by the search + // lowering pass, a backend routes it to the field's lowercased form. + CaseInsensitive bool } // BooleanNode represents a bool value diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go new file mode 100644 index 0000000000..184cb9ff52 --- /dev/null +++ b/services/search/pkg/bleve/backend_test.go @@ -0,0 +1,842 @@ +package bleve_test + +import ( + "context" + "fmt" + + bleveSearch "github.com/blevesearch/bleve/v2" + sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + libregraph "github.com/opencloud-eu/libre-graph-api-go" + "github.com/opencloud-eu/reva/v2/pkg/storagespace" + + "github.com/opencloud-eu/opencloud/pkg/log" + searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" + "github.com/opencloud-eu/opencloud/services/search/pkg/content" + bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func hiddenByID(idx bleveSearch.Index, id string) bool { + GinkgoHelper() + + req := bleveSearch.NewSearchRequest(bleveSearch.NewDocIDQuery([]string{id})) + req.Fields = []string{"Hidden"} + + res, err := idx.Search(req) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits).To(HaveLen(1), "no record for %s", id) + + hidden, _ := res.Hits[0].Fields["Hidden"].(bool) + return hidden +} + +var _ = Describe("Bleve", func() { + var ( + eng *bleve.Backend + idx bleveSearch.Index + + doSearch = func(id string, query, path string) (*searchsvc.SearchIndexResponse, error) { + rID, err := storagespace.ParseID(id) + if err != nil { + return nil, err + } + + return eng.Search(context.Background(), &searchsvc.SearchIndexRequest{ + Query: query, + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: rID.StorageId, + SpaceId: rID.SpaceId, + OpaqueId: rID.OpaqueId, + }, + Path: path, + }, + }) + } + + assertDocCount = func(id string, query string, expectedCount int) []*searchmsg.Match { + res, err := doSearch(id, query, "") + + ExpectWithOffset(1, err).ToNot(HaveOccurred()) + ExpectWithOffset(1, len(res.Matches)).To(Equal(expectedCount), "query returned unexpected number of results: "+query) + return res.Matches + } + + rootResource search.Resource + parentResource search.Resource + childResource search.Resource + childResource2 search.Resource + ) + + BeforeEach(func() { + mapping, err := bleve.NewMapping() + Expect(err).ToNot(HaveOccurred()) + + idx, err = bleveSearch.NewMemOnly(mapping) + Expect(err).ToNot(HaveOccurred()) + + eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}) + Expect(err).ToNot(HaveOccurred()) + + rootResource = search.Resource{ + ID: "1$2!2", + RootID: "1$2!2", + Path: ".", + Document: content.Document{}, + } + + parentResource = search.Resource{ + ID: "1$2!3", + ParentID: rootResource.ID, + RootID: rootResource.ID, + Path: "./parent d!r", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER), + Document: content.Document{Name: "parent d!r"}, + } + + childResource = search.Resource{ + ID: "1$2!4", + ParentID: parentResource.ID, + RootID: rootResource.ID, + Path: "./parent d!r/child.pdf", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{Name: "child.pdf"}, + } + + childResource2 = search.Resource{ + ID: "1$2!5", + ParentID: parentResource.ID, + RootID: rootResource.ID, + Path: "./parent d!r/child2.pdf", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{Name: "child2.pdf"}, + } + }) + + Describe("PurgeSpace", func() { + It("takes every record of that space out of the index", func() { + otherSpace := search.Resource{ + ID: "1$9!9", + RootID: "1$9!9", + Path: ".", + Document: content.Document{Name: "other"}, + } + for _, resource := range []search.Resource{rootResource, parentResource, childResource, otherSpace} { + Expect(eng.Upsert(resource.ID, resource)).To(Succeed()) + } + + Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed()) + + count, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(1)), "only the records of that space are gone") + }) + + It("takes a space out that holds more records than one round", func() { + otherSpace := search.Resource{ + ID: "1$9!9", + RootID: "1$9!9", + Path: ".", + Document: content.Document{Name: "other"}, + } + Expect(eng.Upsert(otherSpace.ID, otherSpace)).To(Succeed()) + + for i := range 120 { + resource := search.Resource{ + ID: fmt.Sprintf("%s!file-%d", rootResource.RootID, i), + RootID: rootResource.RootID, + Path: fmt.Sprintf("./file-%d", i), + Document: content.Document{Name: fmt.Sprintf("file-%d", i)}, + } + Expect(eng.Upsert(resource.ID, resource)).To(Succeed()) + } + + Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed()) + + count, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(1)), "only the record of the other space is left") + }) + }) + + Describe("New", func() { + It("returns a new index instance", func() { + b := bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}) + Expect(b).ToNot(BeNil()) + }) + }) + + Describe("Search", func() { + Context("by other fields than filename", func() { + It("finds files by tags", func() { + parentResource.Document.Tags = []string{"foo", "bar"} + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Tags:foo", 1) + assertDocCount(rootResource.ID, "Tags:bar", 1) + assertDocCount(rootResource.ID, "Tags:foo Tags:bar", 1) + assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1) + assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1) + assertDocCount(rootResource.ID, "Tags:baz", 0) + }) + + It("finds files by size", func() { + parentResource.Document.Size = 12345 + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Size:12345", 1) + assertDocCount(rootResource.ID, "Size:>1000", 1) + assertDocCount(rootResource.ID, "Size:<100000", 1) + assertDocCount(rootResource.ID, "Size:12344", 0) + assertDocCount(rootResource.ID, "Size:<1000", 0) + assertDocCount(rootResource.ID, "Size:>100000", 0) + }) + + It("preserves value case for fields not explicitly marked lowercase", func() { + parentResource.Document.Audio = &libregraph.Audio{ + Artist: libregraph.PtrString("Some Artist"), + } + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `audio.artist:"Some Artist"`, 1) + assertDocCount(rootResource.ID, `audio.artist:"some artist"`, 0) + }) + }) + + Context("by filename", func() { + It("finds files with spaces in the filename", func() { + parentResource.Document.Name = "Foo oo.pdf" + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `name:"foo o*"`, 1) + }) + + It("finds files by digits in the filename", func() { + parentResource.Document.Name = "12345.pdf" + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Name:1234*", 1) + }) + + It("filters hidden files", func() { + childResource.Hidden = true + err := eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Hidden:T", 1) + assertDocCount(rootResource.ID, "Hidden:F", 0) + }) + + Context("with a file in the root of the space", func() { + It("scopes the search to the specified space", func() { + parentResource.Document.Name = "foo.pdf" + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Name:foo.pdf", 1) + assertDocCount("9$8!7", "Name:foo.pdf", 0) + }) + }) + + It("limits the search to the specified fields", func() { + parentResource.Document.Name = "bar.pdf" + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Name:bar.pdf", 1) + assertDocCount(rootResource.ID, "Unknown:field", 0) + }) + + It("returns the total number of hits", func() { + parentResource.Document.Name = "bar.pdf" + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + res, err := doSearch(rootResource.ID, "Name:bar*", "") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(1))) + }) + + It("returns all desired fields", func() { + parentResource.Document.Name = "bar.pdf" + parentResource.Type = 3 + parentResource.MimeType = "application/pdf" + + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + matches := assertDocCount(rootResource.ID, fmt.Sprintf("Name:%s", parentResource.Name), 1) + match := matches[0] + Expect(match.Entity.Ref.Path).To(Equal(parentResource.Path)) + Expect(match.Entity.Name).To(Equal(parentResource.Name)) + Expect(match.Entity.Size).To(Equal(parentResource.Size)) + Expect(match.Entity.Type).To(Equal(parentResource.Type)) + Expect(match.Entity.MimeType).To(Equal(parentResource.MimeType)) + Expect(match.Entity.Deleted).To(BeFalse()) + Expect(match.Score > 0).To(BeTrue()) + }) + + It("finds files by name, prefix or substring match", func() { + parentResource.Document.Name = "foo.pdf" + + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + queries := []string{"foo.pdf", "foo*", "*oo.p*"} + for _, query := range queries { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, query, 1) + } + }) + + It("does a case-insensitive search", func() { + parentResource.Document.Name = "foo.pdf" + + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Name:foo*", 1) + assertDocCount(rootResource.ID, "Name:Foo*", 1) + }) + + Context("and an additional file in a subdirectory", func() { + BeforeEach(func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + }) + + It("finds files living deeper in the tree by filename, prefix or substring match", func() { + queries := []string{"child.pdf", "child*", "*ld.*"} + for _, query := range queries { + assertDocCount(rootResource.ID, query, 1) + } + }) + }) + }) + + Context("by path", func() { + BeforeEach(func() { + for _, r := range []search.Resource{parentResource, childResource, childResource2} { + Expect(eng.Upsert(r.ID, r)).To(Succeed()) + } + }) + + It("matches a folder and its descendants", func() { + assertDocCount(rootResource.ID, `path:"./parent d!r"`, 3) + }) + + It("matches a descendant path only itself", func() { + assertDocCount(rootResource.ID, `path:"./parent d!r/child.pdf"`, 1) + }) + + It("matches case-insensitively", func() { + assertDocCount(rootResource.ID, `path:"./PARENT D!R"`, 3) + }) + }) + + Context("Highlights", func() { + + It("highlights only for content searches", func() { + parentResource.Document.Name = "baz.pdf" + parentResource.Document.Content = "foo bar baz" + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + res, err := doSearch(rootResource.ID, "Name:baz*", "") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(1))) + Expect(res.Matches[0].Entity.Highlights).To(Equal("")) + }) + + It("highlights search terms", func() { + parentResource.Document.Name = "baz.pdf" + parentResource.Document.Content = "foo bar baz" + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + res, err := doSearch(rootResource.ID, "Content:bar", "") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(1))) + Expect(res.Matches[0].Entity.Highlights).To(Equal("foo bar baz")) + }) + + }) + + Context("with a file in the root of the space and folder with a file. all of them have the same name", func() { + BeforeEach(func() { + parentResource := search.Resource{ + ID: "1$2!3", + ParentID: rootResource.ID, + RootID: rootResource.ID, + Path: "./doc", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER), + Document: content.Document{Name: "doc"}, + } + + childResource := search.Resource{ + ID: "1$2!4", + ParentID: parentResource.ID, + RootID: rootResource.ID, + Path: "./doc/doc.pdf", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{Name: "doc.pdf"}, + } + + childResource2 := search.Resource{ + ID: "1$2!7", + ParentID: parentResource.ID, + RootID: rootResource.ID, + Path: "./doc/file.pdf", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{Name: "file.pdf"}, + } + + rootChildResource := search.Resource{ + ID: "1$2!5", + ParentID: rootResource.ID, + RootID: rootResource.ID, + Path: "./doc.pdf", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{Name: "doc.pdf"}, + } + + rootChildResource2 := search.Resource{ + ID: "1$2!6", + ParentID: rootResource.ID, + RootID: rootResource.ID, + Path: "./file.pdf", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{Name: "file.pdf"}, + } + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(rootChildResource.ID, rootChildResource) + Expect(err).ToNot(HaveOccurred()) + err = eng.Upsert(rootChildResource2.ID, rootChildResource2) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + err = eng.Upsert(childResource2.ID, childResource2) + Expect(err).ToNot(HaveOccurred()) + }) + It("search *doc* in a root", func() { + res, err := doSearch(rootResource.ID, "Name:*doc*", "") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(3))) + }) + It("search *doc* in a subfolder", func() { + res, err := doSearch(rootResource.ID, "Name:*doc*", "./doc") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(2))) + }) + It("search *file* in a root", func() { + res, err := doSearch(rootResource.ID, "Name:*file*", "") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(2))) + }) + It("search *file* in a subfolder", func() { + res, err := doSearch(rootResource.ID, "Name:*file*", "./doc") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(1))) + }) + }) + + }) + + Describe("Upsert", func() { + It("adds a resourceInfo to the index", func() { + err := eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + count, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(1))) + + query := bleveSearch.NewMatchQuery("child.pdf") + res, err := idx.Search(bleveSearch.NewSearchRequest(query)) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits.Len()).To(Equal(1)) + }) + + It("updates an existing resource in the index", func() { + + err := eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + countA, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(countA).To(Equal(uint64(1))) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + countB, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(countB).To(Equal(uint64(1))) + }) + }) + + Describe("Delete", func() { + It("marks a resource as deleted", func() { + err := eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Name:*child*", 1) + + err = eng.Delete(childResource.ID) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, "Name:*child*", 0) + }) + + It("marks a child resources as deleted", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) + assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) + + err = eng.Delete(parentResource.ID) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) + assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0) + }) + }) + + Describe("Restore", func() { + It("also marks child resources as restored", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Delete(parentResource.ID) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 0) + assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 0) + + err = eng.Restore(parentResource.ID) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 1) + assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 1) + }) + }) + + Describe("Purge", func() { + It("removes a resource from the index", func() { + err := eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + assertDocCount(rootResource.ID, "Name:child.pdf", 1) + + err = eng.Purge(childResource.ID, false) + + Expect(err).ToNot(HaveOccurred()) + assertDocCount(rootResource.ID, "Name:child.pdf", 0) + }) + It("removes a resource and its children from the index", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) + assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) + + err = eng.Purge(parentResource.ID, false) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) + assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0) + }) + It("removes a resource and ignores its children from the index", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) + + err = eng.Delete(parentResource.ID) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) + + err = eng.Purge(parentResource.ID, true) + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) + assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) + }) + }) + + Describe("Move", func() { + It("renames the parent and its child resources", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + parentResource.Path = "newname" + err = eng.Move(parentResource.ID, parentResource.ParentID, "./my/newname") + Expect(err).ToNot(HaveOccurred()) + + assertDocCount(rootResource.ID, parentResource.Name, 0) + + matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1) + Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3")) + Expect(matches[0].Entity.Ref.Path).To(Equal("./my/newname/child.pdf")) + }) + + DescribeTable("keeps the flag in step with the path", + func(from, target string, hidden bool) { + parentResource.Path = from + parentResource.Hidden = search.IsHidden(from) + childResource.Path = from + "/child.pdf" + childResource.Hidden = parentResource.Hidden + + Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) + Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) + + Expect(eng.Move(parentResource.ID, parentResource.ParentID, target)).To(Succeed()) + + for _, id := range []string{parentResource.ID, childResource.ID} { + Expect(hiddenByID(idx, id)). + To(Equal(hidden), "%s after moving from %s to %s", id, from, target) + } + }, + Entry("into a dot folder", "./parent", "./.trash/parent", true), + Entry("into a plain folder", "./parent", "./archive/parent", false), + Entry("renamed with a leading dot", "./parent", "./.parent", true), + Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false), + Entry("renamed without the leading dot", "./.parent", "./parent", false), + Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true), + ) + + // the trash leaves the path alone, so the flag has to come through untouched + It("carries the flag through the trash and back", func() { + childResource.Path = "./.secret/file.txt" + childResource.Hidden = true + Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) + + Expect(eng.Delete(childResource.ID)).To(Succeed()) + Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after trashing") + + Expect(eng.Restore(childResource.ID)).To(Succeed()) + Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after restoring") + }) + + It("moves the parent and its child resources", func() { + err := eng.Upsert(parentResource.ID, parentResource) + Expect(err).ToNot(HaveOccurred()) + + err = eng.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + parentResource.Path = " " + parentResource.ParentID = "1$2!somewhereopaqueid" + + err = eng.Move(parentResource.ID, parentResource.ParentID, "./somewhere/else/newname") + Expect(err).ToNot(HaveOccurred()) + assertDocCount(rootResource.ID, `parent d!r`, 0) + + matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1) + Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3")) + Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname/child.pdf")) + + matches = assertDocCount(rootResource.ID, `newname`, 1) + Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("somewhereopaqueid")) + Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname")) + + }) + + It("keeps case-insensitive search working after a move", func() { + Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) + Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) + + Expect(eng.Move(parentResource.ID, parentResource.ParentID, "./my/NewName")).To(Succeed()) + + // the lowercased siblings are rebuilt at the new path, so a + // case-insensitive query finds the folder under its new name and path, + // including the descendant, and no longer under the old path. + assertDocCount(rootResource.ID, "name:NEWNAME", 1) + assertDocCount(rootResource.ID, `path:"./MY/NEWNAME"`, 2) + assertDocCount(rootResource.ID, `path:"./parent d!r"`, 0) + }) + }) + + Describe("StartBatch", func() { + It("starts a new batch", func() { + b, err := eng.NewBatch(100) + Expect(err).ToNot(HaveOccurred()) + + err = b.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + count, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(0))) + + err = b.Push() + Expect(err).ToNot(HaveOccurred()) + + count, err = idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(1))) + + query := bleveSearch.NewMatchQuery("child.pdf") + res, err := idx.Search(bleveSearch.NewSearchRequest(query)) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits.Len()).To(Equal(1)) + }) + + It("doesn't intertwine different batches", func() { + b, err := eng.NewBatch(100) + Expect(err).ToNot(HaveOccurred()) + + err = b.Upsert(childResource.ID, childResource) + Expect(err).ToNot(HaveOccurred()) + + count, err := idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(0))) + + b2, err := eng.NewBatch(100) + Expect(err).ToNot(HaveOccurred()) + + err = b2.Upsert(childResource2.ID, childResource2) + Expect(err).ToNot(HaveOccurred()) + + Expect(b.Push()).To(Succeed()) + count, err = idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(1))) + + Expect(b2.Push()).To(Succeed()) + count, err = idx.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(2))) + }) + }) + + Describe("File type specific metadata", func() { + + Context("with audio metadata", func() { + BeforeEach(func() { + resource := search.Resource{ + ID: "1$2!7", + ParentID: rootResource.ID, + RootID: rootResource.ID, + Path: "./some_song.mp3", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{ + Name: "some_song.mp3", + MimeType: "audio/mpeg", + Audio: &libregraph.Audio{ + Album: libregraph.PtrString("Some Album"), + AlbumArtist: libregraph.PtrString("Some AlbumArtist"), + Artist: libregraph.PtrString("Some Artist"), + Bitrate: libregraph.PtrInt64(192), + Composers: libregraph.PtrString("Some Composers"), + Copyright: libregraph.PtrString(""), + Disc: libregraph.PtrInt32(2), + DiscCount: libregraph.PtrInt32(5), + Duration: libregraph.PtrInt64(225000), + Genre: libregraph.PtrString("Some Genre"), + HasDrm: libregraph.PtrBool(false), + IsVariableBitrate: libregraph.PtrBool(true), + Title: libregraph.PtrString("Some Title"), + Track: libregraph.PtrInt32(34), + TrackCount: libregraph.PtrInt32(99), + Year: libregraph.PtrInt32(2004), + }, + }, + } + err := eng.Upsert(resource.ID, resource) + Expect(err).ToNot(HaveOccurred()) + }) + + It("returns audio metadata for search", func() { + matches := assertDocCount(rootResource.ID, `*song*`, 1) + audio := matches[0].Entity.Audio + + Expect(audio).ToNot(BeNil()) + + Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album"))) + Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist"))) + Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist"))) + Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192))) + Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers"))) + Expect(audio.Copyright).To(Equal(libregraph.PtrString(""))) + Expect(audio.Disc).To(Equal(libregraph.PtrInt32(2))) + Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5))) + Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225000))) + Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre"))) + Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false))) + Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true))) + Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title"))) + Expect(audio.Track).To(Equal(libregraph.PtrInt32(34))) + Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(99))) + Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) + }) + }) + + Context("with location metadata", func() { + BeforeEach(func() { + resource := search.Resource{ + ID: "1$2!7", + ParentID: rootResource.ID, + RootID: rootResource.ID, + Path: "./team.jpg", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{ + Name: "team.jpg", + MimeType: "image/jpeg", + Location: &libregraph.GeoCoordinates{ + Altitude: libregraph.PtrFloat64(1047.7), + Latitude: libregraph.PtrFloat64(49.48675890884328), + Longitude: libregraph.PtrFloat64(11.103870357204285), + }, + }, + } + err := eng.Upsert(resource.ID, resource) + Expect(err).ToNot(HaveOccurred()) + }) + + It("returns audio metadata for search", func() { + matches := assertDocCount(rootResource.ID, `*team*`, 1) + location := matches[0].Entity.Location + + Expect(location).ToNot(BeNil()) + + Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(1047.7))) + Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328))) + Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285))) + }) + }) + }) +}) diff --git a/services/search/pkg/bleve/index.go b/services/search/pkg/bleve/index.go index 264ad9893b..76a92625ce 100644 --- a/services/search/pkg/bleve/index.go +++ b/services/search/pkg/bleve/index.go @@ -10,10 +10,8 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" - regexpCharFilter "github.com/blevesearch/bleve/v2/analysis/char/regexp" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" "github.com/blevesearch/bleve/v2/analysis/token/porter" - "github.com/blevesearch/bleve/v2/analysis/tokenizer/single" "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" "github.com/blevesearch/bleve/v2/mapping" storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -60,44 +58,6 @@ func NewMapping() (mapping.IndexMapping, error) { indexMapping := bleve.NewIndexMapping() indexMapping.DefaultAnalyzer = keyword.Name indexMapping.DefaultMapping = docMapping - err = indexMapping.AddCustomCharFilter("dotToSpace", - map[string]any{ - "type": regexpCharFilter.Name, - "regexp": `\.`, - "replace": " ", - }, - ) - if err != nil { - return nil, err - } - - err = indexMapping.AddCustomAnalyzer("lowercaseWords", - map[string]any{ - "type": custom.Name, - "char_filters": []string{"dotToSpace"}, - "tokenizer": unicode.Name, - "token_filters": []string{ - lowercase.Name, - }, - }, - ) - if err != nil { - return nil, err - } - - err = indexMapping.AddCustomAnalyzer("lowercaseKeyword", - map[string]any{ - "type": custom.Name, - "tokenizer": single.Name, - "token_filters": []string{ - lowercase.Name, - }, - }, - ) - if err != nil { - return nil, err - } - err = indexMapping.AddCustomAnalyzer("fulltext", map[string]any{ "type": custom.Name, diff --git a/services/search/pkg/mapping/bleve.go b/services/search/pkg/mapping/bleve.go index f357e4f305..70e1f0949e 100644 --- a/services/search/pkg/mapping/bleve.go +++ b/services/search/pkg/mapping/bleve.go @@ -12,10 +12,8 @@ import ( // struct via reflection. Field names come from json tags; overrides are // keyed by those names (or dotted paths for nested fields). // -// The returned mapping references analyzer names (Analyzer field on the -// FieldOpts, plus "fulltext" / "path_hierarchy" for the corresponding Types); -// the caller is responsible for registering those analyzers on the enclosing -// IndexMapping. +// The returned mapping references the "fulltext" analyzer for Fulltext fields; +// the caller registers it on the enclosing IndexMapping. func BleveBuildMapping(t reflect.Type, overrides map[string]FieldOpts) (*bleveMapping.DocumentMapping, error) { return buildBleveDocMapping(t, overrides, "") } @@ -61,6 +59,16 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix return nil } + if fieldType == TypeKeyword || fieldType == TypePath { + // bleve has no path tokenizer, so a path is a plain keyword here. + base := bleveKeywordMapping(fieldType, opts) + doc.AddFieldMappingsAt(fi.Name, base) + if opts.caseInsensitive() { + doc.AddFieldMappingsAt(fi.Name+LowercaseSuffix, lowercaseSibling(base)) + } + return nil + } + fm, err := bleveFieldMapping(fieldType, opts) if err != nil { return fmt.Errorf("mapping: field %q: %w", key, err) @@ -71,26 +79,46 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix return doc, err } +// bleveKeywordMapping is a case-preserving keyword field; path fields stay out +// of _all by default. +func bleveKeywordMapping(fieldType string, opts FieldOpts) *bleveMapping.FieldMapping { + fm := bleve.NewKeywordFieldMapping() + switch { + case opts.IncludeInAll != nil: + fm.IncludeInAll = *opts.IncludeInAll + case fieldType == TypePath: + fm.IncludeInAll = false + } + return fm +} + +// lowercaseSibling derives the lowercased shadow of a keyword/path field from its +// base mapping: used only for case-insensitive matching, so indexed but never +// stored, kept out of _all, and without doc values, since the case-preserved base +// field is what we return and aggregate on. +func lowercaseSibling(base *bleveMapping.FieldMapping) *bleveMapping.FieldMapping { + fm := *base + fm.Store = false + fm.IncludeInAll = false + fm.DocValues = false + return &fm +} + func bleveFieldMapping(fieldType string, opts FieldOpts) (*bleveMapping.FieldMapping, error) { switch fieldType { case TypeWildcard: // bleve has no wildcard type; fall back to keyword-ish text. fieldType = TypeKeyword fallthrough - case TypeKeyword, TypeFulltext, TypePath: + case TypeKeyword, TypeFulltext: fm := bleve.NewTextFieldMapping() - switch { - case opts.Analyzer != "": - fm.Analyzer = opts.Analyzer - case fieldType == TypeFulltext: + if fieldType == TypeFulltext { fm.Analyzer = "fulltext" - case fieldType == TypePath: - fm.Analyzer = "path_hierarchy" } switch { case opts.IncludeInAll != nil: fm.IncludeInAll = *opts.IncludeInAll - case fieldType == TypeFulltext, fieldType == TypePath: + case fieldType == TypeFulltext: fm.IncludeInAll = false } return fm, nil diff --git a/services/search/pkg/mapping/bleve_test.go b/services/search/pkg/mapping/bleve_test.go index a9cd81954f..4fcb09f7ab 100644 --- a/services/search/pkg/mapping/bleve_test.go +++ b/services/search/pkg/mapping/bleve_test.go @@ -65,21 +65,31 @@ var _ = Describe("BleveBuildMapping", func() { }) It("applies field overrides", func() { - includeInAllFalse := false + True, False := true, false dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{ - "Name": {Analyzer: "lowercaseKeyword"}, + "Name": {CaseInsensitive: &True}, "Content": {Type: TypeFulltext}, - "Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &includeInAllFalse}, + "Tags": {CaseInsensitive: &True, IncludeInAll: &False}, }) Expect(err).ToNot(HaveOccurred()) - nameField := dm.Properties["Name"].Fields[0] - Expect(nameField.Analyzer).To(Equal("lowercaseKeyword"), "Name analyzer") - Expect(nameField.IncludeInAll).To(BeTrue(), "Name IncludeInAll should stay default-true when not overridden") + // Name: case-preserved base keyword + lowercased sibling. + Expect(dm.Properties["Name"]).ToNot(BeNil(), "Name base field") + Expect(dm.Properties["Name"].Fields[0].Analyzer).To(Equal("keyword"), "Name base is a keyword") + Expect(dm.Properties["Name"].Fields[0].Store).To(BeTrue(), "Name base is stored (returned)") + Expect(dm.Properties["Name_lowercase"]).ToNot(BeNil(), "Name_lowercase sibling") + // The sibling is a search-only shadow: indexed but never stored, kept out + // of _all, no doc values (the base is what we return). + sibling := dm.Properties["Name_lowercase"].Fields[0] + Expect(sibling.Index).To(BeTrue(), "Name_lowercase is indexed") + Expect(sibling.Store).To(BeFalse(), "Name_lowercase is not stored") + Expect(sibling.IncludeInAll).To(BeFalse(), "Name_lowercase is out of _all") + Expect(sibling.DocValues).To(BeFalse(), "Name_lowercase has no doc values") contentField := dm.Properties["Content"].Fields[0] Expect(contentField.Analyzer).To(Equal("fulltext"), "Content analyzer") Expect(contentField.IncludeInAll).To(BeFalse(), "Content IncludeInAll should default to false for fulltext type") - tagsField := dm.Properties["Tags"].Fields[0] - Expect(tagsField.IncludeInAll).To(BeFalse(), "Tags IncludeInAll should honor the explicit false override") + // Tags: base + lowercased sibling, both honoring the IncludeInAll override. + Expect(dm.Properties["Tags"].Fields[0].IncludeInAll).To(BeFalse(), "Tags base IncludeInAll honored") + Expect(dm.Properties["Tags_lowercase"].Fields[0].IncludeInAll).To(BeFalse(), "Tags sibling IncludeInAll honored") }) It("builds an object sub-document plus a geopoint sibling", func() { diff --git a/services/search/pkg/mapping/casing.go b/services/search/pkg/mapping/casing.go new file mode 100644 index 0000000000..3ecceda993 --- /dev/null +++ b/services/search/pkg/mapping/casing.go @@ -0,0 +1,57 @@ +package mapping + +import "strings" + +func addLowercaseSiblings(m map[string]any, overrides map[string]FieldOpts) { + for key, opts := range overrides { + if !opts.caseInsensitive() || !isCasedType(opts) { + continue + } + parent, leaf, ok := resolveLeaf(m, key) + if !ok { + continue + } + addLowercaseSibling(parent, leaf) + } +} + +func isCasedType(opts FieldOpts) bool { + return opts.Type == "" || opts.Type == TypeKeyword || opts.Type == TypePath +} + +func resolveLeaf(m map[string]any, dottedPath string) (map[string]any, string, bool) { + parts := strings.Split(dottedPath, ".") + parent := m + for _, p := range parts[:len(parts)-1] { + next, ok := parent[p].(map[string]any) + if !ok { + return nil, "", false + } + parent = next + } + return parent, parts[len(parts)-1], true +} + +// addLowercaseSibling writes a _lowercase sibling; no-op for non-strings. +func addLowercaseSibling(parent map[string]any, leaf string) { + switch v := parent[leaf].(type) { + case string: + parent[leaf+LowercaseSuffix] = strings.ToLower(v) + case []any: + out := make([]any, 0, len(v)) + for _, item := range v { + if s, ok := item.(string); ok { + out = append(out, strings.ToLower(s)) + } + } + if len(out) > 0 { + parent[leaf+LowercaseSuffix] = out + } + case []string: + out := make([]string, len(v)) + for i, s := range v { + out[i] = strings.ToLower(s) + } + parent[leaf+LowercaseSuffix] = out + } +} diff --git a/services/search/pkg/mapping/casing_test.go b/services/search/pkg/mapping/casing_test.go new file mode 100644 index 0000000000..d846158fb8 --- /dev/null +++ b/services/search/pkg/mapping/casing_test.go @@ -0,0 +1,41 @@ +package mapping + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("PrepareForIndex casing", func() { + It("adds lowercased siblings for CaseInsensitive keyword and path fields", func() { + True := true + type doc struct { + Name string `json:"Name"` + Path string `json:"Path"` + Tags []string `json:"Tags"` + } + d := doc{Name: "Report FINAL", Path: "/Foo/Bar", Tags: []string{"Work", "Urgent"}} + m, err := PrepareForIndex(d, map[string]FieldOpts{ + "Name": {CaseInsensitive: &True}, + "Path": {Type: TypePath, CaseInsensitive: &True}, + "Tags": {CaseInsensitive: &True}, + }) + Expect(err).ToNot(HaveOccurred()) + + // Originals stay for the case-preserved base fields and the cascade. + Expect(m["Name"]).To(Equal("Report FINAL")) + Expect(m["Path"]).To(Equal("/Foo/Bar")) + + Expect(m["Name_lowercase"]).To(Equal("report final")) + Expect(m["Path_lowercase"]).To(Equal("/foo/bar")) + Expect(m["Tags_lowercase"]).To(Equal([]any{"work", "urgent"})) + }) + + It("writes no sibling without CaseInsensitive", func() { + type doc struct { + ID string `json:"ID"` + } + m, err := PrepareForIndex(doc{ID: "ABC"}, nil) + Expect(err).ToNot(HaveOccurred()) + Expect(m).ToNot(HaveKey("ID" + LowercaseSuffix)) + }) +}) diff --git a/services/search/pkg/mapping/opensearch.go b/services/search/pkg/mapping/opensearch.go index abeeb96d65..6b6b243ec7 100644 --- a/services/search/pkg/mapping/opensearch.go +++ b/services/search/pkg/mapping/opensearch.go @@ -56,7 +56,20 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p return nil } - fm, err := openSearchFieldMapping(fieldType, opts, fi.GoField.Type) + if fieldType == TypeKeyword || fieldType == TypePath { + // path_hierarchy is case-preserving here; casing lives in the value. + m := map[string]any{"type": "keyword"} + if fieldType == TypePath { + m = map[string]any{"type": "text", "analyzer": "path_hierarchy"} + } + props[fi.Name] = m + if opts.caseInsensitive() { + props[fi.Name+LowercaseSuffix] = m + } + return nil + } + + fm, err := openSearchFieldMapping(fieldType, fi.GoField.Type) if err != nil { return fmt.Errorf("mapping: field %q: %w", key, err) } @@ -66,32 +79,15 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p return props, err } -func openSearchFieldMapping(fieldType string, opts FieldOpts, goType reflect.Type) (map[string]any, error) { +// openSearchFieldMapping handles the non-keyword/path types; keyword and path +// are emitted (with their cased forms) by buildOpenSearchProperties directly. +func openSearchFieldMapping(fieldType string, goType reflect.Type) (map[string]any, error) { switch fieldType { - case TypeKeyword: - m := map[string]any{"type": "keyword"} - if opts.Analyzer != "" { - m["type"] = "text" - m["analyzer"] = opts.Analyzer - } - return m, nil case TypeFulltext: - m := map[string]any{ + return map[string]any{ "type": "text", "term_vector": "with_positions_offsets", - } - if opts.Analyzer != "" { - m["analyzer"] = opts.Analyzer - } - return m, nil - case TypePath: - m := map[string]any{"type": "text"} - if opts.Analyzer != "" { - m["analyzer"] = opts.Analyzer - } else { - m["analyzer"] = "path_hierarchy" - } - return m, nil + }, nil case TypeWildcard: // OpenSearch stores wildcard fields with doc_values=false by // default, so emit it explicitly to keep local and remote diff --git a/services/search/pkg/mapping/opensearch_test.go b/services/search/pkg/mapping/opensearch_test.go index 3ff80f20e3..b64520cb73 100644 --- a/services/search/pkg/mapping/opensearch_test.go +++ b/services/search/pkg/mapping/opensearch_test.go @@ -78,6 +78,7 @@ var _ = Describe("OpenSearchBuildMapping", func() { }) It("applies field overrides", func() { + True := true type doc struct { Name string `json:"Name"` Content string `json:"Content"` @@ -85,23 +86,23 @@ var _ = Describe("OpenSearchBuildMapping", func() { MimeType string `json:"MimeType"` } props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{ - "Name": {Analyzer: "lowercaseKeyword"}, + "Name": {CaseInsensitive: &True}, "Content": {Type: TypeFulltext}, - "Path": {Type: TypePath}, + "Path": {Type: TypePath, CaseInsensitive: &True}, "MimeType": {Type: TypeWildcard}, }) Expect(err).ToNot(HaveOccurred()) - name := props["Name"].(map[string]any) - Expect(name["type"]).To(Equal("text"), "Name: %#v", name) - Expect(name["analyzer"]).To(Equal("lowercaseKeyword"), "Name: %#v", name) + // Name: case-preserved keyword base + lowercased keyword sibling. + Expect(props["Name"]).To(Equal(map[string]any{"type": "keyword"})) + Expect(props["Name_lowercase"]).To(Equal(map[string]any{"type": "keyword"})) content := props["Content"].(map[string]any) Expect(content["type"]).To(Equal("text"), "Content: %#v", content) Expect(content["term_vector"]).To(Equal("with_positions_offsets"), "Content: %#v", content) _, ok := content["analyzer"] Expect(ok).To(BeFalse(), "Content should leave analyzer unset (use OpenSearch default)") - path := props["Path"].(map[string]any) - Expect(path["type"]).To(Equal("text"), "Path: %#v", path) - Expect(path["analyzer"]).To(Equal("path_hierarchy"), "Path: %#v", path) + // Path: path_hierarchy base + lowercased sibling, both case-preserving. + Expect(props["Path"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"})) + Expect(props["Path_lowercase"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"})) mime := props["MimeType"].(map[string]any) Expect(mime["type"]).To(Equal("wildcard"), "MimeType: %#v", mime) }) diff --git a/services/search/pkg/mapping/opts.go b/services/search/pkg/mapping/opts.go index 3daa258a62..9ae7d295a0 100644 --- a/services/search/pkg/mapping/opts.go +++ b/services/search/pkg/mapping/opts.go @@ -17,6 +17,9 @@ const ( TypeGeopoint = "geopoint" ) +// LowercaseSuffix names the lowercased sibling of a keyword/path field. +const LowercaseSuffix = "_lowercase" + // FieldOpts overrides the default type inference for a struct field. Keys in // the override map are json-tag names (e.g. "Name", "location", "audio.artist"), // not Go field names. @@ -24,12 +27,14 @@ type FieldOpts struct { // Type is one of the Type* constants. Empty means "infer from Go type". Type string - // Analyzer is the name of a custom analyzer registered on the bleve - // IndexMapping (e.g. "lowercaseKeyword", "fulltext"). For OpenSearch it - // becomes the analyzer attribute on the field. - Analyzer string + // CaseInsensitive additionally indexes a lowercased _lowercase sibling + // for case-insensitive search; the case-preserved base is always indexed. + // Nil/false means off. Keyword/path only. + CaseInsensitive *bool // IncludeInAll controls bleve's _all field inclusion. Nil means "use the // bleve default for this field type". Has no effect on OpenSearch. IncludeInAll *bool } + +func (o FieldOpts) caseInsensitive() bool { return o.CaseInsensitive != nil && *o.CaseInsensitive } diff --git a/services/search/pkg/mapping/serialize.go b/services/search/pkg/mapping/serialize.go index 027d10da71..ec5420784a 100644 --- a/services/search/pkg/mapping/serialize.go +++ b/services/search/pkg/mapping/serialize.go @@ -19,5 +19,6 @@ func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, err return out, nil } addGeopointSiblings(out, overrides) + addLowercaseSiblings(out, overrides) return out, nil } diff --git a/services/search/pkg/mapping/validate_test.go b/services/search/pkg/mapping/validate_test.go index 278bb3820f..3b15f39caf 100644 --- a/services/search/pkg/mapping/validate_test.go +++ b/services/search/pkg/mapping/validate_test.go @@ -23,9 +23,9 @@ type sample struct { var _ = Describe("Validate", func() { It("accepts known override keys", func() { err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{ - "Name": {Analyzer: "lowercaseKeyword"}, + "Name": {}, "audio": {Type: TypeObject}, - "audio.artist": {Analyzer: "lowercaseKeyword"}, + "audio.artist": {}, "location": {Type: TypeGeopoint}, }) Expect(err).ToNot(HaveOccurred()) diff --git a/services/search/pkg/opensearch/batch.go b/services/search/pkg/opensearch/batch.go index cd1a89e386..1dc0bdf8ca 100644 --- a/services/search/pkg/opensearch/batch.go +++ b/services/search/pkg/opensearch/batch.go @@ -64,27 +64,45 @@ func (b *Batch) Upsert(id string, r search.Resource) error { }) } -func (b *Batch) Move(id string, parentID string, targetPath string) error { +func (b *Batch) Move(id string, parentID string, location string) error { return b.withSizeLimit(func() error { op := func() error { return updateSelfAndDescendants(context.Background(), b.client, b.index, id, func(rootResource search.Resource) *osu.BodyParamScript { + newPath := utils.MakeRelativePath(location) + newName := path.Base(newPath) return &osu.BodyParamScript{ - Source: ` - if (ctx._source.ID == params.id ) { ctx._source.Name = params.newName; ctx._source.ParentID = params.parentID; } - ctx._source.Path = ctx._source.Path.replace(params.oldPath, params.newPath); - boolean hidden = false; - for (String name : ctx._source.Path.splitOnToken('/')) { - if (!name.equals('.') && !name.equals('..') && name.startsWith('.')) { hidden = true; break; } - } - ctx._source.Hidden = hidden; - `, + // Keep the case-preserved base fields and their lowercased + // search siblings in sync: swap the moved prefix in both. The + // lowercased new values come from Go's strings.ToLower via + // params, so the sibling stays byte-identical to what + // PrepareForIndex writes on upsert (painless toLowerCase would + // lowercase differently than Go). + Source: fmt.Sprintf(` + if (ctx._source.ID == params.id) { + ctx._source.Name = params.newName; + ctx._source.ParentID = params.parentID; + if (ctx._source.Name%[1]s != null) { ctx._source.Name%[1]s = params.newNameLower; } + } + ctx._source.Path = ctx._source.Path.replace(params.oldPath, params.newPath); + if (ctx._source.Path%[1]s != null) { + ctx._source.Path%[1]s = ctx._source.Path%[1]s.replace(params.oldPathLower, params.newPathLower); + } + boolean hidden = false; + for (String name : ctx._source.Path.splitOnToken('/')) { + if (!name.equals('.') && !name.equals('..') && name.startsWith('.')) { hidden = true; break; } + } + ctx._source.Hidden = hidden; + `, mapping.LowercaseSuffix), Lang: "painless", Params: map[string]any{ - "id": id, - "parentID": parentID, - "oldPath": rootResource.Path, - "newPath": utils.MakeRelativePath(targetPath), - "newName": path.Base(utils.MakeRelativePath(targetPath)), + "id": id, + "parentID": parentID, + "oldPath": rootResource.Path, + "newPath": newPath, + "newName": newName, + "oldPathLower": strings.ToLower(rootResource.Path), + "newPathLower": strings.ToLower(newPath), + "newNameLower": strings.ToLower(newName), }, } }) diff --git a/services/search/pkg/opensearch/index.go b/services/search/pkg/opensearch/index.go index 120ed6cea6..93d454e53b 100644 --- a/services/search/pkg/opensearch/index.go +++ b/services/search/pkg/opensearch/index.go @@ -61,7 +61,6 @@ func buildResourceMapping() ([]byte, error) { resourceType := reflect.TypeFor[search.Resource]() overrides := maps.Clone(search.Resource{}.SearchFieldOverrides()) overrides["MimeType"] = searchmapping.FieldOpts{Type: searchmapping.TypeWildcard} - overrides["Path"] = searchmapping.FieldOpts{Type: searchmapping.TypePath} if err := searchmapping.Validate(resourceType, overrides); err != nil { return nil, err } @@ -75,16 +74,11 @@ func buildResourceMapping() ([]byte, error) { "number_of_shards": "1", "number_of_replicas": "1", "analysis": map[string]any{ + // path_hierarchy is case-preserving; casing lives in the value. "analyzer": map[string]any{ "path_hierarchy": map[string]any{ "type": "custom", "tokenizer": "path_hierarchy", - "filter": []string{"lowercase"}, - }, - "lowercaseKeyword": map[string]any{ - "type": "custom", - "tokenizer": "keyword", - "filter": []string{"lowercase"}, }, }, "tokenizer": map[string]any{ diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand.go b/services/search/pkg/opensearch/internal/convert/kql_expand.go deleted file mode 100644 index 8d382c8745..0000000000 --- a/services/search/pkg/opensearch/internal/convert/kql_expand.go +++ /dev/null @@ -1,25 +0,0 @@ -package convert - -import ( - "strings" - - "github.com/opencloud-eu/opencloud/pkg/ast" - "github.com/opencloud-eu/opencloud/services/search/pkg/search" -) - -// LowerValues folds restriction values for fields whose index analyzer -// lowercases (search.LowercaseValueFields, shared with bleve); case-preserved -// fields keep their casing. Runs after query.Normalize, so keys are resolved. -func LowerValues(nodes []ast.Node) []ast.Node { - for _, n := range nodes { - switch node := n.(type) { - case *ast.StringNode: - if _, ok := search.LowercaseValueFields()[node.Key]; ok { - node.Value = strings.ToLower(node.Value) - } - case *ast.GroupNode: - LowerValues(node.Nodes) - } - } - return nodes -} diff --git a/services/search/pkg/opensearch/internal/convert/kql_query.go b/services/search/pkg/opensearch/internal/convert/kql_query.go index 18687dda74..711a67b4b4 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_query.go +++ b/services/search/pkg/opensearch/internal/convert/kql_query.go @@ -18,11 +18,10 @@ func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) { return nil, err } - // shared lowering (field resolution + media-type), then value lowercasing. + // shared lowering: field resolution, media-type expansion, value lowercasing. kqlAst = query.Normalize(kqlAst, query.ResolveField) - kqlNodes := LowerValues(kqlAst.Nodes) - builder, err := TranspileKQLToOpenSearch(kqlNodes) + builder, err := TranspileKQLToOpenSearch(kqlAst.Nodes) if err != nil { return nil, fmt.Errorf("failed to compile query: %w", err) } diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index e3fb25a807..8a060a69d8 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -4,12 +4,12 @@ import ( "errors" "fmt" "slices" - "strconv" "strings" "time" "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/pkg/kql" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" ) @@ -99,7 +99,28 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { case *ast.BooleanNode: return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil case *ast.StringNode: - return stringNodeQuery(node), nil + field, value := node.Key, node.Value + if node.CaseInsensitive { + field += mapping.LowercaseSuffix + value = strings.ToLower(value) + } + + isWildcard := strings.Contains(value, "*") + if isWildcard { + return osu.NewWildcardQuery(field).Value(value), nil + } + + totalTerms := strings.Split(value, " ") + isSingleTerm := len(totalTerms) == 1 + isMultiTerm := len(totalTerms) >= 1 + switch { + case isSingleTerm: + return osu.NewTermQuery[string](field).Value(value), nil + case isMultiTerm: + return osu.NewMatchPhraseQuery(field).Query(value), nil + } + + return nil, fmt.Errorf("unsupported string node value: %s", value) case *ast.DateTimeNode: return dateTimeNodeQuery(node) case *ast.NumberNode: @@ -116,67 +137,26 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { return nil, fmt.Errorf("%w: %T", ErrUnsupportedNodeType, node) } -// stringNodeQuery picks the query a string node turns into. -func stringNodeQuery(node *ast.StringNode) osu.Builder { - isWildcard := strings.ContainsAny(node.Value, "*?") - - switch { - // Name: "*oo-bar", "*oo ba*", "*OO*" - // Title: "*rterly rep*" - // Tags: "*spaced tag*" - case isWildcard && slices.Contains([]string{"Name", "Title"}, node.Key): - patterns := []osu.Builder{wildcardOn(node.Key+".wildcard", node.Value)} - if !strings.HasSuffix(node.Value, "*") { - patterns = append(patterns, wildcardOn(node.Key+".wildcard", node.Value+".*")) - } - - return osu.NewBoolQuery(). - Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). - Should(patterns...) - // Tags: "*foo*", "*paced ta*" - case isWildcard && node.Key == "Tags": - return wildcardOn(node.Key+".wildcard", node.Value) - // Path: "./foo*", MimeType: "*plain" - case isWildcard: - return osu.NewWildcardQuery(node.Key).Value(node.Value) - // Name: =new, Title: ="quarterly report" - case node.Exact && slices.Contains([]string{"Name", "Title"}, node.Key): - return osu.NewTermQuery[string](node.Key + ".wildcard"). - Value(node.Value). - Params(&osu.TermQueryParams{CaseInsensitive: true}) - // Tags: "foo-bar", "spaced tag", "FOO-BAR" - case node.Key == "Tags": - return osu.NewTermQuery[string](node.Key + ".wildcard"). - Value(node.Value). - Params(&osu.TermQueryParams{CaseInsensitive: true}) - // Name: "foo-bar", "foo bar" - // Title: "quarterly report" - // Content: "foo bar" - case slices.Contains([]string{"Name", "Title", "Content"}, node.Key): - return osu.NewMatchPhraseQuery(node.Key).Query(node.Value) - // Size: "42", Type: "1" - case slices.Contains([]string{"Size", "Type"}, node.Key): - number, err := strconv.ParseInt(node.Value, 10, 64) - if err != nil { - return osu.NewMatchNoneQuery() - } - - return osu.NewTermQuery[int64](node.Key).Value(number) - // Path: "./foo bar/", the hierarchy tokens carry no trailing slash - case node.Key == "Path": - return osu.NewTermQuery[string](node.Key).Value(strings.TrimSuffix(node.Value, "/")) - // Hidden: "TRUE" arrives lowered, anything that is no bool matches nothing - case node.Key == "Hidden": - value, err := strconv.ParseBool(node.Value) - if err != nil { - return osu.NewMatchNoneQuery() - } - - return osu.NewTermQuery[bool](node.Key).Value(value) - // MimeType: "text/plain" - default: - return osu.NewTermQuery[string](node.Key).Value(node.Value) +// dateTimeNodeQuery turns a date time node into a range query. +func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) { + if node.Operator == nil { + return nil, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType) } + + query := osu.NewRangeQuery[time.Time](node.Key) + + switch node.Operator.Value { + case ">": + return query.Gt(node.Value), nil + case ">=": + return query.Gte(node.Value), nil + case "<": + return query.Lt(node.Value), nil + case "<=": + return query.Lte(node.Value), nil + } + + return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType) } func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) { @@ -203,31 +183,3 @@ func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) { return nil, fmt.Errorf("unsupported operator %s for number node: %w", node.Operator.Value, ErrUnsupportedNodeType) } - -func wildcardOn(field, value string) osu.Builder { - return osu.NewWildcardQuery(field). - Value(value). - Params(&osu.WildcardQueryParams{CaseInsensitive: true}) -} - -// dateTimeNodeQuery turns a date time node into a range query. -func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) { - if node.Operator == nil { - return nil, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType) - } - - query := osu.NewRangeQuery[time.Time](node.Key) - - switch node.Operator.Value { - case ">": - return query.Gt(node.Value), nil - case ">=": - return query.Gte(node.Value), nil - case "<": - return query.Lt(node.Value), nil - case "<=": - return query.Lte(node.Value), nil - } - - return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType) -} diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go index 9e03fd23df..5f29306848 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go @@ -16,13 +16,31 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{ // kql to os dsl - type tests { - Name: "match phrase query - string node on an analyzed field", + Name: "term query - string node", Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "Name", Value: "openCloud"}, }, }, - Want: osu.NewMatchPhraseQuery("Name").Query("openCloud"), + Want: osu.NewTermQuery[string]("Name").Value("openCloud"), + }, + { + Name: "case-insensitive term routes to the lowercased sibling", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "openCloud", CaseInsensitive: true}, + }, + }, + Want: osu.NewTermQuery[string]("Name_lowercase").Value("opencloud"), + }, + { + Name: "case-insensitive wildcard routes to the lowercased sibling", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "Open*", CaseInsensitive: true}, + }, + }, + Want: osu.NewWildcardQuery("Name_lowercase").Value("open*"), }, { Name: "term query - boolean node - true", @@ -58,16 +76,10 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { &ast.StringNode{Key: "Name", Value: "open*"}, }, }, - Want: osu.NewBoolQuery(). - Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). - Should( - osu.NewWildcardQuery("Name.wildcard"). - Value("open*"). - Params(&osu.WildcardQueryParams{CaseInsensitive: true}), - ), + Want: osu.NewWildcardQuery("Name").Value("open*"), }, { - Name: "wildcard query - string node without an unanalyzed sub field", + Name: "wildcard query - fulltext field", Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "Content", Value: "open*"}, @@ -142,8 +154,8 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, }, Want: osu.NewBoolQuery().Must( - osu.NewMatchPhraseQuery("Name").Query("a"), - osu.NewMatchPhraseQuery("Name").Query("b"), + osu.NewTermQuery[string]("Name").Value("a"), + osu.NewTermQuery[string]("Name").Value("b"), ), }, { @@ -155,7 +167,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }}, }, }, - Want: osu.NewMatchPhraseQuery("Name").Query("any"), + Want: osu.NewTermQuery[string]("Name").Value("any"), }, { Name: "range query >", @@ -217,7 +229,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { &ast.StringNode{Key: "Name", Value: "openCloud"}, }, }, - Want: osu.NewMatchPhraseQuery("Name").Query("openCloud"), + Want: osu.NewTermQuery[string]("Name").Value("openCloud"), }, { Name: "[* *]", @@ -229,7 +241,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewBoolQuery(). Must( - osu.NewMatchPhraseQuery("Name").Query("openCloud"), + osu.NewTermQuery[string]("Name").Value("openCloud"), osu.NewTermQuery[string]("age").Value("32"), ), }, @@ -244,7 +256,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewBoolQuery(). Must( - osu.NewMatchPhraseQuery("Name").Query("openCloud"), + osu.NewTermQuery[string]("Name").Value("openCloud"), osu.NewTermQuery[string]("age").Value("32"), ), }, @@ -260,7 +272,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { Want: osu.NewBoolQuery(). Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). Should( - osu.NewMatchPhraseQuery("Name").Query("openCloud"), + osu.NewTermQuery[string]("Name").Value("openCloud"), osu.NewTermQuery[string]("age").Value("32"), ), }, @@ -288,7 +300,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewBoolQuery(). Must( - osu.NewMatchPhraseQuery("Name").Query("openCloud"), + osu.NewTermQuery[string]("Name").Value("openCloud"), ). MustNot( osu.NewTermQuery[string]("age").Value("32"), @@ -308,7 +320,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { Want: osu.NewBoolQuery(). Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). Should( - osu.NewMatchPhraseQuery("Name").Query("openCloud"), + osu.NewTermQuery[string]("Name").Value("openCloud"), osu.NewTermQuery[string]("age").Value("32"), osu.NewTermQuery[string]("age").Value("44"), ), diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index c5b32835b0..51e9ab4f0d 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -12,31 +12,10 @@ import ( bleveQuery "github.com/blevesearch/bleve/v2/search/query" "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/pkg/kql" - "github.com/opencloud-eu/opencloud/services/search/pkg/search" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" + searchQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query" ) -// lowercaseFields holds the fields whose query-side value is pre-lowercased so -// it matches the index-time lowercasing analyzer. Shared with the OpenSearch -// backend via search.LowercaseValueFields; every other field keeps its casing. -var lowercaseFields = search.LowercaseValueFields() - -var _fields = map[string]string{ - "rootid": "RootID", - "path": "Path", - "id": "ID", - "name": "Name", - "size": "Size", - "mtime": "Mtime", - "mediatype": "MimeType", - "type": "Type", - "tag": "Tags", - "tags": "Tags", - "content": "Content", - "title": "Title", - "hidden": "Hidden", - "favorite": "Favorites", -} - // The following quoted string enumerates the characters which may be escaped: "+-=&|>\<\!\(\)\{\}\[\]\^\"\~\:\ `), }), wantErr: false, }, diff --git a/services/search/pkg/query/normalize.go b/services/search/pkg/query/normalize.go index 78a4a55fe9..064a78e2e2 100644 --- a/services/search/pkg/query/normalize.go +++ b/services/search/pkg/query/normalize.go @@ -2,6 +2,7 @@ package query import ( "reflect" + "strings" "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/services/search/pkg/query/mimetype" @@ -31,10 +32,14 @@ func normalizeNodes(nodes []ast.Node, resolve func(string) string, defaultKey st switch node := n.(type) { case *ast.StringNode: node.Key = resolveKey(node.Key) + if FieldValueIsNormalized(node.Key) { + node.Value = strings.ToLower(node.Value) + } if exp := mimetype.Expand(node.Key, node.Value); exp != nil { out = append(out, normalizeNodes(exp, resolve, defaultKey)...) continue } + node.CaseInsensitive = FieldIsCaseInsensitive(node.Key) out = append(out, node) case *ast.DateTimeNode: node.Key = resolveKey(node.Key) diff --git a/services/search/pkg/query/normalize_test.go b/services/search/pkg/query/normalize_test.go index 66ba3243f0..df7568f0c0 100644 --- a/services/search/pkg/query/normalize_test.go +++ b/services/search/pkg/query/normalize_test.go @@ -20,13 +20,24 @@ func norm(nodes ...ast.Node) []ast.Node { func TestResolveField(t *testing.T) { require.Equal(t, "Name", query.ResolveField("")) // empty -> free-text default - require.Equal(t, "Name", query.ResolveField("NAME")) // case-insensitive + require.Equal(t, "Name", query.ResolveField("NAME")) // canonical, case-insensitive key match require.Equal(t, "Tags", query.ResolveField("tag")) // singular alias - require.Equal(t, "MimeType", query.ResolveField("mimetype")) // real field, case-insensitive - require.Equal(t, "photo.cameraMake", query.ResolveField("photo.CAMERAMAKE")) // facet, case-insensitive + require.Equal(t, "MimeType", query.ResolveField("mimetype")) // real field + require.Equal(t, "photo.cameraMake", query.ResolveField("photo.CAMERAMAKE")) // facet, case-insensitive key match require.Equal(t, "unknown.field", query.ResolveField("unknown.field")) // unknown key: unchanged, becomes a dead query } +func TestFieldIsCaseInsensitive(t *testing.T) { + // The four CaseInsensitive override fields (resolved canonical names). + for _, f := range []string{"Name", "Path", "Tags", "Favorites"} { + require.True(t, query.FieldIsCaseInsensitive(f), f) + } + // Case-preserved / non-keyword fields are not. + for _, f := range []string{"MimeType", "ID", "Content", "unknown"} { + require.False(t, query.FieldIsCaseInsensitive(f), f) + } +} + func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) { got := norm( &ast.StringNode{Key: "", Value: "free"}, @@ -40,9 +51,9 @@ func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) { ast.NumberNode{Key: "size", Value: 100}, ) require.Equal(t, []ast.Node{ - &ast.StringNode{Key: "Name", Value: "free"}, + &ast.StringNode{Key: "Name", Value: "free", CaseInsensitive: true}, &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Tags", Value: "x"}, + &ast.StringNode{Key: "Tags", Value: "x", CaseInsensitive: true}, &ast.OperatorNode{Value: "AND"}, &ast.StringNode{Key: "photo.cameraMake", Value: "canon"}, &ast.OperatorNode{Value: "AND"}, @@ -71,11 +82,11 @@ func TestNormalize_GroupKeyDefaulting(t *testing.T) { &ast.GroupNode{Key: "author", Nodes: []ast.Node{ &ast.StringNode{Key: "author", Value: "b"}, &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "Name", Value: "d"}, + &ast.StringNode{Key: "Name", Value: "d", CaseInsensitive: true}, }}, &ast.OperatorNode{Value: "AND"}, &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "e"}, + &ast.StringNode{Key: "Name", Value: "e", CaseInsensitive: true}, }}, }, got) } @@ -87,7 +98,7 @@ func TestNormalize_ConvertsValueNodesToPointers(t *testing.T) { ast.DateTimeNode{Key: "mtime"}, ) require.Equal(t, []ast.Node{ - &ast.StringNode{Key: "Name", Value: "x"}, + &ast.StringNode{Key: "Name", Value: "x", CaseInsensitive: true}, &ast.OperatorNode{Value: "AND"}, &ast.DateTimeNode{Key: "Mtime"}, }, got) diff --git a/services/search/pkg/query/resolver.go b/services/search/pkg/query/resolver.go index 5a4127a817..edc6e15951 100644 --- a/services/search/pkg/query/resolver.go +++ b/services/search/pkg/query/resolver.go @@ -15,27 +15,72 @@ var aliases = map[string]string{ "favorite": "Favorites", } -// fieldIndex maps a lowercased KQL key to the real field name: derived once from -// the resource struct, overlaid with the explicit aliases. +// fieldIndex maps a lowercased KQL key to its canonical field name ("" is the +// bare-search default). var fieldIndex = sync.OnceValue(func() map[string]string { - idx := mapping.FieldNameIndex( - reflect.TypeFor[search.Resource](), - search.Resource{}.SearchFieldOverrides(), - ) + idx := mapping.FieldNameIndex(reflect.TypeFor[search.Resource](), search.Resource{}.SearchFieldOverrides()) for k, v := range aliases { idx[k] = v } + idx[""] = idx["name"] return idx }) -// ResolveField maps a KQL key to the index field name: empty -> Name, a known -// key (case-insensitive) -> its field, anything else unchanged. -func ResolveField(name string) string { - if name == "" { - return "Name" +// caseInsensitiveFields are the fields searched case-insensitively by default, +// derived from the CaseInsensitive overrides. +var caseInsensitiveFields = sync.OnceValue(func() map[string]struct{} { + out := map[string]struct{}{} + for field, opts := range (search.Resource{}).SearchFieldOverrides() { + if opts.CaseInsensitive != nil && *opts.CaseInsensitive { + out[field] = struct{}{} + } } + return out +}) + +// pathFields are hierarchical path fields (TypePath), derived from the overrides. +var pathFields = sync.OnceValue(func() map[string]struct{} { + out := map[string]struct{}{} + for field, opts := range (search.Resource{}).SearchFieldOverrides() { + if opts.Type == mapping.TypePath { + out[field] = struct{}{} + } + } + return out +}) + +// ResolveField maps a KQL key to its canonical field name; unknown keys pass through. +func ResolveField(name string) string { if v, ok := fieldIndex()[strings.ToLower(name)]; ok { return v } return name } + +// normalizedValueFields have their stored values normalized to lowercase at +// index time, so query values fold to match even though the fields themselves +// are case-preserved keywords. +var normalizedValueFields = map[string]struct{}{ + "MimeType": {}, + "Type": {}, + "Hidden": {}, +} + +// FieldValueIsNormalized reports whether a field's stored values are +// normalized lowercase. +func FieldValueIsNormalized(field string) bool { + _, ok := normalizedValueFields[field] + return ok +} + +// FieldIsCaseInsensitive reports whether a field's default search is case-insensitive. +func FieldIsCaseInsensitive(field string) bool { + _, ok := caseInsensitiveFields()[field] + return ok +} + +// FieldIsPath reports whether a field is a hierarchical path field. +func FieldIsPath(field string) bool { + _, ok := pathFields()[field] + return ok +} diff --git a/services/search/pkg/search/search.go b/services/search/pkg/search/search.go index fb4ebef31e..73b07cae01 100644 --- a/services/search/pkg/search/search.go +++ b/services/search/pkg/search/search.go @@ -72,12 +72,13 @@ type Resource struct { // resourceFieldOverrides is built once (it never changes) and reused on hot // paths instead of reallocating per call. var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts { - excludeFromAll := false + True, False := true, false return map[string]mapping.FieldOpts{ - "Name": {Analyzer: "lowercaseKeyword"}, + "Name": {CaseInsensitive: &True}, + "Path": {Type: mapping.TypePath, CaseInsensitive: &True}, "Content": {Type: mapping.TypeFulltext}, - "Tags": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll}, - "Favorites": {Analyzer: "lowercaseKeyword", IncludeInAll: &excludeFromAll}, + "Tags": {CaseInsensitive: &True, IncludeInAll: &False}, + "Favorites": {CaseInsensitive: &True, IncludeInAll: &False}, "location": {Type: mapping.TypeGeopoint}, } }) @@ -89,32 +90,6 @@ func (Resource) SearchFieldOverrides() map[string]mapping.FieldOpts { return resourceFieldOverrides() } -// lowercaseValueFields is the set of index field names whose query values must -// be lowercased to match their index-time lowercasing analyzer (lowercaseKeyword -// or the fulltext type). Built once from the field overrides. -var lowercaseValueFields = sync.OnceValue(func() map[string]struct{} { - out := map[string]struct{}{} - for key, opts := range resourceFieldOverrides() { - if opts.Analyzer == "lowercaseKeyword" || opts.Type == mapping.TypeFulltext { - out[key] = struct{}{} - } - } - // stored values are normalized lowercase, so query values must fold too - // even though the index fields preserve case - for _, key := range []string{"MimeType", "Type", "Hidden"} { - out[key] = struct{}{} - } - return out -}) - -// LowercaseValueFields returns the set of index field names whose query values -// must be lowercased so query-side matching lines up with the index-time -// analyzer. Both search backends use it, so value casing stays consistent; every -// other (case-preserved) field keeps its original case. Read-only, do not mutate. -func LowercaseValueFields() map[string]struct{} { - return lowercaseValueFields() -} - // ResolveReference makes sure the path is relative to the space root func ResolveReference(ctx context.Context, ref *provider.Reference, ri *provider.ResourceInfo, gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) (*provider.Reference, error) { if ref.GetResourceId().GetOpaqueId() == ref.GetResourceId().GetSpaceId() { From e3c45dd1d00b75a50dcb637800fa11223527805f Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 10 Aug 2026 17:26:38 +0200 Subject: [PATCH 16/54] fix(search): analyze OpenSearch full-text queries, stem Content like bleve Single-term `content:` built an unanalyzed term query, so once this branch dropped the blanket query-value lowercasing, `content:Foo` missed on OpenSearch (bleve was unaffected, its query analyzes). Fielded full-text queries now use a match query. OpenSearch `Content` also gets a porter stemming analyzer (it used the default standard analyzer and never stemmed), so full-text search matches bleve on both case and stemming. --- services/search/pkg/bleve/backend_test.go | 12 ++++++++++++ services/search/pkg/mapping/opensearch.go | 1 + services/search/pkg/mapping/opensearch_test.go | 3 +-- services/search/pkg/opensearch/index.go | 7 +++++++ .../internal/convert/kql_transpile.go | 5 +++++ .../internal/convert/kql_transpile_test.go | 9 +++++++++ services/search/pkg/query/resolver.go | 18 ++++++++++++++++++ 7 files changed, 53 insertions(+), 2 deletions(-) diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go index 184cb9ff52..b93c5cd152 100644 --- a/services/search/pkg/bleve/backend_test.go +++ b/services/search/pkg/bleve/backend_test.go @@ -348,6 +348,18 @@ var _ = Describe("Bleve", func() { }) }) + Context("by content", func() { + It("matches full-text case-insensitively and stemmed", func() { + parentResource.Document.Content = "Running Foxes" + Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) + + assertDocCount(rootResource.ID, "content:running", 1) + assertDocCount(rootResource.ID, "content:RUNNING", 1) // case-insensitive + assertDocCount(rootResource.ID, "content:run", 1) // porter stemming + assertDocCount(rootResource.ID, "content:cat", 0) + }) + }) + Context("Highlights", func() { It("highlights only for content searches", func() { diff --git a/services/search/pkg/mapping/opensearch.go b/services/search/pkg/mapping/opensearch.go index 6b6b243ec7..22af1a9831 100644 --- a/services/search/pkg/mapping/opensearch.go +++ b/services/search/pkg/mapping/opensearch.go @@ -87,6 +87,7 @@ func openSearchFieldMapping(fieldType string, goType reflect.Type) (map[string]a return map[string]any{ "type": "text", "term_vector": "with_positions_offsets", + "analyzer": "fulltext", }, nil case TypeWildcard: // OpenSearch stores wildcard fields with doc_values=false by diff --git a/services/search/pkg/mapping/opensearch_test.go b/services/search/pkg/mapping/opensearch_test.go index b64520cb73..1eab516763 100644 --- a/services/search/pkg/mapping/opensearch_test.go +++ b/services/search/pkg/mapping/opensearch_test.go @@ -98,8 +98,7 @@ var _ = Describe("OpenSearchBuildMapping", func() { content := props["Content"].(map[string]any) Expect(content["type"]).To(Equal("text"), "Content: %#v", content) Expect(content["term_vector"]).To(Equal("with_positions_offsets"), "Content: %#v", content) - _, ok := content["analyzer"] - Expect(ok).To(BeFalse(), "Content should leave analyzer unset (use OpenSearch default)") + Expect(content["analyzer"]).To(Equal("fulltext"), "Content uses the stemming fulltext analyzer, like bleve") // Path: path_hierarchy base + lowercased sibling, both case-preserving. Expect(props["Path"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"})) Expect(props["Path_lowercase"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"})) diff --git a/services/search/pkg/opensearch/index.go b/services/search/pkg/opensearch/index.go index 93d454e53b..744dbb8a12 100644 --- a/services/search/pkg/opensearch/index.go +++ b/services/search/pkg/opensearch/index.go @@ -75,11 +75,18 @@ func buildResourceMapping() ([]byte, error) { "number_of_replicas": "1", "analysis": map[string]any{ // path_hierarchy is case-preserving; casing lives in the value. + // fulltext mirrors the bleve fulltext analyzer (lowercase + porter + // stemming) so full-text search behaves the same on both backends. "analyzer": map[string]any{ "path_hierarchy": map[string]any{ "type": "custom", "tokenizer": "path_hierarchy", }, + "fulltext": map[string]any{ + "type": "custom", + "tokenizer": "standard", + "filter": []string{"lowercase", "porter_stem"}, + }, }, "tokenizer": map[string]any{ "path_hierarchy": map[string]any{"type": "path_hierarchy"}, diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index 8a060a69d8..559412247d 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -11,6 +11,7 @@ import ( "github.com/opencloud-eu/opencloud/pkg/kql" "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" + "github.com/opencloud-eu/opencloud/services/search/pkg/query" ) func TranspileKQLToOpenSearch(nodes []ast.Node) (osu.Builder, error) { @@ -105,6 +106,10 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { value = strings.ToLower(value) } + if query.FieldIsFulltext(node.Key) { + return osu.NewMatchPhraseQuery(field).Query(value), nil + } + isWildcard := strings.Contains(value, "*") if isWildcard { return osu.NewWildcardQuery(field).Value(value), nil diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go index 5f29306848..d7255b5b8e 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go @@ -42,6 +42,15 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewWildcardQuery("Name_lowercase").Value("open*"), }, + { + Name: "full-text field uses an analyzed match query, not an unanalyzed term", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.StringNode{Key: "Content", Value: "Running"}, + }, + }, + Want: osu.NewMatchPhraseQuery("Content").Query("Running"), + }, { Name: "term query - boolean node - true", Got: &ast.Ast{ diff --git a/services/search/pkg/query/resolver.go b/services/search/pkg/query/resolver.go index edc6e15951..9a80ac218b 100644 --- a/services/search/pkg/query/resolver.go +++ b/services/search/pkg/query/resolver.go @@ -49,6 +49,18 @@ var pathFields = sync.OnceValue(func() map[string]struct{} { return out }) +// fulltextFields are analyzed full-text fields (TypeFulltext), derived from the +// overrides. +var fulltextFields = sync.OnceValue(func() map[string]struct{} { + out := map[string]struct{}{} + for field, opts := range (search.Resource{}).SearchFieldOverrides() { + if opts.Type == mapping.TypeFulltext { + out[field] = struct{}{} + } + } + return out +}) + // ResolveField maps a KQL key to its canonical field name; unknown keys pass through. func ResolveField(name string) string { if v, ok := fieldIndex()[strings.ToLower(name)]; ok { @@ -84,3 +96,9 @@ func FieldIsPath(field string) bool { _, ok := pathFields()[field] return ok } + +// FieldIsFulltext reports whether a field is an analyzed full-text field. +func FieldIsFulltext(field string) bool { + _, ok := fulltextFields()[field] + return ok +} From 727ca92afed8eee3e745d46fdff6dbd1c769a41b Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 10 Aug 2026 17:53:33 +0200 Subject: [PATCH 17/54] fix(search): review fixes for path AND-term and content wildcard bleve compiled a path restriction to a DisjunctionQuery, which mapBinary redistributes as an OR-chain, so `path:/Foo AND name:bar` matched the folder itself unconditionally. It is now a BooleanQuery (should: folder OR descendants), which mapBinary keeps atomic under an enclosing AND. The OpenSearch full-text branch ran before the wildcard check, so `content:foo*` degraded to a phrase match and diverged from bleve; the wildcard check now comes first. Adds the missing coverage the review flagged: path AND term, content wildcard, case-insensitive tags (the array sibling branch), and a spaced path with descendants on OpenSearch. --- services/search/pkg/bleve/backend_test.go | 19 +++++++++++++++++++ .../internal/convert/kql_transpile.go | 8 ++++---- services/search/pkg/query/bleve/compiler.go | 15 ++++++++------- .../search/pkg/query/bleve/compiler_test.go | 14 ++++++++++---- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go index b93c5cd152..d0756df9ab 100644 --- a/services/search/pkg/bleve/backend_test.go +++ b/services/search/pkg/bleve/backend_test.go @@ -185,6 +185,17 @@ var _ = Describe("Bleve", func() { assertDocCount(rootResource.ID, "Tags:baz", 0) }) + It("finds files by tags case-insensitively", func() { + // exercises the []string/[]any sibling-lowercasing branch end-to-end. + parentResource.Document.Tags = []string{"Work", "Urgent"} + Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) + + assertDocCount(rootResource.ID, "tag:work", 1) // stored "Work", queried lower + assertDocCount(rootResource.ID, "tag:WORK", 1) // queried upper + assertDocCount(rootResource.ID, "Tags:Urgent", 1) + assertDocCount(rootResource.ID, "tag:missing", 0) + }) + It("finds files by size", func() { parentResource.Document.Size = 12345 err := eng.Upsert(parentResource.ID, parentResource) @@ -346,6 +357,13 @@ var _ = Describe("Bleve", func() { It("matches case-insensitively", func() { assertDocCount(rootResource.ID, `path:"./PARENT D!R"`, 3) }) + + It("applies an AND filter to the folder itself, not only descendants", func() { + // regression: the folder-itself clause used to match unconditionally + // under an AND, so the parent leaked in despite the name filter. + matches := assertDocCount(rootResource.ID, `path:"./parent d!r" AND name:child.pdf`, 1) + Expect(matches[0].Entity.Name).To(Equal("child.pdf")) + }) }) Context("by content", func() { @@ -356,6 +374,7 @@ var _ = Describe("Bleve", func() { assertDocCount(rootResource.ID, "content:running", 1) assertDocCount(rootResource.ID, "content:RUNNING", 1) // case-insensitive assertDocCount(rootResource.ID, "content:run", 1) // porter stemming + assertDocCount(rootResource.ID, "content:run*", 1) // wildcard over the stemmed term assertDocCount(rootResource.ID, "content:cat", 0) }) }) diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index 559412247d..7938299bab 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -106,15 +106,15 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { value = strings.ToLower(value) } - if query.FieldIsFulltext(node.Key) { - return osu.NewMatchPhraseQuery(field).Query(value), nil - } - isWildcard := strings.Contains(value, "*") if isWildcard { return osu.NewWildcardQuery(field).Value(value), nil } + if query.FieldIsFulltext(node.Key) { + return osu.NewMatchPhraseQuery(field).Query(value), nil + } + totalTerms := strings.Split(value, " ") isSingleTerm := len(totalTerms) == 1 isMultiTerm := len(totalTerms) >= 1 diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index 51e9ab4f0d..13f7643a69 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -88,13 +88,14 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { var q bleveQuery.Query = bleveQuery.NewQueryStringQuery(k + ":" + v) if searchQuery.FieldIsPath(n.Key) { - // bleve has no path hierarchy analyzer, unlike OpenSearch: match - // the folder itself and its descendants (`\/*`, a trailing - // wildcard on the value). - q = bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{ - q, - bleveQuery.NewQueryStringQuery(k + ":" + v + `\/*`), - }) + // bleve has no path hierarchy analyzer, unlike OpenSearch: match the + // folder itself and its descendants (`\/*`). A BooleanQuery keeps + // this atomic; a DisjunctionQuery would be redistributed by an + // enclosing AND (mapBinary treats a left disjunction as an OR-chain). + bq := bleve.NewBooleanQuery() + bq.AddShould(q, bleveQuery.NewQueryStringQuery(k+":"+v+`\/*`)) + bq.SetMinShould(1) + q = bq } if prev == nil { diff --git a/services/search/pkg/query/bleve/compiler_test.go b/services/search/pkg/query/bleve/compiler_test.go index c54e59f18b..d9a5854a68 100644 --- a/services/search/pkg/query/bleve/compiler_test.go +++ b/services/search/pkg/query/bleve/compiler_test.go @@ -52,10 +52,16 @@ func Test_compile(t *testing.T) { &ast.StringNode{Key: "path", Value: "/Foo"}, }, }, - want: query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Path_lowercase:\/foo`), - query.NewQueryStringQuery(`Path_lowercase:\/foo\/*`), - }), + // a BooleanQuery (should: exact OR descendants), not a DisjunctionQuery, + // so an enclosing AND does not redistribute the folder-itself clause. + want: func() query.Query { + bq := query.NewBooleanQuery(nil, []query.Query{ + query.NewQueryStringQuery(`Path_lowercase:\/foo`), + query.NewQueryStringQuery(`Path_lowercase:\/foo\/*`), + }, nil) + bq.SetMinShould(1) + return query.NewConjunctionQuery([]query.Query{bq}) + }(), wantErr: false, }, { From 1bf15599f70b202eb29c24240dd6dfb3652a61a6 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 10 Aug 2026 17:58:46 +0200 Subject: [PATCH 18/54] fix(search): write a consistent empty _lowercase sibling for empty arrays The []any branch skipped the sibling for an empty array while the []string branch wrote an empty one; both now write it, matching the base field. --- services/search/pkg/mapping/casing.go | 4 +--- services/search/pkg/mapping/casing_test.go | 13 +++++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/services/search/pkg/mapping/casing.go b/services/search/pkg/mapping/casing.go index 3ecceda993..f05413533e 100644 --- a/services/search/pkg/mapping/casing.go +++ b/services/search/pkg/mapping/casing.go @@ -44,9 +44,7 @@ func addLowercaseSibling(parent map[string]any, leaf string) { out = append(out, strings.ToLower(s)) } } - if len(out) > 0 { - parent[leaf+LowercaseSuffix] = out - } + parent[leaf+LowercaseSuffix] = out case []string: out := make([]string, len(v)) for i, s := range v { diff --git a/services/search/pkg/mapping/casing_test.go b/services/search/pkg/mapping/casing_test.go index d846158fb8..5fba3e9eae 100644 --- a/services/search/pkg/mapping/casing_test.go +++ b/services/search/pkg/mapping/casing_test.go @@ -38,4 +38,17 @@ var _ = Describe("PrepareForIndex casing", func() { Expect(err).ToNot(HaveOccurred()) Expect(m).ToNot(HaveKey("ID" + LowercaseSuffix)) }) + + It("writes an empty sibling for an empty array, like a non-empty one", func() { + True := true + type doc struct { + Tags []string `json:"Tags"` + } + m, err := PrepareForIndex(doc{Tags: []string{}}, map[string]FieldOpts{ + "Tags": {CaseInsensitive: &True}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(m).To(HaveKey("Tags" + LowercaseSuffix)) + Expect(m["Tags"+LowercaseSuffix]).To(BeEmpty()) + }) }) From c680a90e492092f4903e8362a8be800e4f44ad33 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 10 Aug 2026 18:01:28 +0200 Subject: [PATCH 19/54] fix(search): reject CaseInsensitive on non-keyword/path fields CaseInsensitive routes queries to a _lowercase sibling that is only generated for keyword/path fields, so marking any other type CaseInsensitive would silently match nothing. Validate now rejects it up front. --- services/search/pkg/mapping/validate.go | 23 +++++++++++++++----- services/search/pkg/mapping/validate_test.go | 10 +++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/services/search/pkg/mapping/validate.go b/services/search/pkg/mapping/validate.go index 761309992a..f0a8e462e2 100644 --- a/services/search/pkg/mapping/validate.go +++ b/services/search/pkg/mapping/validate.go @@ -17,17 +17,28 @@ func Validate(t reflect.Type, overrides map[string]FieldOpts) error { return nil } names := collectNames(t, "") - var unknown []string - for k := range overrides { + var unknown, miscased []string + for k, opts := range overrides { if _, ok := names[k]; !ok { unknown = append(unknown, k) + continue + } + // CaseInsensitive routes queries to a _lowercase sibling, which is + // only generated for keyword/path fields; on any other type the query + // would target a non-existent field and silently match nothing. + if opts.caseInsensitive() && !isCasedType(opts) { + miscased = append(miscased, k) } } - if len(unknown) == 0 { - return nil + if len(unknown) > 0 { + sort.Strings(unknown) + return fmt.Errorf("mapping: unknown override keys: %s", strings.Join(unknown, ", ")) } - sort.Strings(unknown) - return fmt.Errorf("mapping: unknown override keys: %s", strings.Join(unknown, ", ")) + if len(miscased) > 0 { + sort.Strings(miscased) + return fmt.Errorf("mapping: CaseInsensitive is only valid on keyword/path fields: %s", strings.Join(miscased, ", ")) + } + return nil } func collectNames(t reflect.Type, prefix string) map[string]struct{} { diff --git a/services/search/pkg/mapping/validate_test.go b/services/search/pkg/mapping/validate_test.go index 3b15f39caf..69a96b0a01 100644 --- a/services/search/pkg/mapping/validate_test.go +++ b/services/search/pkg/mapping/validate_test.go @@ -44,4 +44,14 @@ var _ = Describe("Validate", func() { It("accepts empty overrides", func() { Expect(Validate(reflect.TypeFor[sample](), nil)).To(Succeed()) }) + + It("rejects CaseInsensitive on a non-keyword/path field", func() { + True := true + err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{ + "Name": {Type: TypeFulltext, CaseInsensitive: &True}, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Name")) + Expect(err.Error()).To(ContainSubstring("CaseInsensitive")) + }) }) From 8cb973d695c4b6b0aa3fc0a7b184b4dfa0e82afc Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 10 Aug 2026 18:47:46 +0200 Subject: [PATCH 20/54] test(search): cover mediatype and direct MimeType search on both backends Adds bleve and OpenSearch coverage for category (image), literal MIME (image/svg+xml, with + and /), and raw MimeType: queries. Documents why MimeType skips the bleve escaper: it is not a bug, bleve treats / and + as literals mid-term, so a literal MIME still matches exactly while the category wildcard image/* keeps its *. --- services/search/pkg/bleve/backend_test.go | 19 +++++++++++++++++++ services/search/pkg/query/bleve/compiler.go | 6 ++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go index d0756df9ab..cdccb327dd 100644 --- a/services/search/pkg/bleve/backend_test.go +++ b/services/search/pkg/bleve/backend_test.go @@ -379,6 +379,25 @@ var _ = Describe("Bleve", func() { }) }) + Context("by mediatype", func() { + It("matches categories and literal MIME types (incl. + and /)", func() { + childResource.Document.MimeType = "image/svg+xml" + childResource2.Document.MimeType = "image/png" + for _, r := range []search.Resource{childResource, childResource2} { + Expect(eng.Upsert(r.ID, r)).To(Succeed()) + } + + assertDocCount(rootResource.ID, "mediatype:image", 2) // image/* wildcard -> both + assertDocCount(rootResource.ID, "mediatype:pdf", 0) + // literal MIME with + and /, must hit only the svg doc, not the png + assertDocCount(rootResource.ID, "mediatype:image/svg+xml", 1) + assertDocCount(rootResource.ID, "mediatype:image/png", 1) + // the same literal via the raw field name (no mediatype alias) + assertDocCount(rootResource.ID, "MimeType:image/svg+xml", 1) + assertDocCount(rootResource.ID, "MimeType:image/png", 1) + }) + }) + Context("Highlights", func() { It("highlights only for content searches", func() { diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index 13f7643a69..5a57d2bc78 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -74,8 +74,10 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { for i := offset; i < len(nodes); i++ { switch n := nodes[i].(type) { case *ast.StringNode: - // keys are resolved and media-type expanded by normalize; MimeType - // values are literal MIME types, so they skip the escaper. + // keys are resolved and media-type expanded by normalize. MimeType + // skips the escaper so the category wildcards (image/*) keep their `*`; + // bleve treats `/` and `+` as literals mid-term, so a literal MIME like + // image/svg+xml still matches exactly. k := n.Key v := n.Value if k != "ID" && k != "Size" && k != "MimeType" { From dad99e85b26164031300885eaac606238347406b Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 10 Aug 2026 18:54:13 +0200 Subject: [PATCH 21/54] fix(search): make mediatype categories and MIME types case-insensitive mediatype:Folder / mediatype:IMAGE resolved to a literal MimeType search and matched nothing because Expand switched on the raw value. The value is now lowercased in the lowering pass, so categories and literal MIME types match regardless of case, consistently on both backends. --- services/search/pkg/bleve/backend_test.go | 1 + services/search/pkg/query/mimetype/mimetype.go | 4 +++- services/search/pkg/query/mimetype/mimetype_test.go | 11 +++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go index cdccb327dd..43a8e5f0d3 100644 --- a/services/search/pkg/bleve/backend_test.go +++ b/services/search/pkg/bleve/backend_test.go @@ -388,6 +388,7 @@ var _ = Describe("Bleve", func() { } assertDocCount(rootResource.ID, "mediatype:image", 2) // image/* wildcard -> both + assertDocCount(rootResource.ID, "mediatype:IMAGE", 2) // categories are case-insensitive assertDocCount(rootResource.ID, "mediatype:pdf", 0) // literal MIME with + and /, must hit only the svg doc, not the png assertDocCount(rootResource.ID, "mediatype:image/svg+xml", 1) diff --git a/services/search/pkg/query/mimetype/mimetype.go b/services/search/pkg/query/mimetype/mimetype.go index 98692d9524..ca61e651ac 100644 --- a/services/search/pkg/query/mimetype/mimetype.go +++ b/services/search/pkg/query/mimetype/mimetype.go @@ -14,11 +14,13 @@ const field = "MimeType" // Expand turns mediatype: into the MimeType query it stands for: category // values (file/document/image/...) expand to their MIME set, anything else is a -// literal MimeType:. Returns nil for non-mediatype keys. +// literal MimeType:. Returns nil for non-mediatype keys. Categories and +// MIME types are case-insensitive, so the value is lowercased. func Expand(key, value string) []ast.Node { if strings.ToLower(key) != "mediatype" { return nil } + value = strings.ToLower(value) switch value { case "file": return []ast.Node{ diff --git a/services/search/pkg/query/mimetype/mimetype_test.go b/services/search/pkg/query/mimetype/mimetype_test.go index 0b25844e28..d4b78671b1 100644 --- a/services/search/pkg/query/mimetype/mimetype_test.go +++ b/services/search/pkg/query/mimetype/mimetype_test.go @@ -22,6 +22,17 @@ func TestExpand_keyIsCaseInsensitive(t *testing.T) { require.NotNil(t, mimetype.Expand("MediaType", "file")) } +func TestExpand_valueIsCaseInsensitive(t *testing.T) { + // a category matches regardless of case + require.Equal(t, []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + }, mimetype.Expand("mediatype", "Folder")) + // a literal MIME type is lowercased too (MIME types are case-insensitive) + require.Equal(t, []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "image/svg+xml"}, + }, mimetype.Expand("mediatype", "Image/SVG+XML")) +} + // A non-category value is a literal MIME type and targets the MimeType field. func TestExpand_literalValuePassesThroughToMimeType(t *testing.T) { require.Equal(t, []ast.Node{ From 69c517a8b2bf9f87646578520f2d4f48931cb4e7 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 10 Aug 2026 19:12:24 +0200 Subject: [PATCH 22/54] fix(search): nest json-tagged embedded structs instead of flattening them resolveField marked every anonymous field embedded, so walkFields (mapping, field index, validate) and fillStruct (deserializer) flattened a json-tagged embedded struct, while conversions.To/encoding/json on the write path nests it under the tag, mapping and deserializing it at the wrong path. An anonymous field is now embedded only without a json tag name, matching encoding/json; fillStruct also recurses into a value nested struct. No current type has a tagged embedded struct, so runtime behavior is unchanged; this hardens the reflection walker. --- services/search/pkg/mapping/deserialize.go | 8 +++++++ .../search/pkg/mapping/deserialize_test.go | 21 +++++++++++++++++++ .../search/pkg/mapping/fieldindex_test.go | 19 +++++++++++++++++ services/search/pkg/mapping/infer.go | 11 +++++++--- 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/services/search/pkg/mapping/deserialize.go b/services/search/pkg/mapping/deserialize.go index f8c2f844a1..a247de659e 100644 --- a/services/search/pkg/mapping/deserialize.go +++ b/services/search/pkg/mapping/deserialize.go @@ -87,6 +87,14 @@ func fillStruct[V any](v reflect.Value, fields map[string]V, prefix string, setL } } + // Value nested struct (e.g. a tagged embedded struct): recurse under key. + if fv.Kind() == reflect.Struct && fv.Type() != timeType && fv.Type() != timestampType { + if fillStruct(fv, fields, key, setLeaf) { + touched = true + } + continue + } + if raw, ok := fields[key]; ok && setLeaf(fv, raw) == nil { touched = true } diff --git a/services/search/pkg/mapping/deserialize_test.go b/services/search/pkg/mapping/deserialize_test.go index 55824aeaa5..11c6110d5b 100644 --- a/services/search/pkg/mapping/deserialize_test.go +++ b/services/search/pkg/mapping/deserialize_test.go @@ -32,6 +32,13 @@ type embedded struct { Photo *photo `json:"photo,omitempty"` } +// taggedEmbedded embeds Leaf with a json tag, so encoding/json nests it under +// "leaf" rather than flattening it onto the parent. +type taggedEmbedded struct { + Leaf `json:"leaf"` + Top string `json:"top"` +} + var _ = Describe("Deserialize", func() { It("panics for a non-struct type at a prefix", func() { Expect(func() { @@ -115,4 +122,18 @@ var _ = Describe("Deserialize", func() { Expect(r.Year).ToNot(BeNil()) Expect(*r.Year).To(Equal(int32(2024))) }) + + It("nests a tagged embedded struct instead of flattening it", func() { + // matches encoding/json: a tagged embedded struct is read under its tag. + r := Deserialize[taggedEmbedded](map[string]any{ + "leaf.Name": "n", + "top": "t", + }) + Expect(r.Leaf.Name).To(Equal("n")) + Expect(r.Top).To(Equal("t")) + + // the flattened top-level key must NOT populate the nested field. + flat := Deserialize[taggedEmbedded](map[string]any{"Name": "flat"}) + Expect(flat.Leaf.Name).To(BeEmpty()) + }) }) diff --git a/services/search/pkg/mapping/fieldindex_test.go b/services/search/pkg/mapping/fieldindex_test.go index d0e47dcb2e..56e6b0119b 100644 --- a/services/search/pkg/mapping/fieldindex_test.go +++ b/services/search/pkg/mapping/fieldindex_test.go @@ -77,3 +77,22 @@ func TestFieldNameIndex_CoversAllTopLevelFields(t *testing.T) { require.Equalf(t, want, resolve(idx, in), "derived should cover %q", in) } } + +// NestInner is embedded with a json tag below, so it must nest, not flatten. +type NestInner struct { + A string `json:"A"` +} + +type taggedOuter struct { + NestInner `json:"inner"` + Top string `json:"top"` +} + +// A json-tagged embedded struct nests under its tag in the derived index too +// (walkFields must match encoding/json), so its fields are "inner.A", not "A". +func TestFieldNameIndex_TaggedEmbeddedNests(t *testing.T) { + idx := mapping.FieldNameIndex(reflect.TypeFor[taggedOuter](), nil) + require.Equal(t, "inner.A", resolve(idx, "inner.a")) // nested under the tag + require.Equal(t, "top", resolve(idx, "top")) + require.Equal(t, "a", resolve(idx, "a")) // not flattened: bare "a" is not a key +} diff --git a/services/search/pkg/mapping/infer.go b/services/search/pkg/mapping/infer.go index 76a49bd669..5524703f8d 100644 --- a/services/search/pkg/mapping/infer.go +++ b/services/search/pkg/mapping/infer.go @@ -58,6 +58,7 @@ func resolveField(sf reflect.StructField) fieldInfo { return fieldInfo{Skip: true} } name := sf.Name + named := false tag := sf.Tag.Get("json") if tag != "" { first, _, _ := strings.Cut(tag, ",") @@ -66,12 +67,16 @@ func resolveField(sf reflect.StructField) fieldInfo { } if first != "" { name = first + named = true } } return fieldInfo{ - Name: name, - GoField: sf, - Embedded: sf.Anonymous, + Name: name, + GoField: sf, + // An anonymous field is embedded (flattened onto the parent) only when it + // has no json tag name, matching encoding/json: a tag name nests it as a + // regular field instead. + Embedded: sf.Anonymous && !named, } } From c94059fb85394e2a2f183049064f26cf4e2d769d Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 10 Aug 2026 19:40:31 +0200 Subject: [PATCH 23/54] fix(search): keep mediatype:file atomic so it composes with other terms mediatype:file expands to a NOT restriction. Spliced inline as `NOT MimeType:httpd/unix-directory`, the bleve compiler's NOT branch left a stale operand, so `mediatype:file AND name:x` dropped `name:x` and matched nothing (the web Files filter). It is now wrapped in a group so the negation stays atomic; verified fixing both bleve and OpenSearch. --- services/search/pkg/bleve/backend_test.go | 17 +++++- .../internal/convert/kql_expand_test.go | 57 ------------------- .../search/pkg/query/bleve/compiler_test.go | 4 +- .../search/pkg/query/mimetype/mimetype.go | 9 ++- .../pkg/query/mimetype/mimetype_test.go | 7 ++- services/search/pkg/query/normalize_test.go | 6 +- 6 files changed, 33 insertions(+), 67 deletions(-) delete mode 100644 services/search/pkg/opensearch/internal/convert/kql_expand_test.go diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go index 43a8e5f0d3..16422b809b 100644 --- a/services/search/pkg/bleve/backend_test.go +++ b/services/search/pkg/bleve/backend_test.go @@ -190,8 +190,8 @@ var _ = Describe("Bleve", func() { parentResource.Document.Tags = []string{"Work", "Urgent"} Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) - assertDocCount(rootResource.ID, "tag:work", 1) // stored "Work", queried lower - assertDocCount(rootResource.ID, "tag:WORK", 1) // queried upper + assertDocCount(rootResource.ID, "tag:work", 1) // stored "Work", queried lower + assertDocCount(rootResource.ID, "tag:WORK", 1) // queried upper assertDocCount(rootResource.ID, "Tags:Urgent", 1) assertDocCount(rootResource.ID, "tag:missing", 0) }) @@ -397,6 +397,19 @@ var _ = Describe("Bleve", func() { assertDocCount(rootResource.ID, "MimeType:image/svg+xml", 1) assertDocCount(rootResource.ID, "MimeType:image/png", 1) }) + + It("combines mediatype:file with another term", func() { + // regression: mediatype:file (a NOT) next to an operator dropped the + // other operand, so mediatype:file AND name:x matched nothing. + parentResource.Document.MimeType = "httpd/unix-directory" // a folder + childResource.Document.MimeType = "image/png" // a file + for _, r := range []search.Resource{parentResource, childResource} { + Expect(eng.Upsert(r.ID, r)).To(Succeed()) + } + assertDocCount(rootResource.ID, "mediatype:file", 1) // only the file + assertDocCount(rootResource.ID, "mediatype:file AND name:child.pdf", 1) // file AND its name + assertDocCount(rootResource.ID, "mediatype:file AND name:nope", 0) + }) }) Context("Highlights", func() { diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go deleted file mode 100644 index ef1927b48d..0000000000 --- a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package convert_test - -import ( - "testing" - - "github.com/stretchr/testify/require" - - "github.com/opencloud-eu/opencloud/pkg/ast" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" -) - -// LowerValues runs after the shared query.Normalize pass, so it operates on -// already-resolved pointer nodes. Field resolution, media-type expansion and -// group-key defaulting are tested once at the query.Normalize level (see -// pkg/query normalize_test), not here. Only lowercase-analyzed fields get their -// value folded; case-preserved fields keep their casing. -func TestLowerValues(t *testing.T) { - tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ - { - Name: "lowercase-analyzed field: value is folded, recursing into groups", - Got: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "StringNode"}, - &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "StringNode"}, - }}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "stringnode"}, - &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "stringnode"}, - }}, - }, - }, - { - Name: "case-preserved field: value keeps its casing", - Got: []ast.Node{ - &ast.StringNode{Key: "aBc", Value: "StringNode"}, - &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "aBc", Value: "StringNode"}, - }}, - }, - Want: []ast.Node{ - &ast.StringNode{Key: "aBc", Value: "StringNode"}, - &ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{ - &ast.StringNode{Key: "aBc", Value: "StringNode"}, - }}, - }, - }, - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - require.Equal(t, test.Want, convert.LowerValues(test.Got)) - }) - } -} diff --git a/services/search/pkg/query/bleve/compiler_test.go b/services/search/pkg/query/bleve/compiler_test.go index d9a5854a68..23d189f689 100644 --- a/services/search/pkg/query/bleve/compiler_test.go +++ b/services/search/pkg/query/bleve/compiler_test.go @@ -321,8 +321,8 @@ func Test_compile(t *testing.T) { }, want: query.NewConjunctionQuery([]query.Query{ query.NewQueryStringQuery(`Name_lowercase:john\ smith`), - query.NewQueryStringQuery(`Hidden:T`), - query.NewQueryStringQuery(`Hidden:T`), + query.NewQueryStringQuery(`Hidden:t`), + query.NewQueryStringQuery(`Hidden:t`), }), wantErr: false, }, diff --git a/services/search/pkg/query/mimetype/mimetype.go b/services/search/pkg/query/mimetype/mimetype.go index ca61e651ac..06700049e9 100644 --- a/services/search/pkg/query/mimetype/mimetype.go +++ b/services/search/pkg/query/mimetype/mimetype.go @@ -23,9 +23,14 @@ func Expand(key, value string) []ast.Node { value = strings.ToLower(value) switch value { case "file": + // Group the negation so it stays atomic next to an operator: a bare + // `NOT ` sequence spliced inline miscompiles as the left of an AND + // (mediatype:file AND name:x would drop name:x). return []ast.Node{ - &ast.OperatorNode{Value: kql.BoolNOT}, - &ast.StringNode{Key: field, Value: "httpd/unix-directory"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.OperatorNode{Value: kql.BoolNOT}, + &ast.StringNode{Key: field, Value: "httpd/unix-directory"}, + }}, } case "folder": return term("httpd/unix-directory") diff --git a/services/search/pkg/query/mimetype/mimetype_test.go b/services/search/pkg/query/mimetype/mimetype_test.go index d4b78671b1..c9d9a7cf6c 100644 --- a/services/search/pkg/query/mimetype/mimetype_test.go +++ b/services/search/pkg/query/mimetype/mimetype_test.go @@ -45,9 +45,12 @@ func TestExpand_literalValuePassesThroughToMimeType(t *testing.T) { } func TestExpand_fileIsNotAFolder(t *testing.T) { + // grouped so the negation stays atomic next to an operator. require.Equal(t, []ast.Node{ - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + }}, }, mimetype.Expand("mediatype", "file")) } diff --git a/services/search/pkg/query/normalize_test.go b/services/search/pkg/query/normalize_test.go index df7568f0c0..8af9cf7435 100644 --- a/services/search/pkg/query/normalize_test.go +++ b/services/search/pkg/query/normalize_test.go @@ -57,8 +57,10 @@ func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) { &ast.OperatorNode{Value: "AND"}, &ast.StringNode{Key: "photo.cameraMake", Value: "canon"}, &ast.OperatorNode{Value: "AND"}, - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + }}, &ast.OperatorNode{Value: "AND"}, &ast.NumberNode{Key: "Size", Value: 100}, }, got) From fc41995a47f9a94de3d8dba22095fba86279061c Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 10 Aug 2026 19:42:08 +0200 Subject: [PATCH 24/54] fix(search): validate CaseInsensitive against the effective field type The guard only rejected CaseInsensitive when a non-keyword/path Type was set explicitly. With no Type, isCasedType treated the field as cased, so CaseInsensitive on an inferred numeric/bool/datetime field passed validation but produced no _lowercase sibling, and the query would silently match nothing. Validate now falls back to the inferred Go type. --- services/search/pkg/mapping/validate.go | 33 ++++++++++++++------ services/search/pkg/mapping/validate_test.go | 24 ++++++++++++++ 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/services/search/pkg/mapping/validate.go b/services/search/pkg/mapping/validate.go index f0a8e462e2..fa6a8b222e 100644 --- a/services/search/pkg/mapping/validate.go +++ b/services/search/pkg/mapping/validate.go @@ -16,17 +16,20 @@ func Validate(t reflect.Type, overrides map[string]FieldOpts) error { if len(overrides) == 0 { return nil } - names := collectNames(t, "") + fields := collectFields(t, "") var unknown, miscased []string for k, opts := range overrides { - if _, ok := names[k]; !ok { + goType, ok := fields[k] + if !ok { unknown = append(unknown, k) continue } // CaseInsensitive routes queries to a _lowercase sibling, which is // only generated for keyword/path fields; on any other type the query - // would target a non-existent field and silently match nothing. - if opts.caseInsensitive() && !isCasedType(opts) { + // would target a non-existent field and silently match nothing. Use the + // effective type (override, else the inferred Go type), since an override + // with no explicit Type still infers keyword/numeric/... from the field. + if opts.caseInsensitive() && !effectivelyCased(opts, goType) { miscased = append(miscased, k) } } @@ -41,17 +44,29 @@ func Validate(t reflect.Type, overrides map[string]FieldOpts) error { return nil } -func collectNames(t reflect.Type, prefix string) map[string]struct{} { - out := map[string]struct{}{} +// effectivelyCased reports whether a field is keyword/path (the only types that +// get a _lowercase sibling), from the override type or the inferred Go type. +func effectivelyCased(opts FieldOpts, goType reflect.Type) bool { + eff := opts.Type + if eff == "" && goType != nil { + eff = inferType(goType) + } + return eff == TypeKeyword || eff == TypePath +} + +// collectFields maps every known field name (nested as "parent.child") to its Go +// type. Embedded structs are flattened, matching encoding/json. +func collectFields(t reflect.Type, prefix string) map[string]reflect.Type { + out := map[string]reflect.Type{} _ = walkFields(t, func(fi fieldInfo) error { key := fi.Name if prefix != "" { key = prefix + "." + fi.Name } - out[key] = struct{}{} + out[key] = fi.GoField.Type if sub := structType(fi.GoField.Type); sub != nil { - for k := range collectNames(sub, key) { - out[k] = struct{}{} + for k, v := range collectFields(sub, key) { + out[k] = v } } return nil diff --git a/services/search/pkg/mapping/validate_test.go b/services/search/pkg/mapping/validate_test.go index 69a96b0a01..7a34183a4b 100644 --- a/services/search/pkg/mapping/validate_test.go +++ b/services/search/pkg/mapping/validate_test.go @@ -54,4 +54,28 @@ var _ = Describe("Validate", func() { Expect(err.Error()).To(ContainSubstring("Name")) Expect(err.Error()).To(ContainSubstring("CaseInsensitive")) }) + + It("rejects CaseInsensitive on an inferred non-keyword field (empty Type)", func() { + True := true + type doc struct { + Size uint64 `json:"Size"` + } + err := Validate(reflect.TypeFor[doc](), map[string]FieldOpts{ + "Size": {CaseInsensitive: &True}, // no explicit Type -> inferred numeric + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Size")) + }) + + It("accepts CaseInsensitive on an inferred keyword field (empty Type)", func() { + True := true + type doc struct { + Name string `json:"Name"` + Tags []string `json:"Tags"` + } + Expect(Validate(reflect.TypeFor[doc](), map[string]FieldOpts{ + "Name": {CaseInsensitive: &True}, + "Tags": {CaseInsensitive: &True}, + })).To(Succeed()) + }) }) From 0284224787cb6b61747608dd16a524fde31920a4 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 10 Aug 2026 19:45:24 +0200 Subject: [PATCH 25/54] refactor(search): anchor the OpenSearch move prefix replace to the path start The move script rewrote Path/Path_lowercase with painless String.replace, which replaces every occurrence of the old path, not just the leading prefix. OpenCloud paths are ./-prefixed so the full old path only occurs at the start and the result is byte-identical, but startsWith + substring makes the prefix-only intent explicit and robust to any path format. Not a live bug fix, a hardening. --- services/search/pkg/opensearch/batch.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/services/search/pkg/opensearch/batch.go b/services/search/pkg/opensearch/batch.go index 1dc0bdf8ca..04ba0c9a04 100644 --- a/services/search/pkg/opensearch/batch.go +++ b/services/search/pkg/opensearch/batch.go @@ -72,7 +72,10 @@ func (b *Batch) Move(id string, parentID string, location string) error { newName := path.Base(newPath) return &osu.BodyParamScript{ // Keep the case-preserved base fields and their lowercased - // search siblings in sync: swap the moved prefix in both. The + // search siblings in sync: swap the moved prefix in both. Only + // the leading oldPath is replaced (startsWith + substring, not + // String.replace, which would also rewrite a repeated segment + // deeper in a descendant's path, e.g. /Music/Music.m3u). The // lowercased new values come from Go's strings.ToLower via // params, so the sibling stays byte-identical to what // PrepareForIndex writes on upsert (painless toLowerCase would @@ -83,9 +86,11 @@ func (b *Batch) Move(id string, parentID string, location string) error { ctx._source.ParentID = params.parentID; if (ctx._source.Name%[1]s != null) { ctx._source.Name%[1]s = params.newNameLower; } } - ctx._source.Path = ctx._source.Path.replace(params.oldPath, params.newPath); - if (ctx._source.Path%[1]s != null) { - ctx._source.Path%[1]s = ctx._source.Path%[1]s.replace(params.oldPathLower, params.newPathLower); + if (ctx._source.Path != null && ctx._source.Path.startsWith(params.oldPath)) { + ctx._source.Path = params.newPath + ctx._source.Path.substring(params.oldPath.length()); + } + if (ctx._source.Path%[1]s != null && ctx._source.Path%[1]s.startsWith(params.oldPathLower)) { + ctx._source.Path%[1]s = params.newPathLower + ctx._source.Path%[1]s.substring(params.oldPathLower.length()); } boolean hidden = false; for (String name : ctx._source.Path.splitOnToken('/')) { From 17144d47912c2b4e5b48dd44adf76a8541b5b41d Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 10 Aug 2026 21:16:35 +0200 Subject: [PATCH 26/54] fix(search): keep OpenSearch path queries as unanalyzed term queries A path value with spaces went through a match_phrase query, which analyzes the query with the path_hierarchy analyzer; the resulting "." prefix token matches every document in the space, breaking descendant matching and the stale-path check after a move. --- .../internal/convert/kql_transpile.go | 7 +++ .../internal/convert/kql_transpile_test.go | 52 +++---------------- 2 files changed, 15 insertions(+), 44 deletions(-) diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index 7938299bab..368f1ad9be 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -115,6 +115,13 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { return osu.NewMatchPhraseQuery(field).Query(value), nil } + // a path value is a single term in the path_hierarchy token stream; a + // phrase match would analyze the query into its path prefixes and match + // everything under the root, so paths with spaces must stay term queries. + if query.FieldIsPath(node.Key) { + return osu.NewTermQuery[string](field).Value(value), nil + } + totalTerms := strings.Split(value, " ") isSingleTerm := len(totalTerms) == 1 isMultiTerm := len(totalTerms) >= 1 diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go index d7255b5b8e..8cfbf54472 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go @@ -97,60 +97,24 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { Want: osu.NewWildcardQuery("Content").Value("open*"), }, { - Name: "wildcard query - a question mark counts as a wildcard", + // a phrase match would analyze the query with path_hierarchy and match + // everything under the root + Name: "path with spaces stays an unanalyzed term query", Got: &ast.Ast{ Nodes: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "fo?o"}, + &ast.StringNode{Key: "Path", Value: "./parent d!r/child.pdf"}, }, }, - Want: osu.NewBoolQuery(). - Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). - Should( - osu.NewWildcardQuery("Name.wildcard"). - Value("fo?o"). - Params(&osu.WildcardQueryParams{CaseInsensitive: true}), - osu.NewWildcardQuery("Name.wildcard"). - Value("fo?o.*"). - Params(&osu.WildcardQueryParams{CaseInsensitive: true}), - ), + Want: osu.NewTermQuery[string]("Path").Value("./parent d!r/child.pdf"), }, { - Name: "term query - an equals restriction matches the whole name", + Name: "case-insensitive path with spaces routes to the lowercased sibling as a term query", Got: &ast.Ast{ Nodes: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "foo bar.txt", Exact: true}, + &ast.StringNode{Key: "Path", Value: "./Parent Dir", CaseInsensitive: true}, }, }, - Want: osu.NewTermQuery[string]("Name.wildcard"). - Value("foo bar.txt"). - Params(&osu.TermQueryParams{CaseInsensitive: true}), - }, - { - Name: "term query - a path loses its trailing slash", - Got: &ast.Ast{ - Nodes: []ast.Node{ - &ast.StringNode{Key: "Path", Value: "./Documents/"}, - }, - }, - Want: osu.NewTermQuery[string]("Path").Value("./Documents"), - }, - { - Name: "term query - a hidden string turns into a bool", - Got: &ast.Ast{ - Nodes: []ast.Node{ - &ast.StringNode{Key: "Hidden", Value: "true"}, - }, - }, - Want: osu.NewTermQuery[bool]("Hidden").Value(true), - }, - { - Name: "match-none query - a hidden string that is no bool", - Got: &ast.Ast{ - Nodes: []ast.Node{ - &ast.StringNode{Key: "Hidden", Value: "banana"}, - }, - }, - Want: osu.NewMatchNoneQuery(), + Want: osu.NewTermQuery[string]("Path_lowercase").Value("./parent dir"), }, { Name: "bool query", From 5d9a255b1625699c3e00fd55a07314562ae5c53c Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 10 Aug 2026 20:14:04 +0000 Subject: [PATCH 27/54] test(search): port fieldindex, mimetype and normalize tests to ginkgo --- .../search/pkg/mapping/fieldindex_test.go | 129 +++++++------ .../pkg/query/mimetype/mimetype_suite_test.go | 13 ++ .../pkg/query/mimetype/mimetype_test.go | 180 +++++++++--------- services/search/pkg/query/normalize_test.go | 178 ++++++++--------- services/search/pkg/query/query_suite_test.go | 13 ++ 5 files changed, 274 insertions(+), 239 deletions(-) create mode 100644 services/search/pkg/query/mimetype/mimetype_suite_test.go create mode 100644 services/search/pkg/query/query_suite_test.go diff --git a/services/search/pkg/mapping/fieldindex_test.go b/services/search/pkg/mapping/fieldindex_test.go index 56e6b0119b..a5c0fb0f5b 100644 --- a/services/search/pkg/mapping/fieldindex_test.go +++ b/services/search/pkg/mapping/fieldindex_test.go @@ -3,15 +3,15 @@ package mapping_test import ( "reflect" "strings" - "testing" - "github.com/stretchr/testify/require" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) -func resourceFieldIndex(testing.TB) map[string]string { +func resourceFieldIndex() map[string]string { return mapping.FieldNameIndex( reflect.TypeFor[search.Resource](), search.Resource{}.SearchFieldOverrides(), @@ -26,58 +26,6 @@ func resolve(idx map[string]string, key string) string { return key } -func TestFieldNameIndex_TopLevelCaseInsensitive(t *testing.T) { - idx := resourceFieldIndex(t) - for in, want := range map[string]string{ - "rootid": "RootID", "ROOTID": "RootID", "RootID": "RootID", - "name": "Name", "NAME": "Name", - "mimetype": "MimeType", "MimeType": "MimeType", - "tags": "Tags", "favorites": "Favorites", - "mtime": "Mtime", "parentid": "ParentID", "id": "ID", - } { - require.Equalf(t, want, resolve(idx, in), "resolve(%q)", in) - } -} - -// Facet sub-fields are lowerCamelCase in the index (from libregraph json tags); -// the derived index resolves them case-insensitively. -func TestFieldNameIndex_FacetsCaseInsensitive(t *testing.T) { - idx := resourceFieldIndex(t) - for in, want := range map[string]string{ - // case-insensitive: same field, different casings - "photo.cameramake": "photo.cameraMake", - "photo.CAMERAMAKE": "photo.cameraMake", - // a representative sub-field across each facet - "photo.takendatetime": "photo.takenDateTime", - "audio.artist": "audio.artist", - "audio.albumartist": "audio.albumArtist", - "image.width": "image.width", - "location.latitude": "location.latitude", - } { - require.Equalf(t, want, resolve(idx, in), "resolve(%q)", in) - } -} - -func TestFieldNameIndex_UnknownPassesThrough(t *testing.T) { - idx := resourceFieldIndex(t) - require.Equal(t, "nope.field", resolve(idx, "nope.field")) - require.Equal(t, "custom", resolve(idx, "custom")) -} - -// All top-level fields are covered from one derived source, so both backends -// resolve them the same way. -func TestFieldNameIndex_CoversAllTopLevelFields(t *testing.T) { - idx := resourceFieldIndex(t) - for in, want := range map[string]string{ - "rootid": "RootID", "path": "Path", "id": "ID", "name": "Name", - "size": "Size", "mtime": "Mtime", "type": "Type", - "content": "Content", "hidden": "Hidden", "tags": "Tags", - "favorites": "Favorites", - } { - require.Equalf(t, want, resolve(idx, in), "derived should cover %q", in) - } -} - // NestInner is embedded with a json tag below, so it must nest, not flatten. type NestInner struct { A string `json:"A"` @@ -88,11 +36,66 @@ type taggedOuter struct { Top string `json:"top"` } -// A json-tagged embedded struct nests under its tag in the derived index too -// (walkFields must match encoding/json), so its fields are "inner.A", not "A". -func TestFieldNameIndex_TaggedEmbeddedNests(t *testing.T) { - idx := mapping.FieldNameIndex(reflect.TypeFor[taggedOuter](), nil) - require.Equal(t, "inner.A", resolve(idx, "inner.a")) // nested under the tag - require.Equal(t, "top", resolve(idx, "top")) - require.Equal(t, "a", resolve(idx, "a")) // not flattened: bare "a" is not a key -} +var _ = Describe("FieldNameIndex", func() { + It("resolves top-level fields case-insensitively", func() { + idx := resourceFieldIndex() + for in, want := range map[string]string{ + "rootid": "RootID", "ROOTID": "RootID", "RootID": "RootID", + "name": "Name", "NAME": "Name", + "mimetype": "MimeType", "MimeType": "MimeType", + "tags": "Tags", "favorites": "Favorites", + "mtime": "Mtime", "parentid": "ParentID", "id": "ID", + } { + Expect(resolve(idx, in)).To(Equal(want), "resolve(%q)", in) + } + }) + + // Facet sub-fields are lowerCamelCase in the index (from libregraph json + // tags); the derived index resolves them case-insensitively. + It("resolves facet sub-fields case-insensitively", func() { + idx := resourceFieldIndex() + for in, want := range map[string]string{ + // case-insensitive: same field, different casings + "photo.cameramake": "photo.cameraMake", + "photo.CAMERAMAKE": "photo.cameraMake", + // a representative sub-field across each facet + "photo.takendatetime": "photo.takenDateTime", + "audio.artist": "audio.artist", + "audio.albumartist": "audio.albumArtist", + "image.width": "image.width", + "location.latitude": "location.latitude", + } { + Expect(resolve(idx, in)).To(Equal(want), "resolve(%q)", in) + } + }) + + It("passes unknown keys through unchanged", func() { + idx := resourceFieldIndex() + Expect(resolve(idx, "nope.field")).To(Equal("nope.field")) + Expect(resolve(idx, "custom")).To(Equal("custom")) + }) + + // All top-level fields are covered from one derived source, so both + // backends resolve them the same way. + It("covers all top-level fields", func() { + idx := resourceFieldIndex() + for in, want := range map[string]string{ + "rootid": "RootID", "path": "Path", "id": "ID", "name": "Name", + "size": "Size", "mtime": "Mtime", "type": "Type", + "content": "Content", "hidden": "Hidden", "tags": "Tags", + "favorites": "Favorites", + } { + Expect(resolve(idx, in)).To(Equal(want), "derived should cover %q", in) + } + }) + + // A json-tagged embedded struct nests under its tag in the derived index + // too (walkFields must match encoding/json), so its fields are "inner.A", + // not "A". + It("nests json-tagged embedded structs", func() { + idx := mapping.FieldNameIndex(reflect.TypeFor[taggedOuter](), nil) + Expect(resolve(idx, "inner.a")).To(Equal("inner.A")) // nested under the tag + Expect(resolve(idx, "top")).To(Equal("top")) + Expect(resolve(idx, "a")).To(Equal("a")) // not flattened: bare "a" is not a key + }) +}) diff --git a/services/search/pkg/query/mimetype/mimetype_suite_test.go b/services/search/pkg/query/mimetype/mimetype_suite_test.go new file mode 100644 index 0000000000..321ff2e8d9 --- /dev/null +++ b/services/search/pkg/query/mimetype/mimetype_suite_test.go @@ -0,0 +1,13 @@ +package mimetype_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMimetype(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Mimetype Suite") +} diff --git a/services/search/pkg/query/mimetype/mimetype_test.go b/services/search/pkg/query/mimetype/mimetype_test.go index c9d9a7cf6c..51fe24e357 100644 --- a/services/search/pkg/query/mimetype/mimetype_test.go +++ b/services/search/pkg/query/mimetype/mimetype_test.go @@ -1,9 +1,8 @@ package mimetype_test import ( - "testing" - - "github.com/stretchr/testify/require" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/services/search/pkg/query/mimetype" @@ -12,93 +11,6 @@ import ( // This is the single place the mediatype -> MimeType mapping is tested. The // query pipeline consumes Expand via query.Normalize and must NOT re-test it. -func TestExpand_onlyTriggersOnMediatype(t *testing.T) { - require.Nil(t, mimetype.Expand("Name", "document")) - require.Nil(t, mimetype.Expand("MimeType", "file")) // the real field name is not the trigger - require.Nil(t, mimetype.Expand("Tags", "file")) -} - -func TestExpand_keyIsCaseInsensitive(t *testing.T) { - require.NotNil(t, mimetype.Expand("MediaType", "file")) -} - -func TestExpand_valueIsCaseInsensitive(t *testing.T) { - // a category matches regardless of case - require.Equal(t, []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - }, mimetype.Expand("mediatype", "Folder")) - // a literal MIME type is lowercased too (MIME types are case-insensitive) - require.Equal(t, []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "image/svg+xml"}, - }, mimetype.Expand("mediatype", "Image/SVG+XML")) -} - -// A non-category value is a literal MIME type and targets the MimeType field. -func TestExpand_literalValuePassesThroughToMimeType(t *testing.T) { - require.Equal(t, []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "application/pdf"}, - }, mimetype.Expand("mediatype", "application/pdf")) - - require.Equal(t, []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "image/jpeg"}, - }, mimetype.Expand("mediatype", "image/jpeg")) -} - -func TestExpand_fileIsNotAFolder(t *testing.T) { - // grouped so the negation stays atomic next to an operator. - require.Equal(t, []ast.Node{ - &ast.GroupNode{Nodes: []ast.Node{ - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - }}, - }, mimetype.Expand("mediatype", "file")) -} - -func TestExpand_folderIsASingleTerm(t *testing.T) { - require.Equal(t, []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - }, mimetype.Expand("mediatype", "folder")) -} - -func TestExpand_wildcardCategories(t *testing.T) { - for value, mime := range map[string]string{ - "image": "image/*", "video": "video/*", "audio": "audio/*", "pdf": "application/pdf", - } { - require.Equal(t, []ast.Node{ - &ast.StringNode{Key: "MimeType", Value: mime}, - }, mimetype.Expand("mediatype", value), value) - } -} - -func TestExpand_documentGroup(t *testing.T) { - got := mimetype.Expand("mediatype", "document") - require.Len(t, got, 1) - group, ok := got[0].(*ast.GroupNode) - require.True(t, ok) - require.Equal(t, mimeValues(group), []string{ - "application/msword", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "application/vnd.openxmlformats-officedocument.wordprocessingml.form", - "application/vnd.oasis.opendocument.text", - "text/plain", - "text/markdown", - "application/rtf", - "application/vnd.apple.pages", - }) -} - -// spreadsheet asserts the exact MIME set, in order, with no duplicate entry. -func TestExpand_spreadsheet(t *testing.T) { - group := mimetype.Expand("mediatype", "spreadsheet")[0].(*ast.GroupNode) - require.Equal(t, mimeValues(group), []string{ - "application/vnd.ms-excel", - "application/vnd.oasis.opendocument.spreadsheet", - "text/csv", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "application/vnd.apple.numbers", - }) -} - // mimeValues extracts the StringNode values from an OR group, dropping operators. func mimeValues(group *ast.GroupNode) []string { var out []string @@ -109,3 +21,91 @@ func mimeValues(group *ast.GroupNode) []string { } return out } + +var _ = Describe("Expand", func() { + It("only triggers on the mediatype key", func() { + Expect(mimetype.Expand("Name", "document")).To(BeNil()) + Expect(mimetype.Expand("MimeType", "file")).To(BeNil()) // the real field name is not the trigger + Expect(mimetype.Expand("Tags", "file")).To(BeNil()) + }) + + It("matches the key case-insensitively", func() { + Expect(mimetype.Expand("MediaType", "file")).ToNot(BeNil()) + }) + + It("matches the value case-insensitively", func() { + // a category matches regardless of case + Expect(mimetype.Expand("mediatype", "Folder")).To(Equal([]ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + })) + // a literal MIME type is lowercased too (MIME types are case-insensitive) + Expect(mimetype.Expand("mediatype", "Image/SVG+XML")).To(Equal([]ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "image/svg+xml"}, + })) + }) + + // A non-category value is a literal MIME type and targets the MimeType field. + It("passes literal values through to MimeType", func() { + Expect(mimetype.Expand("mediatype", "application/pdf")).To(Equal([]ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "application/pdf"}, + })) + Expect(mimetype.Expand("mediatype", "image/jpeg")).To(Equal([]ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "image/jpeg"}, + })) + }) + + It("expands file to not-a-folder", func() { + // grouped so the negation stays atomic next to an operator. + Expect(mimetype.Expand("mediatype", "file")).To(Equal([]ast.Node{ + &ast.GroupNode{Nodes: []ast.Node{ + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + }}, + })) + }) + + It("expands folder to a single term", func() { + Expect(mimetype.Expand("mediatype", "folder")).To(Equal([]ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + })) + }) + + It("expands wildcard categories", func() { + for value, mime := range map[string]string{ + "image": "image/*", "video": "video/*", "audio": "audio/*", "pdf": "application/pdf", + } { + Expect(mimetype.Expand("mediatype", value)).To(Equal([]ast.Node{ + &ast.StringNode{Key: "MimeType", Value: mime}, + }), value) + } + }) + + It("expands the document group", func() { + got := mimetype.Expand("mediatype", "document") + Expect(got).To(HaveLen(1)) + group, ok := got[0].(*ast.GroupNode) + Expect(ok).To(BeTrue()) + Expect(mimeValues(group)).To(Equal([]string{ + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.wordprocessingml.form", + "application/vnd.oasis.opendocument.text", + "text/plain", + "text/markdown", + "application/rtf", + "application/vnd.apple.pages", + })) + }) + + // spreadsheet asserts the exact MIME set, in order, with no duplicate entry. + It("expands the spreadsheet group", func() { + group := mimetype.Expand("mediatype", "spreadsheet")[0].(*ast.GroupNode) + Expect(mimeValues(group)).To(Equal([]string{ + "application/vnd.ms-excel", + "application/vnd.oasis.opendocument.spreadsheet", + "text/csv", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.apple.numbers", + })) + }) +}) diff --git a/services/search/pkg/query/normalize_test.go b/services/search/pkg/query/normalize_test.go index 8af9cf7435..617283277a 100644 --- a/services/search/pkg/query/normalize_test.go +++ b/services/search/pkg/query/normalize_test.go @@ -1,9 +1,8 @@ package query_test import ( - "testing" - - "github.com/stretchr/testify/require" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/services/search/pkg/query" @@ -18,90 +17,97 @@ func norm(nodes ...ast.Node) []ast.Node { return query.Normalize(&ast.Ast{Nodes: nodes}, query.ResolveField).Nodes } -func TestResolveField(t *testing.T) { - require.Equal(t, "Name", query.ResolveField("")) // empty -> free-text default - require.Equal(t, "Name", query.ResolveField("NAME")) // canonical, case-insensitive key match - require.Equal(t, "Tags", query.ResolveField("tag")) // singular alias - require.Equal(t, "MimeType", query.ResolveField("mimetype")) // real field - require.Equal(t, "photo.cameraMake", query.ResolveField("photo.CAMERAMAKE")) // facet, case-insensitive key match - require.Equal(t, "unknown.field", query.ResolveField("unknown.field")) // unknown key: unchanged, becomes a dead query -} +var _ = Describe("ResolveField", func() { + It("resolves keys to canonical field names", func() { + Expect(query.ResolveField("")).To(Equal("Name")) // empty -> free-text default + Expect(query.ResolveField("NAME")).To(Equal("Name")) // canonical, case-insensitive key match + Expect(query.ResolveField("tag")).To(Equal("Tags")) // singular alias + Expect(query.ResolveField("mimetype")).To(Equal("MimeType")) // real field + Expect(query.ResolveField("photo.CAMERAMAKE")).To(Equal("photo.cameraMake")) // facet, case-insensitive key match + Expect(query.ResolveField("unknown.field")).To(Equal("unknown.field")) // unknown key: unchanged, becomes a dead query + }) +}) -func TestFieldIsCaseInsensitive(t *testing.T) { - // The four CaseInsensitive override fields (resolved canonical names). - for _, f := range []string{"Name", "Path", "Tags", "Favorites"} { - require.True(t, query.FieldIsCaseInsensitive(f), f) - } - // Case-preserved / non-keyword fields are not. - for _, f := range []string{"MimeType", "ID", "Content", "unknown"} { - require.False(t, query.FieldIsCaseInsensitive(f), f) - } -} +var _ = Describe("FieldIsCaseInsensitive", func() { + It("reports the CaseInsensitive override fields", func() { + // The four CaseInsensitive override fields (resolved canonical names). + for _, f := range []string{"Name", "Path", "Tags", "Favorites"} { + Expect(query.FieldIsCaseInsensitive(f)).To(BeTrue(), f) + } + // Case-preserved / non-keyword fields are not. + for _, f := range []string{"MimeType", "ID", "Content", "unknown"} { + Expect(query.FieldIsCaseInsensitive(f)).To(BeFalse(), f) + } + }) +}) -func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) { - got := norm( - &ast.StringNode{Key: "", Value: "free"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "TAG", Value: "x"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "photo.cameramake", Value: "canon"}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "mediatype", Value: "file"}, - &ast.OperatorNode{Value: "AND"}, - ast.NumberNode{Key: "size", Value: 100}, - ) - require.Equal(t, []ast.Node{ - &ast.StringNode{Key: "Name", Value: "free", CaseInsensitive: true}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "Tags", Value: "x", CaseInsensitive: true}, - &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "photo.cameraMake", Value: "canon"}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.NumberNode{Key: "Size", Value: 100}, - }, got) -} +var _ = Describe("Normalize", func() { + It("resolves fields and expands mediatype", func() { + got := norm( + &ast.StringNode{Key: "", Value: "free"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "TAG", Value: "x"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "photo.cameramake", Value: "canon"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "mediatype", Value: "file"}, + &ast.OperatorNode{Value: "AND"}, + ast.NumberNode{Key: "size", Value: 100}, + ) + Expect(got).To(Equal([]ast.Node{ + &ast.StringNode{Key: "Name", Value: "free", CaseInsensitive: true}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Tags", Value: "x", CaseInsensitive: true}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "photo.cameraMake", Value: "canon"}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + &ast.OperatorNode{Value: "AND"}, + &ast.NumberNode{Key: "Size", Value: 100}, + }}, + })) + }) -// A bare restriction inside a named group inherits the group key; a keyed child -// keeps its own key; a bare restriction in an unnamed group falls back to Name. -func TestNormalize_GroupKeyDefaulting(t *testing.T) { - got := norm( - &ast.GroupNode{Key: "author", Nodes: []ast.Node{ - &ast.StringNode{Value: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "name", Value: "d"}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Value: "e"}, - }}, - ) - require.Equal(t, []ast.Node{ - &ast.GroupNode{Key: "author", Nodes: []ast.Node{ - &ast.StringNode{Key: "author", Value: "b"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: "Name", Value: "d", CaseInsensitive: true}, - }}, - &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "e", CaseInsensitive: true}, - }}, - }, got) -} + // A bare restriction inside a named group inherits the group key; a keyed + // child keeps its own key; a bare restriction in an unnamed group falls + // back to Name. + It("defaults group keys", func() { + got := norm( + &ast.GroupNode{Key: "author", Nodes: []ast.Node{ + &ast.StringNode{Value: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "name", Value: "d"}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Value: "e"}, + }}, + ) + Expect(got).To(Equal([]ast.Node{ + &ast.GroupNode{Key: "author", Nodes: []ast.Node{ + &ast.StringNode{Key: "author", Value: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "Name", Value: "d", CaseInsensitive: true}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "e", CaseInsensitive: true}, + }}, + })) + }) -func TestNormalize_ConvertsValueNodesToPointers(t *testing.T) { - got := norm( - ast.StringNode{Key: "name", Value: "x"}, - ast.OperatorNode{Value: "AND"}, - ast.DateTimeNode{Key: "mtime"}, - ) - require.Equal(t, []ast.Node{ - &ast.StringNode{Key: "Name", Value: "x", CaseInsensitive: true}, - &ast.OperatorNode{Value: "AND"}, - &ast.DateTimeNode{Key: "Mtime"}, - }, got) -} + It("converts value nodes to pointers", func() { + got := norm( + ast.StringNode{Key: "name", Value: "x"}, + ast.OperatorNode{Value: "AND"}, + ast.DateTimeNode{Key: "mtime"}, + ) + Expect(got).To(Equal([]ast.Node{ + &ast.StringNode{Key: "Name", Value: "x", CaseInsensitive: true}, + &ast.OperatorNode{Value: "AND"}, + &ast.DateTimeNode{Key: "Mtime"}, + })) + }) +}) diff --git a/services/search/pkg/query/query_suite_test.go b/services/search/pkg/query/query_suite_test.go new file mode 100644 index 0000000000..9d9ddf7487 --- /dev/null +++ b/services/search/pkg/query/query_suite_test.go @@ -0,0 +1,13 @@ +package query_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestQuery(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Query Suite") +} From f6739f552d53d20eb3b321d867b01a3fb22c6244 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Tue, 11 Aug 2026 09:48:05 +0200 Subject: [PATCH 28/54] fix(search): bind a leading NOT to the term right after it, on both backends A leading NOT next to an operator was miscompiled: the bleve compiler left the consumed term in `next`, so `NOT x AND y` dropped `y` and produced a self-contradicting clause; the OpenSearch transpiler checked nextOp==AND before prevOp==NOT, so the negated term landed in `must` instead of `must_not`. NOT is unary and binds to the node directly after it regardless of what follows. This also fixes `mediatype:file AND ` (the web Files filter) at the root, so the earlier mediatype:file group workaround is dropped. --- services/search/pkg/bleve/backend_test.go | 12 +++++++++++ .../internal/convert/kql_transpile.go | 14 ++++++++++--- .../internal/convert/kql_transpile_test.go | 20 +++++++++++++++++++ services/search/pkg/query/bleve/compiler.go | 5 ++++- .../search/pkg/query/mimetype/mimetype.go | 9 ++------- .../pkg/query/mimetype/mimetype_test.go | 7 ++----- services/search/pkg/query/normalize_test.go | 10 ++++------ 7 files changed, 55 insertions(+), 22 deletions(-) diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go index 16422b809b..417c6c50ad 100644 --- a/services/search/pkg/bleve/backend_test.go +++ b/services/search/pkg/bleve/backend_test.go @@ -196,6 +196,18 @@ var _ = Describe("Bleve", func() { assertDocCount(rootResource.ID, "tag:missing", 0) }) + It("binds a leading NOT to the term right after it, combined with AND", func() { + // regression: a leading NOT next to AND dropped the AND'd term, so + // `NOT tag:x AND name:y` matched nothing (a self-contradicting clause). + parentResource.Document.Tags = []string{"physik"} + childResource.Document.Tags = []string{"mathe"} + Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) + Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) + + assertDocCount(rootResource.ID, "NOT tag:physik AND name:child.pdf", 1) // the mathe child + assertDocCount(rootResource.ID, "NOT tag:mathe AND name:parent*", 1) // the physik parent + }) + It("finds files by size", func() { parentResource.Document.Size = 12345 err := eng.Upsert(parentResource.ID, parentResource) diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index 368f1ad9be..0e74dd96c0 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -49,13 +49,21 @@ func (t kqlOpensearchTranspiler) transpile(nodes []ast.Node) (osu.Builder, error nextOp := t.getOperatorValueAt(nodes, i+1) prevOp := t.getOperatorValueAt(nodes, i-1) + // A preceding NOT negates this node regardless of what follows (NOT x AND y + // is (NOT x) AND y), so it must win over nextOp. The prevOp AND/OR cases + // give the right operand its own bucket instead of inheriting the previous + // one, which matters right after a NOT (its MustNot must not carry over). switch { + case prevOp == kql.BoolNOT: + boolQueryAdd = boolQuery.MustNot case nextOp == kql.BoolOR: boolQueryAdd = boolQuery.Should case nextOp == kql.BoolAND: boolQueryAdd = boolQuery.Must - case prevOp == kql.BoolNOT: - boolQueryAdd = boolQuery.MustNot + case prevOp == kql.BoolOR: + boolQueryAdd = boolQuery.Should + case prevOp == kql.BoolAND: + boolQueryAdd = boolQuery.Must } builder, err := t.toBuilder(node) @@ -72,7 +80,7 @@ func (t kqlOpensearchTranspiler) transpile(nodes []ast.Node) (osu.Builder, error continue } - if nextOp == kql.BoolOR { + if nextOp == kql.BoolOR || prevOp == kql.BoolOR { // if there are should clauses, we set the minimum should match to 1 boolQueryParams.MinimumShouldMatch = 1 } diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go index 8cfbf54472..85996976d7 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go @@ -279,6 +279,26 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { osu.NewTermQuery[string]("age").Value("32"), ), }, + { + // NOT binds to the node directly after it, not to whatever operator + // follows that node: NOT x AND y is (NOT x) AND y. + Name: "[NOT * AND *]", + Got: &ast.Ast{ + Nodes: []ast.Node{ + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "age", Value: "32"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Name", Value: "openCloud"}, + }, + }, + Want: osu.NewBoolQuery(). + MustNot( + osu.NewTermQuery[string]("age").Value("32"), + ). + Must( + osu.NewTermQuery[string]("Name").Value("openCloud"), + ), + }, { Name: "[* OR * OR *]", Got: &ast.Ast{ diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index 5a57d2bc78..4a536d3776 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -190,8 +190,11 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { q := bleve.NewBooleanQuery() q.AddMustNot(next) if prev == nil { - // unary in the beginning + // unary at the beginning: the term was consumed into the + // MustNot via nextNode, so clear next, otherwise a following + // operator would bind the stale term (NOT x AND y drops y). prev = q + next = nil } else { next = q } diff --git a/services/search/pkg/query/mimetype/mimetype.go b/services/search/pkg/query/mimetype/mimetype.go index 06700049e9..ca61e651ac 100644 --- a/services/search/pkg/query/mimetype/mimetype.go +++ b/services/search/pkg/query/mimetype/mimetype.go @@ -23,14 +23,9 @@ func Expand(key, value string) []ast.Node { value = strings.ToLower(value) switch value { case "file": - // Group the negation so it stays atomic next to an operator: a bare - // `NOT ` sequence spliced inline miscompiles as the left of an AND - // (mediatype:file AND name:x would drop name:x). return []ast.Node{ - &ast.GroupNode{Nodes: []ast.Node{ - &ast.OperatorNode{Value: kql.BoolNOT}, - &ast.StringNode{Key: field, Value: "httpd/unix-directory"}, - }}, + &ast.OperatorNode{Value: kql.BoolNOT}, + &ast.StringNode{Key: field, Value: "httpd/unix-directory"}, } case "folder": return term("httpd/unix-directory") diff --git a/services/search/pkg/query/mimetype/mimetype_test.go b/services/search/pkg/query/mimetype/mimetype_test.go index 51fe24e357..d48fb44e6a 100644 --- a/services/search/pkg/query/mimetype/mimetype_test.go +++ b/services/search/pkg/query/mimetype/mimetype_test.go @@ -55,12 +55,9 @@ var _ = Describe("Expand", func() { }) It("expands file to not-a-folder", func() { - // grouped so the negation stays atomic next to an operator. Expect(mimetype.Expand("mediatype", "file")).To(Equal([]ast.Node{ - &ast.GroupNode{Nodes: []ast.Node{ - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - }}, + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, })) }) diff --git a/services/search/pkg/query/normalize_test.go b/services/search/pkg/query/normalize_test.go index 617283277a..3fec574da1 100644 --- a/services/search/pkg/query/normalize_test.go +++ b/services/search/pkg/query/normalize_test.go @@ -61,12 +61,10 @@ var _ = Describe("Normalize", func() { &ast.OperatorNode{Value: "AND"}, &ast.StringNode{Key: "photo.cameraMake", Value: "canon"}, &ast.OperatorNode{Value: "AND"}, - &ast.GroupNode{Nodes: []ast.Node{ - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, - &ast.OperatorNode{Value: "AND"}, - &ast.NumberNode{Key: "Size", Value: 100}, - }}, + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + &ast.OperatorNode{Value: "AND"}, + &ast.NumberNode{Key: "Size", Value: 100}, })) }) From 9c46713a654057dfac1f7b77c208e44ffede5ea1 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Thu, 13 Aug 2026 01:37:17 +0200 Subject: [PATCH 29/54] refactor(search): make paths case-sensitive and scope searches at query level Paths act as references (location scoping, deep links): /Foo and /foo are distinct siblings, so path: matching must be exact. Case-insensitive folder discovery is served by name: and its lowercase sibling. This also matches bleve on main, where path queries have always been case-sensitive. Dropping the sibling removes its biggest maintenance cost: Path is the one mutable sibling field, a move rewrites the paths of a whole subtree and the OpenSearch move script had to rebuild Path_lowercase alongside the base field. With paths case-sensitive, the ref path scope moves into the query itself: bleve as a term/prefix disjunction on the keyword Path, OpenSearch as a term filter on its path_hierarchy tokens. The post-filter that used to drop out-of-scope hits after the query ran is gone; totals and paging now respect the scope instead of being computed over the whole space, and a wrong-cased scope simply matches nothing. --- services/search/pkg/bleve/backend.go | 22 +++--- services/search/pkg/bleve/backend_test.go | 70 +++++++++++++++++-- services/search/pkg/opensearch/backend.go | 21 +++--- services/search/pkg/opensearch/batch.go | 15 ++-- .../search/pkg/query/bleve/compiler_test.go | 4 +- services/search/pkg/query/normalize_test.go | 6 +- services/search/pkg/search/search.go | 2 +- 7 files changed, 94 insertions(+), 46 deletions(-) diff --git a/services/search/pkg/bleve/backend.go b/services/search/pkg/bleve/backend.go index e325d81261..a15491cb3c 100644 --- a/services/search/pkg/bleve/backend.go +++ b/services/search/pkg/bleve/backend.go @@ -3,7 +3,6 @@ package bleve import ( "context" "math" - "strings" "time" "github.com/blevesearch/bleve/v2" @@ -75,6 +74,16 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques ), }, ) + // Scope below the space root: restrict at query level so totals and + // paging respect the path too. Path is a case-preserving keyword + // (paths act as references, /Foo and /foo are distinct), so the exact + // folder or the folder prefix matches all of, and only, the scope. + if requestedPath := utils.MakeRelativePath(sir.Ref.Path); requestedPath != "." { + q.Conjuncts = append(q.Conjuncts, query.NewDisjunctionQuery([]query.Query{ + &query.TermQuery{FieldVal: "Path", Term: requestedPath}, + &query.PrefixQuery{FieldVal: "Path", Prefix: requestedPath + "/"}, + })) + } } bleveReq := bleve.NewSearchRequest(q) @@ -98,17 +107,6 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques matches := make([]*searchMessage.Match, 0, len(res.Hits)) totalMatches := res.Total for _, hit := range res.Hits { - if sir.Ref != nil { - hitPath := strings.TrimSuffix(getFieldValue[string](hit.Fields, "Path"), "/") - requestedPath := utils.MakeRelativePath(sir.Ref.Path) - isRoot := hitPath == requestedPath - - if !isRoot && requestedPath != "." && !strings.HasPrefix(hitPath, requestedPath+"/") { - totalMatches-- - continue - } - } - rootID, err := storagespace.ParseID(getFieldValue[string](hit.Fields, "RootID")) if err != nil { return nil, err diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go index 417c6c50ad..a40572c615 100644 --- a/services/search/pkg/bleve/backend_test.go +++ b/services/search/pkg/bleve/backend_test.go @@ -366,8 +366,10 @@ var _ = Describe("Bleve", func() { assertDocCount(rootResource.ID, `path:"./parent d!r/child.pdf"`, 1) }) - It("matches case-insensitively", func() { - assertDocCount(rootResource.ID, `path:"./PARENT D!R"`, 3) + It("matches case-sensitively", func() { + // paths act as references: /Foo and /foo are distinct siblings, + // so a wrong-cased path must not match + assertDocCount(rootResource.ID, `path:"./PARENT D!R"`, 0) }) It("applies an AND filter to the folder itself, not only descendants", func() { @@ -535,6 +537,61 @@ var _ = Describe("Bleve", func() { }) + Describe("path scoped searches", func() { + BeforeEach(func() { + Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) + Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) + Expect(eng.Upsert(childResource2.ID, childResource2)).To(Succeed()) + outside := search.Resource{ + ID: "1$2!6", + ParentID: rootResource.ID, + RootID: rootResource.ID, + Path: "./other/child3.pdf", + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{Name: "child3.pdf"}, + } + Expect(eng.Upsert(outside.ID, outside)).To(Succeed()) + }) + + It("restricts hits and totals to the scope at query level", func() { + // without the scope all three children match + res, err := doSearch(rootResource.ID, "name:child*", "") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(3))) + + res, err = doSearch(rootResource.ID, "name:child*", "./parent d!r") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(2))) + Expect(len(res.Matches)).To(Equal(2)) + }) + + It("keeps totals right on a small page", func() { + // the scope is part of the query, so totals cover the full scope + // even when the page holds a single hit + rID, err := storagespace.ParseID(rootResource.ID) + Expect(err).ToNot(HaveOccurred()) + res, err := eng.Search(context.Background(), &searchsvc.SearchIndexRequest{ + Query: "name:child*", + PageSize: 1, + Ref: &searchmsg.Reference{ + ResourceId: &searchmsg.ResourceID{ + StorageId: rID.StorageId, SpaceId: rID.SpaceId, OpaqueId: rID.OpaqueId, + }, + Path: "./parent d!r", + }, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(2))) + Expect(len(res.Matches)).To(Equal(1)) + }) + + It("matches the scope case-sensitively", func() { + res, err := doSearch(rootResource.ID, "name:child*", "./PARENT D!R") + Expect(err).ToNot(HaveOccurred()) + Expect(res.TotalMatches).To(Equal(int32(0))) + }) + }) + Describe("Upsert", func() { It("adds a resourceInfo to the index", func() { err := eng.Upsert(childResource.ID, childResource) @@ -756,11 +813,12 @@ var _ = Describe("Bleve", func() { Expect(eng.Move(parentResource.ID, parentResource.ParentID, "./my/NewName")).To(Succeed()) - // the lowercased siblings are rebuilt at the new path, so a - // case-insensitive query finds the folder under its new name and path, - // including the descendant, and no longer under the old path. + // the lowercased name sibling is rebuilt, so a case-insensitive + // name query still works; the path is case-sensitive by design, so + // only the exact new path matches (and the old one no longer does). assertDocCount(rootResource.ID, "name:NEWNAME", 1) - assertDocCount(rootResource.ID, `path:"./MY/NEWNAME"`, 2) + assertDocCount(rootResource.ID, `path:"./my/NewName"`, 2) + assertDocCount(rootResource.ID, `path:"./MY/NEWNAME"`, 0) assertDocCount(rootResource.ID, `path:"./parent d!r"`, 0) }) }) diff --git a/services/search/pkg/opensearch/backend.go b/services/search/pkg/opensearch/backend.go index 93433f502a..49cf67fc54 100644 --- a/services/search/pkg/opensearch/backend.go +++ b/services/search/pkg/opensearch/backend.go @@ -3,7 +3,6 @@ package opensearch import ( "context" "fmt" - "strings" "time" storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -98,6 +97,15 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ ), ), ) + // Scope below the space root: restrict at query level so totals and + // paging respect the path too. Path uses the case-preserving + // path_hierarchy analyzer, so the folder path is an indexed token of + // the folder itself and every descendant. + if requestedPath := utils.MakeRelativePath(sir.Ref.Path); requestedPath != "." { + boolQuery.Filter( + osu.NewTermQuery[string]("Path").Value(requestedPath), + ) + } } searchParams := opensearchgoAPI.SearchParams{ @@ -150,17 +158,6 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ return nil, fmt.Errorf("failed to convert hit to match: %w", err) } - if sir.Ref != nil { - hitPath := strings.TrimSuffix(match.GetEntity().GetRef().GetPath(), "/") - requestedPath := utils.MakeRelativePath(sir.Ref.Path) - isRoot := hitPath == requestedPath - - if !isRoot && requestedPath != "." && !strings.HasPrefix(hitPath, requestedPath+"/") { - totalMatches-- - continue - } - } - matches = append(matches, match) } diff --git a/services/search/pkg/opensearch/batch.go b/services/search/pkg/opensearch/batch.go index 04ba0c9a04..637764bff2 100644 --- a/services/search/pkg/opensearch/batch.go +++ b/services/search/pkg/opensearch/batch.go @@ -64,19 +64,19 @@ func (b *Batch) Upsert(id string, r search.Resource) error { }) } -func (b *Batch) Move(id string, parentID string, location string) error { +func (b *Batch) Move(id, parentID, location string) error { return b.withSizeLimit(func() error { op := func() error { return updateSelfAndDescendants(context.Background(), b.client, b.index, id, func(rootResource search.Resource) *osu.BodyParamScript { newPath := utils.MakeRelativePath(location) newName := path.Base(newPath) return &osu.BodyParamScript{ - // Keep the case-preserved base fields and their lowercased - // search siblings in sync: swap the moved prefix in both. Only - // the leading oldPath is replaced (startsWith + substring, not + // Keep Name and its lowercased search sibling in sync; Path has + // no sibling (case-sensitive by design). Only the leading + // oldPath is replaced (startsWith + substring, not // String.replace, which would also rewrite a repeated segment // deeper in a descendant's path, e.g. /Music/Music.m3u). The - // lowercased new values come from Go's strings.ToLower via + // lowercased new name comes from Go's strings.ToLower via // params, so the sibling stays byte-identical to what // PrepareForIndex writes on upsert (painless toLowerCase would // lowercase differently than Go). @@ -89,9 +89,6 @@ func (b *Batch) Move(id string, parentID string, location string) error { if (ctx._source.Path != null && ctx._source.Path.startsWith(params.oldPath)) { ctx._source.Path = params.newPath + ctx._source.Path.substring(params.oldPath.length()); } - if (ctx._source.Path%[1]s != null && ctx._source.Path%[1]s.startsWith(params.oldPathLower)) { - ctx._source.Path%[1]s = params.newPathLower + ctx._source.Path%[1]s.substring(params.oldPathLower.length()); - } boolean hidden = false; for (String name : ctx._source.Path.splitOnToken('/')) { if (!name.equals('.') && !name.equals('..') && name.startsWith('.')) { hidden = true; break; } @@ -105,8 +102,6 @@ func (b *Batch) Move(id string, parentID string, location string) error { "oldPath": rootResource.Path, "newPath": newPath, "newName": newName, - "oldPathLower": strings.ToLower(rootResource.Path), - "newPathLower": strings.ToLower(newPath), "newNameLower": strings.ToLower(newName), }, } diff --git a/services/search/pkg/query/bleve/compiler_test.go b/services/search/pkg/query/bleve/compiler_test.go index 23d189f689..c1662bbf19 100644 --- a/services/search/pkg/query/bleve/compiler_test.go +++ b/services/search/pkg/query/bleve/compiler_test.go @@ -56,8 +56,8 @@ func Test_compile(t *testing.T) { // so an enclosing AND does not redistribute the folder-itself clause. want: func() query.Query { bq := query.NewBooleanQuery(nil, []query.Query{ - query.NewQueryStringQuery(`Path_lowercase:\/foo`), - query.NewQueryStringQuery(`Path_lowercase:\/foo\/*`), + query.NewQueryStringQuery(`Path:\/Foo`), + query.NewQueryStringQuery(`Path:\/Foo\/*`), }, nil) bq.SetMinShould(1) return query.NewConjunctionQuery([]query.Query{bq}) diff --git a/services/search/pkg/query/normalize_test.go b/services/search/pkg/query/normalize_test.go index 3fec574da1..d9421fdc0f 100644 --- a/services/search/pkg/query/normalize_test.go +++ b/services/search/pkg/query/normalize_test.go @@ -30,12 +30,12 @@ var _ = Describe("ResolveField", func() { var _ = Describe("FieldIsCaseInsensitive", func() { It("reports the CaseInsensitive override fields", func() { - // The four CaseInsensitive override fields (resolved canonical names). - for _, f := range []string{"Name", "Path", "Tags", "Favorites"} { + // The three CaseInsensitive override fields (resolved canonical names). + for _, f := range []string{"Name", "Tags", "Favorites"} { Expect(query.FieldIsCaseInsensitive(f)).To(BeTrue(), f) } // Case-preserved / non-keyword fields are not. - for _, f := range []string{"MimeType", "ID", "Content", "unknown"} { + for _, f := range []string{"MimeType", "ID", "Content", "Path", "unknown"} { Expect(query.FieldIsCaseInsensitive(f)).To(BeFalse(), f) } }) diff --git a/services/search/pkg/search/search.go b/services/search/pkg/search/search.go index 73b07cae01..81175fc0ad 100644 --- a/services/search/pkg/search/search.go +++ b/services/search/pkg/search/search.go @@ -75,7 +75,7 @@ var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts True, False := true, false return map[string]mapping.FieldOpts{ "Name": {CaseInsensitive: &True}, - "Path": {Type: mapping.TypePath, CaseInsensitive: &True}, + "Path": {Type: mapping.TypePath}, "Content": {Type: mapping.TypeFulltext}, "Tags": {CaseInsensitive: &True, IncludeInAll: &False}, "Favorites": {CaseInsensitive: &True, IncludeInAll: &False}, From e64692ce9ea4f3fd339198c60b0abd1c22347530 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Tue, 18 Aug 2026 17:12:23 +0200 Subject: [PATCH 30/54] test(search): re-apply opensearch backend coverage under ginkgo --- .../search/pkg/opensearch/backend_test.go | 850 ++++++++++++++++++ 1 file changed, 850 insertions(+) diff --git a/services/search/pkg/opensearch/backend_test.go b/services/search/pkg/opensearch/backend_test.go index ecf728fe12..6fad3a5eee 100644 --- a/services/search/pkg/opensearch/backend_test.go +++ b/services/search/pkg/opensearch/backend_test.go @@ -8,6 +8,10 @@ import ( opensearchgo "github.com/opensearch-project/opensearch-go/v4" opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" + "github.com/opencloud-eu/reva/v2/pkg/errtypes" + + searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" ) @@ -31,4 +35,850 @@ var _ = Describe("Backend", func() { Expect(err).To(MatchError(opensearch.ErrUnhealthyCluster)) }) }) + + Describe("Search", func() { + const indexName = "opencloud-test-engine-search" + + var ( + tc *opensearchtest.TestClient + backend *opensearch.Backend + document search.Resource + ) + + BeforeEach(func() { + tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, nil, 0) + deleteIndexOnCleanup(tc, indexName) + + var err error + backend, err = opensearch.NewBackend(indexName, tc.Client()) + Expect(err).ToNot(HaveOccurred()) + + document = opensearchtest.Testdata.Resources.File + Expect(backend.Upsert(document.ID, document)).To(Succeed()) + tc.Require.IndicesRefresh([]string{indexName}, nil) + tc.Require.IndicesCount([]string{indexName}, nil, 1) + }) + + It("performs the most simple search", func() { + resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ + Query: fmt.Sprintf(`"%s"`, document.Name), + }) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Matches).To(HaveLen(1)) + Expect(resp.TotalMatches).To(Equal(int32(1))) + Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID)) + }) + + It("ignores files that are marked as deleted", func() { + deletedDocument := opensearchtest.Testdata.Resources.File + deletedDocument.ID = "1$2!4" + deletedDocument.Deleted = true + + Expect(backend.Upsert(deletedDocument.ID, deletedDocument)).To(Succeed()) + tc.Require.IndicesRefresh([]string{indexName}, nil) + tc.Require.IndicesCount([]string{indexName}, nil, 2) + + resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ + Query: fmt.Sprintf(`"%s"`, document.Name), + }) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Matches).To(HaveLen(1)) + Expect(resp.TotalMatches).To(Equal(int32(1))) + Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID)) + }) + + It("restricts hits and totals to the path scope", func() { + outside := opensearchtest.Testdata.Resources.File + outside.ID = "1$1!5" + outside.Path = "./other folder/else.jpg" + Expect(backend.Upsert(outside.ID, outside)).To(Succeed()) + tc.Require.IndicesRefresh([]string{indexName}, nil) + + scoped := &searchMessage.Reference{ + ResourceId: &searchMessage.ResourceID{StorageId: "1", SpaceId: "1", OpaqueId: "1"}, + Path: "./parent d!r", + } + resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ + Query: fmt.Sprintf(`"%s"`, document.Name), + Ref: scoped, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Matches).To(HaveLen(1)) + Expect(resp.TotalMatches).To(Equal(int32(1))) + Expect(resp.Matches[0].Entity.Ref.Path).To(Equal("./parent d!r/child.jpg")) + + // the scope is a reference and matches case-sensitively + wrongCase := &searchMessage.Reference{ + ResourceId: &searchMessage.ResourceID{StorageId: "1", SpaceId: "1", OpaqueId: "1"}, + Path: "./PARENT D!R", + } + respWrongCase, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ + Query: fmt.Sprintf(`"%s"`, document.Name), + Ref: wrongCase, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(respWrongCase.Matches).To(HaveLen(0)) + Expect(respWrongCase.TotalMatches).To(Equal(int32(0))) + }) + }) + + Describe("FullTextSearch", func() { + const indexName = "opencloud-test-engine-fulltext" + + var ( + tc *opensearchtest.TestClient + backend *opensearch.Backend + ) + + BeforeEach(func() { + tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, nil, 0) + deleteIndexOnCleanup(tc, indexName) + + var err error + backend, err = opensearch.NewBackend(indexName, tc.Client()) + Expect(err).ToNot(HaveOccurred()) + + document := opensearchtest.Testdata.Resources.File + document.Content = "Running Foxes" + Expect(backend.Upsert(document.ID, document)).To(Succeed()) + tc.Require.IndicesRefresh([]string{indexName}, nil) + }) + + It("searches content case-insensitively and stemmed, like bleve", func() { + // case-folded and porter-stemmed by the fulltext analyzer; the match + // query analyzes the query value the same way. "content:run*" is an + // unanalyzed wildcard over the stemmed term "run", so it must still + // route to a wildcard query (not degrade to a phrase match). + for _, q := range []string{"content:running", "content:RUNNING", "content:run", "content:run*"} { + resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: q}) + Expect(err).ToNot(HaveOccurred(), q) + Expect(resp.Matches).To(HaveLen(1), q) + } + + resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: "content:cat"}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Matches).To(HaveLen(0)) + }) + }) + + Describe("CaseInsensitiveSearch", func() { + const indexName = "opencloud-test-engine-ci" + + var ( + tc *opensearchtest.TestClient + backend *opensearch.Backend + ) + + BeforeEach(func() { + tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, nil, 0) + deleteIndexOnCleanup(tc, indexName) + + var err error + backend, err = opensearch.NewBackend(indexName, tc.Client()) + Expect(err).ToNot(HaveOccurred()) + + folder := opensearchtest.Testdata.Resources.Folder + folder.ID = "1$2!cifolder" + folder.Path = "./My Dir" + folder.Tags = []string{"Work", "Urgent"} + Expect(backend.Upsert(folder.ID, folder)).To(Succeed()) + + child := opensearchtest.Testdata.Resources.File + child.ID = "1$2!cichild" + child.ParentID = folder.ID + child.Path = "./My Dir/report.pdf" + child.Tags = nil + Expect(backend.Upsert(child.ID, child)).To(Succeed()) + + // a doc outside the folder, so the path assertions below discriminate: + // a phrase-matched path query would analyze into the "." prefix and + // match this one too + outside := opensearchtest.Testdata.Resources.File + outside.ID = "1$2!cioutside" + outside.Path = "./other.pdf" + outside.Tags = nil + Expect(backend.Upsert(outside.ID, outside)).To(Succeed()) + tc.Require.IndicesRefresh([]string{indexName}, nil) + }) + + It("matches tags case-insensitively (array sibling)", func() { + for _, q := range []string{"tag:work", "tag:WORK", "Tags:Urgent"} { + resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: q}) + Expect(err).ToNot(HaveOccurred(), q) + Expect(resp.Matches).To(HaveLen(1), q) + } + }) + + It("matches a spaced path on the folder and its descendants case-sensitively", func() { + resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./My Dir"`}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Matches).To(HaveLen(2)) // folder itself + the descendant, not the outside doc + + // paths act as references, a wrong-cased path must not match + respWrongCase, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./MY DIR"`}) + Expect(err).ToNot(HaveOccurred()) + Expect(respWrongCase.Matches).To(HaveLen(0)) + }) + + It("matches a spaced descendant path only itself", func() { + resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./My Dir/report.pdf"`}) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Matches).To(HaveLen(1)) + }) + }) + + Describe("MediaTypeSearch", func() { + const indexName = "opencloud-test-engine-mediatype" + + var ( + tc *opensearchtest.TestClient + backend *opensearch.Backend + ) + + BeforeEach(func() { + tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, nil, 0) + deleteIndexOnCleanup(tc, indexName) + + var err error + backend, err = opensearch.NewBackend(indexName, tc.Client()) + Expect(err).ToNot(HaveOccurred()) + + svg := opensearchtest.Testdata.Resources.File + svg.ID = "1$2!svg" + svg.MimeType = "image/svg+xml" + Expect(backend.Upsert(svg.ID, svg)).To(Succeed()) + + png := opensearchtest.Testdata.Resources.File + png.ID = "1$2!png" + png.MimeType = "image/png" + Expect(backend.Upsert(png.ID, png)).To(Succeed()) + + folder := opensearchtest.Testdata.Resources.Folder + folder.ID = "1$2!dir" + folder.MimeType = "httpd/unix-directory" + Expect(backend.Upsert(folder.ID, folder)).To(Succeed()) + tc.Require.IndicesRefresh([]string{indexName}, nil) + }) + + DescribeTable("resolves the media type query", + func(query string, want int) { + resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query}) + Expect(err).ToNot(HaveOccurred(), query) + Expect(resp.Matches).To(HaveLen(want), query) + }, + Entry("image/* wildcard matches both files", "mediatype:image", 2), + Entry("categories are case-insensitive", "mediatype:IMAGE", 2), + Entry("literal MIME (+ and /) via mediatype", "mediatype:image/svg+xml", 1), + Entry("same literal via the raw field name", "MimeType:image/svg+xml", 1), + Entry("literal png MIME", "mediatype:image/png", 1), + Entry("no pdf documents", "mediatype:pdf", 0), + Entry("folder category matches the directory only", "mediatype:folder", 1), + Entry("file category matches both files, not the directory", "mediatype:file", 2), + Entry("file category combined with a term", "mediatype:file AND MimeType:image/png", 1), + ) + }) + + Describe("Upsert", func() { + const indexName = "opencloud-test-engine-upsert" + + var ( + tc *opensearchtest.TestClient + backend *opensearch.Backend + ) + + BeforeEach(func() { + tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, nil, 0) + deleteIndexOnCleanup(tc, indexName) + + var err error + backend, err = opensearch.NewBackend(indexName, tc.Client()) + Expect(err).ToNot(HaveOccurred()) + }) + + It("upserts a full document", func() { + document := opensearchtest.Testdata.Resources.File + Expect(backend.Upsert(document.ID, document)).To(Succeed()) + + tc.Require.IndicesCount([]string{indexName}, nil, 1) + }) + + It("upserts a document without an mtime", func() { + // content.Extract leaves Mtime nil when the resource info carries none + document := opensearchtest.Testdata.Resources.File + document.ID = "1$1!4" + document.Mtime = nil + Expect(backend.Upsert(document.ID, document)).To(Succeed()) + + tc.Require.IndicesCount([]string{indexName}, nil, 1) + }) + }) + + Describe("Move", func() { + const indexName = "opencloud-test-engine-move" + + var ( + tc *opensearchtest.TestClient + backend *opensearch.Backend + ) + + BeforeEach(func() { + tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, nil, 0) + deleteIndexOnCleanup(tc, indexName) + + var err error + backend, err = opensearch.NewBackend(indexName, tc.Client()) + Expect(err).ToNot(HaveOccurred()) + }) + + It("moves the document to a new path", func() { + document := opensearchtest.Testdata.Resources.File + tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) + tc.Require.IndicesCount([]string{indexName}, nil, 1) + + body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ + "query": map[string]any{ + "ids": map[string]any{ + "values": []string{document.ID}, + }, + }, + }) + + resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits) + Expect(resources).To(HaveLen(1)) + Expect(resources[0].Path).To(Equal(document.Path)) + + document.Path = "./new/path/to/resource" + Expect(backend.Move(document.ID, document.ParentID, document.Path)).To(Succeed()) + + resources = opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits) + Expect(resources).To(HaveLen(1)) + Expect(resources[0].Path).To(Equal(document.Path)) + }) + + It("keeps case-sensitive path search working after a move", func() { + // Spaced paths so the queries only stay exact as term queries; a phrase + // match would analyze into the "." prefix and match regardless. + document := opensearchtest.Testdata.Resources.File + document.ID = "1$2!cimove" + document.Path = "./Foo Dir/Bar" + Expect(backend.Upsert(document.ID, document)).To(Succeed()) + tc.Require.IndicesRefresh([]string{indexName}, nil) + + document.Path = "./Moved Dir/Bar" + Expect(backend.Move(document.ID, document.ParentID, document.Path)).To(Succeed()) + tc.Require.IndicesRefresh([]string{indexName}, nil) + + // Path is case-sensitive by design: the exact new path matches, a + // wrong-cased query does not, and the old path no longer matches. + respNew, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./Moved Dir/Bar"`}) + Expect(err).ToNot(HaveOccurred()) + Expect(respNew.Matches).To(HaveLen(1)) + + respWrongCase, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./MOVED DIR/BAR"`}) + Expect(err).ToNot(HaveOccurred()) + Expect(respWrongCase.Matches).To(HaveLen(0)) + + respOld, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./Foo Dir/Bar"`}) + Expect(err).ToNot(HaveOccurred()) + Expect(respOld.Matches).To(HaveLen(0)) + }) + }) + + Describe("WriteVisibility", func() { + const indexName = "opencloud-test-engine-write-visibility" + + It("deletes a record that was just written", func() { + document := opensearchtest.Testdata.Resources.File + document.ID = "1$1!95" + document.Name = "textfile.txt" + document.Path = "./textfile.txt" + + backend, tc := newBackend(indexName) + deleteIndexOnCleanup(tc, indexName) + + Expect(backend.Upsert(document.ID, document)).To(Succeed()) + Expect(backend.Delete(document.ID)).To(Succeed()) + + resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ + Query: fmt.Sprintf(`name:"%s"`, document.Name), + }) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.Matches).To(BeEmpty()) + }) + }) + + Describe("Delete", func() { + const indexName = "opencloud-test-engine-delete" + + var ( + tc *opensearchtest.TestClient + backend *opensearch.Backend + ) + + BeforeEach(func() { + tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, nil, 0) + deleteIndexOnCleanup(tc, indexName) + + var err error + backend, err = opensearch.NewBackend(indexName, tc.Client()) + Expect(err).ToNot(HaveOccurred()) + }) + + It("marks the document as deleted", func() { + document := opensearchtest.Testdata.Resources.File + tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) + tc.Require.IndicesCount([]string{indexName}, nil, 1) + + body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ + "query": map[string]any{ + "term": map[string]any{ + "Deleted": map[string]any{ + "value": true, + }, + }, + }, + }) + + tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0) + + Expect(backend.Delete(document.ID)).To(Succeed()) + tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1) + }) + }) + + Describe("Restore", func() { + const indexName = "opencloud-test-engine-restore" + + var ( + tc *opensearchtest.TestClient + backend *opensearch.Backend + ) + + BeforeEach(func() { + tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, nil, 0) + deleteIndexOnCleanup(tc, indexName) + + var err error + backend, err = opensearch.NewBackend(indexName, tc.Client()) + Expect(err).ToNot(HaveOccurred()) + }) + + It("marks the document as not deleted", func() { + document := opensearchtest.Testdata.Resources.File + document.Deleted = true + tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) + tc.Require.IndicesCount([]string{indexName}, nil, 1) + + body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ + "query": map[string]any{ + "term": map[string]any{ + "Deleted": map[string]any{ + "value": true, + }, + }, + }, + }) + + tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1) + + Expect(backend.Restore(document.ID)).To(Succeed()) + tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0) + }) + }) + + Describe("Purge", func() { + const indexName = "opencloud-test-engine-purge" + + var ( + tc *opensearchtest.TestClient + backend *opensearch.Backend + ) + + BeforeEach(func() { + tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, nil, 0) + deleteIndexOnCleanup(tc, indexName) + + var err error + backend, err = opensearch.NewBackend(indexName, tc.Client()) + Expect(err).ToNot(HaveOccurred()) + }) + + It("purges a full document", func() { + document := opensearchtest.Testdata.Resources.File + tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) + tc.Require.IndicesCount([]string{indexName}, nil, 1) + + Expect(backend.Purge(document.ID, false)).To(Succeed()) + + tc.Require.IndicesCount([]string{indexName}, nil, 0) + }) + + It("purges resource trees", func() { + resourceFolder := opensearchtest.Testdata.Resources.Folder + tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder))) + + resourceFile := opensearchtest.Testdata.Resources.File + tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile))) + + tc.Require.IndicesCount([]string{indexName}, nil, 2) + + Expect(backend.Purge(resourceFolder.ID, false)).To(Succeed()) + + tc.Require.IndicesCount([]string{indexName}, nil, 0) + }) + + It("purges resource trees and ignores undeleted resources", func() { + resourceFolder := opensearchtest.Testdata.Resources.Folder + tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder))) + + resourceFile := opensearchtest.Testdata.Resources.File + tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile))) + + tc.Require.IndicesCount([]string{indexName}, nil, 2) + + Expect(backend.Delete(resourceFile.ID)).To(Succeed()) + tc.Require.IndicesRefresh([]string{indexName}, nil) + Expect(backend.Purge(resourceFolder.ID, true)).To(Succeed()) + + tc.Require.IndicesCount([]string{indexName}, nil, 1) + }) + }) + + Describe("PurgeSpace", func() { + const indexName = "opencloud-test-engine-purge-space" + + var ( + tc *opensearchtest.TestClient + backend *opensearch.Backend + ) + + BeforeEach(func() { + tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, nil, 0) + deleteIndexOnCleanup(tc, indexName) + + var err error + backend, err = opensearch.NewBackend(indexName, tc.Client()) + Expect(err).ToNot(HaveOccurred()) + }) + + It("takes every record of that space out of the index", func() { + gone := opensearchtest.Testdata.Resources.File + tc.Require.DocumentCreate(indexName, gone.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), gone))) + + stays := opensearchtest.Testdata.Resources.File + stays.ID = "1$2!3" + stays.RootID = "1$2!2" + tc.Require.DocumentCreate(indexName, stays.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), stays))) + + tc.Require.IndicesCount([]string{indexName}, nil, 2) + + Expect(backend.PurgeSpace(gone.RootID)).To(Succeed()) + + tc.Require.IndicesRefresh([]string{indexName}, nil) + left := opensearchtest.SearchHitsMustBeConverted[search.Resource]( + GinkgoTB(), + tc.Require.Search(indexName, strings.NewReader(`{"query":{"match_all":{}}}`)).Hits, + ) + Expect(left).To(HaveLen(1), "only the records of that space are gone") + Expect(left[0].ID).To(Equal(stays.ID)) + }) + }) + + Describe("Hidden", func() { + const indexName = "opencloud-test-engine-hidden" + + DescribeTable("keeps the flag in step with the path", + func(from, target string, hidden bool) { + folder := opensearchtest.Testdata.Resources.Folder + folder.ID = "1$1!30" + folder.Name = "parent" + folder.Path = from + folder.Hidden = search.IsHidden(from) + + child := opensearchtest.Testdata.Resources.File + child.ID = "1$1!31" + child.Name = "child.txt" + child.Path = from + "/child.txt" + child.ParentID = folder.ID + child.Hidden = folder.Hidden + + backend, tc := newBackend(indexName, folder, child) + deleteIndexOnCleanup(tc, indexName) + tc.Require.IndicesRefresh([]string{indexName}, nil) + + Expect(backend.Move(folder.ID, folder.ParentID, target)).To(Succeed()) + tc.Require.IndicesRefresh([]string{indexName}, nil) + + for _, id := range []string{folder.ID, child.ID} { + Expect(resourceByID(tc, indexName, id).Hidden). + To(Equal(hidden), "%s after moving from %s to %s", id, from, target) + } + }, + Entry("into a dot folder", "./parent", "./.trash/parent", true), + Entry("into a plain folder", "./parent", "./archive/parent", false), + Entry("renamed with a leading dot", "./parent", "./.parent", true), + Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false), + Entry("renamed without the leading dot", "./.parent", "./parent", false), + Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true), + ) + + It("carries the flag through the trash and back", func() { + hidden := opensearchtest.Testdata.Resources.File + hidden.ID = "1$1!32" + hidden.Path = "./.secret/file.txt" + hidden.Hidden = true + + backend, tc := newBackend(indexName, hidden) + deleteIndexOnCleanup(tc, indexName) + tc.Require.IndicesRefresh([]string{indexName}, nil) + + Expect(backend.Delete(hidden.ID)).To(Succeed()) + tc.Require.IndicesRefresh([]string{indexName}, nil) + Expect(resourceByID(tc, indexName, hidden.ID).Hidden).To(BeTrue(), "after trashing") + + Expect(backend.Restore(hidden.ID)).To(Succeed()) + tc.Require.IndicesRefresh([]string{indexName}, nil) + Expect(resourceByID(tc, indexName, hidden.ID).Hidden).To(BeTrue(), "after restoring") + }) + }) + + Describe("DocCount", func() { + const indexName = "opencloud-test-engine-doc-count" + + var ( + tc *opensearchtest.TestClient + backend *opensearch.Backend + ) + + BeforeEach(func() { + tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) + tc.Require.IndicesReset([]string{indexName}) + tc.Require.IndicesCount([]string{indexName}, nil, 0) + deleteIndexOnCleanup(tc, indexName) + + var err error + backend, err = opensearch.NewBackend(indexName, tc.Client()) + Expect(err).ToNot(HaveOccurred()) + }) + + It("ignores deleted documents", func() { + document := opensearchtest.Testdata.Resources.File + tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) + tc.Require.IndicesCount([]string{indexName}, nil, 1) + + count, err := backend.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(1))) + + tc.Require.Update(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ + "doc": map[string]any{ + "Deleted": true, + }, + }))) + + tc.Require.IndicesCount([]string{indexName}, nil, 1) + + count, err = backend.DocCount() + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(uint64(0))) + }) + }) + + // The following specs ensure that updates which affect a resource and its descendants + // (Delete, Restore, Move) are scoped to the root (space) of the target resource. Two + // resources living in different roots may share the exact same path, so matching by + // path alone would incorrectly update the wrong resource. + Describe("updateSelfAndDescendants root scope", func() { + It("deletes only the resource in the target root", func() { + const indexName = "opencloud-test-engine-root-scope-delete" + + target := opensearchtest.Testdata.Resources.File + other := otherRoot(target) + + backend, tc := newBackend(indexName, target, other) + deleteIndexOnCleanup(tc, indexName) + + Expect(backend.Delete(target.ID)).To(Succeed()) + + Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeTrue(), "target resource should be marked as deleted") + Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeFalse(), "resource in a different root must not be affected") + }) + + It("restores only the resource in the target root", func() { + const indexName = "opencloud-test-engine-root-scope-restore" + + target := opensearchtest.Testdata.Resources.File + target.Deleted = true + other := otherRoot(target) + + backend, tc := newBackend(indexName, target, other) + deleteIndexOnCleanup(tc, indexName) + + Expect(backend.Restore(target.ID)).To(Succeed()) + + Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeFalse(), "target resource should be restored") + Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeTrue(), "resource in a different root must not be affected") + }) + + It("moves only the resource in the target root", func() { + const indexName = "opencloud-test-engine-root-scope-move" + + target := opensearchtest.Testdata.Resources.File + other := otherRoot(target) + + backend, tc := newBackend(indexName, target, other) + deleteIndexOnCleanup(tc, indexName) + + Expect(backend.Move(target.ID, target.ParentID, "./new/path/to/resource")).To(Succeed()) + + Expect(resourceByID(tc, indexName, target.ID).Path).To(Equal("./new/path/to/resource"), "target resource should be moved") + Expect(resourceByID(tc, indexName, other.ID).Path).To(Equal(other.Path), "resource in a different root must not be moved") + }) + }) + + Describe("SearchInAnalyzedFields", func() { + const indexName = "opencloud-test-engine-search-analyzed-fields" + + var ( + tc *opensearchtest.TestClient + backend *opensearch.Backend + ) + + BeforeEach(func() { + dashed := opensearchtest.Testdata.Resources.Folder + dashed.ID = "1$1!10" + dashed.Name = "new-folder" + dashed.Path = "./new-folder" + dashed.Title = "quarterly report" + + plain := opensearchtest.Testdata.Resources.Folder + plain.ID = "1$1!11" + plain.Name = "documents" + plain.Path = "./documents" + plain.Title = "notes" + + spaced := opensearchtest.Testdata.Resources.Folder + spaced.ID = "1$1!12" + spaced.Name = "foo bar" + spaced.Path = "./foo bar" + spaced.Title = "spaced out" + + backend, tc = newBackend(indexName) + deleteIndexOnCleanup(tc, indexName) + for _, r := range []search.Resource{dashed, plain, spaced} { + Expect(backend.Upsert(r.ID, r)).To(Succeed()) + } + tc.Require.IndicesRefresh([]string{indexName}, nil) + tc.Require.IndicesCount([]string{indexName}, nil, 3) + }) + + DescribeTable("finds what the analyzer made of the value", + func(query string, want []string) { + resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query}) + Expect(err).ToNot(HaveOccurred()) + + names := make([]string, 0, len(resp.Matches)) + for _, match := range resp.Matches { + names = append(names, match.Entity.Name) + } + Expect(names).To(ConsistOf(want)) + }, + Entry("the full name with the dash", "new-folder", []string{"new-folder"}), + Entry("one token of it", "new", []string{"new-folder"}), + Entry("a name without a dash", "documents", []string{"documents"}), + Entry("a wildcard", "*folder*", []string{"new-folder"}), + // the shape the web client sends for every name search + Entry("a wildcard around the whole dashed name", `name:"*new-folder*"`, []string{"new-folder"}), + Entry("a wildcard spanning the dash", `name:"*w-fol*"`, []string{"new-folder"}), + Entry("a wildcard in a different case", `name:"*NEW-FOLDER*"`, []string{"new-folder"}), + Entry("a wildcard spanning a space", `name:"*oo ba*"`, []string{"foo bar"}), + Entry("a wildcard around a name with a space", `name:"*foo bar*"`, []string{"foo bar"}), + Entry("a name with a space", `name:"foo bar"`, []string{"foo bar"}), + Entry("a title of two words", `Title:"quarterly report"`, []string{"new-folder"}), + Entry("one token of a title", "Title:quarterly", []string{"new-folder"}), + ) + }) + + Describe("SearchByTag", func() { + const indexName = "opencloud-test-engine-search-by-tag" + + var ( + tc *opensearchtest.TestClient + backend *opensearch.Backend + ) + + BeforeEach(func() { + tagged := opensearchtest.Testdata.Resources.Folder + tagged.ID = "1$1!20" + tagged.Name = "tagged" + tagged.Path = "./tagged" + tagged.Tags = []string{"foo-bar"} + + other := opensearchtest.Testdata.Resources.Folder + other.ID = "1$1!21" + other.Name = "other" + other.Path = "./other" + other.Tags = []string{"foo"} + + backend, tc = newBackend(indexName) + deleteIndexOnCleanup(tc, indexName) + for _, r := range []search.Resource{tagged, other} { + Expect(backend.Upsert(r.ID, r)).To(Succeed()) + } + tc.Require.IndicesRefresh([]string{indexName}, nil) + tc.Require.IndicesCount([]string{indexName}, nil, 2) + }) + + // a tag is one label, not prose, so it matches as a whole or not at all + DescribeTable("matches a tag as a whole", + func(query string, want []string) { + resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query}) + Expect(err).ToNot(HaveOccurred()) + + names := make([]string, 0, len(resp.Matches)) + for _, match := range resp.Matches { + names = append(names, match.Entity.Name) + } + Expect(names).To(ConsistOf(want)) + }, + Entry("the whole tag", `tag:("foo-bar")`, []string{"tagged"}), + Entry("a token of a tag does not match it", `tag:("foo")`, []string{"other"}), + Entry("a tag in a different case", `tag:("FOO-BAR")`, []string{"tagged"}), + Entry("a wildcard reaches both", `tag:("*foo*")`, []string{"tagged", "other"}), + ) + }) + + Describe("SearchWithAnInvalidQuery", func() { + const indexName = "opencloud-test-engine-search-invalid-query" + + It("answers with a bad request", func() { + backend, tc := newBackend(indexName) + deleteIndexOnCleanup(tc, indexName) + + _, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: "AND mediatype:document"}) + Expect(err).To(HaveOccurred()) + Expect(err).To(BeAssignableToTypeOf(errtypes.BadRequest(""))) + Expect(err.Error()).To(Equal(`error: bad request: the expression can't begin from a binary operator: 'AND'`)) + }) + }) }) From 6b07ea745c1baf85c2a5ea2663de8db45baca5f7 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Sat, 29 Aug 2026 09:46:30 +0200 Subject: [PATCH 31/54] feat(search): split names and titles into words on both engines A single word finds the names and titles that contain it, `report` finds Report.txt, on bleve as well as on OpenSearch, which so far only did it by accident of its dynamic mapping. Modelled like SharePoint's NoWordBreaker: a keyword field is one whole value unless the override switches that off, which adds a search-only _words sibling next to _lowercase, analyzed into lowercased words (a dot is a word boundary, no stemming). The base stays the whole value for returning and aggregating, wildcards and whole values keep using _lowercase. Quotes do not change the meaning, a phrase is a phrase either way, and there is no exact-match operator yet. --- services/search/pkg/bleve/backend_test.go | 2 ++ services/search/pkg/bleve/index.go | 23 +++++++++++++ services/search/pkg/mapping/bleve.go | 17 ++++++---- services/search/pkg/mapping/bleve_test.go | 16 ++++++++++ services/search/pkg/mapping/casing.go | 22 +++++++++++-- services/search/pkg/mapping/casing_test.go | 15 +++++++++ services/search/pkg/mapping/opensearch.go | 3 ++ .../search/pkg/mapping/opensearch_test.go | 15 +++++++++ services/search/pkg/mapping/opts.go | 16 ++++++++++ services/search/pkg/mapping/serialize.go | 2 +- services/search/pkg/mapping/validate.go | 19 ++++++++++- services/search/pkg/mapping/validate_test.go | 10 ++++++ services/search/pkg/opensearch/batch.go | 5 +-- services/search/pkg/opensearch/index.go | 15 +++++++++ services/search/pkg/opensearch/index_test.go | 1 - .../internal/convert/kql_transpile.go | 6 ++++ .../internal/convert/kql_transpile_test.go | 30 ++++++++--------- services/search/pkg/query/bleve/compiler.go | 7 ++++ .../search/pkg/query/bleve/compiler_test.go | 32 +++++++++---------- services/search/pkg/query/normalize_test.go | 16 ++++++++-- services/search/pkg/query/resolver.go | 19 +++++++++++ services/search/pkg/search/search.go | 5 ++- vendor/modules.txt | 1 + 23 files changed, 249 insertions(+), 48 deletions(-) diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go index a40572c615..6df0975a9d 100644 --- a/services/search/pkg/bleve/backend_test.go +++ b/services/search/pkg/bleve/backend_test.go @@ -602,6 +602,7 @@ var _ = Describe("Bleve", func() { Expect(count).To(Equal(uint64(1))) query := bleveSearch.NewMatchQuery("child.pdf") + query.SetField("Name") res, err := idx.Search(bleveSearch.NewSearchRequest(query)) Expect(err).ToNot(HaveOccurred()) Expect(res.Hits.Len()).To(Equal(1)) @@ -843,6 +844,7 @@ var _ = Describe("Bleve", func() { Expect(count).To(Equal(uint64(1))) query := bleveSearch.NewMatchQuery("child.pdf") + query.SetField("Name") res, err := idx.Search(bleveSearch.NewSearchRequest(query)) Expect(err).ToNot(HaveOccurred()) Expect(res.Hits.Len()).To(Equal(1)) diff --git a/services/search/pkg/bleve/index.go b/services/search/pkg/bleve/index.go index 76a92625ce..c65373a55f 100644 --- a/services/search/pkg/bleve/index.go +++ b/services/search/pkg/bleve/index.go @@ -10,6 +10,7 @@ import ( "github.com/blevesearch/bleve/v2" "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2/analysis/char/regexp" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" "github.com/blevesearch/bleve/v2/analysis/token/porter" "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" @@ -72,6 +73,28 @@ func NewMapping() (mapping.IndexMapping, error) { return nil, err } + // words: split into lowercased words, a dot is a word boundary too so that + // "report" finds "Report.txt"; no stemming, a name is not prose + err = indexMapping.AddCustomCharFilter("dot_to_space", map[string]any{ + "type": regexp.Name, + "regexp": `\.`, + "replace": " ", + }) + if err != nil { + return nil, err + } + err = indexMapping.AddCustomAnalyzer(searchmapping.WordsAnalyzer, + map[string]any{ + "type": custom.Name, + "char_filters": []string{"dot_to_space"}, + "tokenizer": unicode.Name, + "token_filters": []string{lowercase.Name}, + }, + ) + if err != nil { + return nil, err + } + return indexMapping, nil } diff --git a/services/search/pkg/mapping/bleve.go b/services/search/pkg/mapping/bleve.go index 70e1f0949e..150a5a427a 100644 --- a/services/search/pkg/mapping/bleve.go +++ b/services/search/pkg/mapping/bleve.go @@ -64,7 +64,12 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix base := bleveKeywordMapping(fieldType, opts) doc.AddFieldMappingsAt(fi.Name, base) if opts.caseInsensitive() { - doc.AddFieldMappingsAt(fi.Name+LowercaseSuffix, lowercaseSibling(base)) + doc.AddFieldMappingsAt(fi.Name+LowercaseSuffix, searchSibling(base)) + } + if opts.wordBroken() { + words := searchSibling(base) + words.Analyzer = WordsAnalyzer + doc.AddFieldMappingsAt(fi.Name+WordsSuffix, words) } return nil } @@ -92,11 +97,11 @@ func bleveKeywordMapping(fieldType string, opts FieldOpts) *bleveMapping.FieldMa return fm } -// lowercaseSibling derives the lowercased shadow of a keyword/path field from its -// base mapping: used only for case-insensitive matching, so indexed but never -// stored, kept out of _all, and without doc values, since the case-preserved base -// field is what we return and aggregate on. -func lowercaseSibling(base *bleveMapping.FieldMapping) *bleveMapping.FieldMapping { +// searchSibling derives a search-only shadow of a keyword/path field from its +// base mapping (the _lowercase and _words siblings): indexed but never stored, +// kept out of _all, and without doc values, since the case-preserved base field +// is what we return and aggregate on. +func searchSibling(base *bleveMapping.FieldMapping) *bleveMapping.FieldMapping { fm := *base fm.Store = false fm.IncludeInAll = false diff --git a/services/search/pkg/mapping/bleve_test.go b/services/search/pkg/mapping/bleve_test.go index 4fcb09f7ab..dc76c6bd90 100644 --- a/services/search/pkg/mapping/bleve_test.go +++ b/services/search/pkg/mapping/bleve_test.go @@ -92,6 +92,22 @@ var _ = Describe("BleveBuildMapping", func() { Expect(dm.Properties["Tags_lowercase"].Fields[0].IncludeInAll).To(BeFalse(), "Tags sibling IncludeInAll honored") }) + It("splits a keyword into words when NoWordBreaker is false", func() { + True, False := true, false + dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{ + "Name": {NoWordBreaker: &False, CaseInsensitive: &True}, + }) + Expect(err).ToNot(HaveOccurred()) + // the base stays a keyword, the words go to a search-only sibling + Expect(dm.Properties["Name"].Fields[0].Analyzer).To(Equal("keyword"), "Name base stays a keyword") + Expect(dm.Properties["Name"].Fields[0].Store).To(BeTrue(), "Name base is stored (returned)") + Expect(dm.Properties["Name_lowercase"].Fields[0].Analyzer).To(Equal("keyword"), "Name_lowercase stays a keyword") + words := dm.Properties["Name_words"].Fields[0] + Expect(words.Analyzer).To(Equal(WordsAnalyzer), "Name_words is split into words") + Expect(words.Store).To(BeFalse(), "Name_words is not stored") + Expect(words.IncludeInAll).To(BeFalse(), "Name_words is out of _all") + }) + It("builds an object sub-document plus a geopoint sibling", func() { type geoDoc struct { Location *struct { diff --git a/services/search/pkg/mapping/casing.go b/services/search/pkg/mapping/casing.go index f05413533e..abe9348477 100644 --- a/services/search/pkg/mapping/casing.go +++ b/services/search/pkg/mapping/casing.go @@ -2,16 +2,32 @@ package mapping import "strings" -func addLowercaseSiblings(m map[string]any, overrides map[string]FieldOpts) { +// addSearchSiblings writes the _lowercase and _words siblings the overrides ask +// for next to their base values. +func addSearchSiblings(m map[string]any, overrides map[string]FieldOpts) { for key, opts := range overrides { - if !opts.caseInsensitive() || !isCasedType(opts) { + if !isCasedType(opts) || (!opts.caseInsensitive() && !opts.wordBroken()) { continue } parent, leaf, ok := resolveLeaf(m, key) if !ok { continue } - addLowercaseSibling(parent, leaf) + if opts.caseInsensitive() { + addLowercaseSibling(parent, leaf) + } + if opts.wordBroken() { + addWordsSibling(parent, leaf) + } + } +} + +// addWordsSibling copies the value to a _words sibling; the words +// analyzer does the splitting and lowercasing. No-op for non-strings. +func addWordsSibling(parent map[string]any, leaf string) { + switch v := parent[leaf].(type) { + case string, []any, []string: + parent[leaf+WordsSuffix] = v } } diff --git a/services/search/pkg/mapping/casing_test.go b/services/search/pkg/mapping/casing_test.go index 5fba3e9eae..255eb36ef4 100644 --- a/services/search/pkg/mapping/casing_test.go +++ b/services/search/pkg/mapping/casing_test.go @@ -30,6 +30,21 @@ var _ = Describe("PrepareForIndex casing", func() { Expect(m["Tags_lowercase"]).To(Equal([]any{"work", "urgent"})) }) + It("copies the value to a words sibling when NoWordBreaker is off", func() { + True, False := true, false + type doc struct { + Name string `json:"Name"` + } + m, err := PrepareForIndex(doc{Name: "Report FINAL"}, map[string]FieldOpts{ + "Name": {NoWordBreaker: &False, CaseInsensitive: &True}, + }) + Expect(err).ToNot(HaveOccurred()) + // the analyzer splits and lowercases, the value goes over as is + Expect(m["Name"]).To(Equal("Report FINAL")) + Expect(m["Name_lowercase"]).To(Equal("report final")) + Expect(m["Name_words"]).To(Equal("Report FINAL")) + }) + It("writes no sibling without CaseInsensitive", func() { type doc struct { ID string `json:"ID"` diff --git a/services/search/pkg/mapping/opensearch.go b/services/search/pkg/mapping/opensearch.go index 22af1a9831..5254e48423 100644 --- a/services/search/pkg/mapping/opensearch.go +++ b/services/search/pkg/mapping/opensearch.go @@ -66,6 +66,9 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p if opts.caseInsensitive() { props[fi.Name+LowercaseSuffix] = m } + if opts.wordBroken() { + props[fi.Name+WordsSuffix] = map[string]any{"type": "text", "analyzer": WordsAnalyzer} + } return nil } diff --git a/services/search/pkg/mapping/opensearch_test.go b/services/search/pkg/mapping/opensearch_test.go index 1eab516763..4ad05548bb 100644 --- a/services/search/pkg/mapping/opensearch_test.go +++ b/services/search/pkg/mapping/opensearch_test.go @@ -106,6 +106,21 @@ var _ = Describe("OpenSearchBuildMapping", func() { Expect(mime["type"]).To(Equal("wildcard"), "MimeType: %#v", mime) }) + It("splits a keyword into words when NoWordBreaker is false", func() { + True, False := true, false + type doc struct { + Name string `json:"Name"` + } + props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{ + "Name": {NoWordBreaker: &False, CaseInsensitive: &True}, + }) + Expect(err).ToNot(HaveOccurred()) + // the base stays a keyword, the words go to their own sibling + Expect(props["Name"]).To(Equal(map[string]any{"type": "keyword"})) + Expect(props["Name_lowercase"]).To(Equal(map[string]any{"type": "keyword"})) + Expect(props["Name_words"]).To(Equal(map[string]any{"type": "text", "analyzer": WordsAnalyzer})) + }) + It("builds an object plus a geo_point sibling for geopoints", func() { type doc struct { Location *struct { diff --git a/services/search/pkg/mapping/opts.go b/services/search/pkg/mapping/opts.go index 9ae7d295a0..f2589fc865 100644 --- a/services/search/pkg/mapping/opts.go +++ b/services/search/pkg/mapping/opts.go @@ -20,6 +20,12 @@ const ( // LowercaseSuffix names the lowercased sibling of a keyword/path field. const LowercaseSuffix = "_lowercase" +// WordsSuffix names the word-broken sibling of a keyword field. +const WordsSuffix = "_words" + +// WordsAnalyzer names the analyzer both engines register for the words sibling. +const WordsAnalyzer = "words" + // FieldOpts overrides the default type inference for a struct field. Keys in // the override map are json-tag names (e.g. "Name", "location", "audio.artist"), // not Go field names. @@ -32,9 +38,19 @@ type FieldOpts struct { // Nil/false means off. Keyword/path only. CaseInsensitive *bool + // NoWordBreaker is SharePoint's switch: nil or true leaves a keyword field + // one whole value, false additionally indexes a _words sibling split + // into lowercased words (no stemming), so a single word matches a value + // that contains it: "report" finds "Report.txt". The base stays the whole + // value for returning and aggregating; wildcards and whole-value matches + // use the _lowercase sibling, so it wants CaseInsensitive alongside. + // Keyword only. + NoWordBreaker *bool + // IncludeInAll controls bleve's _all field inclusion. Nil means "use the // bleve default for this field type". Has no effect on OpenSearch. IncludeInAll *bool } func (o FieldOpts) caseInsensitive() bool { return o.CaseInsensitive != nil && *o.CaseInsensitive } +func (o FieldOpts) wordBroken() bool { return o.NoWordBreaker != nil && !*o.NoWordBreaker } diff --git a/services/search/pkg/mapping/serialize.go b/services/search/pkg/mapping/serialize.go index ec5420784a..b4e5310989 100644 --- a/services/search/pkg/mapping/serialize.go +++ b/services/search/pkg/mapping/serialize.go @@ -19,6 +19,6 @@ func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, err return out, nil } addGeopointSiblings(out, overrides) - addLowercaseSiblings(out, overrides) + addSearchSiblings(out, overrides) return out, nil } diff --git a/services/search/pkg/mapping/validate.go b/services/search/pkg/mapping/validate.go index fa6a8b222e..3147eb20ed 100644 --- a/services/search/pkg/mapping/validate.go +++ b/services/search/pkg/mapping/validate.go @@ -17,13 +17,16 @@ func Validate(t reflect.Type, overrides map[string]FieldOpts) error { return nil } fields := collectFields(t, "") - var unknown, miscased []string + var unknown, miscased, unbroken []string for k, opts := range overrides { goType, ok := fields[k] if !ok { unknown = append(unknown, k) continue } + if opts.wordBroken() && !effectivelyKeyword(opts, goType) { + unbroken = append(unbroken, k) + } // CaseInsensitive routes queries to a _lowercase sibling, which is // only generated for keyword/path fields; on any other type the query // would target a non-existent field and silently match nothing. Use the @@ -41,6 +44,10 @@ func Validate(t reflect.Type, overrides map[string]FieldOpts) error { sort.Strings(miscased) return fmt.Errorf("mapping: CaseInsensitive is only valid on keyword/path fields: %s", strings.Join(miscased, ", ")) } + if len(unbroken) > 0 { + sort.Strings(unbroken) + return fmt.Errorf("mapping: NoWordBreaker is only valid on keyword fields: %s", strings.Join(unbroken, ", ")) + } return nil } @@ -54,6 +61,16 @@ func effectivelyCased(opts FieldOpts, goType reflect.Type) bool { return eff == TypeKeyword || eff == TypePath } +// effectivelyKeyword reports whether a field is a keyword, the only type +// NoWordBreaker applies to. +func effectivelyKeyword(opts FieldOpts, goType reflect.Type) bool { + eff := opts.Type + if eff == "" && goType != nil { + eff = inferType(goType) + } + return eff == TypeKeyword +} + // collectFields maps every known field name (nested as "parent.child") to its Go // type. Embedded structs are flattened, matching encoding/json. func collectFields(t reflect.Type, prefix string) map[string]reflect.Type { diff --git a/services/search/pkg/mapping/validate_test.go b/services/search/pkg/mapping/validate_test.go index 7a34183a4b..39bf5034d5 100644 --- a/services/search/pkg/mapping/validate_test.go +++ b/services/search/pkg/mapping/validate_test.go @@ -55,6 +55,16 @@ var _ = Describe("Validate", func() { Expect(err.Error()).To(ContainSubstring("CaseInsensitive")) }) + It("rejects NoWordBreaker on a non-keyword field", func() { + False := false + err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{ + "Name": {Type: TypeFulltext, NoWordBreaker: &False}, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Name")) + Expect(err.Error()).To(ContainSubstring("NoWordBreaker")) + }) + It("rejects CaseInsensitive on an inferred non-keyword field (empty Type)", func() { True := true type doc struct { diff --git a/services/search/pkg/opensearch/batch.go b/services/search/pkg/opensearch/batch.go index 637764bff2..9cc9fb852a 100644 --- a/services/search/pkg/opensearch/batch.go +++ b/services/search/pkg/opensearch/batch.go @@ -71,7 +71,7 @@ func (b *Batch) Move(id, parentID, location string) error { newPath := utils.MakeRelativePath(location) newName := path.Base(newPath) return &osu.BodyParamScript{ - // Keep Name and its lowercased search sibling in sync; Path has + // Keep Name and its search siblings in sync; Path has // no sibling (case-sensitive by design). Only the leading // oldPath is replaced (startsWith + substring, not // String.replace, which would also rewrite a repeated segment @@ -85,6 +85,7 @@ func (b *Batch) Move(id, parentID, location string) error { ctx._source.Name = params.newName; ctx._source.ParentID = params.parentID; if (ctx._source.Name%[1]s != null) { ctx._source.Name%[1]s = params.newNameLower; } + if (ctx._source.Name%[2]s != null) { ctx._source.Name%[2]s = params.newName; } } if (ctx._source.Path != null && ctx._source.Path.startsWith(params.oldPath)) { ctx._source.Path = params.newPath + ctx._source.Path.substring(params.oldPath.length()); @@ -94,7 +95,7 @@ func (b *Batch) Move(id, parentID, location string) error { if (!name.equals('.') && !name.equals('..') && name.startsWith('.')) { hidden = true; break; } } ctx._source.Hidden = hidden; - `, mapping.LowercaseSuffix), + `, mapping.LowercaseSuffix, mapping.WordsSuffix), Lang: "painless", Params: map[string]any{ "id": id, diff --git a/services/search/pkg/opensearch/index.go b/services/search/pkg/opensearch/index.go index 744dbb8a12..8049cc4bd0 100644 --- a/services/search/pkg/opensearch/index.go +++ b/services/search/pkg/opensearch/index.go @@ -87,6 +87,21 @@ func buildResourceMapping() ([]byte, error) { "tokenizer": "standard", "filter": []string{"lowercase", "porter_stem"}, }, + // words: split into lowercased words, a dot is a word boundary + // too so that "report" finds "Report.txt"; no stemming, a name + // is not prose + searchmapping.WordsAnalyzer: map[string]any{ + "type": "custom", + "char_filter": []string{"dot_to_space"}, + "tokenizer": "standard", + "filter": []string{"lowercase"}, + }, + }, + "char_filter": map[string]any{ + "dot_to_space": map[string]any{ + "type": "mapping", + "mappings": []string{`. => \u0020`}, + }, }, "tokenizer": map[string]any{ "path_hierarchy": map[string]any{"type": "path_hierarchy"}, diff --git a/services/search/pkg/opensearch/index_test.go b/services/search/pkg/opensearch/index_test.go index 3ba5a38cec..5837189e82 100644 --- a/services/search/pkg/opensearch/index_test.go +++ b/services/search/pkg/opensearch/index_test.go @@ -10,7 +10,6 @@ import ( "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index 0e74dd96c0..6ff8016001 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -119,6 +119,12 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { return osu.NewWildcardQuery(field).Value(value), nil } + // a word-broken field matches the value as a phrase of its words on the + // _words sibling, whose analyzer lowercases; wildcards stay on _lowercase + if query.FieldIsWordBroken(node.Key) { + return osu.NewMatchPhraseQuery(node.Key + mapping.WordsSuffix).Query(node.Value), nil + } + if query.FieldIsFulltext(node.Key) { return osu.NewMatchPhraseQuery(field).Query(value), nil } diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go index 85996976d7..b16085a9b4 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go @@ -16,22 +16,22 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{ // kql to os dsl - type tests { - Name: "term query - string node", + Name: "word-broken field matches the value as a phrase on its words sibling", Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "Name", Value: "openCloud"}, }, }, - Want: osu.NewTermQuery[string]("Name").Value("openCloud"), + Want: osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), }, { Name: "case-insensitive term routes to the lowercased sibling", Got: &ast.Ast{ Nodes: []ast.Node{ - &ast.StringNode{Key: "Name", Value: "openCloud", CaseInsensitive: true}, + &ast.StringNode{Key: "Tags", Value: "openCloud", CaseInsensitive: true}, }, }, - Want: osu.NewTermQuery[string]("Name_lowercase").Value("opencloud"), + Want: osu.NewTermQuery[string]("Tags_lowercase").Value("opencloud"), }, { Name: "case-insensitive wildcard routes to the lowercased sibling", @@ -76,7 +76,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { &ast.StringNode{Key: "Name", Value: "open cloud"}, }, }, - Want: osu.NewMatchPhraseQuery("Name").Query(`open cloud`), + Want: osu.NewMatchPhraseQuery("Name_words").Query(`open cloud`), }, { Name: "wildcard query - string node", @@ -127,8 +127,8 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, }, Want: osu.NewBoolQuery().Must( - osu.NewTermQuery[string]("Name").Value("a"), - osu.NewTermQuery[string]("Name").Value("b"), + osu.NewMatchPhraseQuery("Name_words").Query("a"), + osu.NewMatchPhraseQuery("Name_words").Query("b"), ), }, { @@ -140,7 +140,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }}, }, }, - Want: osu.NewTermQuery[string]("Name").Value("any"), + Want: osu.NewMatchPhraseQuery("Name_words").Query("any"), }, { Name: "range query >", @@ -202,7 +202,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { &ast.StringNode{Key: "Name", Value: "openCloud"}, }, }, - Want: osu.NewTermQuery[string]("Name").Value("openCloud"), + Want: osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), }, { Name: "[* *]", @@ -214,7 +214,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewBoolQuery(). Must( - osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), osu.NewTermQuery[string]("age").Value("32"), ), }, @@ -229,7 +229,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewBoolQuery(). Must( - osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), osu.NewTermQuery[string]("age").Value("32"), ), }, @@ -245,7 +245,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { Want: osu.NewBoolQuery(). Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). Should( - osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), osu.NewTermQuery[string]("age").Value("32"), ), }, @@ -273,7 +273,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { }, Want: osu.NewBoolQuery(). Must( - osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), ). MustNot( osu.NewTermQuery[string]("age").Value("32"), @@ -296,7 +296,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { osu.NewTermQuery[string]("age").Value("32"), ). Must( - osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), ), }, { @@ -313,7 +313,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) { Want: osu.NewBoolQuery(). Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). Should( - osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewMatchPhraseQuery("Name_words").Query("openCloud"), osu.NewTermQuery[string]("age").Value("32"), osu.NewTermQuery[string]("age").Value("44"), ), diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index 4a536d3776..8536acc78b 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -88,6 +88,13 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { v = strings.ToLower(v) } + // a word-broken field matches the value as a phrase of its words on the + // _words sibling (a quoted query string term is a match phrase query + // run through the field's analyzer); wildcards stay on _lowercase + if searchQuery.FieldIsWordBroken(n.Key) && !strings.Contains(n.Value, "*") { + k, v = n.Key+mapping.WordsSuffix, `"`+strings.ReplaceAll(n.Value, `"`, `\"`)+`"` + } + var q bleveQuery.Query = bleveQuery.NewQueryStringQuery(k + ":" + v) if searchQuery.FieldIsPath(n.Key) { // bleve has no path hierarchy analyzer, unlike OpenSearch: match the diff --git a/services/search/pkg/query/bleve/compiler_test.go b/services/search/pkg/query/bleve/compiler_test.go index c1662bbf19..a7ff7ab9dd 100644 --- a/services/search/pkg/query/bleve/compiler_test.go +++ b/services/search/pkg/query/bleve/compiler_test.go @@ -39,7 +39,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:federated`), + query.NewQueryStringQuery(`Name_words:"federated"`), }), wantErr: false, }, @@ -72,7 +72,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:john\ smith`), + query.NewQueryStringQuery(`Name_words:"John Smith"`), }), wantErr: false, }, @@ -86,8 +86,8 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:john\ smith`), - query.NewQueryStringQuery(`Name_lowercase:jane`), + query.NewQueryStringQuery(`Name_words:"John Smith"`), + query.NewQueryStringQuery(`Name_words:"Jane"`), }), wantErr: false, }, @@ -139,10 +139,10 @@ func Test_compile(t *testing.T) { }, want: query.NewDisjunctionQuery([]query.Query{ query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:a`), - query.NewQueryStringQuery(`Name_lowercase:b`), + query.NewQueryStringQuery(`Name_words:"a"`), + query.NewQueryStringQuery(`Name_words:"b"`), }), - query.NewQueryStringQuery(`Name_lowercase:c`), + query.NewQueryStringQuery(`Name_words:"c"`), }), wantErr: false, }, @@ -158,10 +158,10 @@ func Test_compile(t *testing.T) { }, }, want: query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:a`), + query.NewQueryStringQuery(`Name_words:"a"`), query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:b`), - query.NewQueryStringQuery(`Name_lowercase:c`), + query.NewQueryStringQuery(`Name_words:"b"`), + query.NewQueryStringQuery(`Name_words:"c"`), }), }), wantErr: false, @@ -183,11 +183,11 @@ func Test_compile(t *testing.T) { }, want: query.NewConjunctionQuery([]query.Query{ query.NewDisjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:a`), - query.NewQueryStringQuery(`Name_lowercase:b`), - query.NewQueryStringQuery(`Name_lowercase:c`), + query.NewQueryStringQuery(`Name_words:"a"`), + query.NewQueryStringQuery(`Name_words:"b"`), + query.NewQueryStringQuery(`Name_words:"c"`), }), - query.NewQueryStringQuery(`Name_lowercase:d`), + query.NewQueryStringQuery(`Name_words:"d"`), }), wantErr: false, }, @@ -320,7 +320,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:john\ smith`), + query.NewQueryStringQuery(`Name_words:"John Smith"`), query.NewQueryStringQuery(`Hidden:t`), query.NewQueryStringQuery(`Hidden:t`), }), @@ -548,7 +548,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_lowercase:john\ smith\ \+\-\=\&\|\>\<\!\(\)\{\}\[\]\^\"\~\:\ `), + query.NewQueryStringQuery(`Name_words:"John Smith +-=&|> Date: Sat, 29 Aug 2026 13:10:00 +0200 Subject: [PATCH 32/54] feat(search): search every keyword field case-insensitively by default KQL searches case-insensitively, so every keyword field gets its lowercased search sibling unless it opts out: ids are opaque, paths are POSIX, the mime type is normalized already. That takes the facets along, artist or camera model match regardless of case, while the case-preserved base still answers and aggregates. Which siblings a field carries is decided once, from the struct and the overrides, and the renderers, the document writer and the query lowering all read it from there. --- services/search/pkg/bleve/backend_test.go | 4 +- services/search/pkg/mapping/casing.go | 57 ++++++++++++++----- services/search/pkg/mapping/casing_test.go | 12 +++- services/search/pkg/mapping/opts.go | 5 +- services/search/pkg/mapping/serialize.go | 3 +- services/search/pkg/mapping/serialize_test.go | 9 +-- services/search/pkg/mapping/validate.go | 2 +- services/search/pkg/opensearch/index.go | 4 +- services/search/pkg/query/normalize_test.go | 10 ++-- services/search/pkg/query/resolver.go | 32 ++--------- services/search/pkg/search/search.go | 23 +++++--- 11 files changed, 95 insertions(+), 66 deletions(-) diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go index 6df0975a9d..a98e768941 100644 --- a/services/search/pkg/bleve/backend_test.go +++ b/services/search/pkg/bleve/backend_test.go @@ -221,7 +221,7 @@ var _ = Describe("Bleve", func() { assertDocCount(rootResource.ID, "Size:>100000", 0) }) - It("preserves value case for fields not explicitly marked lowercase", func() { + It("matches facet values case-insensitively", func() { parentResource.Document.Audio = &libregraph.Audio{ Artist: libregraph.PtrString("Some Artist"), } @@ -229,7 +229,7 @@ var _ = Describe("Bleve", func() { Expect(err).ToNot(HaveOccurred()) assertDocCount(rootResource.ID, `audio.artist:"Some Artist"`, 1) - assertDocCount(rootResource.ID, `audio.artist:"some artist"`, 0) + assertDocCount(rootResource.ID, `audio.artist:"some artist"`, 1) }) }) diff --git a/services/search/pkg/mapping/casing.go b/services/search/pkg/mapping/casing.go index abe9348477..6eb6824fea 100644 --- a/services/search/pkg/mapping/casing.go +++ b/services/search/pkg/mapping/casing.go @@ -1,27 +1,60 @@ package mapping -import "strings" +import ( + "reflect" + "strings" +) -// addSearchSiblings writes the _lowercase and _words siblings the overrides ask -// for next to their base values. -func addSearchSiblings(m map[string]any, overrides map[string]FieldOpts) { - for key, opts := range overrides { - if !isCasedType(opts) || (!opts.caseInsensitive() && !opts.wordBroken()) { - continue - } +// addSearchSiblings writes the _lowercase and _words siblings next to their +// base values, for every keyword/path field of t that has them (see +// SearchSiblings). +func addSearchSiblings(m map[string]any, t reflect.Type, overrides map[string]FieldOpts) { + for key, siblings := range SearchSiblings(t, overrides) { parent, leaf, ok := resolveLeaf(m, key) if !ok { continue } - if opts.caseInsensitive() { + if siblings.Lowercase { addLowercaseSibling(parent, leaf) } - if opts.wordBroken() { + if siblings.Words { addWordsSibling(parent, leaf) } } } +// Siblings says which search-only siblings a field carries. +type Siblings struct { + Lowercase bool + Words bool +} + +// SearchSiblings lists the fields of t (json names, nested as parent.child) +// that carry a _lowercase or _words sibling, from the effective field type and +// the overrides. It is the one place that decides, the renderers, the +// document writer and the query lowering all follow it. +func SearchSiblings(t reflect.Type, overrides map[string]FieldOpts) map[string]Siblings { + out := map[string]Siblings{} + for key, goType := range collectFields(t, "") { + opts := overrides[key] + eff := opts.Type + if eff == "" { + eff = inferType(goType) + } + if eff != TypeKeyword && eff != TypePath { + continue + } + siblings := Siblings{ + Lowercase: opts.caseInsensitive(), + Words: eff == TypeKeyword && opts.wordBroken(), + } + if siblings.Lowercase || siblings.Words { + out[key] = siblings + } + } + return out +} + // addWordsSibling copies the value to a _words sibling; the words // analyzer does the splitting and lowercasing. No-op for non-strings. func addWordsSibling(parent map[string]any, leaf string) { @@ -31,10 +64,6 @@ func addWordsSibling(parent map[string]any, leaf string) { } } -func isCasedType(opts FieldOpts) bool { - return opts.Type == "" || opts.Type == TypeKeyword || opts.Type == TypePath -} - func resolveLeaf(m map[string]any, dottedPath string) (map[string]any, string, bool) { parts := strings.Split(dottedPath, ".") parent := m diff --git a/services/search/pkg/mapping/casing_test.go b/services/search/pkg/mapping/casing_test.go index 255eb36ef4..4f77eb4bc0 100644 --- a/services/search/pkg/mapping/casing_test.go +++ b/services/search/pkg/mapping/casing_test.go @@ -45,12 +45,22 @@ var _ = Describe("PrepareForIndex casing", func() { Expect(m["Name_words"]).To(Equal("Report FINAL")) }) - It("writes no sibling without CaseInsensitive", func() { + It("writes the lowercased sibling by default", func() { type doc struct { ID string `json:"ID"` } m, err := PrepareForIndex(doc{ID: "ABC"}, nil) Expect(err).ToNot(HaveOccurred()) + Expect(m["ID"+LowercaseSuffix]).To(Equal("abc")) + }) + + It("writes no sibling with CaseInsensitive off", func() { + False := false + type doc struct { + ID string `json:"ID"` + } + m, err := PrepareForIndex(doc{ID: "ABC"}, map[string]FieldOpts{"ID": {CaseInsensitive: &False}}) + Expect(err).ToNot(HaveOccurred()) Expect(m).ToNot(HaveKey("ID" + LowercaseSuffix)) }) diff --git a/services/search/pkg/mapping/opts.go b/services/search/pkg/mapping/opts.go index f2589fc865..6d2495ae0c 100644 --- a/services/search/pkg/mapping/opts.go +++ b/services/search/pkg/mapping/opts.go @@ -35,7 +35,8 @@ type FieldOpts struct { // CaseInsensitive additionally indexes a lowercased _lowercase sibling // for case-insensitive search; the case-preserved base is always indexed. - // Nil/false means off. Keyword/path only. + // On by default for keyword/path fields, KQL searches case-insensitively; + // false opts a field out (ids, paths). CaseInsensitive *bool // NoWordBreaker is SharePoint's switch: nil or true leaves a keyword field @@ -52,5 +53,5 @@ type FieldOpts struct { IncludeInAll *bool } -func (o FieldOpts) caseInsensitive() bool { return o.CaseInsensitive != nil && *o.CaseInsensitive } +func (o FieldOpts) caseInsensitive() bool { return o.CaseInsensitive == nil || *o.CaseInsensitive } func (o FieldOpts) wordBroken() bool { return o.NoWordBreaker != nil && !*o.NoWordBreaker } diff --git a/services/search/pkg/mapping/serialize.go b/services/search/pkg/mapping/serialize.go index b4e5310989..e115e95e65 100644 --- a/services/search/pkg/mapping/serialize.go +++ b/services/search/pkg/mapping/serialize.go @@ -2,6 +2,7 @@ package mapping import ( "fmt" + "reflect" "github.com/opencloud-eu/opencloud/pkg/conversions" ) @@ -19,6 +20,6 @@ func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, err return out, nil } addGeopointSiblings(out, overrides) - addSearchSiblings(out, overrides) + addSearchSiblings(out, deref(reflect.TypeOf(v)), overrides) return out, nil } diff --git a/services/search/pkg/mapping/serialize_test.go b/services/search/pkg/mapping/serialize_test.go index 18051639fa..39fc75c05d 100644 --- a/services/search/pkg/mapping/serialize_test.go +++ b/services/search/pkg/mapping/serialize_test.go @@ -23,17 +23,18 @@ var _ = Describe("PrepareForIndex serialization", func() { }) It("flattens embedded structs", func() { - type inner struct { + type Inner struct { Name string `json:"Name"` Size uint64 `json:"Size"` } type outer struct { - inner + Inner ID string `json:"ID"` } - m, err := PrepareForIndex(outer{inner: inner{Name: "a", Size: 7}, ID: "x"}, nil) + m, err := PrepareForIndex(outer{Inner: Inner{Name: "a", Size: 7}, ID: "x"}, nil) Expect(err).ToNot(HaveOccurred()) - want := map[string]any{"Name": "a", "Size": float64(7), "ID": "x"} + // keywords carry their lowercased search sibling by default + want := map[string]any{"Name": "a", "Name_lowercase": "a", "Size": float64(7), "ID": "x", "ID_lowercase": "x"} Expect(m).To(Equal(want)) }) diff --git a/services/search/pkg/mapping/validate.go b/services/search/pkg/mapping/validate.go index 3147eb20ed..c7b59e9e57 100644 --- a/services/search/pkg/mapping/validate.go +++ b/services/search/pkg/mapping/validate.go @@ -32,7 +32,7 @@ func Validate(t reflect.Type, overrides map[string]FieldOpts) error { // would target a non-existent field and silently match nothing. Use the // effective type (override, else the inferred Go type), since an override // with no explicit Type still infers keyword/numeric/... from the field. - if opts.caseInsensitive() && !effectivelyCased(opts, goType) { + if opts.CaseInsensitive != nil && *opts.CaseInsensitive && !effectivelyCased(opts, goType) { miscased = append(miscased, k) } } diff --git a/services/search/pkg/opensearch/index.go b/services/search/pkg/opensearch/index.go index 8049cc4bd0..17ef45a4ba 100644 --- a/services/search/pkg/opensearch/index.go +++ b/services/search/pkg/opensearch/index.go @@ -60,7 +60,9 @@ func (m IndexManager) MarshalJSON() ([]byte, error) { func buildResourceMapping() ([]byte, error) { resourceType := reflect.TypeFor[search.Resource]() overrides := maps.Clone(search.Resource{}.SearchFieldOverrides()) - overrides["MimeType"] = searchmapping.FieldOpts{Type: searchmapping.TypeWildcard} + mimeType := overrides["MimeType"] + mimeType.Type = searchmapping.TypeWildcard + overrides["MimeType"] = mimeType if err := searchmapping.Validate(resourceType, overrides); err != nil { return nil, err } diff --git a/services/search/pkg/query/normalize_test.go b/services/search/pkg/query/normalize_test.go index 281289a908..8bb723b082 100644 --- a/services/search/pkg/query/normalize_test.go +++ b/services/search/pkg/query/normalize_test.go @@ -30,12 +30,12 @@ var _ = Describe("ResolveField", func() { var _ = Describe("FieldIsCaseInsensitive", func() { It("reports the CaseInsensitive override fields", func() { - // The CaseInsensitive override fields (resolved canonical names). - for _, f := range []string{"Name", "Title", "Tags", "Favorites"} { + // keyword fields are case-insensitive by default, facets included + for _, f := range []string{"Name", "Title", "Tags", "Favorites", "audio.artist", "photo.cameraMake"} { Expect(query.FieldIsCaseInsensitive(f)).To(BeTrue(), f) } - // Case-preserved / non-keyword fields are not. - for _, f := range []string{"MimeType", "ID", "Content", "Path", "unknown"} { + // opted out (ids, path, mime type) or not a keyword at all + for _, f := range []string{"MimeType", "ID", "RootID", "ParentID", "Content", "Path", "Size", "unknown"} { Expect(query.FieldIsCaseInsensitive(f)).To(BeFalse(), f) } }) @@ -71,7 +71,7 @@ var _ = Describe("Normalize", func() { &ast.OperatorNode{Value: "AND"}, &ast.StringNode{Key: "Tags", Value: "x", CaseInsensitive: true}, &ast.OperatorNode{Value: "AND"}, - &ast.StringNode{Key: "photo.cameraMake", Value: "canon"}, + &ast.StringNode{Key: "photo.cameraMake", Value: "canon", CaseInsensitive: true}, &ast.OperatorNode{Value: "AND"}, &ast.OperatorNode{Value: "NOT"}, &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, diff --git a/services/search/pkg/query/resolver.go b/services/search/pkg/query/resolver.go index 2ab2f69530..6f46d65952 100644 --- a/services/search/pkg/query/resolver.go +++ b/services/search/pkg/query/resolver.go @@ -26,16 +26,10 @@ var fieldIndex = sync.OnceValue(func() map[string]string { return idx }) -// caseInsensitiveFields are the fields searched case-insensitively by default, -// derived from the CaseInsensitive overrides. -var caseInsensitiveFields = sync.OnceValue(func() map[string]struct{} { - out := map[string]struct{}{} - for field, opts := range (search.Resource{}).SearchFieldOverrides() { - if opts.CaseInsensitive != nil && *opts.CaseInsensitive { - out[field] = struct{}{} - } - } - return out +// siblingFields lists which search siblings every field carries, from the +// resource struct and its overrides. +var siblingFields = sync.OnceValue(func() map[string]mapping.Siblings { + return mapping.SearchSiblings(reflect.TypeFor[search.Resource](), search.Resource{}.SearchFieldOverrides()) }) // pathFields are hierarchical path fields (TypePath), derived from the overrides. @@ -87,8 +81,7 @@ func FieldValueIsNormalized(field string) bool { // FieldIsCaseInsensitive reports whether a field's default search is case-insensitive. func FieldIsCaseInsensitive(field string) bool { - _, ok := caseInsensitiveFields()[field] - return ok + return siblingFields()[field].Lowercase } // FieldIsPath reports whether a field is a hierarchical path field. @@ -103,21 +96,8 @@ func FieldIsFulltext(field string) bool { return ok } -// wordBrokenFields are the keyword fields split into words (NoWordBreaker set -// to false), derived from the overrides. -var wordBrokenFields = sync.OnceValue(func() map[string]struct{} { - out := map[string]struct{}{} - for field, opts := range (search.Resource{}).SearchFieldOverrides() { - if opts.NoWordBreaker != nil && !*opts.NoWordBreaker { - out[field] = struct{}{} - } - } - return out -}) - // FieldIsWordBroken reports whether a field is split into words, so a value // without a wildcard matches it as a phrase of those words instead of as a whole. func FieldIsWordBroken(field string) bool { - _, ok := wordBrokenFields()[field] - return ok + return siblingFields()[field].Words } diff --git a/services/search/pkg/search/search.go b/services/search/pkg/search/search.go index e49652ff84..c21dfe4ef4 100644 --- a/services/search/pkg/search/search.go +++ b/services/search/pkg/search/search.go @@ -72,16 +72,21 @@ type Resource struct { // resourceFieldOverrides is built once (it never changes) and reused on hot // paths instead of reallocating per call. var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts { - True, False := true, false + False := false return map[string]mapping.FieldOpts{ - // a single word finds names and titles that contain it; wildcards and - // whole values go through the lowercased sibling - "Name": {NoWordBreaker: &False, CaseInsensitive: &True}, - "Title": {NoWordBreaker: &False, CaseInsensitive: &True}, - "Path": {Type: mapping.TypePath}, - "Content": {Type: mapping.TypeFulltext}, - "Tags": {CaseInsensitive: &True, IncludeInAll: &False}, - "Favorites": {CaseInsensitive: &True, IncludeInAll: &False}, + // every keyword field searches case-insensitively unless opted out: + // ids are opaque, paths are POSIX, the mime type is normalized already + "ID": {CaseInsensitive: &False}, + "RootID": {CaseInsensitive: &False}, + "ParentID": {CaseInsensitive: &False}, + "Path": {Type: mapping.TypePath, CaseInsensitive: &False}, + "MimeType": {CaseInsensitive: &False}, + "Content": {Type: mapping.TypeFulltext}, + // a single word finds names and titles that contain it + "Name": {NoWordBreaker: &False}, + "Title": {NoWordBreaker: &False}, + "Tags": {IncludeInAll: &False}, + "Favorites": {IncludeInAll: &False}, "location": {Type: mapping.TypeGeopoint}, } }) From 903ef4521b7e51df48a9f2ddb68a2e9f0a569aac Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Sat, 29 Aug 2026 13:25:34 +0200 Subject: [PATCH 33/54] feat(search): search facets by word too, whole values only by opt-out SharePoint's default for a text property is word breaking, so ours is too: every keyword field gets the _words sibling unless it opts out with NoWordBreaker, which now carries SharePoint's polarity as well. Artist, album, camera model and the other facets match by word like name and title; tags and favorites stay one label, ids, paths and the mime type one value. --- services/search/pkg/mapping/bleve_test.go | 7 ++--- services/search/pkg/mapping/casing_test.go | 7 ++--- .../search/pkg/mapping/opensearch_test.go | 20 ++++++++++---- services/search/pkg/mapping/opts.go | 16 ++++++------ services/search/pkg/mapping/serialize_test.go | 8 ++++-- services/search/pkg/mapping/validate.go | 2 +- services/search/pkg/mapping/validate_test.go | 2 +- services/search/pkg/query/normalize_test.go | 6 ++--- services/search/pkg/search/search.go | 26 +++++++++---------- 9 files changed, 50 insertions(+), 44 deletions(-) diff --git a/services/search/pkg/mapping/bleve_test.go b/services/search/pkg/mapping/bleve_test.go index dc76c6bd90..ed8310bf3f 100644 --- a/services/search/pkg/mapping/bleve_test.go +++ b/services/search/pkg/mapping/bleve_test.go @@ -92,11 +92,8 @@ var _ = Describe("BleveBuildMapping", func() { Expect(dm.Properties["Tags_lowercase"].Fields[0].IncludeInAll).To(BeFalse(), "Tags sibling IncludeInAll honored") }) - It("splits a keyword into words when NoWordBreaker is false", func() { - True, False := true, false - dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{ - "Name": {NoWordBreaker: &False, CaseInsensitive: &True}, - }) + It("gives a keyword its lowercase and words siblings by default", func() { + dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil) Expect(err).ToNot(HaveOccurred()) // the base stays a keyword, the words go to a search-only sibling Expect(dm.Properties["Name"].Fields[0].Analyzer).To(Equal("keyword"), "Name base stays a keyword") diff --git a/services/search/pkg/mapping/casing_test.go b/services/search/pkg/mapping/casing_test.go index 4f77eb4bc0..862c310c53 100644 --- a/services/search/pkg/mapping/casing_test.go +++ b/services/search/pkg/mapping/casing_test.go @@ -30,14 +30,11 @@ var _ = Describe("PrepareForIndex casing", func() { Expect(m["Tags_lowercase"]).To(Equal([]any{"work", "urgent"})) }) - It("copies the value to a words sibling when NoWordBreaker is off", func() { - True, False := true, false + It("copies the value to a words sibling by default", func() { type doc struct { Name string `json:"Name"` } - m, err := PrepareForIndex(doc{Name: "Report FINAL"}, map[string]FieldOpts{ - "Name": {NoWordBreaker: &False, CaseInsensitive: &True}, - }) + m, err := PrepareForIndex(doc{Name: "Report FINAL"}, nil) Expect(err).ToNot(HaveOccurred()) // the analyzer splits and lowercases, the value goes over as is Expect(m["Name"]).To(Equal("Report FINAL")) diff --git a/services/search/pkg/mapping/opensearch_test.go b/services/search/pkg/mapping/opensearch_test.go index 4ad05548bb..7e89a3e75b 100644 --- a/services/search/pkg/mapping/opensearch_test.go +++ b/services/search/pkg/mapping/opensearch_test.go @@ -106,14 +106,11 @@ var _ = Describe("OpenSearchBuildMapping", func() { Expect(mime["type"]).To(Equal("wildcard"), "MimeType: %#v", mime) }) - It("splits a keyword into words when NoWordBreaker is false", func() { - True, False := true, false + It("gives a keyword its lowercase and words siblings by default", func() { type doc struct { Name string `json:"Name"` } - props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{ - "Name": {NoWordBreaker: &False, CaseInsensitive: &True}, - }) + props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), nil) Expect(err).ToNot(HaveOccurred()) // the base stays a keyword, the words go to their own sibling Expect(props["Name"]).To(Equal(map[string]any{"type": "keyword"})) @@ -121,6 +118,19 @@ var _ = Describe("OpenSearchBuildMapping", func() { Expect(props["Name_words"]).To(Equal(map[string]any{"type": "text", "analyzer": WordsAnalyzer})) }) + It("leaves a keyword one whole value with NoWordBreaker", func() { + True := true + type doc struct { + Tag string `json:"Tag"` + } + props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{ + "Tag": {NoWordBreaker: &True}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(props).To(HaveKey("Tag_lowercase")) + Expect(props).ToNot(HaveKey("Tag_words")) + }) + It("builds an object plus a geo_point sibling for geopoints", func() { type doc struct { Location *struct { diff --git a/services/search/pkg/mapping/opts.go b/services/search/pkg/mapping/opts.go index 6d2495ae0c..d8af5f17bf 100644 --- a/services/search/pkg/mapping/opts.go +++ b/services/search/pkg/mapping/opts.go @@ -39,13 +39,13 @@ type FieldOpts struct { // false opts a field out (ids, paths). CaseInsensitive *bool - // NoWordBreaker is SharePoint's switch: nil or true leaves a keyword field - // one whole value, false additionally indexes a _words sibling split - // into lowercased words (no stemming), so a single word matches a value - // that contains it: "report" finds "Report.txt". The base stays the whole - // value for returning and aggregating; wildcards and whole-value matches - // use the _lowercase sibling, so it wants CaseInsensitive alongside. - // Keyword only. + // NoWordBreaker is SharePoint's switch, with its default: a keyword field + // additionally indexes a _words sibling split into lowercased words + // (no stemming), so a single word matches a value that contains it, + // "report" finds "Report.txt"; true opts a field out and leaves it one + // whole value (tags, ids, paths). The base stays the whole value for + // returning and aggregating; wildcards and whole-value matches use the + // _lowercase sibling. Keyword only. NoWordBreaker *bool // IncludeInAll controls bleve's _all field inclusion. Nil means "use the @@ -54,4 +54,4 @@ type FieldOpts struct { } func (o FieldOpts) caseInsensitive() bool { return o.CaseInsensitive == nil || *o.CaseInsensitive } -func (o FieldOpts) wordBroken() bool { return o.NoWordBreaker != nil && !*o.NoWordBreaker } +func (o FieldOpts) wordBroken() bool { return o.NoWordBreaker == nil || !*o.NoWordBreaker } diff --git a/services/search/pkg/mapping/serialize_test.go b/services/search/pkg/mapping/serialize_test.go index 39fc75c05d..b2a220f190 100644 --- a/services/search/pkg/mapping/serialize_test.go +++ b/services/search/pkg/mapping/serialize_test.go @@ -33,8 +33,12 @@ var _ = Describe("PrepareForIndex serialization", func() { } m, err := PrepareForIndex(outer{Inner: Inner{Name: "a", Size: 7}, ID: "x"}, nil) Expect(err).ToNot(HaveOccurred()) - // keywords carry their lowercased search sibling by default - want := map[string]any{"Name": "a", "Name_lowercase": "a", "Size": float64(7), "ID": "x", "ID_lowercase": "x"} + // keywords carry their lowercase and words search siblings by default + want := map[string]any{ + "Name": "a", "Name_lowercase": "a", "Name_words": "a", + "Size": float64(7), + "ID": "x", "ID_lowercase": "x", "ID_words": "x", + } Expect(m).To(Equal(want)) }) diff --git a/services/search/pkg/mapping/validate.go b/services/search/pkg/mapping/validate.go index c7b59e9e57..0b6c259be2 100644 --- a/services/search/pkg/mapping/validate.go +++ b/services/search/pkg/mapping/validate.go @@ -24,7 +24,7 @@ func Validate(t reflect.Type, overrides map[string]FieldOpts) error { unknown = append(unknown, k) continue } - if opts.wordBroken() && !effectivelyKeyword(opts, goType) { + if opts.NoWordBreaker != nil && !*opts.NoWordBreaker && !effectivelyKeyword(opts, goType) { unbroken = append(unbroken, k) } // CaseInsensitive routes queries to a _lowercase sibling, which is diff --git a/services/search/pkg/mapping/validate_test.go b/services/search/pkg/mapping/validate_test.go index 39bf5034d5..cc76f1e831 100644 --- a/services/search/pkg/mapping/validate_test.go +++ b/services/search/pkg/mapping/validate_test.go @@ -55,7 +55,7 @@ var _ = Describe("Validate", func() { Expect(err.Error()).To(ContainSubstring("CaseInsensitive")) }) - It("rejects NoWordBreaker on a non-keyword field", func() { + It("rejects switching NoWordBreaker off on a non-keyword field", func() { False := false err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{ "Name": {Type: TypeFulltext, NoWordBreaker: &False}, diff --git a/services/search/pkg/query/normalize_test.go b/services/search/pkg/query/normalize_test.go index 8bb723b082..5260eae383 100644 --- a/services/search/pkg/query/normalize_test.go +++ b/services/search/pkg/query/normalize_test.go @@ -42,11 +42,11 @@ var _ = Describe("FieldIsCaseInsensitive", func() { }) var _ = Describe("FieldIsWordBroken", func() { - It("reports the fields with NoWordBreaker switched off", func() { - for _, f := range []string{"Name", "Title"} { + It("reports the word-broken fields, keywords unless opted out", func() { + for _, f := range []string{"Name", "Title", "audio.artist", "photo.cameraMake"} { Expect(query.FieldIsWordBroken(f)).To(BeTrue(), f) } - // whole-value keywords, paths and full text are not + // opted out (labels, ids, mime type), paths and full text are not for _, f := range []string{"Tags", "Favorites", "MimeType", "ID", "Content", "Path", "unknown"} { Expect(query.FieldIsWordBroken(f)).To(BeFalse(), f) } diff --git a/services/search/pkg/search/search.go b/services/search/pkg/search/search.go index c21dfe4ef4..23d28a1148 100644 --- a/services/search/pkg/search/search.go +++ b/services/search/pkg/search/search.go @@ -72,21 +72,19 @@ type Resource struct { // resourceFieldOverrides is built once (it never changes) and reused on hot // paths instead of reallocating per call. var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts { - False := false + True, False := true, false return map[string]mapping.FieldOpts{ - // every keyword field searches case-insensitively unless opted out: - // ids are opaque, paths are POSIX, the mime type is normalized already - "ID": {CaseInsensitive: &False}, - "RootID": {CaseInsensitive: &False}, - "ParentID": {CaseInsensitive: &False}, - "Path": {Type: mapping.TypePath, CaseInsensitive: &False}, - "MimeType": {CaseInsensitive: &False}, - "Content": {Type: mapping.TypeFulltext}, - // a single word finds names and titles that contain it - "Name": {NoWordBreaker: &False}, - "Title": {NoWordBreaker: &False}, - "Tags": {IncludeInAll: &False}, - "Favorites": {IncludeInAll: &False}, + // every keyword field searches case-insensitively and by word (name, + // title, the facets) unless opted out: ids are opaque, paths are POSIX, + // the mime type is normalized already, a tag is one label + "ID": {CaseInsensitive: &False, NoWordBreaker: &True}, + "RootID": {CaseInsensitive: &False, NoWordBreaker: &True}, + "ParentID": {CaseInsensitive: &False, NoWordBreaker: &True}, + "Path": {Type: mapping.TypePath, CaseInsensitive: &False}, + "MimeType": {CaseInsensitive: &False, NoWordBreaker: &True}, + "Content": {Type: mapping.TypeFulltext}, + "Tags": {NoWordBreaker: &True, IncludeInAll: &False}, + "Favorites": {NoWordBreaker: &True, IncludeInAll: &False}, "location": {Type: mapping.TypeGeopoint}, } }) From b5693d7058845c33c1ca71b2049e44fa7369b335 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 12:51:18 +0200 Subject: [PATCH 34/54] chore(search): one schema generation, bump it to 4 The hand-written v3 of the interim mapping is taken; both engines derive index name and directory from search.SchemaVersion. --- services/search/pkg/bleve/index.go | 1 - services/search/pkg/bleve/index_test.go | 8 +++++--- services/search/pkg/search/search.go | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/services/search/pkg/bleve/index.go b/services/search/pkg/bleve/index.go index c65373a55f..03eafe0e11 100644 --- a/services/search/pkg/bleve/index.go +++ b/services/search/pkg/bleve/index.go @@ -23,7 +23,6 @@ import ( const ( wildcardSuffix = ".wildcard" - indexVersion = "v2" ) func NewIndex(root string) (bleve.Index, error) { diff --git a/services/search/pkg/bleve/index_test.go b/services/search/pkg/bleve/index_test.go index a62dfc2c28..b09d2da12a 100644 --- a/services/search/pkg/bleve/index_test.go +++ b/services/search/pkg/bleve/index_test.go @@ -1,12 +1,14 @@ package bleve_test import ( + "fmt" "path/filepath" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) var _ = Describe("Index", func() { @@ -18,8 +20,8 @@ var _ = Describe("Index", func() { Expect(err).ToNot(HaveOccurred()) DeferCleanup(index.Close) - Expect(index.Name()).To(Equal(filepath.Join(root, "bleve-v2"))) - Expect(filepath.Join(root, "bleve-v2")).To(BeADirectory()) + Expect(index.Name()).To(Equal(filepath.Join(root, fmt.Sprintf("bleve-v%d", search.SchemaVersion)))) + Expect(filepath.Join(root, fmt.Sprintf("bleve-v%d", search.SchemaVersion))).To(BeADirectory()) }) It("opens the index that is already there", func() { @@ -33,7 +35,7 @@ var _ = Describe("Index", func() { Expect(err).ToNot(HaveOccurred()) DeferCleanup(reopened.Close) - Expect(reopened.Name()).To(Equal(filepath.Join(root, "bleve-v2"))) + Expect(reopened.Name()).To(Equal(filepath.Join(root, fmt.Sprintf("bleve-v%d", search.SchemaVersion)))) }) }) }) diff --git a/services/search/pkg/search/search.go b/services/search/pkg/search/search.go index 23d28a1148..4302a440b9 100644 --- a/services/search/pkg/search/search.go +++ b/services/search/pkg/search/search.go @@ -27,7 +27,7 @@ import ( // on a breaking mapping change: each version gets its own index (OpenSearch name // suffix, bleve path suffix), so the service builds a fresh index instead of // colliding with the old one. No migration; reindex to populate. -const SchemaVersion = 3 +const SchemaVersion = 4 var scopeRegex = regexp.MustCompile(`scope:\s*([^" "\n\r]*)`) From ba210232ab95019279a760675c52c163f50b6818 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 12:52:56 +0200 Subject: [PATCH 35/54] feat(search): analyze Content by words on both engines, no stemming Adopts the parity-pinned semantics from #3408: 'report' does not match 'reports'; the porter fulltext analyzer is gone. --- services/search/pkg/bleve/backend_test.go | 6 +++--- services/search/pkg/bleve/index.go | 15 --------------- services/search/pkg/mapping/bleve.go | 4 ++-- services/search/pkg/mapping/bleve_test.go | 4 ++-- services/search/pkg/mapping/opensearch.go | 2 +- services/search/pkg/mapping/opensearch_test.go | 2 +- services/search/pkg/opensearch/index.go | 7 ------- 7 files changed, 9 insertions(+), 31 deletions(-) diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go index a98e768941..21603ee343 100644 --- a/services/search/pkg/bleve/backend_test.go +++ b/services/search/pkg/bleve/backend_test.go @@ -381,14 +381,14 @@ var _ = Describe("Bleve", func() { }) Context("by content", func() { - It("matches full-text case-insensitively and stemmed", func() { + It("matches full-text case-insensitively, without stemming", func() { parentResource.Document.Content = "Running Foxes" Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) assertDocCount(rootResource.ID, "content:running", 1) assertDocCount(rootResource.ID, "content:RUNNING", 1) // case-insensitive - assertDocCount(rootResource.ID, "content:run", 1) // porter stemming - assertDocCount(rootResource.ID, "content:run*", 1) // wildcard over the stemmed term + assertDocCount(rootResource.ID, "content:run", 0) // no stemming + assertDocCount(rootResource.ID, "content:run*", 1) // wildcard over the word assertDocCount(rootResource.ID, "content:cat", 0) }) }) diff --git a/services/search/pkg/bleve/index.go b/services/search/pkg/bleve/index.go index 03eafe0e11..3ff18c517a 100644 --- a/services/search/pkg/bleve/index.go +++ b/services/search/pkg/bleve/index.go @@ -12,7 +12,6 @@ import ( "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" "github.com/blevesearch/bleve/v2/analysis/char/regexp" "github.com/blevesearch/bleve/v2/analysis/token/lowercase" - "github.com/blevesearch/bleve/v2/analysis/token/porter" "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" "github.com/blevesearch/bleve/v2/mapping" storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -58,20 +57,6 @@ func NewMapping() (mapping.IndexMapping, error) { indexMapping := bleve.NewIndexMapping() indexMapping.DefaultAnalyzer = keyword.Name indexMapping.DefaultMapping = docMapping - err = indexMapping.AddCustomAnalyzer("fulltext", - map[string]any{ - "type": custom.Name, - "tokenizer": unicode.Name, - "token_filters": []string{ - lowercase.Name, - porter.Name, - }, - }, - ) - if err != nil { - return nil, err - } - // words: split into lowercased words, a dot is a word boundary too so that // "report" finds "Report.txt"; no stemming, a name is not prose err = indexMapping.AddCustomCharFilter("dot_to_space", map[string]any{ diff --git a/services/search/pkg/mapping/bleve.go b/services/search/pkg/mapping/bleve.go index 150a5a427a..c02177caae 100644 --- a/services/search/pkg/mapping/bleve.go +++ b/services/search/pkg/mapping/bleve.go @@ -12,7 +12,7 @@ import ( // struct via reflection. Field names come from json tags; overrides are // keyed by those names (or dotted paths for nested fields). // -// The returned mapping references the "fulltext" analyzer for Fulltext fields; +// The returned mapping references the words analyzer for Fulltext fields; // the caller registers it on the enclosing IndexMapping. func BleveBuildMapping(t reflect.Type, overrides map[string]FieldOpts) (*bleveMapping.DocumentMapping, error) { return buildBleveDocMapping(t, overrides, "") @@ -118,7 +118,7 @@ func bleveFieldMapping(fieldType string, opts FieldOpts) (*bleveMapping.FieldMap case TypeKeyword, TypeFulltext: fm := bleve.NewTextFieldMapping() if fieldType == TypeFulltext { - fm.Analyzer = "fulltext" + fm.Analyzer = WordsAnalyzer } switch { case opts.IncludeInAll != nil: diff --git a/services/search/pkg/mapping/bleve_test.go b/services/search/pkg/mapping/bleve_test.go index ed8310bf3f..9ba1344a5c 100644 --- a/services/search/pkg/mapping/bleve_test.go +++ b/services/search/pkg/mapping/bleve_test.go @@ -85,8 +85,8 @@ var _ = Describe("BleveBuildMapping", func() { Expect(sibling.IncludeInAll).To(BeFalse(), "Name_lowercase is out of _all") Expect(sibling.DocValues).To(BeFalse(), "Name_lowercase has no doc values") contentField := dm.Properties["Content"].Fields[0] - Expect(contentField.Analyzer).To(Equal("fulltext"), "Content analyzer") - Expect(contentField.IncludeInAll).To(BeFalse(), "Content IncludeInAll should default to false for fulltext type") + Expect(contentField.Analyzer).To(Equal(WordsAnalyzer), "Content analyzer") + Expect(contentField.IncludeInAll).To(BeFalse(), "Content IncludeInAll should default to false for the fulltext type") // Tags: base + lowercased sibling, both honoring the IncludeInAll override. Expect(dm.Properties["Tags"].Fields[0].IncludeInAll).To(BeFalse(), "Tags base IncludeInAll honored") Expect(dm.Properties["Tags_lowercase"].Fields[0].IncludeInAll).To(BeFalse(), "Tags sibling IncludeInAll honored") diff --git a/services/search/pkg/mapping/opensearch.go b/services/search/pkg/mapping/opensearch.go index 5254e48423..87319d4bfe 100644 --- a/services/search/pkg/mapping/opensearch.go +++ b/services/search/pkg/mapping/opensearch.go @@ -90,7 +90,7 @@ func openSearchFieldMapping(fieldType string, goType reflect.Type) (map[string]a return map[string]any{ "type": "text", "term_vector": "with_positions_offsets", - "analyzer": "fulltext", + "analyzer": WordsAnalyzer, }, nil case TypeWildcard: // OpenSearch stores wildcard fields with doc_values=false by diff --git a/services/search/pkg/mapping/opensearch_test.go b/services/search/pkg/mapping/opensearch_test.go index 7e89a3e75b..f55b2de640 100644 --- a/services/search/pkg/mapping/opensearch_test.go +++ b/services/search/pkg/mapping/opensearch_test.go @@ -98,7 +98,7 @@ var _ = Describe("OpenSearchBuildMapping", func() { content := props["Content"].(map[string]any) Expect(content["type"]).To(Equal("text"), "Content: %#v", content) Expect(content["term_vector"]).To(Equal("with_positions_offsets"), "Content: %#v", content) - Expect(content["analyzer"]).To(Equal("fulltext"), "Content uses the stemming fulltext analyzer, like bleve") + Expect(content["analyzer"]).To(Equal(WordsAnalyzer), "Content uses the words analyzer, like bleve") // Path: path_hierarchy base + lowercased sibling, both case-preserving. Expect(props["Path"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"})) Expect(props["Path_lowercase"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"})) diff --git a/services/search/pkg/opensearch/index.go b/services/search/pkg/opensearch/index.go index 17ef45a4ba..b611db71a9 100644 --- a/services/search/pkg/opensearch/index.go +++ b/services/search/pkg/opensearch/index.go @@ -77,18 +77,11 @@ func buildResourceMapping() ([]byte, error) { "number_of_replicas": "1", "analysis": map[string]any{ // path_hierarchy is case-preserving; casing lives in the value. - // fulltext mirrors the bleve fulltext analyzer (lowercase + porter - // stemming) so full-text search behaves the same on both backends. "analyzer": map[string]any{ "path_hierarchy": map[string]any{ "type": "custom", "tokenizer": "path_hierarchy", }, - "fulltext": map[string]any{ - "type": "custom", - "tokenizer": "standard", - "filter": []string{"lowercase", "porter_stem"}, - }, // words: split into lowercased words, a dot is a word boundary // too so that "report" finds "Report.txt"; no stemming, a name // is not prose From 283de0d29aac05b951ff7284ba601b18e806d848 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 12:54:52 +0200 Subject: [PATCH 36/54] feat(search): adopt the parity-pinned query semantics on the sibling routing From #3408: hidden takes bool words only, type categories map to the stored value in the shared pass, ? counts as a wildcard, a non-suffix wildcard on a word-broken field forgives the extension, = matches the whole value on the lowercased sibling, paths lose their trailing slash. Dead compiler helpers removed. --- .../internal/convert/kql_transpile.go | 32 +++- services/search/pkg/query/bleve/compiler.go | 156 ++++++------------ .../search/pkg/query/bleve/compiler_test.go | 21 ++- services/search/pkg/query/normalize.go | 19 +++ 4 files changed, 110 insertions(+), 118 deletions(-) diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index 6ff8016001..9747c7e2b3 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "slices" + "strconv" "strings" "time" @@ -108,17 +109,44 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { case *ast.BooleanNode: return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil case *ast.StringNode: + // hidden takes bool words only; anything else matches nothing + if node.Key == "Hidden" { + b, err := strconv.ParseBool(node.Value) + if err != nil { + return osu.NewMatchNoneQuery(), nil + } + return osu.NewTermQuery[bool](node.Key).Value(b), nil + } + field, value := node.Key, node.Value + if query.FieldIsPath(node.Key) { + value = strings.TrimSuffix(value, "/") + } if node.CaseInsensitive { field += mapping.LowercaseSuffix value = strings.ToLower(value) } - isWildcard := strings.Contains(value, "*") - if isWildcard { + if isWildcard := strings.ContainsAny(value, "*?"); isWildcard { + // a wildcard on a word-broken field forgives a missing extension: + // *report also matches Report.txt + if query.FieldIsWordBroken(node.Key) && !strings.HasSuffix(value, "*") { + return osu.NewBoolQuery(). + Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}). + Should( + osu.NewWildcardQuery(field).Value(value), + osu.NewWildcardQuery(field).Value(value+".*"), + ), nil + } return osu.NewWildcardQuery(field).Value(value), nil } + // = matches the whole value, on the lowercased sibling for + // case-insensitive fields + if node.Exact { + return osu.NewTermQuery[string](field).Value(value), nil + } + // a word-broken field matches the value as a phrase of its words on the // _words sibling, whose analyzer lowercases; wildcards stay on _lowercase if query.FieldIsWordBroken(node.Key) { diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index 8536acc78b..d2f049724a 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -6,8 +6,6 @@ import ( "strconv" "strings" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/blevesearch/bleve/v2" bleveQuery "github.com/blevesearch/bleve/v2/search/query" "github.com/opencloud-eu/opencloud/pkg/ast" @@ -74,28 +72,71 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { for i := offset; i < len(nodes); i++ { switch n := nodes[i].(type) { case *ast.StringNode: + // hidden takes bool words only; anything else matches nothing + if n.Key == "Hidden" { + var q bleveQuery.Query + if b, err := strconv.ParseBool(n.Value); err == nil { + bq := bleveQuery.NewBoolFieldQuery(b) + bq.SetField(n.Key) + q = bq + } else { + q = bleveQuery.NewMatchNoneQuery() + } + if prev == nil { + prev = q + } else { + next = q + } + break + } + // keys are resolved and media-type expanded by normalize. MimeType // skips the escaper so the category wildcards (image/*) keep their `*`; // bleve treats `/` and `+` as literals mid-term, so a literal MIME like // image/svg+xml still matches exactly. + val := n.Value + if searchQuery.FieldIsPath(n.Key) { + val = strings.TrimSuffix(val, "/") + } k := n.Key - v := n.Value + v := val if k != "ID" && k != "Size" && k != "MimeType" { - v = bleveEscaper.Replace(n.Value) + v = bleveEscaper.Replace(val) } if n.CaseInsensitive { k += mapping.LowercaseSuffix v = strings.ToLower(v) + val = strings.ToLower(val) } + isWildcard := strings.ContainsAny(val, "*?") + // a word-broken field matches the value as a phrase of its words on the // _words sibling (a quoted query string term is a match phrase query // run through the field's analyzer); wildcards stay on _lowercase - if searchQuery.FieldIsWordBroken(n.Key) && !strings.Contains(n.Value, "*") { - k, v = n.Key+mapping.WordsSuffix, `"`+strings.ReplaceAll(n.Value, `"`, `\"`)+`"` + if searchQuery.FieldIsWordBroken(n.Key) && !isWildcard && !n.Exact { + k, v = n.Key+mapping.WordsSuffix, `"`+strings.ReplaceAll(val, `"`, `\"`)+`"` } var q bleveQuery.Query = bleveQuery.NewQueryStringQuery(k + ":" + v) + switch { + case n.Exact && !isWildcard: + // = matches the whole value, on the lowercased sibling for + // case-insensitive fields + tq := bleveQuery.NewTermQuery(val) + tq.SetField(k) + q = tq + case isWildcard && searchQuery.FieldIsWordBroken(n.Key) && !strings.HasSuffix(val, "*"): + // a wildcard on a word-broken field forgives a missing extension: + // *report also matches Report.txt + bq := bleve.NewBooleanQuery() + bq.AddShould( + bleveQuery.NewQueryStringQuery(k+":"+v), + bleveQuery.NewQueryStringQuery(k+":"+v+".*"), + ) + bq.SetMinShould(1) + q = bq + } if searchQuery.FieldIsPath(n.Key) { // bleve has no path hierarchy analyzer, unlike OpenSearch: match the // folder itself and its descendants (`\/*`). A BooleanQuery keeps @@ -316,31 +357,6 @@ func numberRange(field string, operator *ast.OperatorNode, value float64) bleveQ return q } -func pathAndBelow(field, path string) bleveQuery.Query { - path = strings.TrimSuffix(path, "/") - - self := bleveQuery.NewTermQuery(path) - self.SetField(field) - - below := bleveQuery.NewPrefixQuery(path + "/") - below.SetField(field) - - return closed(bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{self, below})) -} - -func closed(q bleveQuery.Query) bleveQuery.Query { - // a bare disjunction reads as an open OR chain to mapBinary, a later OR - // would merge into it and widen the group - return bleveQuery.NewConjunctionQuery([]bleveQuery.Query{q}) -} - -func phrase(field, value string) bleveQuery.Query { - q := bleveQuery.NewMatchPhraseQuery(value) - q.SetField(field) - - return q -} - func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode { for _, n := range group.Nodes { if onode, ok := n.(*ast.StringNode); ok { @@ -349,81 +365,3 @@ func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode { } return group } - -func resourceType(value string) string { - switch strings.ToLower(value) { - case "file": - return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10) - case "folder": - return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10) - default: - return value - } -} - -func mimeType(k, v string) (bleveQuery.Query, bool) { - switch v { - case "file": - q := bleve.NewBooleanQuery() - q.AddMustNot(bleveQuery.NewQueryStringQuery(k + ":httpd/unix-directory")) - return q, false - case "folder": - return bleveQuery.NewQueryStringQuery(k + ":httpd/unix-directory"), false - case "document": - return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k, - "application/msword", - "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "application/vnd.openxmlformats-officedocument.wordprocessingml.form", - "application/vnd.oasis.opendocument.text", - "text/plain", - "text/markdown", - "application/rtf", - "application/vnd.apple.pages", - )), true - case "spreadsheet": - return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k, - "application/vnd.ms-excel", - "application/vnd.oasis.opendocument.spreadsheet", - "text/csv", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "application/vnd.apple.numbers", - )), true - case "presentation": - return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k, - "application/vnd.openxmlformats-officedocument.presentationml.presentation", - "application/vnd.oasis.opendocument.presentation", - "application/vnd.ms-powerpoint", - "application/vnd.apple.keynote", - )), true - case "pdf": - return bleveQuery.NewQueryStringQuery(k + ":application/pdf"), false - case "image": - return bleveQuery.NewQueryStringQuery(k + ":image/*"), false - case "video": - return bleveQuery.NewQueryStringQuery(k + ":video/*"), false - case "audio": - return bleveQuery.NewQueryStringQuery(k + ":audio/*"), false - case "archive": - return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k, - "application/zip", - "application/gzip", - "application/x-gzip", - "application/x-7z-compressed", - "application/x-rar-compressed", - "application/x-tar", - "application/x-bzip2", - "application/x-bzip", - "application/x-tgz", - )), true - default: - return bleveQuery.NewQueryStringQuery(k + ":" + v), false - } -} - -func newQueryStringQueryList(k string, v ...string) []bleveQuery.Query { - list := make([]bleveQuery.Query, len(v)) - for i := 0; i < len(v); i++ { - list[i] = bleveQuery.NewQueryStringQuery(k + ":" + v[i]) - } - return list -} diff --git a/services/search/pkg/query/bleve/compiler_test.go b/services/search/pkg/query/bleve/compiler_test.go index a7ff7ab9dd..c4becff38f 100644 --- a/services/search/pkg/query/bleve/compiler_test.go +++ b/services/search/pkg/query/bleve/compiler_test.go @@ -24,6 +24,13 @@ var timeMustParse = func(t *testing.T, ts string) time.Time { // canonical ASTs (real field names, media-type already expanded) and call // compile() directly, dropping the query.Normalize wrapper and the mediatype // cases. +func boolFieldQuery(field string, value bool) query.Query { + q := query.NewBoolFieldQuery(value) + q.SetField(field) + + return q +} + func Test_compile(t *testing.T) { tests := []struct { name string @@ -72,7 +79,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_words:"John Smith"`), + query.NewQueryStringQuery(`Name_words:"john smith"`), }), wantErr: false, }, @@ -86,8 +93,8 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_words:"John Smith"`), - query.NewQueryStringQuery(`Name_words:"Jane"`), + query.NewQueryStringQuery(`Name_words:"john smith"`), + query.NewQueryStringQuery(`Name_words:"jane"`), }), wantErr: false, }, @@ -320,9 +327,9 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_words:"John Smith"`), - query.NewQueryStringQuery(`Hidden:t`), - query.NewQueryStringQuery(`Hidden:t`), + query.NewQueryStringQuery(`Name_words:"john smith"`), + boolFieldQuery("Hidden", true), + boolFieldQuery("Hidden", true), }), wantErr: false, }, @@ -548,7 +555,7 @@ func Test_compile(t *testing.T) { }, }, want: query.NewConjunctionQuery([]query.Query{ - query.NewQueryStringQuery(`Name_words:"John Smith +-=&|> Date: Mon, 31 Aug 2026 12:57:47 +0200 Subject: [PATCH 37/54] fix(search): match fulltext phrases in order on bleve Content:"reports monthly" must not match 'monthly reports'; the phrase runs on the analyzed field like on OpenSearch. --- services/search/pkg/query/bleve/compiler.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index d2f049724a..e4be2d50c9 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -113,9 +113,12 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { // a word-broken field matches the value as a phrase of its words on the // _words sibling (a quoted query string term is a match phrase query - // run through the field's analyzer); wildcards stay on _lowercase + // run through the field's analyzer); wildcards stay on _lowercase. + // A fulltext field is its own words field, the phrase runs on it. if searchQuery.FieldIsWordBroken(n.Key) && !isWildcard && !n.Exact { k, v = n.Key+mapping.WordsSuffix, `"`+strings.ReplaceAll(val, `"`, `\"`)+`"` + } else if searchQuery.FieldIsFulltext(n.Key) && !isWildcard && !n.Exact { + v = `"` + strings.ReplaceAll(val, `"`, `\"`) + `"` } var q bleveQuery.Query = bleveQuery.NewQueryStringQuery(k + ":" + v) From 523d377a088ace759b48111e3f4f9a1d53dd3640 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 12:57:47 +0200 Subject: [PATCH 38/54] test(search): carry the parity suite over to the rebased semantics Typed Mtime fixtures, VersionedIndexName, and audio.artist matches case-insensitively now (facets search case-insensitively by default). --- services/search/pkg/parity/README.md | 2 +- services/search/pkg/parity/engines_test.go | 2 +- services/search/pkg/parity/fixtures_test.go | 13 +++++++++++-- services/search/pkg/parity/matrix_test.go | 5 +++-- services/search/pkg/parity/query_fields_test.go | 2 +- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index 5487239b60..194397e9ff 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -232,7 +232,7 @@ Fixtures: | FIELDS-11 | `id:"1$1!AB-23"` | cased.txt | cased.txt | cased.txt | ✅ | | FIELDS-12 | `id:"1$1!ab-23"` | no match | no match | no match | ✅ | | FIELDS-13 | `audio.artist:"Some Artist"` | song.mp3 | song.mp3 | song.mp3 | ✅ | -| FIELDS-14 | `audio.artist:"some artist"` | no match | no match | no match | ✅ | +| FIELDS-14 | `audio.artist:"some artist"` | song.mp3 | song.mp3 | song.mp3 | ✅ | ### deleted diff --git a/services/search/pkg/parity/engines_test.go b/services/search/pkg/parity/engines_test.go index 71fd99033a..7568dc7409 100644 --- a/services/search/pkg/parity/engines_test.go +++ b/services/search/pkg/parity/engines_test.go @@ -92,7 +92,7 @@ func newOpenSearch(name string, fixtures []search.Resource) testEngine { GinkgoHelper() tc := opensearchtest.NewDefaultTestClient(GinkgoTB(), openSearchClient) - index := opensearch.IndexName(name) + index := opensearch.VersionedIndexName(name) if err := tc.IndicesReset(context.Background(), []string{index}); err != nil { return testEngine{name: "opensearch", unavailable: err.Error()} diff --git a/services/search/pkg/parity/fixtures_test.go b/services/search/pkg/parity/fixtures_test.go index 4bfebd5e8c..330fddc3dd 100644 --- a/services/search/pkg/parity/fixtures_test.go +++ b/services/search/pkg/parity/fixtures_test.go @@ -24,7 +24,15 @@ func withMime(mime string) fixtureOption { return func(r *search.Resource) { r.M func withTitle(t string) fixtureOption { return func(r *search.Resource) { r.Title = t } } func withContent(c string) fixtureOption { return func(r *search.Resource) { r.Content = c } } func withSize(s uint64) fixtureOption { return func(r *search.Resource) { r.Size = s } } -func withMtime(m string) fixtureOption { return func(r *search.Resource) { r.Mtime = m } } +func withMtime(m string) fixtureOption { + return func(r *search.Resource) { + t, err := time.Parse(time.RFC3339Nano, m) + if err != nil { + panic(err) + } + r.Mtime = &t + } +} func withID(id string) fixtureOption { return func(r *search.Resource) { r.ID = id } } func withParent(id string) fixtureOption { return func(r *search.Resource) { r.ParentID = id } } func withRoot(id string) fixtureOption { return func(r *search.Resource) { r.RootID = id } } @@ -47,6 +55,7 @@ func withLocation(location *libregraph.GeoCoordinates) fixtureOption { } func fixtureDoc(name string, opts ...fixtureOption) search.Resource { + mtime := fixtureNow r := search.Resource{ ID: "1$1!" + name, RootID: "1$1!1", @@ -56,7 +65,7 @@ func fixtureDoc(name string, opts ...fixtureOption) search.Resource { Document: content.Document{ Name: name, MimeType: "text/plain", - Mtime: fixtureNow.Format(time.RFC3339Nano), + Mtime: &mtime, Size: 1000, }, } diff --git a/services/search/pkg/parity/matrix_test.go b/services/search/pkg/parity/matrix_test.go index be0c538597..5630ac0cfb 100644 --- a/services/search/pkg/parity/matrix_test.go +++ b/services/search/pkg/parity/matrix_test.go @@ -6,6 +6,7 @@ import ( "os" "sort" "strings" + "time" sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" . "github.com/onsi/ginkgo/v2" @@ -336,8 +337,8 @@ func fixtureFields(f search.Resource, withID bool) string { add("Size = %d", f.Size) } - if !strings.HasPrefix(f.Mtime, fixtureNow.Format("2006-01-02")) { - add("Mtime = %s", f.Mtime) + if f.Mtime == nil || f.Mtime.Format("2006-01-02") != fixtureNow.Format("2006-01-02") { + add("Mtime = %s", f.Mtime.Format(time.RFC3339)) } if f.Hidden { diff --git a/services/search/pkg/parity/query_fields_test.go b/services/search/pkg/parity/query_fields_test.go index e5600893fb..2d80fb4f8f 100644 --- a/services/search/pkg/parity/query_fields_test.go +++ b/services/search/pkg/parity/query_fields_test.go @@ -35,7 +35,7 @@ func fieldsGroup() queryGroup { {id: 12, query: `id:"1$1!ab-23"`}, // a facet value keeps its case, the field is not marked lowercase {id: 13, query: `audio.artist:"Some Artist"`, want: []string{"song.mp3"}}, - {id: 14, query: `audio.artist:"some artist"`}, + {id: 14, query: `audio.artist:"some artist"`, want: []string{"song.mp3"}}, // facets search case-insensitively }, } } From b92d671c02be0fd8c01d27dfa6328be35a562659 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 13:01:22 +0200 Subject: [PATCH 39/54] chore(search): drop the orphaned hand-written v3 mapping leftovers --- services/search/pkg/bleve/index.go | 4 - .../internal/indexes/resource_v3.json | 122 ------------------ 2 files changed, 126 deletions(-) delete mode 100644 services/search/pkg/opensearch/internal/indexes/resource_v3.json diff --git a/services/search/pkg/bleve/index.go b/services/search/pkg/bleve/index.go index 3ff18c517a..9dd7d290ed 100644 --- a/services/search/pkg/bleve/index.go +++ b/services/search/pkg/bleve/index.go @@ -20,10 +20,6 @@ import ( "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) -const ( - wildcardSuffix = ".wildcard" -) - func NewIndex(root string) (bleve.Index, error) { destination := filepath.Join(root, fmt.Sprintf("bleve-v%d", search.SchemaVersion)) index, err := bleve.Open(destination) diff --git a/services/search/pkg/opensearch/internal/indexes/resource_v3.json b/services/search/pkg/opensearch/internal/indexes/resource_v3.json deleted file mode 100644 index 5180f6ad56..0000000000 --- a/services/search/pkg/opensearch/internal/indexes/resource_v3.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "settings": { - "number_of_shards": "1", - "number_of_replicas": "1", - "analysis": { - "analyzer": { - "path_hierarchy": { - "tokenizer": "path_hierarchy", - "type": "custom" - }, - "name_words": { - "type": "custom", - "char_filter": [ - "dot_to_space" - ], - "tokenizer": "standard", - "filter": [ - "lowercase" - ] - } - }, - "tokenizer": { - "path_hierarchy": { - "type": "path_hierarchy" - } - }, - "normalizer": { - "lowercase": { - "type": "custom", - "filter": [ - "lowercase" - ] - } - }, - "char_filter": { - "dot_to_space": { - "type": "pattern_replace", - "pattern": "\\.", - "replacement": " " - } - } - } - }, - "mappings": { - "properties": { - "Content": { - "type": "text", - "analyzer": "name_words", - "term_vector": "with_positions_offsets" - }, - "ID": { - "type": "keyword" - }, - "ParentID": { - "type": "keyword" - }, - "RootID": { - "type": "keyword" - }, - "MimeType": { - "type": "wildcard" - }, - "Path": { - "type": "text", - "analyzer": "path_hierarchy" - }, - "Deleted": { - "type": "boolean" - }, - "Hidden": { - "type": "boolean" - }, - "Favorites": { - "type": "keyword" - }, - "Tags": { - "type": "text", - "fields": { - "wildcard": { - "type": "wildcard", - "normalizer": "lowercase" - } - } - }, - "Name": { - "type": "text", - "analyzer": "name_words", - "fields": { - "wildcard": { - "type": "wildcard", - "normalizer": "lowercase" - } - } - }, - "Title": { - "type": "text", - "analyzer": "name_words", - "fields": { - "wildcard": { - "type": "wildcard", - "normalizer": "lowercase" - } - } - }, - "Mtime": { - "type": "date", - "ignore_malformed": true - } - }, - "dynamic_templates": [ - { - "audio_facets": { - "path_match": "audio.*", - "match_mapping_type": "string", - "mapping": { - "type": "keyword" - } - } - } - ] - } -} From 2383bcddcc2a90edcd60c69ec1ada51ca7e76ddf Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 13:11:36 +0200 Subject: [PATCH 40/54] test(search): leave behavior to the parity suite The engine suites keep what is engine-specific (index setup, health, purge-space batching); every behavior answer lives in the parity suite once. The shared test client learns IndicesCount, and FIELDS-15 pins the Size/Type gate for number queries. --- services/search/internal/opensearchtest/os.go | 19 + services/search/pkg/bleve/backend_test.go | 860 ------------------ .../search/pkg/opensearch/backend_test.go | 832 +---------------- services/search/pkg/parity/README.md | 1 + .../search/pkg/parity/query_fields_test.go | 3 +- 5 files changed, 37 insertions(+), 1678 deletions(-) diff --git a/services/search/internal/opensearchtest/os.go b/services/search/internal/opensearchtest/os.go index 16c3c10025..e8308dedf4 100644 --- a/services/search/internal/opensearchtest/os.go +++ b/services/search/internal/opensearchtest/os.go @@ -137,6 +137,19 @@ func (tc *TestClient) IndicesCreate(ctx context.Context, index string, body io.R } } +// IndicesCount returns the number of documents in the given indices. +func (tc *TestClient) IndicesCount(ctx context.Context, indices []string, body io.Reader) (int, error) { + resp, err := tc.c.Indices.Count(ctx, &opensearchgoAPI.IndicesCountReq{ + Indices: indices, + Body: body, + }) + if err != nil { + return 0, fmt.Errorf("failed to count documents in %v: %w", indices, err) + } + + return resp.Count, nil +} + type testRequireClient struct { tc *TestClient t testing.TB @@ -157,3 +170,9 @@ func (trc *testRequireClient) IndicesCreate(index string, body io.Reader) { func (trc *testRequireClient) IndicesDelete(indices []string) { require.NoError(trc.t, trc.tc.IndicesDelete(trc.t.Context(), indices)) } + +func (trc *testRequireClient) IndicesCount(indices []string, body io.Reader, want int) { + got, err := trc.tc.IndicesCount(trc.t.Context(), indices, body) + require.NoError(trc.t, err) + require.Equal(trc.t, want, got) +} diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go index 21603ee343..a0ff3092b5 100644 --- a/services/search/pkg/bleve/backend_test.go +++ b/services/search/pkg/bleve/backend_test.go @@ -1,75 +1,28 @@ package bleve_test import ( - "context" "fmt" bleveSearch "github.com/blevesearch/bleve/v2" sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - libregraph "github.com/opencloud-eu/libre-graph-api-go" - "github.com/opencloud-eu/reva/v2/pkg/storagespace" "github.com/opencloud-eu/opencloud/pkg/log" - searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" - searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" "github.com/opencloud-eu/opencloud/services/search/pkg/content" bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) -func hiddenByID(idx bleveSearch.Index, id string) bool { - GinkgoHelper() - - req := bleveSearch.NewSearchRequest(bleveSearch.NewDocIDQuery([]string{id})) - req.Fields = []string{"Hidden"} - - res, err := idx.Search(req) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Hits).To(HaveLen(1), "no record for %s", id) - - hidden, _ := res.Hits[0].Fields["Hidden"].(bool) - return hidden -} - var _ = Describe("Bleve", func() { var ( eng *bleve.Backend idx bleveSearch.Index - doSearch = func(id string, query, path string) (*searchsvc.SearchIndexResponse, error) { - rID, err := storagespace.ParseID(id) - if err != nil { - return nil, err - } - - return eng.Search(context.Background(), &searchsvc.SearchIndexRequest{ - Query: query, - Ref: &searchmsg.Reference{ - ResourceId: &searchmsg.ResourceID{ - StorageId: rID.StorageId, - SpaceId: rID.SpaceId, - OpaqueId: rID.OpaqueId, - }, - Path: path, - }, - }) - } - - assertDocCount = func(id string, query string, expectedCount int) []*searchmsg.Match { - res, err := doSearch(id, query, "") - - ExpectWithOffset(1, err).ToNot(HaveOccurred()) - ExpectWithOffset(1, len(res.Matches)).To(Equal(expectedCount), "query returned unexpected number of results: "+query) - return res.Matches - } - rootResource search.Resource parentResource search.Resource childResource search.Resource - childResource2 search.Resource ) BeforeEach(func() { @@ -107,14 +60,6 @@ var _ = Describe("Bleve", func() { Document: content.Document{Name: "child.pdf"}, } - childResource2 = search.Resource{ - ID: "1$2!5", - ParentID: parentResource.ID, - RootID: rootResource.ID, - Path: "./parent d!r/child2.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "child2.pdf"}, - } }) Describe("PurgeSpace", func() { @@ -170,809 +115,4 @@ var _ = Describe("Bleve", func() { }) }) - Describe("Search", func() { - Context("by other fields than filename", func() { - It("finds files by tags", func() { - parentResource.Document.Tags = []string{"foo", "bar"} - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Tags:foo", 1) - assertDocCount(rootResource.ID, "Tags:bar", 1) - assertDocCount(rootResource.ID, "Tags:foo Tags:bar", 1) - assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1) - assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1) - assertDocCount(rootResource.ID, "Tags:baz", 0) - }) - - It("finds files by tags case-insensitively", func() { - // exercises the []string/[]any sibling-lowercasing branch end-to-end. - parentResource.Document.Tags = []string{"Work", "Urgent"} - Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) - - assertDocCount(rootResource.ID, "tag:work", 1) // stored "Work", queried lower - assertDocCount(rootResource.ID, "tag:WORK", 1) // queried upper - assertDocCount(rootResource.ID, "Tags:Urgent", 1) - assertDocCount(rootResource.ID, "tag:missing", 0) - }) - - It("binds a leading NOT to the term right after it, combined with AND", func() { - // regression: a leading NOT next to AND dropped the AND'd term, so - // `NOT tag:x AND name:y` matched nothing (a self-contradicting clause). - parentResource.Document.Tags = []string{"physik"} - childResource.Document.Tags = []string{"mathe"} - Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) - Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) - - assertDocCount(rootResource.ID, "NOT tag:physik AND name:child.pdf", 1) // the mathe child - assertDocCount(rootResource.ID, "NOT tag:mathe AND name:parent*", 1) // the physik parent - }) - - It("finds files by size", func() { - parentResource.Document.Size = 12345 - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Size:12345", 1) - assertDocCount(rootResource.ID, "Size:>1000", 1) - assertDocCount(rootResource.ID, "Size:<100000", 1) - assertDocCount(rootResource.ID, "Size:12344", 0) - assertDocCount(rootResource.ID, "Size:<1000", 0) - assertDocCount(rootResource.ID, "Size:>100000", 0) - }) - - It("matches facet values case-insensitively", func() { - parentResource.Document.Audio = &libregraph.Audio{ - Artist: libregraph.PtrString("Some Artist"), - } - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `audio.artist:"Some Artist"`, 1) - assertDocCount(rootResource.ID, `audio.artist:"some artist"`, 1) - }) - }) - - Context("by filename", func() { - It("finds files with spaces in the filename", func() { - parentResource.Document.Name = "Foo oo.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `name:"foo o*"`, 1) - }) - - It("finds files by digits in the filename", func() { - parentResource.Document.Name = "12345.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:1234*", 1) - }) - - It("filters hidden files", func() { - childResource.Hidden = true - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Hidden:T", 1) - assertDocCount(rootResource.ID, "Hidden:F", 0) - }) - - Context("with a file in the root of the space", func() { - It("scopes the search to the specified space", func() { - parentResource.Document.Name = "foo.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:foo.pdf", 1) - assertDocCount("9$8!7", "Name:foo.pdf", 0) - }) - }) - - It("limits the search to the specified fields", func() { - parentResource.Document.Name = "bar.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:bar.pdf", 1) - assertDocCount(rootResource.ID, "Unknown:field", 0) - }) - - It("returns the total number of hits", func() { - parentResource.Document.Name = "bar.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - res, err := doSearch(rootResource.ID, "Name:bar*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - }) - - It("returns all desired fields", func() { - parentResource.Document.Name = "bar.pdf" - parentResource.Type = 3 - parentResource.MimeType = "application/pdf" - - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - matches := assertDocCount(rootResource.ID, fmt.Sprintf("Name:%s", parentResource.Name), 1) - match := matches[0] - Expect(match.Entity.Ref.Path).To(Equal(parentResource.Path)) - Expect(match.Entity.Name).To(Equal(parentResource.Name)) - Expect(match.Entity.Size).To(Equal(parentResource.Size)) - Expect(match.Entity.Type).To(Equal(parentResource.Type)) - Expect(match.Entity.MimeType).To(Equal(parentResource.MimeType)) - Expect(match.Entity.Deleted).To(BeFalse()) - Expect(match.Score > 0).To(BeTrue()) - }) - - It("finds files by name, prefix or substring match", func() { - parentResource.Document.Name = "foo.pdf" - - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - queries := []string{"foo.pdf", "foo*", "*oo.p*"} - for _, query := range queries { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, query, 1) - } - }) - - It("does a case-insensitive search", func() { - parentResource.Document.Name = "foo.pdf" - - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:foo*", 1) - assertDocCount(rootResource.ID, "Name:Foo*", 1) - }) - - Context("and an additional file in a subdirectory", func() { - BeforeEach(func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - }) - - It("finds files living deeper in the tree by filename, prefix or substring match", func() { - queries := []string{"child.pdf", "child*", "*ld.*"} - for _, query := range queries { - assertDocCount(rootResource.ID, query, 1) - } - }) - }) - }) - - Context("by path", func() { - BeforeEach(func() { - for _, r := range []search.Resource{parentResource, childResource, childResource2} { - Expect(eng.Upsert(r.ID, r)).To(Succeed()) - } - }) - - It("matches a folder and its descendants", func() { - assertDocCount(rootResource.ID, `path:"./parent d!r"`, 3) - }) - - It("matches a descendant path only itself", func() { - assertDocCount(rootResource.ID, `path:"./parent d!r/child.pdf"`, 1) - }) - - It("matches case-sensitively", func() { - // paths act as references: /Foo and /foo are distinct siblings, - // so a wrong-cased path must not match - assertDocCount(rootResource.ID, `path:"./PARENT D!R"`, 0) - }) - - It("applies an AND filter to the folder itself, not only descendants", func() { - // regression: the folder-itself clause used to match unconditionally - // under an AND, so the parent leaked in despite the name filter. - matches := assertDocCount(rootResource.ID, `path:"./parent d!r" AND name:child.pdf`, 1) - Expect(matches[0].Entity.Name).To(Equal("child.pdf")) - }) - }) - - Context("by content", func() { - It("matches full-text case-insensitively, without stemming", func() { - parentResource.Document.Content = "Running Foxes" - Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) - - assertDocCount(rootResource.ID, "content:running", 1) - assertDocCount(rootResource.ID, "content:RUNNING", 1) // case-insensitive - assertDocCount(rootResource.ID, "content:run", 0) // no stemming - assertDocCount(rootResource.ID, "content:run*", 1) // wildcard over the word - assertDocCount(rootResource.ID, "content:cat", 0) - }) - }) - - Context("by mediatype", func() { - It("matches categories and literal MIME types (incl. + and /)", func() { - childResource.Document.MimeType = "image/svg+xml" - childResource2.Document.MimeType = "image/png" - for _, r := range []search.Resource{childResource, childResource2} { - Expect(eng.Upsert(r.ID, r)).To(Succeed()) - } - - assertDocCount(rootResource.ID, "mediatype:image", 2) // image/* wildcard -> both - assertDocCount(rootResource.ID, "mediatype:IMAGE", 2) // categories are case-insensitive - assertDocCount(rootResource.ID, "mediatype:pdf", 0) - // literal MIME with + and /, must hit only the svg doc, not the png - assertDocCount(rootResource.ID, "mediatype:image/svg+xml", 1) - assertDocCount(rootResource.ID, "mediatype:image/png", 1) - // the same literal via the raw field name (no mediatype alias) - assertDocCount(rootResource.ID, "MimeType:image/svg+xml", 1) - assertDocCount(rootResource.ID, "MimeType:image/png", 1) - }) - - It("combines mediatype:file with another term", func() { - // regression: mediatype:file (a NOT) next to an operator dropped the - // other operand, so mediatype:file AND name:x matched nothing. - parentResource.Document.MimeType = "httpd/unix-directory" // a folder - childResource.Document.MimeType = "image/png" // a file - for _, r := range []search.Resource{parentResource, childResource} { - Expect(eng.Upsert(r.ID, r)).To(Succeed()) - } - assertDocCount(rootResource.ID, "mediatype:file", 1) // only the file - assertDocCount(rootResource.ID, "mediatype:file AND name:child.pdf", 1) // file AND its name - assertDocCount(rootResource.ID, "mediatype:file AND name:nope", 0) - }) - }) - - Context("Highlights", func() { - - It("highlights only for content searches", func() { - parentResource.Document.Name = "baz.pdf" - parentResource.Document.Content = "foo bar baz" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - res, err := doSearch(rootResource.ID, "Name:baz*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - Expect(res.Matches[0].Entity.Highlights).To(Equal("")) - }) - - It("highlights search terms", func() { - parentResource.Document.Name = "baz.pdf" - parentResource.Document.Content = "foo bar baz" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - res, err := doSearch(rootResource.ID, "Content:bar", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - Expect(res.Matches[0].Entity.Highlights).To(Equal("foo bar baz")) - }) - - }) - - Context("with a file in the root of the space and folder with a file. all of them have the same name", func() { - BeforeEach(func() { - parentResource := search.Resource{ - ID: "1$2!3", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./doc", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER), - Document: content.Document{Name: "doc"}, - } - - childResource := search.Resource{ - ID: "1$2!4", - ParentID: parentResource.ID, - RootID: rootResource.ID, - Path: "./doc/doc.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "doc.pdf"}, - } - - childResource2 := search.Resource{ - ID: "1$2!7", - ParentID: parentResource.ID, - RootID: rootResource.ID, - Path: "./doc/file.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "file.pdf"}, - } - - rootChildResource := search.Resource{ - ID: "1$2!5", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./doc.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "doc.pdf"}, - } - - rootChildResource2 := search.Resource{ - ID: "1$2!6", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./file.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "file.pdf"}, - } - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(rootChildResource.ID, rootChildResource) - Expect(err).ToNot(HaveOccurred()) - err = eng.Upsert(rootChildResource2.ID, rootChildResource2) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - err = eng.Upsert(childResource2.ID, childResource2) - Expect(err).ToNot(HaveOccurred()) - }) - It("search *doc* in a root", func() { - res, err := doSearch(rootResource.ID, "Name:*doc*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(3))) - }) - It("search *doc* in a subfolder", func() { - res, err := doSearch(rootResource.ID, "Name:*doc*", "./doc") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(2))) - }) - It("search *file* in a root", func() { - res, err := doSearch(rootResource.ID, "Name:*file*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(2))) - }) - It("search *file* in a subfolder", func() { - res, err := doSearch(rootResource.ID, "Name:*file*", "./doc") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - }) - }) - - }) - - Describe("path scoped searches", func() { - BeforeEach(func() { - Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) - Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) - Expect(eng.Upsert(childResource2.ID, childResource2)).To(Succeed()) - outside := search.Resource{ - ID: "1$2!6", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./other/child3.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "child3.pdf"}, - } - Expect(eng.Upsert(outside.ID, outside)).To(Succeed()) - }) - - It("restricts hits and totals to the scope at query level", func() { - // without the scope all three children match - res, err := doSearch(rootResource.ID, "name:child*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(3))) - - res, err = doSearch(rootResource.ID, "name:child*", "./parent d!r") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(2))) - Expect(len(res.Matches)).To(Equal(2)) - }) - - It("keeps totals right on a small page", func() { - // the scope is part of the query, so totals cover the full scope - // even when the page holds a single hit - rID, err := storagespace.ParseID(rootResource.ID) - Expect(err).ToNot(HaveOccurred()) - res, err := eng.Search(context.Background(), &searchsvc.SearchIndexRequest{ - Query: "name:child*", - PageSize: 1, - Ref: &searchmsg.Reference{ - ResourceId: &searchmsg.ResourceID{ - StorageId: rID.StorageId, SpaceId: rID.SpaceId, OpaqueId: rID.OpaqueId, - }, - Path: "./parent d!r", - }, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(2))) - Expect(len(res.Matches)).To(Equal(1)) - }) - - It("matches the scope case-sensitively", func() { - res, err := doSearch(rootResource.ID, "name:child*", "./PARENT D!R") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(0))) - }) - }) - - Describe("Upsert", func() { - It("adds a resourceInfo to the index", func() { - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - query := bleveSearch.NewMatchQuery("child.pdf") - query.SetField("Name") - res, err := idx.Search(bleveSearch.NewSearchRequest(query)) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Hits.Len()).To(Equal(1)) - }) - - It("updates an existing resource in the index", func() { - - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - countA, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(countA).To(Equal(uint64(1))) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - countB, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(countB).To(Equal(uint64(1))) - }) - }) - - Describe("Delete", func() { - It("marks a resource as deleted", func() { - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:*child*", 1) - - err = eng.Delete(childResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:*child*", 0) - }) - - It("marks a child resources as deleted", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - - err = eng.Delete(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0) - }) - }) - - Describe("Restore", func() { - It("also marks child resources as restored", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Delete(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 0) - - err = eng.Restore(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 1) - assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 1) - }) - }) - - Describe("Purge", func() { - It("removes a resource from the index", func() { - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - assertDocCount(rootResource.ID, "Name:child.pdf", 1) - - err = eng.Purge(childResource.ID, false) - - Expect(err).ToNot(HaveOccurred()) - assertDocCount(rootResource.ID, "Name:child.pdf", 0) - }) - It("removes a resource and its children from the index", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - - err = eng.Purge(parentResource.ID, false) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0) - }) - It("removes a resource and ignores its children from the index", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) - - err = eng.Delete(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - - err = eng.Purge(parentResource.ID, true) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - }) - }) - - Describe("Move", func() { - It("renames the parent and its child resources", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - parentResource.Path = "newname" - err = eng.Move(parentResource.ID, parentResource.ParentID, "./my/newname") - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, parentResource.Name, 0) - - matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1) - Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3")) - Expect(matches[0].Entity.Ref.Path).To(Equal("./my/newname/child.pdf")) - }) - - DescribeTable("keeps the flag in step with the path", - func(from, target string, hidden bool) { - parentResource.Path = from - parentResource.Hidden = search.IsHidden(from) - childResource.Path = from + "/child.pdf" - childResource.Hidden = parentResource.Hidden - - Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) - Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) - - Expect(eng.Move(parentResource.ID, parentResource.ParentID, target)).To(Succeed()) - - for _, id := range []string{parentResource.ID, childResource.ID} { - Expect(hiddenByID(idx, id)). - To(Equal(hidden), "%s after moving from %s to %s", id, from, target) - } - }, - Entry("into a dot folder", "./parent", "./.trash/parent", true), - Entry("into a plain folder", "./parent", "./archive/parent", false), - Entry("renamed with a leading dot", "./parent", "./.parent", true), - Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false), - Entry("renamed without the leading dot", "./.parent", "./parent", false), - Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true), - ) - - // the trash leaves the path alone, so the flag has to come through untouched - It("carries the flag through the trash and back", func() { - childResource.Path = "./.secret/file.txt" - childResource.Hidden = true - Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) - - Expect(eng.Delete(childResource.ID)).To(Succeed()) - Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after trashing") - - Expect(eng.Restore(childResource.ID)).To(Succeed()) - Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after restoring") - }) - - It("moves the parent and its child resources", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - parentResource.Path = " " - parentResource.ParentID = "1$2!somewhereopaqueid" - - err = eng.Move(parentResource.ID, parentResource.ParentID, "./somewhere/else/newname") - Expect(err).ToNot(HaveOccurred()) - assertDocCount(rootResource.ID, `parent d!r`, 0) - - matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1) - Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3")) - Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname/child.pdf")) - - matches = assertDocCount(rootResource.ID, `newname`, 1) - Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("somewhereopaqueid")) - Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname")) - - }) - - It("keeps case-insensitive search working after a move", func() { - Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) - Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) - - Expect(eng.Move(parentResource.ID, parentResource.ParentID, "./my/NewName")).To(Succeed()) - - // the lowercased name sibling is rebuilt, so a case-insensitive - // name query still works; the path is case-sensitive by design, so - // only the exact new path matches (and the old one no longer does). - assertDocCount(rootResource.ID, "name:NEWNAME", 1) - assertDocCount(rootResource.ID, `path:"./my/NewName"`, 2) - assertDocCount(rootResource.ID, `path:"./MY/NEWNAME"`, 0) - assertDocCount(rootResource.ID, `path:"./parent d!r"`, 0) - }) - }) - - Describe("StartBatch", func() { - It("starts a new batch", func() { - b, err := eng.NewBatch(100) - Expect(err).ToNot(HaveOccurred()) - - err = b.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(0))) - - err = b.Push() - Expect(err).ToNot(HaveOccurred()) - - count, err = idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - query := bleveSearch.NewMatchQuery("child.pdf") - query.SetField("Name") - res, err := idx.Search(bleveSearch.NewSearchRequest(query)) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Hits.Len()).To(Equal(1)) - }) - - It("doesn't intertwine different batches", func() { - b, err := eng.NewBatch(100) - Expect(err).ToNot(HaveOccurred()) - - err = b.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(0))) - - b2, err := eng.NewBatch(100) - Expect(err).ToNot(HaveOccurred()) - - err = b2.Upsert(childResource2.ID, childResource2) - Expect(err).ToNot(HaveOccurred()) - - Expect(b.Push()).To(Succeed()) - count, err = idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - Expect(b2.Push()).To(Succeed()) - count, err = idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(2))) - }) - }) - - Describe("File type specific metadata", func() { - - Context("with audio metadata", func() { - BeforeEach(func() { - resource := search.Resource{ - ID: "1$2!7", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./some_song.mp3", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{ - Name: "some_song.mp3", - MimeType: "audio/mpeg", - Audio: &libregraph.Audio{ - Album: libregraph.PtrString("Some Album"), - AlbumArtist: libregraph.PtrString("Some AlbumArtist"), - Artist: libregraph.PtrString("Some Artist"), - Bitrate: libregraph.PtrInt64(192), - Composers: libregraph.PtrString("Some Composers"), - Copyright: libregraph.PtrString(""), - Disc: libregraph.PtrInt32(2), - DiscCount: libregraph.PtrInt32(5), - Duration: libregraph.PtrInt64(225000), - Genre: libregraph.PtrString("Some Genre"), - HasDrm: libregraph.PtrBool(false), - IsVariableBitrate: libregraph.PtrBool(true), - Title: libregraph.PtrString("Some Title"), - Track: libregraph.PtrInt32(34), - TrackCount: libregraph.PtrInt32(99), - Year: libregraph.PtrInt32(2004), - }, - }, - } - err := eng.Upsert(resource.ID, resource) - Expect(err).ToNot(HaveOccurred()) - }) - - It("returns audio metadata for search", func() { - matches := assertDocCount(rootResource.ID, `*song*`, 1) - audio := matches[0].Entity.Audio - - Expect(audio).ToNot(BeNil()) - - Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album"))) - Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist"))) - Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist"))) - Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192))) - Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers"))) - Expect(audio.Copyright).To(Equal(libregraph.PtrString(""))) - Expect(audio.Disc).To(Equal(libregraph.PtrInt32(2))) - Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5))) - Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225000))) - Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre"))) - Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false))) - Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true))) - Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title"))) - Expect(audio.Track).To(Equal(libregraph.PtrInt32(34))) - Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(99))) - Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) - }) - }) - - Context("with location metadata", func() { - BeforeEach(func() { - resource := search.Resource{ - ID: "1$2!7", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./team.jpg", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{ - Name: "team.jpg", - MimeType: "image/jpeg", - Location: &libregraph.GeoCoordinates{ - Altitude: libregraph.PtrFloat64(1047.7), - Latitude: libregraph.PtrFloat64(49.48675890884328), - Longitude: libregraph.PtrFloat64(11.103870357204285), - }, - }, - } - err := eng.Upsert(resource.ID, resource) - Expect(err).ToNot(HaveOccurred()) - }) - - It("returns audio metadata for search", func() { - matches := assertDocCount(rootResource.ID, `*team*`, 1) - location := matches[0].Entity.Location - - Expect(location).ToNot(BeNil()) - - Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(1047.7))) - Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328))) - Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285))) - }) - }) - }) }) diff --git a/services/search/pkg/opensearch/backend_test.go b/services/search/pkg/opensearch/backend_test.go index 6fad3a5eee..dc83e057b8 100644 --- a/services/search/pkg/opensearch/backend_test.go +++ b/services/search/pkg/opensearch/backend_test.go @@ -1,6 +1,7 @@ package opensearch_test import ( + "context" "testing" . "github.com/onsi/ginkgo/v2" @@ -8,10 +9,7 @@ import ( opensearchgo "github.com/opensearch-project/opensearch-go/v4" opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - - searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" - searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" ) @@ -20,6 +18,12 @@ func TestOpenSearchBackend(t *testing.T) { RunSpecs(t, "OpenSearch Backend Suite") } +func deleteIndexOnCleanup(tc *opensearchtest.TestClient, indexName string) { + DeferCleanup(func() { + Expect(tc.IndicesDelete(context.Background(), []string{indexName})).To(Succeed()) + }) +} + var _ = Describe("Backend", func() { Describe("NewBackend", func() { It("fails to create if the cluster is not healthy", func() { @@ -36,256 +40,6 @@ var _ = Describe("Backend", func() { }) }) - Describe("Search", func() { - const indexName = "opencloud-test-engine-search" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - document search.Resource - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - - document = opensearchtest.Testdata.Resources.File - Expect(backend.Upsert(document.ID, document)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - }) - - It("performs the most simple search", func() { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ - Query: fmt.Sprintf(`"%s"`, document.Name), - }) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(HaveLen(1)) - Expect(resp.TotalMatches).To(Equal(int32(1))) - Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID)) - }) - - It("ignores files that are marked as deleted", func() { - deletedDocument := opensearchtest.Testdata.Resources.File - deletedDocument.ID = "1$2!4" - deletedDocument.Deleted = true - - Expect(backend.Upsert(deletedDocument.ID, deletedDocument)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ - Query: fmt.Sprintf(`"%s"`, document.Name), - }) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(HaveLen(1)) - Expect(resp.TotalMatches).To(Equal(int32(1))) - Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID)) - }) - - It("restricts hits and totals to the path scope", func() { - outside := opensearchtest.Testdata.Resources.File - outside.ID = "1$1!5" - outside.Path = "./other folder/else.jpg" - Expect(backend.Upsert(outside.ID, outside)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - - scoped := &searchMessage.Reference{ - ResourceId: &searchMessage.ResourceID{StorageId: "1", SpaceId: "1", OpaqueId: "1"}, - Path: "./parent d!r", - } - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ - Query: fmt.Sprintf(`"%s"`, document.Name), - Ref: scoped, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(HaveLen(1)) - Expect(resp.TotalMatches).To(Equal(int32(1))) - Expect(resp.Matches[0].Entity.Ref.Path).To(Equal("./parent d!r/child.jpg")) - - // the scope is a reference and matches case-sensitively - wrongCase := &searchMessage.Reference{ - ResourceId: &searchMessage.ResourceID{StorageId: "1", SpaceId: "1", OpaqueId: "1"}, - Path: "./PARENT D!R", - } - respWrongCase, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ - Query: fmt.Sprintf(`"%s"`, document.Name), - Ref: wrongCase, - }) - Expect(err).ToNot(HaveOccurred()) - Expect(respWrongCase.Matches).To(HaveLen(0)) - Expect(respWrongCase.TotalMatches).To(Equal(int32(0))) - }) - }) - - Describe("FullTextSearch", func() { - const indexName = "opencloud-test-engine-fulltext" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - - document := opensearchtest.Testdata.Resources.File - document.Content = "Running Foxes" - Expect(backend.Upsert(document.ID, document)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - }) - - It("searches content case-insensitively and stemmed, like bleve", func() { - // case-folded and porter-stemmed by the fulltext analyzer; the match - // query analyzes the query value the same way. "content:run*" is an - // unanalyzed wildcard over the stemmed term "run", so it must still - // route to a wildcard query (not degrade to a phrase match). - for _, q := range []string{"content:running", "content:RUNNING", "content:run", "content:run*"} { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: q}) - Expect(err).ToNot(HaveOccurred(), q) - Expect(resp.Matches).To(HaveLen(1), q) - } - - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: "content:cat"}) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(HaveLen(0)) - }) - }) - - Describe("CaseInsensitiveSearch", func() { - const indexName = "opencloud-test-engine-ci" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - - folder := opensearchtest.Testdata.Resources.Folder - folder.ID = "1$2!cifolder" - folder.Path = "./My Dir" - folder.Tags = []string{"Work", "Urgent"} - Expect(backend.Upsert(folder.ID, folder)).To(Succeed()) - - child := opensearchtest.Testdata.Resources.File - child.ID = "1$2!cichild" - child.ParentID = folder.ID - child.Path = "./My Dir/report.pdf" - child.Tags = nil - Expect(backend.Upsert(child.ID, child)).To(Succeed()) - - // a doc outside the folder, so the path assertions below discriminate: - // a phrase-matched path query would analyze into the "." prefix and - // match this one too - outside := opensearchtest.Testdata.Resources.File - outside.ID = "1$2!cioutside" - outside.Path = "./other.pdf" - outside.Tags = nil - Expect(backend.Upsert(outside.ID, outside)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - }) - - It("matches tags case-insensitively (array sibling)", func() { - for _, q := range []string{"tag:work", "tag:WORK", "Tags:Urgent"} { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: q}) - Expect(err).ToNot(HaveOccurred(), q) - Expect(resp.Matches).To(HaveLen(1), q) - } - }) - - It("matches a spaced path on the folder and its descendants case-sensitively", func() { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./My Dir"`}) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(HaveLen(2)) // folder itself + the descendant, not the outside doc - - // paths act as references, a wrong-cased path must not match - respWrongCase, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./MY DIR"`}) - Expect(err).ToNot(HaveOccurred()) - Expect(respWrongCase.Matches).To(HaveLen(0)) - }) - - It("matches a spaced descendant path only itself", func() { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./My Dir/report.pdf"`}) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(HaveLen(1)) - }) - }) - - Describe("MediaTypeSearch", func() { - const indexName = "opencloud-test-engine-mediatype" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - - svg := opensearchtest.Testdata.Resources.File - svg.ID = "1$2!svg" - svg.MimeType = "image/svg+xml" - Expect(backend.Upsert(svg.ID, svg)).To(Succeed()) - - png := opensearchtest.Testdata.Resources.File - png.ID = "1$2!png" - png.MimeType = "image/png" - Expect(backend.Upsert(png.ID, png)).To(Succeed()) - - folder := opensearchtest.Testdata.Resources.Folder - folder.ID = "1$2!dir" - folder.MimeType = "httpd/unix-directory" - Expect(backend.Upsert(folder.ID, folder)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - }) - - DescribeTable("resolves the media type query", - func(query string, want int) { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query}) - Expect(err).ToNot(HaveOccurred(), query) - Expect(resp.Matches).To(HaveLen(want), query) - }, - Entry("image/* wildcard matches both files", "mediatype:image", 2), - Entry("categories are case-insensitive", "mediatype:IMAGE", 2), - Entry("literal MIME (+ and /) via mediatype", "mediatype:image/svg+xml", 1), - Entry("same literal via the raw field name", "MimeType:image/svg+xml", 1), - Entry("literal png MIME", "mediatype:image/png", 1), - Entry("no pdf documents", "mediatype:pdf", 0), - Entry("folder category matches the directory only", "mediatype:folder", 1), - Entry("file category matches both files, not the directory", "mediatype:file", 2), - Entry("file category combined with a term", "mediatype:file AND MimeType:image/png", 1), - ) - }) - Describe("Upsert", func() { const indexName = "opencloud-test-engine-upsert" @@ -295,10 +49,12 @@ var _ = Describe("Backend", func() { ) BeforeEach(func() { + // the backend versions the physical index by schema generation + physical := opensearch.VersionedIndexName(indexName) + tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) + tc.Require.IndicesReset([]string{physical}) + deleteIndexOnCleanup(tc, physical) var err error backend, err = opensearch.NewBackend(indexName, tc.Client()) @@ -309,7 +65,7 @@ var _ = Describe("Backend", func() { document := opensearchtest.Testdata.Resources.File Expect(backend.Upsert(document.ID, document)).To(Succeed()) - tc.Require.IndicesCount([]string{indexName}, nil, 1) + tc.Require.IndicesCount([]string{opensearch.VersionedIndexName(indexName)}, nil, 1) }) It("upserts a document without an mtime", func() { @@ -319,566 +75,8 @@ var _ = Describe("Backend", func() { document.Mtime = nil Expect(backend.Upsert(document.ID, document)).To(Succeed()) - tc.Require.IndicesCount([]string{indexName}, nil, 1) + tc.Require.IndicesCount([]string{opensearch.VersionedIndexName(indexName)}, nil, 1) }) }) - Describe("Move", func() { - const indexName = "opencloud-test-engine-move" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("moves the document to a new path", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "query": map[string]any{ - "ids": map[string]any{ - "values": []string{document.ID}, - }, - }, - }) - - resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits) - Expect(resources).To(HaveLen(1)) - Expect(resources[0].Path).To(Equal(document.Path)) - - document.Path = "./new/path/to/resource" - Expect(backend.Move(document.ID, document.ParentID, document.Path)).To(Succeed()) - - resources = opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits) - Expect(resources).To(HaveLen(1)) - Expect(resources[0].Path).To(Equal(document.Path)) - }) - - It("keeps case-sensitive path search working after a move", func() { - // Spaced paths so the queries only stay exact as term queries; a phrase - // match would analyze into the "." prefix and match regardless. - document := opensearchtest.Testdata.Resources.File - document.ID = "1$2!cimove" - document.Path = "./Foo Dir/Bar" - Expect(backend.Upsert(document.ID, document)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - - document.Path = "./Moved Dir/Bar" - Expect(backend.Move(document.ID, document.ParentID, document.Path)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - - // Path is case-sensitive by design: the exact new path matches, a - // wrong-cased query does not, and the old path no longer matches. - respNew, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./Moved Dir/Bar"`}) - Expect(err).ToNot(HaveOccurred()) - Expect(respNew.Matches).To(HaveLen(1)) - - respWrongCase, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./MOVED DIR/BAR"`}) - Expect(err).ToNot(HaveOccurred()) - Expect(respWrongCase.Matches).To(HaveLen(0)) - - respOld, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: `path:"./Foo Dir/Bar"`}) - Expect(err).ToNot(HaveOccurred()) - Expect(respOld.Matches).To(HaveLen(0)) - }) - }) - - Describe("WriteVisibility", func() { - const indexName = "opencloud-test-engine-write-visibility" - - It("deletes a record that was just written", func() { - document := opensearchtest.Testdata.Resources.File - document.ID = "1$1!95" - document.Name = "textfile.txt" - document.Path = "./textfile.txt" - - backend, tc := newBackend(indexName) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Upsert(document.ID, document)).To(Succeed()) - Expect(backend.Delete(document.ID)).To(Succeed()) - - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ - Query: fmt.Sprintf(`name:"%s"`, document.Name), - }) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(BeEmpty()) - }) - }) - - Describe("Delete", func() { - const indexName = "opencloud-test-engine-delete" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("marks the document as deleted", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "query": map[string]any{ - "term": map[string]any{ - "Deleted": map[string]any{ - "value": true, - }, - }, - }, - }) - - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0) - - Expect(backend.Delete(document.ID)).To(Succeed()) - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1) - }) - }) - - Describe("Restore", func() { - const indexName = "opencloud-test-engine-restore" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("marks the document as not deleted", func() { - document := opensearchtest.Testdata.Resources.File - document.Deleted = true - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "query": map[string]any{ - "term": map[string]any{ - "Deleted": map[string]any{ - "value": true, - }, - }, - }, - }) - - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1) - - Expect(backend.Restore(document.ID)).To(Succeed()) - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0) - }) - }) - - Describe("Purge", func() { - const indexName = "opencloud-test-engine-purge" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("purges a full document", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - Expect(backend.Purge(document.ID, false)).To(Succeed()) - - tc.Require.IndicesCount([]string{indexName}, nil, 0) - }) - - It("purges resource trees", func() { - resourceFolder := opensearchtest.Testdata.Resources.Folder - tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder))) - - resourceFile := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile))) - - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - Expect(backend.Purge(resourceFolder.ID, false)).To(Succeed()) - - tc.Require.IndicesCount([]string{indexName}, nil, 0) - }) - - It("purges resource trees and ignores undeleted resources", func() { - resourceFolder := opensearchtest.Testdata.Resources.Folder - tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder))) - - resourceFile := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile))) - - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - Expect(backend.Delete(resourceFile.ID)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - Expect(backend.Purge(resourceFolder.ID, true)).To(Succeed()) - - tc.Require.IndicesCount([]string{indexName}, nil, 1) - }) - }) - - Describe("PurgeSpace", func() { - const indexName = "opencloud-test-engine-purge-space" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("takes every record of that space out of the index", func() { - gone := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, gone.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), gone))) - - stays := opensearchtest.Testdata.Resources.File - stays.ID = "1$2!3" - stays.RootID = "1$2!2" - tc.Require.DocumentCreate(indexName, stays.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), stays))) - - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - Expect(backend.PurgeSpace(gone.RootID)).To(Succeed()) - - tc.Require.IndicesRefresh([]string{indexName}, nil) - left := opensearchtest.SearchHitsMustBeConverted[search.Resource]( - GinkgoTB(), - tc.Require.Search(indexName, strings.NewReader(`{"query":{"match_all":{}}}`)).Hits, - ) - Expect(left).To(HaveLen(1), "only the records of that space are gone") - Expect(left[0].ID).To(Equal(stays.ID)) - }) - }) - - Describe("Hidden", func() { - const indexName = "opencloud-test-engine-hidden" - - DescribeTable("keeps the flag in step with the path", - func(from, target string, hidden bool) { - folder := opensearchtest.Testdata.Resources.Folder - folder.ID = "1$1!30" - folder.Name = "parent" - folder.Path = from - folder.Hidden = search.IsHidden(from) - - child := opensearchtest.Testdata.Resources.File - child.ID = "1$1!31" - child.Name = "child.txt" - child.Path = from + "/child.txt" - child.ParentID = folder.ID - child.Hidden = folder.Hidden - - backend, tc := newBackend(indexName, folder, child) - deleteIndexOnCleanup(tc, indexName) - tc.Require.IndicesRefresh([]string{indexName}, nil) - - Expect(backend.Move(folder.ID, folder.ParentID, target)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - - for _, id := range []string{folder.ID, child.ID} { - Expect(resourceByID(tc, indexName, id).Hidden). - To(Equal(hidden), "%s after moving from %s to %s", id, from, target) - } - }, - Entry("into a dot folder", "./parent", "./.trash/parent", true), - Entry("into a plain folder", "./parent", "./archive/parent", false), - Entry("renamed with a leading dot", "./parent", "./.parent", true), - Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false), - Entry("renamed without the leading dot", "./.parent", "./parent", false), - Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true), - ) - - It("carries the flag through the trash and back", func() { - hidden := opensearchtest.Testdata.Resources.File - hidden.ID = "1$1!32" - hidden.Path = "./.secret/file.txt" - hidden.Hidden = true - - backend, tc := newBackend(indexName, hidden) - deleteIndexOnCleanup(tc, indexName) - tc.Require.IndicesRefresh([]string{indexName}, nil) - - Expect(backend.Delete(hidden.ID)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - Expect(resourceByID(tc, indexName, hidden.ID).Hidden).To(BeTrue(), "after trashing") - - Expect(backend.Restore(hidden.ID)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - Expect(resourceByID(tc, indexName, hidden.ID).Hidden).To(BeTrue(), "after restoring") - }) - }) - - Describe("DocCount", func() { - const indexName = "opencloud-test-engine-doc-count" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("ignores deleted documents", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - count, err := backend.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - tc.Require.Update(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "doc": map[string]any{ - "Deleted": true, - }, - }))) - - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - count, err = backend.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(0))) - }) - }) - - // The following specs ensure that updates which affect a resource and its descendants - // (Delete, Restore, Move) are scoped to the root (space) of the target resource. Two - // resources living in different roots may share the exact same path, so matching by - // path alone would incorrectly update the wrong resource. - Describe("updateSelfAndDescendants root scope", func() { - It("deletes only the resource in the target root", func() { - const indexName = "opencloud-test-engine-root-scope-delete" - - target := opensearchtest.Testdata.Resources.File - other := otherRoot(target) - - backend, tc := newBackend(indexName, target, other) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Delete(target.ID)).To(Succeed()) - - Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeTrue(), "target resource should be marked as deleted") - Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeFalse(), "resource in a different root must not be affected") - }) - - It("restores only the resource in the target root", func() { - const indexName = "opencloud-test-engine-root-scope-restore" - - target := opensearchtest.Testdata.Resources.File - target.Deleted = true - other := otherRoot(target) - - backend, tc := newBackend(indexName, target, other) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Restore(target.ID)).To(Succeed()) - - Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeFalse(), "target resource should be restored") - Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeTrue(), "resource in a different root must not be affected") - }) - - It("moves only the resource in the target root", func() { - const indexName = "opencloud-test-engine-root-scope-move" - - target := opensearchtest.Testdata.Resources.File - other := otherRoot(target) - - backend, tc := newBackend(indexName, target, other) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Move(target.ID, target.ParentID, "./new/path/to/resource")).To(Succeed()) - - Expect(resourceByID(tc, indexName, target.ID).Path).To(Equal("./new/path/to/resource"), "target resource should be moved") - Expect(resourceByID(tc, indexName, other.ID).Path).To(Equal(other.Path), "resource in a different root must not be moved") - }) - }) - - Describe("SearchInAnalyzedFields", func() { - const indexName = "opencloud-test-engine-search-analyzed-fields" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - dashed := opensearchtest.Testdata.Resources.Folder - dashed.ID = "1$1!10" - dashed.Name = "new-folder" - dashed.Path = "./new-folder" - dashed.Title = "quarterly report" - - plain := opensearchtest.Testdata.Resources.Folder - plain.ID = "1$1!11" - plain.Name = "documents" - plain.Path = "./documents" - plain.Title = "notes" - - spaced := opensearchtest.Testdata.Resources.Folder - spaced.ID = "1$1!12" - spaced.Name = "foo bar" - spaced.Path = "./foo bar" - spaced.Title = "spaced out" - - backend, tc = newBackend(indexName) - deleteIndexOnCleanup(tc, indexName) - for _, r := range []search.Resource{dashed, plain, spaced} { - Expect(backend.Upsert(r.ID, r)).To(Succeed()) - } - tc.Require.IndicesRefresh([]string{indexName}, nil) - tc.Require.IndicesCount([]string{indexName}, nil, 3) - }) - - DescribeTable("finds what the analyzer made of the value", - func(query string, want []string) { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query}) - Expect(err).ToNot(HaveOccurred()) - - names := make([]string, 0, len(resp.Matches)) - for _, match := range resp.Matches { - names = append(names, match.Entity.Name) - } - Expect(names).To(ConsistOf(want)) - }, - Entry("the full name with the dash", "new-folder", []string{"new-folder"}), - Entry("one token of it", "new", []string{"new-folder"}), - Entry("a name without a dash", "documents", []string{"documents"}), - Entry("a wildcard", "*folder*", []string{"new-folder"}), - // the shape the web client sends for every name search - Entry("a wildcard around the whole dashed name", `name:"*new-folder*"`, []string{"new-folder"}), - Entry("a wildcard spanning the dash", `name:"*w-fol*"`, []string{"new-folder"}), - Entry("a wildcard in a different case", `name:"*NEW-FOLDER*"`, []string{"new-folder"}), - Entry("a wildcard spanning a space", `name:"*oo ba*"`, []string{"foo bar"}), - Entry("a wildcard around a name with a space", `name:"*foo bar*"`, []string{"foo bar"}), - Entry("a name with a space", `name:"foo bar"`, []string{"foo bar"}), - Entry("a title of two words", `Title:"quarterly report"`, []string{"new-folder"}), - Entry("one token of a title", "Title:quarterly", []string{"new-folder"}), - ) - }) - - Describe("SearchByTag", func() { - const indexName = "opencloud-test-engine-search-by-tag" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tagged := opensearchtest.Testdata.Resources.Folder - tagged.ID = "1$1!20" - tagged.Name = "tagged" - tagged.Path = "./tagged" - tagged.Tags = []string{"foo-bar"} - - other := opensearchtest.Testdata.Resources.Folder - other.ID = "1$1!21" - other.Name = "other" - other.Path = "./other" - other.Tags = []string{"foo"} - - backend, tc = newBackend(indexName) - deleteIndexOnCleanup(tc, indexName) - for _, r := range []search.Resource{tagged, other} { - Expect(backend.Upsert(r.ID, r)).To(Succeed()) - } - tc.Require.IndicesRefresh([]string{indexName}, nil) - tc.Require.IndicesCount([]string{indexName}, nil, 2) - }) - - // a tag is one label, not prose, so it matches as a whole or not at all - DescribeTable("matches a tag as a whole", - func(query string, want []string) { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query}) - Expect(err).ToNot(HaveOccurred()) - - names := make([]string, 0, len(resp.Matches)) - for _, match := range resp.Matches { - names = append(names, match.Entity.Name) - } - Expect(names).To(ConsistOf(want)) - }, - Entry("the whole tag", `tag:("foo-bar")`, []string{"tagged"}), - Entry("a token of a tag does not match it", `tag:("foo")`, []string{"other"}), - Entry("a tag in a different case", `tag:("FOO-BAR")`, []string{"tagged"}), - Entry("a wildcard reaches both", `tag:("*foo*")`, []string{"tagged", "other"}), - ) - }) - - Describe("SearchWithAnInvalidQuery", func() { - const indexName = "opencloud-test-engine-search-invalid-query" - - It("answers with a bad request", func() { - backend, tc := newBackend(indexName) - deleteIndexOnCleanup(tc, indexName) - - _, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: "AND mediatype:document"}) - Expect(err).To(HaveOccurred()) - Expect(err).To(BeAssignableToTypeOf(errtypes.BadRequest(""))) - Expect(err.Error()).To(Equal(`error: bad request: the expression can't begin from a binary operator: 'AND'`)) - }) - }) }) diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index 194397e9ff..51cb469234 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -233,6 +233,7 @@ Fixtures: | FIELDS-12 | `id:"1$1!ab-23"` | no match | no match | no match | ✅ | | FIELDS-13 | `audio.artist:"Some Artist"` | song.mp3 | song.mp3 | song.mp3 | ✅ | | FIELDS-14 | `audio.artist:"some artist"` | song.mp3 | song.mp3 | song.mp3 | ✅ | +| FIELDS-15 | `audio.duration>100` | no match | no match | no match | ✅ | ### deleted diff --git a/services/search/pkg/parity/query_fields_test.go b/services/search/pkg/parity/query_fields_test.go index 2d80fb4f8f..5e3120b688 100644 --- a/services/search/pkg/parity/query_fields_test.go +++ b/services/search/pkg/parity/query_fields_test.go @@ -18,7 +18,7 @@ func fieldsGroup() queryGroup { fixtureDoc("plain.txt"), fixtureFolder("box"), fixtureDoc("boxed.txt", withParent("1$1!box"), withPath("./box/boxed.txt")), - fixtureDoc("song.mp3", withMime("audio/mpeg"), withAudio(&libregraph.Audio{Artist: libregraph.PtrString("Some Artist")})), + fixtureDoc("song.mp3", withMime("audio/mpeg"), withAudio(&libregraph.Audio{Artist: libregraph.PtrString("Some Artist"), Duration: libregraph.PtrInt64(200)})), }, cases: []queryCase{ {id: 1, query: `size:42`, want: []string{"small.txt"}}, @@ -36,6 +36,7 @@ func fieldsGroup() queryGroup { // a facet value keeps its case, the field is not marked lowercase {id: 13, query: `audio.artist:"Some Artist"`, want: []string{"song.mp3"}}, {id: 14, query: `audio.artist:"some artist"`, want: []string{"song.mp3"}}, // facets search case-insensitively + {id: 15, query: `audio.duration>100`}, // number queries are gated to Size and Type on both engines }, } } From 95038844725baee7d6c082fc945e7c45d13bd5be Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 13:11:36 +0200 Subject: [PATCH 41/54] docs(search): migration notes for schema v4 Also make the reindex copy safe to run after the service already indexed (op_type create, conflicts proceed). --- services/search/MIGRATION.md | 17 ++++++++++------- services/search/README.md | 5 ++++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/services/search/MIGRATION.md b/services/search/MIGRATION.md index a0cb646e61..17c51e693b 100644 --- a/services/search/MIGRATION.md +++ b/services/search/MIGRATION.md @@ -9,9 +9,11 @@ until you remove it. ### OpenSearch -The new index is `opencloud-resource-v3`. Fill it in one of two ways: +The new index is `opencloud-resource-v4`. Fill it in one of two ways: -- copy the old index, fast and keeps the extracted file contents, or +- copy the old index, fast and keeps the extracted file contents; run it soon + after the upgrade, documents the service has indexed since win over copied + ones (`op_type: create`), or - index all spaces again, slower since every file is read once more, but drops documents that no longer have a resource. @@ -21,12 +23,13 @@ The address below is the one from `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ADDRESSES`, ```shell # either copy the old index -curl -X POST "https://opensearch.example.com:9200/_reindex?wait_for_completion=false" \ +curl -X POST "https://os.example.com:9200/_reindex?wait_for_completion=false" \ -H 'Content-Type: application/json' -d ' - {"source":{"index":"opencloud-resource"},"dest":{"index":"opencloud-resource-v3"}}' + {"source":{"index":"opencloud-resource","conflicts":"proceed"}, + "dest":{"index":"opencloud-resource-v4","op_type":"create"}}' # the answer carries a task id, watch it while it runs -curl "https://opensearch.example.com:9200/_tasks/" +curl "https://os.example.com:9200/_tasks/" # or index all spaces again, the service keeps running while it happens opencloud search index --all-spaces @@ -35,12 +38,12 @@ opencloud search index --all-spaces Once the new index is filled, remove the old one: ```shell -curl -X DELETE "https://opensearch.example.com:9200/opencloud-resource" +curl -X DELETE "https://os.example.com:9200/opencloud-resource" ``` ### bleve -The new index is the `bleve-v2` directory next to the old `bleve` one, both in +The new index is the `bleve-v4` directory next to the old `bleve` one, both in `$OC_BASE_DATA_PATH/search` by default (`SEARCH_ENGINE_BLEVE_DATA_PATH`). A bleve index cannot be copied, index all spaces again: diff --git a/services/search/README.md b/services/search/README.md index f0938d14e9..984684ab3c 100644 --- a/services/search/README.md +++ b/services/search/README.md @@ -39,7 +39,10 @@ To enable OpenSearch as a backend, the following settings must be set: Additionally, the following optional settings can be set: -* `SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME=val` (default: `opencloud-resource`): Base name of the OpenSearch index. The running index is suffixed with the schema version (e.g. `opencloud-resource-v3`); a breaking schema change targets a fresh index and leaves the old one in place. +* `SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME=val` (default: + `opencloud-resource`): base name of the OpenSearch index. The running + index is suffixed with the schema version (`opencloud-resource-v4`); a + breaking schema change targets a fresh index, the old one stays in place. * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_USERNAME=val`: Username for HTTP Basic Authentication. * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_PASSWORD=val`: Password for HTTP Basic Authentication. * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_HEADER=val`: HTTP headers to include in requests. From 9e8bb474b49715ee18eb3a070dc9f248b4e9e794 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 13:43:19 +0200 Subject: [PATCH 42/54] fix(search): compose the mediatype:file negation correctly The file expansion is grouped again so its NOT stays atomic next to other terms (OpenSearch turned 'mediatype:file OR x' into '(NOT dir) AND x'), and the bleve compiler no longer re-keys resolved groups (a negated mediatype group targeted the raw 'mediatype' field and matched everything). Pinned as MEDIATYPE-07..10. --- services/search/pkg/parity/query_mediatype_test.go | 4 ++++ services/search/pkg/query/bleve/compiler.go | 14 +------------- services/search/pkg/query/mimetype/mimetype.go | 8 ++++++-- .../search/pkg/query/mimetype/mimetype_test.go | 6 ++++-- services/search/pkg/query/normalize_test.go | 6 ++++-- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/services/search/pkg/parity/query_mediatype_test.go b/services/search/pkg/parity/query_mediatype_test.go index cd43ff89d5..32a79a89fa 100644 --- a/services/search/pkg/parity/query_mediatype_test.go +++ b/services/search/pkg/parity/query_mediatype_test.go @@ -20,6 +20,10 @@ func mediatypeGroup() queryGroup { {id: 4, query: `mediatype:*jpeg`, want: []string{"photo.jpg"}}, {id: 5, query: `mediatype:image`, want: []string{"photo.jpg"}}, {id: 6, query: `mediatype:folder`, want: []string{"albums", "drafts"}}, + {id: 7, query: `mediatype:file`, want: []string{"notes.md", "photo.jpg"}}, + {id: 8, query: `NOT mediatype:file`, want: []string{"albums", "drafts"}}, + {id: 9, query: `mediatype:file OR mediatype:image`, want: []string{"notes.md", "photo.jpg"}}, + {id: 10, query: `NOT mediatype:(image OR folder)`, want: []string{"notes.md"}}, }, } } diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index e4be2d50c9..1ca00dbe7e 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -269,10 +269,7 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { func nextNode(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { if n, ok := nodes[offset].(*ast.GroupNode); ok { - if n.Key != "" { - n = normalizeGroupingProperty(n) - } - + // keys are resolved and group keys propagated by normalize gq, _, err := walk(0, n.Nodes) if err != nil { return nil, 0, err @@ -359,12 +356,3 @@ func numberRange(field string, operator *ast.OperatorNode, value float64) bleveQ return q } - -func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode { - for _, n := range group.Nodes { - if onode, ok := n.(*ast.StringNode); ok { - onode.Key = group.Key - } - } - return group -} diff --git a/services/search/pkg/query/mimetype/mimetype.go b/services/search/pkg/query/mimetype/mimetype.go index ca61e651ac..64b901beb5 100644 --- a/services/search/pkg/query/mimetype/mimetype.go +++ b/services/search/pkg/query/mimetype/mimetype.go @@ -23,9 +23,13 @@ func Expand(key, value string) []ast.Node { value = strings.ToLower(value) switch value { case "file": + // grouped so the negation stays atomic when it composes with other + // terms (mediatype:file OR ...) return []ast.Node{ - &ast.OperatorNode{Value: kql.BoolNOT}, - &ast.StringNode{Key: field, Value: "httpd/unix-directory"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.OperatorNode{Value: kql.BoolNOT}, + &ast.StringNode{Key: field, Value: "httpd/unix-directory"}, + }}, } case "folder": return term("httpd/unix-directory") diff --git a/services/search/pkg/query/mimetype/mimetype_test.go b/services/search/pkg/query/mimetype/mimetype_test.go index d48fb44e6a..b21c5befbe 100644 --- a/services/search/pkg/query/mimetype/mimetype_test.go +++ b/services/search/pkg/query/mimetype/mimetype_test.go @@ -56,8 +56,10 @@ var _ = Describe("Expand", func() { It("expands file to not-a-folder", func() { Expect(mimetype.Expand("mediatype", "file")).To(Equal([]ast.Node{ - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + }}, })) }) diff --git a/services/search/pkg/query/normalize_test.go b/services/search/pkg/query/normalize_test.go index 5260eae383..cb77be7c12 100644 --- a/services/search/pkg/query/normalize_test.go +++ b/services/search/pkg/query/normalize_test.go @@ -73,8 +73,10 @@ var _ = Describe("Normalize", func() { &ast.OperatorNode{Value: "AND"}, &ast.StringNode{Key: "photo.cameraMake", Value: "canon", CaseInsensitive: true}, &ast.OperatorNode{Value: "AND"}, - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + }}, &ast.OperatorNode{Value: "AND"}, &ast.NumberNode{Key: "Size", Value: 100}, })) From dd6b3548f92ffab26d0eeb74da59b132502cb781 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 13:44:47 +0200 Subject: [PATCH 43/54] fix(search): tighten the generated search siblings The OpenSearch _lowercase siblings are search-only like bleve's (no doc_values, own map instance instead of aliasing the base), and the dead Path_words field is gone: _words exists for keyword fields only, as SearchSiblings declares. --- services/search/pkg/mapping/bleve.go | 2 +- services/search/pkg/mapping/opensearch.go | 11 +++++++++-- services/search/pkg/mapping/opensearch_test.go | 6 +++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/services/search/pkg/mapping/bleve.go b/services/search/pkg/mapping/bleve.go index c02177caae..9ee2d9cb82 100644 --- a/services/search/pkg/mapping/bleve.go +++ b/services/search/pkg/mapping/bleve.go @@ -66,7 +66,7 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix if opts.caseInsensitive() { doc.AddFieldMappingsAt(fi.Name+LowercaseSuffix, searchSibling(base)) } - if opts.wordBroken() { + if fieldType == TypeKeyword && opts.wordBroken() { words := searchSibling(base) words.Analyzer = WordsAnalyzer doc.AddFieldMappingsAt(fi.Name+WordsSuffix, words) diff --git a/services/search/pkg/mapping/opensearch.go b/services/search/pkg/mapping/opensearch.go index 87319d4bfe..ddd6249b6f 100644 --- a/services/search/pkg/mapping/opensearch.go +++ b/services/search/pkg/mapping/opensearch.go @@ -2,6 +2,7 @@ package mapping import ( "fmt" + "maps" "reflect" ) @@ -64,9 +65,15 @@ func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, p } props[fi.Name] = m if opts.caseInsensitive() { - props[fi.Name+LowercaseSuffix] = m + // search-only, like the bleve sibling: never sorted or + // aggregated on, so no doc_values + sibling := maps.Clone(m) + if sibling["type"] == "keyword" { + sibling["doc_values"] = false + } + props[fi.Name+LowercaseSuffix] = sibling } - if opts.wordBroken() { + if fieldType == TypeKeyword && opts.wordBroken() { props[fi.Name+WordsSuffix] = map[string]any{"type": "text", "analyzer": WordsAnalyzer} } return nil diff --git a/services/search/pkg/mapping/opensearch_test.go b/services/search/pkg/mapping/opensearch_test.go index f55b2de640..cfe1dea8a6 100644 --- a/services/search/pkg/mapping/opensearch_test.go +++ b/services/search/pkg/mapping/opensearch_test.go @@ -92,9 +92,9 @@ var _ = Describe("OpenSearchBuildMapping", func() { "MimeType": {Type: TypeWildcard}, }) Expect(err).ToNot(HaveOccurred()) - // Name: case-preserved keyword base + lowercased keyword sibling. + // Name: case-preserved keyword base + lowercased search-only sibling. Expect(props["Name"]).To(Equal(map[string]any{"type": "keyword"})) - Expect(props["Name_lowercase"]).To(Equal(map[string]any{"type": "keyword"})) + Expect(props["Name_lowercase"]).To(Equal(map[string]any{"type": "keyword", "doc_values": false})) content := props["Content"].(map[string]any) Expect(content["type"]).To(Equal("text"), "Content: %#v", content) Expect(content["term_vector"]).To(Equal("with_positions_offsets"), "Content: %#v", content) @@ -114,7 +114,7 @@ var _ = Describe("OpenSearchBuildMapping", func() { Expect(err).ToNot(HaveOccurred()) // the base stays a keyword, the words go to their own sibling Expect(props["Name"]).To(Equal(map[string]any{"type": "keyword"})) - Expect(props["Name_lowercase"]).To(Equal(map[string]any{"type": "keyword"})) + Expect(props["Name_lowercase"]).To(Equal(map[string]any{"type": "keyword", "doc_values": false})) Expect(props["Name_words"]).To(Equal(map[string]any{"type": "text", "analyzer": WordsAnalyzer})) }) From 98bdce156b0d85c424ffbd82bc817b4693b17829 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 13:44:47 +0200 Subject: [PATCH 44/54] chore: drop the duplicated vendor entry for analysis/char/regexp --- vendor/modules.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/vendor/modules.txt b/vendor/modules.txt index a82d979bd1..8079ad739c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -127,7 +127,6 @@ github.com/blevesearch/bleve/v2/analysis/analyzer/keyword github.com/blevesearch/bleve/v2/analysis/analyzer/standard github.com/blevesearch/bleve/v2/analysis/char/regexp github.com/blevesearch/bleve/v2/analysis/datetime/flexible -github.com/blevesearch/bleve/v2/analysis/char/regexp github.com/blevesearch/bleve/v2/analysis/datetime/optional github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds From e14bef7ef35b540e5dd5097212b8c37c9e8060ab Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 13:44:47 +0200 Subject: [PATCH 45/54] chore(search): version-free schema examples in docs strings --- services/search/pkg/config/engine.go | 2 +- services/search/pkg/opensearch/index.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/search/pkg/config/engine.go b/services/search/pkg/config/engine.go index 9c9b47997d..55827a47bb 100644 --- a/services/search/pkg/config/engine.go +++ b/services/search/pkg/config/engine.go @@ -25,7 +25,7 @@ type EngineOpenSearch struct { // EngineOpenSearchResourceIndex defines the OpenSearch index for resources type EngineOpenSearchResourceIndex struct { - Name string `yaml:"name" env:"SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME" desc:"The base name of the OpenSearch index for resources. The running index is suffixed with the schema version, e.g. opencloud-resource-v3." introductionVersion:"4.0.0"` + Name string `yaml:"name" env:"SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME" desc:"The base name of the OpenSearch index for resources. The running index is suffixed with the current schema version." introductionVersion:"4.0.0"` } // EngineOpenSearchClient configures the OpenSearch client diff --git a/services/search/pkg/opensearch/index.go b/services/search/pkg/opensearch/index.go index b611db71a9..bbc38640f4 100644 --- a/services/search/pkg/opensearch/index.go +++ b/services/search/pkg/opensearch/index.go @@ -25,7 +25,7 @@ var ( ) // VersionedIndexName suffixes the base index name with the schema version, e.g. -// "opencloud-resource" -> "opencloud-resource-v3". +// "opencloud-resource" -> the name suffixed with the current schema version. func VersionedIndexName(base string) string { return fmt.Sprintf("%s-v%d", base, search.SchemaVersion) } From 5abef9448258e3aa5831c5cc60ead0d7560f03cd Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 13:45:42 +0200 Subject: [PATCH 46/54] test(search): regenerate the parity matrix for the mediatype:file rows --- services/search/pkg/parity/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index 51cb469234..140545306a 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -181,6 +181,10 @@ Fixtures: | MEDIATYPE-04 | `mediatype:*jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ | | MEDIATYPE-05 | `mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ | | MEDIATYPE-06 | `mediatype:folder` | albums, drafts | albums, drafts | albums, drafts | ✅ | +| MEDIATYPE-07 | `mediatype:file` | notes.md, photo.jpg | notes.md, photo.jpg | notes.md, photo.jpg | ✅ | +| MEDIATYPE-08 | `NOT mediatype:file` | albums, drafts | albums, drafts | albums, drafts | ✅ | +| MEDIATYPE-09 | `mediatype:file OR mediatype:image` | notes.md, photo.jpg | notes.md, photo.jpg | notes.md, photo.jpg | ✅ | +| MEDIATYPE-10 | `NOT mediatype:(image OR folder)` | notes.md | notes.md | notes.md | ✅ | ### path From d4f1ae910ccbd484f672ad907a6807065c946421 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 13:50:36 +0200 Subject: [PATCH 47/54] fix(search): scope the OpenSearch purge to the resource's space The delete-by-query matched the bare Path, so a purge could take same-path documents in other spaces with it; bleve was already RootID-scoped. Pinned as rootscope-04. --- services/search/pkg/opensearch/batch.go | 6 +++++- services/search/pkg/parity/README.md | 2 ++ services/search/pkg/parity/lifecycle_rootscope_test.go | 8 ++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/services/search/pkg/opensearch/batch.go b/services/search/pkg/opensearch/batch.go index 9cc9fb852a..5df34d78b7 100644 --- a/services/search/pkg/opensearch/batch.go +++ b/services/search/pkg/opensearch/batch.go @@ -168,7 +168,11 @@ func (b *Batch) Purge(id string, onlyDeleted bool) error { return fmt.Errorf("failed to get resource: %w", err) } - query := osu.NewBoolQuery().Must(osu.NewTermQuery[string]("Path").Value(resource.Path)) + // scope to the resource's space: the same path exists in other spaces + query := osu.NewBoolQuery().Must( + osu.NewTermQuery[string]("RootID").Value(resource.RootID), + osu.NewTermQuery[string]("Path").Value(resource.Path), + ) if onlyDeleted { query.Must(osu.NewTermQuery[bool]("Deleted").Value(true)) } diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index 140545306a..67c02ab784 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -504,6 +504,8 @@ Fixtures: | ROOTSCOPE-01 | deletes only the one in the target root, then `name:"*twin*"` | twin.txt | twin.txt | twin.txt | ✅ | | ROOTSCOPE-02 | restores only the one in the target root, then `name:"*target*"` | target.txt | target.txt | target.txt | ✅ | | ROOTSCOPE-02 | restores only the one in the target root, then `name:"*twin*"` | no match | no match | no match | ✅ | +| ROOTSCOPE-04 | purges only the one in the target root, then `name:"*target*"` | no match | no match | no match | ✅ | +| ROOTSCOPE-04 | purges only the one in the target root, then `name:"*twin*"` | twin.txt | twin.txt | twin.txt | ✅ | | ROOTSCOPE-03 | moves only the one in the target root, then `path:"./moved.txt"` | moved.txt | moved.txt | moved.txt | ✅ | | ROOTSCOPE-03 | moves only the one in the target root, then `path:"./same/path.txt"` | twin.txt | twin.txt | twin.txt | ✅ | diff --git a/services/search/pkg/parity/lifecycle_rootscope_test.go b/services/search/pkg/parity/lifecycle_rootscope_test.go index 163df7d51f..36140e8bd0 100644 --- a/services/search/pkg/parity/lifecycle_rootscope_test.go +++ b/services/search/pkg/parity/lifecycle_rootscope_test.go @@ -37,6 +37,14 @@ func rootScopeLifecycle() lifecycleGroup { {`name:"*twin*"`, nil}, }, }, + { + id: 4, title: "purges only the one in the target root", + do: func(e search.Engine) error { return e.Purge(target.ID, false) }, + expect: []expectation{ + {`name:"*target*"`, nil}, + {`name:"*twin*"`, []string{"twin.txt"}}, + }, + }, { id: 3, title: "moves only the one in the target root", do: func(e search.Engine) error { return e.Move(target.ID, target.ParentID, "./moved.txt") }, From 2458d51ed143c2b0660550275259220b4e5830dd Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 13:50:36 +0200 Subject: [PATCH 48/54] feat(search): favorites are opaque user ids, no lowercase sibling Same rule as the other ids; removing the sibling later would take another schema generation. --- services/search/pkg/query/normalize_test.go | 6 +++--- services/search/pkg/search/search.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/search/pkg/query/normalize_test.go b/services/search/pkg/query/normalize_test.go index cb77be7c12..519e979a83 100644 --- a/services/search/pkg/query/normalize_test.go +++ b/services/search/pkg/query/normalize_test.go @@ -31,11 +31,11 @@ var _ = Describe("ResolveField", func() { var _ = Describe("FieldIsCaseInsensitive", func() { It("reports the CaseInsensitive override fields", func() { // keyword fields are case-insensitive by default, facets included - for _, f := range []string{"Name", "Title", "Tags", "Favorites", "audio.artist", "photo.cameraMake"} { + for _, f := range []string{"Name", "Title", "Tags", "audio.artist", "photo.cameraMake"} { Expect(query.FieldIsCaseInsensitive(f)).To(BeTrue(), f) } - // opted out (ids, path, mime type) or not a keyword at all - for _, f := range []string{"MimeType", "ID", "RootID", "ParentID", "Content", "Path", "Size", "unknown"} { + // opted out (ids, favorites, path, mime type) or not a keyword at all + for _, f := range []string{"MimeType", "ID", "RootID", "ParentID", "Favorites", "Content", "Path", "Size", "unknown"} { Expect(query.FieldIsCaseInsensitive(f)).To(BeFalse(), f) } }) diff --git a/services/search/pkg/search/search.go b/services/search/pkg/search/search.go index 4302a440b9..fd7cb6c0d6 100644 --- a/services/search/pkg/search/search.go +++ b/services/search/pkg/search/search.go @@ -84,7 +84,7 @@ var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts "MimeType": {CaseInsensitive: &False, NoWordBreaker: &True}, "Content": {Type: mapping.TypeFulltext}, "Tags": {NoWordBreaker: &True, IncludeInAll: &False}, - "Favorites": {NoWordBreaker: &True, IncludeInAll: &False}, + "Favorites": {NoWordBreaker: &True, IncludeInAll: &False, CaseInsensitive: &False}, // opaque user ids "location": {Type: mapping.TypeGeopoint}, } }) From 5d3c67b5877247123488626a6b00f41ca0475003 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 14:06:12 +0200 Subject: [PATCH 49/54] docs(search): the only migration path is a reindex Copying the old index over misses the search sibling fields, copied documents would be unfindable. --- services/search/MIGRATION.md | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/services/search/MIGRATION.md b/services/search/MIGRATION.md index 17c51e693b..1176484e80 100644 --- a/services/search/MIGRATION.md +++ b/services/search/MIGRATION.md @@ -9,29 +9,12 @@ until you remove it. ### OpenSearch -The new index is `opencloud-resource-v4`. Fill it in one of two ways: - -- copy the old index, fast and keeps the extracted file contents; run it soon - after the upgrade, documents the service has indexed since win over copied - ones (`op_type: create`), or -- index all spaces again, slower since every file is read once more, but drops - documents that no longer have a resource. - -The address below is the one from `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ADDRESSES`, -`opencloud-resource` the name from -`SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME`. +The new index is `opencloud-resource-v4`. Fill it by indexing all spaces +again; copying the old index over would miss the search sibling fields the +service writes at index time, so copied documents would not be found. ```shell -# either copy the old index -curl -X POST "https://os.example.com:9200/_reindex?wait_for_completion=false" \ - -H 'Content-Type: application/json' -d ' - {"source":{"index":"opencloud-resource","conflicts":"proceed"}, - "dest":{"index":"opencloud-resource-v4","op_type":"create"}}' - -# the answer carries a task id, watch it while it runs -curl "https://os.example.com:9200/_tasks/" - -# or index all spaces again, the service keeps running while it happens +# the service keeps running while it happens opencloud search index --all-spaces ``` From 3a7d6ca54b03d65e6171de88ddd53981f620dd93 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 14:12:38 +0200 Subject: [PATCH 50/54] docs(search): no index names in the migration steps, admins never type them --- services/search/MIGRATION.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/search/MIGRATION.md b/services/search/MIGRATION.md index 1176484e80..21ddbb0d40 100644 --- a/services/search/MIGRATION.md +++ b/services/search/MIGRATION.md @@ -9,9 +9,9 @@ until you remove it. ### OpenSearch -The new index is `opencloud-resource-v4`. Fill it by indexing all spaces -again; copying the old index over would miss the search sibling fields the -service writes at index time, so copied documents would not be found. +Fill the new index by indexing all spaces again; do not copy the old index +over (`_reindex`), the service writes search fields at index time that a copy +would miss, copied documents would not be found. ```shell # the service keeps running while it happens @@ -26,7 +26,7 @@ curl -X DELETE "https://os.example.com:9200/opencloud-resource" ### bleve -The new index is the `bleve-v4` directory next to the old `bleve` one, both in +The new index is a directory next to the old `bleve` one, both in `$OC_BASE_DATA_PATH/search` by default (`SEARCH_ENGINE_BLEVE_DATA_PATH`). A bleve index cannot be copied, index all spaces again: From 73bfc1cf48699c7a6eb9b053f2143a928d6b118e Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 14:12:52 +0200 Subject: [PATCH 51/54] docs(search): just the reindex step --- services/search/MIGRATION.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/search/MIGRATION.md b/services/search/MIGRATION.md index 21ddbb0d40..eae6ccad44 100644 --- a/services/search/MIGRATION.md +++ b/services/search/MIGRATION.md @@ -9,9 +9,7 @@ until you remove it. ### OpenSearch -Fill the new index by indexing all spaces again; do not copy the old index -over (`_reindex`), the service writes search fields at index time that a copy -would miss, copied documents would not be found. +Fill the new index by indexing all spaces again: ```shell # the service keeps running while it happens From 0188b18d2b5fb12006fd419047c4d4720fdd2cde Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 14:14:19 +0200 Subject: [PATCH 52/54] docs(search): everything but the current index generation can go --- services/search/MIGRATION.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/services/search/MIGRATION.md b/services/search/MIGRATION.md index eae6ccad44..5f3018c7e9 100644 --- a/services/search/MIGRATION.md +++ b/services/search/MIGRATION.md @@ -16,9 +16,11 @@ Fill the new index by indexing all spaces again: opencloud search index --all-spaces ``` -Once the new index is filled, remove the old one: +Once the new index is filled, every index but the one with the highest +`-v` suffix can go (the v7.x index has no suffix): ```shell +curl "https://os.example.com:9200/_cat/indices/opencloud-resource*" curl -X DELETE "https://os.example.com:9200/opencloud-resource" ``` @@ -32,7 +34,8 @@ bleve index cannot be copied, index all spaces again: opencloud search index --all-spaces ``` -Once the new index is filled, remove the old one: +Once the new index is filled, every directory but the one with the highest +`bleve-v` suffix can go (the v7.x directory has no suffix): ```shell rm -r "$OC_BASE_DATA_PATH/search/bleve" From aa2f7da0297692074e026e577671750dde109634 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 14:15:17 +0200 Subject: [PATCH 53/54] docs(search): the unversioned index is the one up to 7.4 --- services/search/MIGRATION.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/search/MIGRATION.md b/services/search/MIGRATION.md index 5f3018c7e9..5bd8c48c3b 100644 --- a/services/search/MIGRATION.md +++ b/services/search/MIGRATION.md @@ -17,7 +17,7 @@ opencloud search index --all-spaces ``` Once the new index is filled, every index but the one with the highest -`-v` suffix can go (the v7.x index has no suffix): +`-v` suffix can go (indexes up to 7.4 have no suffix): ```shell curl "https://os.example.com:9200/_cat/indices/opencloud-resource*" @@ -35,7 +35,7 @@ opencloud search index --all-spaces ``` Once the new index is filled, every directory but the one with the highest -`bleve-v` suffix can go (the v7.x directory has no suffix): +`bleve-v` suffix can go (directories up to 7.4 have no suffix): ```shell rm -r "$OC_BASE_DATA_PATH/search/bleve" From 1bf813068e7823c5b4857bb5208419f694b6b9f9 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Mon, 31 Aug 2026 14:17:29 +0200 Subject: [PATCH 54/54] docs(search): version-free index example in the README too --- services/search/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/search/README.md b/services/search/README.md index 984684ab3c..026f4fc5ae 100644 --- a/services/search/README.md +++ b/services/search/README.md @@ -41,8 +41,8 @@ Additionally, the following optional settings can be set: * `SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME=val` (default: `opencloud-resource`): base name of the OpenSearch index. The running - index is suffixed with the schema version (`opencloud-resource-v4`); a - breaking schema change targets a fresh index, the old one stays in place. + index is suffixed with the current schema version; a breaking schema + change targets a fresh index, the old one stays in place. * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_USERNAME=val`: Username for HTTP Basic Authentication. * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_PASSWORD=val`: Password for HTTP Basic Authentication. * `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_HEADER=val`: HTTP headers to include in requests.