diff --git a/services/search/pkg/bleve/batch.go b/services/search/pkg/bleve/batch.go index a6966a8a02..aedd588023 100644 --- a/services/search/pkg/bleve/batch.go +++ b/services/search/pkg/bleve/batch.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/document" storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/opencloud-eu/reva/v2/pkg/utils" @@ -45,11 +46,18 @@ func (b *Batch) Upsert(id string, r search.Resource) error { // type-specific adaptations via the mapping package) and appends it to the // batch under id. func (b *Batch) indexResource(id string, r search.Resource) error { - doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides()) + data, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides()) if err != nil { return err } - return b.batch.Index(id, doc) + doc := document.NewDocument(id) + if err := b.index.Mapping().MapDocument(doc, data); err != nil { + return err + } + if err := addOpenExtensionFields(doc, b.index.Mapping(), r.OpenExtensions); err != nil { + return err + } + return b.batch.IndexAdvanced(doc) } func (b *Batch) Move(id, parentID, location string) error { diff --git a/services/search/pkg/bleve/bleve.go b/services/search/pkg/bleve/bleve.go index 6c244a542a..2e0d8b1152 100644 --- a/services/search/pkg/bleve/bleve.go +++ b/services/search/pkg/bleve/bleve.go @@ -78,5 +78,7 @@ func hitToFacet[T any](fields map[string]any, prefix string) *T { // for per-field parse errors, so corrupted hit values surface as zero // values on individual fields instead of dropping the whole record. func matchToResource(match *bleveSearch.DocumentMatch) *search.Resource { - return mapping.Deserialize[search.Resource](match.Fields) + r := mapping.Deserialize[search.Resource](match.Fields) + r.OpenExtensions = openExtensionsFromHit(match.Fields) + return r } diff --git a/services/search/pkg/bleve/openextensions.go b/services/search/pkg/bleve/openextensions.go new file mode 100644 index 0000000000..0568eb10d3 --- /dev/null +++ b/services/search/pkg/bleve/openextensions.go @@ -0,0 +1,80 @@ +package bleve + +import ( + "strings" + "time" + + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" + "github.com/blevesearch/bleve/v2/document" + bleveMapping "github.com/blevesearch/bleve/v2/mapping" + index "github.com/blevesearch/bleve_index_api" + + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" +) + +// Open extensions bypass the mapping: it is fixed when the index is created +// and can only infer a type from a Go value. The typed siblings are built as +// document fields and indexed with IndexAdvanced, so the stored mapping never +// changes. The stored values are stored, not indexed, so Move, Delete and +// Restore can re-index a hit without losing them. + +const openExtensionIndexOptions = index.IndexField | index.DocValues + +func addOpenExtensionFields(doc *document.Document, im bleveMapping.IndexMapping, values map[string]string) error { + analyzer := im.AnalyzerNamed(keyword.Name) + for key, raw := range values { + doc.AddField(document.NewTextFieldCustom(mapping.OpenExtensionsStoredField+"."+key, nil, []byte(raw), index.StoreField, nil)) + } + for _, leaf := range mapping.OpenExtensionLeaves(values) { + positions := func(i, n int) []uint64 { + if n == 1 { + return nil + } + return []uint64{uint64(i)} //nolint:gosec // a loop index is never negative + } + switch leaf.Sibling { + case mapping.SiblingKeyword, mapping.SiblingLower: + for i, s := range leaf.Strings { + doc.AddField(document.NewTextFieldCustom(leaf.Field, positions(i, len(leaf.Strings)), []byte(s), openExtensionIndexOptions, analyzer)) + } + case mapping.SiblingNumber: + for i, f := range leaf.Numbers { + doc.AddField(document.NewNumericFieldWithIndexingOptions(leaf.Field, positions(i, len(leaf.Numbers)), f, openExtensionIndexOptions)) + } + case mapping.SiblingBool: + for i, b := range leaf.Bools { + doc.AddField(document.NewBooleanFieldWithIndexingOptions(leaf.Field, positions(i, len(leaf.Bools)), b, openExtensionIndexOptions)) + } + case mapping.SiblingDate: + for i, t := range leaf.Times { + f, err := document.NewDateTimeFieldWithIndexingOptions(leaf.Field, positions(i, len(leaf.Times)), t, time.RFC3339Nano, openExtensionIndexOptions) + if err != nil { + return err + } + doc.AddField(f) + } + case mapping.SiblingGeo: + doc.AddField(document.NewGeoPointFieldWithIndexingOptions(leaf.Field, nil, leaf.Geo.Longitude, leaf.Geo.Latitude, openExtensionIndexOptions)) + } + } + return nil +} + +func openExtensionsFromHit(fields map[string]any) map[string]string { + var values map[string]string + prefix := mapping.OpenExtensionsStoredField + "." + for field, value := range fields { + if !strings.HasPrefix(field, prefix) { + continue + } + raw, ok := value.(string) + if !ok { + continue + } + if values == nil { + values = map[string]string{} + } + values[field[len(prefix):]] = raw + } + return values +} diff --git a/services/search/pkg/bleve/openextensions_test.go b/services/search/pkg/bleve/openextensions_test.go new file mode 100644 index 0000000000..a02ba72039 --- /dev/null +++ b/services/search/pkg/bleve/openextensions_test.go @@ -0,0 +1,163 @@ +package bleve_test + +import ( + "context" + "time" + + bleveSearch "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search/query" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/opencloud-eu/opencloud/pkg/log" + searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" + "github.com/opencloud-eu/opencloud/services/search/pkg/content" + bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +// Open extensions bypass the mapping (IndexAdvanced), so the invariants the +// rest of the engine relies on are pinned here: the typed siblings exist as +// fields, they answer typed queries, the mapping is untouched, and the stored +// values survive a Move, which re-indexes the resource from a hit. +var _ = Describe("Open extensions", func() { + var ( + idx bleveSearch.Index + eng *bleve.Backend + ) + + key := func(property string) string { + return "http://opencloud.eu/ns/extensions/com.example.project/" + property + } + stored := map[string]string{ + key("state"): "s:Open", + key("priority"): "n:3", + key("done"): "b:false", + key("due"): "d:2026-10-01T00:00:00Z", + key("site"): "g:52.5,13.4", + } + + resource := func(id, path string, exts map[string]string) search.Resource { + return search.Resource{ + ID: id, + RootID: "1$2!2", + ParentID: "1$2!2", + Path: path, + Type: 1, + Document: content.Document{Name: path[2:], OpenExtensions: exts}, + } + } + + hits := func(q query.Query) []string { + GinkgoHelper() + req := bleveSearch.NewSearchRequest(q) + req.Fields = []string{"*"} + res, err := idx.Search(req) + Expect(err).ToNot(HaveOccurred()) + ids := make([]string, 0, len(res.Hits)) + for _, h := range res.Hits { + ids = append(ids, h.ID) + } + return ids + } + + term := func(field, value string) query.Query { + q := query.NewTermQuery(value) + q.SetField(field) + return q + } + + BeforeEach(func() { + m, err := bleve.NewMapping() + Expect(err).ToNot(HaveOccurred()) + idx, err = bleveSearch.NewMemOnly(m) + Expect(err).ToNot(HaveOccurred()) + eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}) + + Expect(eng.Upsert("1$2!a", resource("1$2!a", "./a.txt", stored))).To(Succeed()) + Expect(eng.Upsert("1$2!b", resource("1$2!b", "./b.txt", map[string]string{key("state"): "s:closed", key("priority"): "s:high"}))).To(Succeed()) + Expect(eng.Upsert("1$2!c", resource("1$2!c", "./c.txt", nil))).To(Succeed()) + }) + + It("indexes every property under the sibling of its kind", func() { + fields, err := idx.Fields() + Expect(err).ToNot(HaveOccurred()) + Expect(fields).To(ContainElements( + "ext.com.example.project.state.@keyword", + "ext.com.example.project.state.@lower", + "ext.com.example.project.priority.@number", + "ext.com.example.project.priority.@keyword", + "ext.com.example.project.done.@bool", + "ext.com.example.project.due.@date", + "ext.com.example.project.site.@geo", + )) + }) + + It("answers typed queries on the siblings", func() { + Expect(hits(term("ext.com.example.project.state.@lower", "open"))).To(ConsistOf("1$2!a")) + Expect(hits(term("ext.com.example.project.state.@keyword", "Open"))).To(ConsistOf("1$2!a")) + Expect(hits(term("ext.com.example.project.state.@keyword", "open"))).To(BeEmpty(), "the keyword sibling is case-sensitive") + + lo, hi := 2.0, 4.0 + inclusive := true + nq := query.NewNumericRangeInclusiveQuery(&lo, &hi, &inclusive, &inclusive) + nq.SetField("ext.com.example.project.priority.@number") + Expect(hits(nq)).To(ConsistOf("1$2!a"), "b's priority is a string and lives in the keyword sibling") + Expect(hits(term("ext.com.example.project.priority.@lower", "high"))).To(ConsistOf("1$2!b")) + + bq := query.NewBoolFieldQuery(false) + bq.SetField("ext.com.example.project.done.@bool") + Expect(hits(bq)).To(ConsistOf("1$2!a")) + + start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + end := time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC) + dq := query.NewDateRangeQuery(start, end) + dq.SetField("ext.com.example.project.due.@date") + Expect(hits(dq)).To(ConsistOf("1$2!a")) + + gq := query.NewGeoDistanceQuery(13.4, 52.5, "1km") + gq.SetField("ext.com.example.project.site.@geo") + Expect(hits(gq)).To(ConsistOf("1$2!a")) + }) + + It("stores the values with the hit and keeps them through a Move", func() { + req := bleveSearch.NewSearchRequest(bleveSearch.NewDocIDQuery([]string{"1$2!a"})) + req.Fields = []string{"*"} + res, err := idx.Search(req) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Hits).To(HaveLen(1)) + Expect(res.Hits[0].Fields).To(HaveKeyWithValue("OpenExtensions."+key("state"), "s:Open")) + Expect(res.Hits[0].Fields).To(HaveKeyWithValue("OpenExtensions."+key("site"), "g:52.5,13.4")) + + Expect(eng.Move("1$2!a", "1$2!2", "./moved.txt")).To(Succeed()) + Expect(hits(term("ext.com.example.project.state.@lower", "open"))).To(ConsistOf("1$2!a"), "the re-indexed resource keeps its extensions") + + sr, err := eng.Search(context.Background(), &searchService.SearchIndexRequest{Query: `name:moved.txt`}) + Expect(err).ToNot(HaveOccurred()) + Expect(sr.Matches).To(HaveLen(1)) + }) + + It("replaces the siblings on upsert instead of accumulating them", func() { + Expect(eng.Upsert("1$2!a", resource("1$2!a", "./a.txt", map[string]string{key("state"): "s:closed"}))).To(Succeed()) + Expect(hits(term("ext.com.example.project.state.@lower", "open"))).To(BeEmpty()) + Expect(hits(term("ext.com.example.project.state.@lower", "closed"))).To(ConsistOf("1$2!a", "1$2!b")) + + lo, hi := 2.0, 4.0 + inclusive := true + nq := query.NewNumericRangeInclusiveQuery(&lo, &hi, &inclusive, &inclusive) + nq.SetField("ext.com.example.project.priority.@number") + Expect(hits(nq)).To(BeEmpty(), "a property that is gone leaves no sibling behind") + }) + + It("leaves the stored mapping untouched", func() { + fields, err := idx.Fields() + Expect(err).ToNot(HaveOccurred()) + Expect(fields).To(ContainElement("ext.com.example.project.state.@lower")) + + // the mapping only knows the resource struct; an extension field never + // becomes part of it, so the reconciler never sees one + m := idx.Mapping() + Expect(m.AnalyzerNameForPath("ext.com.example.project.state.@lower")).To(Equal(m.AnalyzerNameForPath("does.not.exist"))) + }) +}) diff --git a/services/search/pkg/content/basic.go b/services/search/pkg/content/basic.go index 7ba8638a18..64d28ca7a0 100644 --- a/services/search/pkg/content/basic.go +++ b/services/search/pkg/content/basic.go @@ -7,6 +7,7 @@ import ( storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/opencloud-eu/opencloud/pkg/conversions" "github.com/opencloud-eu/opencloud/pkg/log" + "github.com/opencloud-eu/reva/v2/pkg/openextension" "github.com/opencloud-eu/reva/v2/pkg/tags" "github.com/opencloud-eu/reva/v2/pkg/utils" ) @@ -33,6 +34,15 @@ func (b Basic) Extract(_ context.Context, ri *storageProvider.ResourceInfo) (Doc if t, ok := m["tags"]; ok { doc.Tags = tags.New(t).AsSlice() } + for key, value := range m { + if _, _, ok := openextension.SplitKey(key); !ok { + continue + } + if doc.OpenExtensions == nil { + doc.OpenExtensions = map[string]string{} + } + doc.OpenExtensions[key] = value + } } if m := ri.Opaque.GetMap(); m != nil && m["favorites"] != nil { diff --git a/services/search/pkg/content/content.go b/services/search/pkg/content/content.go index 6a945b0fd3..ae6d02030f 100644 --- a/services/search/pkg/content/content.go +++ b/services/search/pkg/content/content.go @@ -30,6 +30,10 @@ type Document struct { Video *libregraph.Video `json:"video,omitempty"` MotionPhoto *libregraph.MotionPhoto `json:"motionPhoto,omitempty"` LivePhoto *libregraph.LivePhoto `json:"livePhoto,omitempty"` + + // OpenExtensions are the stored open extension properties by metadata key; + // the engines index them as typed siblings, see mapping.OpenExtensionLeaves. + OpenExtensions map[string]string `json:"-"` } func CleanString(content, langCode string) string { diff --git a/services/search/pkg/mapping/openextensions.go b/services/search/pkg/mapping/openextensions.go new file mode 100644 index 0000000000..f5ec8d2c16 --- /dev/null +++ b/services/search/pkg/mapping/openextensions.go @@ -0,0 +1,147 @@ +package mapping + +import ( + "sort" + "strings" + "time" + + "github.com/opencloud-eu/reva/v2/pkg/openextension" +) + +// An open extension property is indexed under the sibling of its value's +// kind, ext...@, so it can change its kind +// between writes without a mapping change. The sibling segment starts with +// "@", which neither a name nor a property may, so a leaf never collides with +// an object of another extension in the dotted tree OpenSearch builds. +const ( + OpenExtensionsRoot = "ext" + // OpenExtensionsStoredField holds the stored values by metadata key, so an + // engine can rebuild the document from a hit. + OpenExtensionsStoredField = "OpenExtensions" + // OpenExtensionsQueryPrefix: KQL addresses extensions.. + OpenExtensionsQueryPrefix = "extensions." + + SiblingKeyword = "@keyword" + SiblingLower = "@lower" + SiblingNumber = "@number" + SiblingBool = "@bool" + SiblingDate = "@date" + SiblingGeo = "@geo" +) + +// OpenExtensionField is the indexed field name of one typed sibling. +func OpenExtensionField(name, property, sibling string) string { + return OpenExtensionsRoot + "." + name + "." + property + "." + sibling +} + +// OpenExtensionFieldFromQuery turns a KQL key into the sibling's field name. The +// prefix is matched case-insensitively, name and property are taken as written. +func OpenExtensionFieldFromQuery(key, sibling string) (string, bool) { + rest, ok := openExtensionPath(key) + if !ok { + return "", false + } + return OpenExtensionsRoot + "." + rest + "." + sibling, true +} + +// IsOpenExtensionQueryField reports whether a KQL key addresses an extension property. +func IsOpenExtensionQueryField(key string) bool { + _, ok := openExtensionPath(key) + return ok +} + +func openExtensionPath(key string) (string, bool) { + if len(key) <= len(OpenExtensionsQueryPrefix) || !strings.EqualFold(key[:len(OpenExtensionsQueryPrefix)], OpenExtensionsQueryPrefix) { + return "", false + } + rest := key[len(OpenExtensionsQueryPrefix):] + // the property is the last segment, the name needs at least two more + if strings.Count(rest, ".") < 2 || strings.HasPrefix(rest, ".") || strings.HasSuffix(rest, ".") { + return "", false + } + return rest, true +} + +// OpenExtensionLeaf is one typed sibling value, the unit both engines index. +type OpenExtensionLeaf struct { + Field string + Sibling string + + Strings []string + Numbers []float64 + Bools []bool + Times []time.Time + Geo *openextension.GeoPoint +} + +// OpenExtensionLeaves flattens stored extension properties (metadata key to +// stored value) into the leaves to index, sorted by field. Unreadable values +// are skipped, the index must never fail on user data. +func OpenExtensionLeaves(values map[string]string) []OpenExtensionLeaf { + var leaves []OpenExtensionLeaf + for key, raw := range values { + name, property, ok := openextension.SplitKey(key) + if !ok { + continue + } + v, err := openextension.DecodeValue(raw) + if err != nil { + continue + } + leaves = append(leaves, propertyLeaves(name, property, v)...) + } + sort.Slice(leaves, func(i, j int) bool { return leaves[i].Field < leaves[j].Field }) + return leaves +} + +func propertyLeaves(name, property string, v openextension.Value) []OpenExtensionLeaf { + leaf := func(sibling string) OpenExtensionLeaf { + return OpenExtensionLeaf{Field: OpenExtensionField(name, property, sibling), Sibling: sibling} + } + switch v.Kind { + case openextension.KindString: + values := v.Strings() + if len(values) == 0 { + return nil + } + lower := make([]string, len(values)) + for i, s := range values { + lower[i] = strings.ToLower(s) + } + keyword := leaf(SiblingKeyword) + keyword.Strings = values + lowered := leaf(SiblingLower) + lowered.Strings = lower + return []OpenExtensionLeaf{keyword, lowered} + case openextension.KindNumber: + l := leaf(SiblingNumber) + l.Numbers = v.Numbers() + if len(l.Numbers) == 0 { + return nil + } + return []OpenExtensionLeaf{l} + case openextension.KindBool: + l := leaf(SiblingBool) + l.Bools = v.Bools() + if len(l.Bools) == 0 { + return nil + } + return []OpenExtensionLeaf{l} + case openextension.KindDate: + l := leaf(SiblingDate) + l.Times = v.Times() + if len(l.Times) == 0 { + return nil + } + return []OpenExtensionLeaf{l} + case openextension.KindGeo: + p, ok := v.Geo() + if !ok { + return nil + } + l := leaf(SiblingGeo) + l.Geo = &p + return []OpenExtensionLeaf{l} + } + return nil +} diff --git a/services/search/pkg/mapping/openextensions_test.go b/services/search/pkg/mapping/openextensions_test.go new file mode 100644 index 0000000000..ac8aa46c7a --- /dev/null +++ b/services/search/pkg/mapping/openextensions_test.go @@ -0,0 +1,74 @@ +package mapping + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("extension fields", func() { + key := func(name, property string) string { + return "http://opencloud.eu/ns/extensions/" + name + "/" + property + } + + It("names a sibling and resolves the KQL spelling", func() { + Expect(OpenExtensionField("com.example.project", "priority", SiblingNumber)).To(Equal("ext.com.example.project.priority.@number")) + + field, ok := OpenExtensionFieldFromQuery("extensions.com.example.project.priority", SiblingNumber) + Expect(ok).To(BeTrue()) + Expect(field).To(Equal("ext.com.example.project.priority.@number")) + + field, ok = OpenExtensionFieldFromQuery("Extensions.com.example.project.Priority", SiblingLower) + Expect(ok).To(BeTrue(), "the prefix is case-insensitive, the rest is taken as written") + Expect(field).To(Equal("ext.com.example.project.Priority.@lower")) + }) + + It("rejects keys that are not an extension property", func() { + for _, key := range []string{"Name", "extensions", "extensions.", "extensions.priority", "extensions.project.priority", "extensions.com.example.", "extension.com.example.x"} { + Expect(IsOpenExtensionQueryField(key)).To(BeFalse(), key) + } + Expect(IsOpenExtensionQueryField("extensions.com.example.priority")).To(BeTrue()) + }) + + It("flattens stored extension properties into typed leaves", func() { + const project = "com.example.project" + leaves := OpenExtensionLeaves(map[string]string{ + key(project, "state"): "s:Open", + key(project, "priority"): "n:3", + key(project, "done"): "b:false", + key(project, "tags"): `S:["A","b"]`, + key(project, "due"): "d:2026-10-01T00:00:00Z", + key(project, "site"): "g:52.5,13.4", + key(project, "broken"): "n:not a number", + "tags": "a,b", + }) + + byField := map[string]OpenExtensionLeaf{} + for _, l := range leaves { + byField[l.Field] = l + } + Expect(byField).To(HaveLen(8)) + Expect(byField["ext.com.example.project.state.@keyword"].Strings).To(Equal([]string{"Open"})) + Expect(byField["ext.com.example.project.state.@lower"].Strings).To(Equal([]string{"open"})) + Expect(byField["ext.com.example.project.tags.@keyword"].Strings).To(Equal([]string{"A", "b"})) + Expect(byField["ext.com.example.project.tags.@lower"].Strings).To(Equal([]string{"a", "b"})) + Expect(byField["ext.com.example.project.priority.@number"].Numbers).To(Equal([]float64{3})) + Expect(byField["ext.com.example.project.done.@bool"].Bools).To(Equal([]bool{false})) + Expect(byField["ext.com.example.project.due.@date"].Times).To(Equal([]time.Time{time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)})) + Expect(byField["ext.com.example.project.site.@geo"].Geo.Latitude).To(Equal(52.5)) + Expect(byField["ext.com.example.project.site.@geo"].Geo.Longitude).To(Equal(13.4)) + + Expect(leaves[0].Field < leaves[len(leaves)-1].Field).To(BeTrue(), "leaves are sorted by field") + }) + + It("indexes a property under the sibling of its current kind only", func() { + asNumber := OpenExtensionLeaves(map[string]string{key("x.y", "p"): "n:3"}) + asString := OpenExtensionLeaves(map[string]string{key("x.y", "p"): "s:3"}) + Expect(asNumber).To(HaveLen(1)) + Expect(asNumber[0].Field).To(Equal("ext.x.y.p.@number")) + Expect(asString).To(HaveLen(2)) + Expect(asString[0].Field).To(Equal("ext.x.y.p.@keyword")) + Expect(asString[1].Field).To(Equal("ext.x.y.p.@lower")) + }) +}) diff --git a/services/search/pkg/opensearch/batch.go b/services/search/pkg/opensearch/batch.go index 5df34d78b7..c97fe745aa 100644 --- a/services/search/pkg/opensearch/batch.go +++ b/services/search/pkg/opensearch/batch.go @@ -48,6 +48,7 @@ func (b *Batch) Upsert(id string, r search.Resource) error { if err != nil { return fmt.Errorf("failed to marshal resource: %w", err) } + addOpenExtensionSource(body, r.OpenExtensions) op := func() []map[string]any { return []map[string]any{ diff --git a/services/search/pkg/opensearch/index.go b/services/search/pkg/opensearch/index.go index b68b71f302..a1e305a733 100644 --- a/services/search/pkg/opensearch/index.go +++ b/services/search/pkg/opensearch/index.go @@ -74,6 +74,7 @@ func buildResourceMapping() ([]byte, error) { if err != nil { return nil, err } + maps.Copy(props, openExtensionProperties()) index := map[string]any{ "settings": map[string]any{ @@ -108,7 +109,8 @@ func buildResourceMapping() ([]byte, error) { }, }, "mappings": map[string]any{ - "properties": props, + "dynamic_templates": openExtensionTemplates(), + "properties": props, }, } return json.Marshal(index) @@ -198,15 +200,38 @@ func (r *osReconciler) Classify() (searchmapping.Classification, error) { reasons = append(reasons, fmt.Sprintf("settings.analysis changed: index %s, code %s", rawOrUnset(rv), rawOrUnset(lv))) } - classification := searchmapping.Classify( - propertiesMap(r.remote.Get("mappings.properties").Raw), - propertiesMap(r.local.Get("mappings.properties").Raw), - nil, - ) + remoteProps := propertiesMap(r.remote.Get("mappings.properties").Raw) + localProps := propertiesMap(r.local.Get("mappings.properties").Raw) + reasons = append(reasons, classifyOpenExtensionMapping(localProps, remoteProps, + r.local.Get("mappings.dynamic_templates").Raw, r.remote.Get("mappings.dynamic_templates").Raw)...) + + classification := searchmapping.Classify(remoteProps, localProps, nil) classification.AddBreaking(reasons...) return classification, nil } +// classifyOpenExtensionMapping takes the ext tree out of the property comparison: +// the index grows concrete properties under it that the code never declares. +// Its dynamic flag and the templates are the rule, a changed rule is breaking. +func classifyOpenExtensionMapping(local, remote map[string]any, localTemplates, remoteTemplates string) []string { + var reasons []string + localExt, _ := local[searchmapping.OpenExtensionsRoot].(map[string]any) + remoteExt, _ := remote[searchmapping.OpenExtensionsRoot].(map[string]any) + if localExt == nil || remoteExt == nil { + return nil // an addition or a removal, Classify reports either + } + delete(local, searchmapping.OpenExtensionsRoot) + delete(remote, searchmapping.OpenExtensionsRoot) + // the flag comes back as a string + if lv, rv := fmt.Sprint(localExt["dynamic"]), fmt.Sprint(remoteExt["dynamic"]); lv != rv { + reasons = append(reasons, fmt.Sprintf("field %s: dynamic changed: index %s, code %s", searchmapping.OpenExtensionsRoot, rv, lv)) + } + if !jsonEqual(localTemplates, remoteTemplates) { + reasons = append(reasons, fmt.Sprintf("mappings.dynamic_templates changed: index %s, code %s", rawOrUnset(remoteTemplates), rawOrUnset(localTemplates))) + } + return reasons +} + // ApplyAdditive puts the full code properties (only additions, per the // classifier). The PUT is atomic, so persisted is true only on success. func (r *osReconciler) ApplyAdditive() (bool, error) { diff --git a/services/search/pkg/opensearch/openextensions.go b/services/search/pkg/opensearch/openextensions.go new file mode 100644 index 0000000000..876ceb18b4 --- /dev/null +++ b/services/search/pkg/opensearch/openextensions.go @@ -0,0 +1,91 @@ +package opensearch + +import ( + "strings" + "time" + + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" +) + +// The ext object is dynamic and its siblings are typed by name through dynamic +// templates, so a new property needs no mapping change on our side. + +func openExtensionTemplates() []map[string]any { + template := func(sibling string, m map[string]any) map[string]any { + return map[string]any{ + "ext_" + strings.TrimPrefix(sibling, "@"): map[string]any{ + "path_match": mapping.OpenExtensionsRoot + ".*", + "match": sibling, + "mapping": m, + }, + } + } + return []map[string]any{ + template(mapping.SiblingKeyword, map[string]any{"type": "keyword"}), + template(mapping.SiblingLower, map[string]any{"type": "keyword", "doc_values": false}), + template(mapping.SiblingNumber, map[string]any{"type": "double"}), + template(mapping.SiblingBool, map[string]any{"type": "boolean"}), + template(mapping.SiblingDate, map[string]any{"type": "date"}), + template(mapping.SiblingGeo, map[string]any{"type": "geo_point"}), + } +} + +func openExtensionProperties() map[string]any { + return map[string]any{ + mapping.OpenExtensionsStoredField: map[string]any{"type": "object", "enabled": false}, + mapping.OpenExtensionsRoot: map[string]any{"type": "object", "dynamic": true}, + } +} + +func addOpenExtensionSource(body map[string]any, values map[string]string) { + if len(values) == 0 { + return + } + stored := make(map[string]any, len(values)) + for key, raw := range values { + stored[key] = raw + } + body[mapping.OpenExtensionsStoredField] = stored + + tree := map[string]any{} + for _, leaf := range mapping.OpenExtensionLeaves(values) { + var value any + switch leaf.Sibling { + case mapping.SiblingKeyword, mapping.SiblingLower: + value = single(leaf.Strings) + case mapping.SiblingNumber: + value = single(leaf.Numbers) + case mapping.SiblingBool: + value = single(leaf.Bools) + case mapping.SiblingDate: + formatted := make([]string, len(leaf.Times)) + for i, t := range leaf.Times { + formatted[i] = t.UTC().Format(time.RFC3339Nano) + } + value = single(formatted) + case mapping.SiblingGeo: + value = map[string]any{"lat": leaf.Geo.Latitude, "lon": leaf.Geo.Longitude} + } + setPath(tree, strings.Split(strings.TrimPrefix(leaf.Field, mapping.OpenExtensionsRoot+"."), "."), value) + } + body[mapping.OpenExtensionsRoot] = tree +} + +func single[T any](values []T) any { + if len(values) == 1 { + return values[0] + } + return values +} + +func setPath(tree map[string]any, path []string, value any) { + for _, segment := range path[:len(path)-1] { + next, ok := tree[segment].(map[string]any) + if !ok { + next = map[string]any{} + tree[segment] = next + } + tree = next + } + tree[path[len(path)-1]] = value +} diff --git a/services/search/pkg/opensearch/openextensions_internal_test.go b/services/search/pkg/opensearch/openextensions_internal_test.go new file mode 100644 index 0000000000..ec15867a10 --- /dev/null +++ b/services/search/pkg/opensearch/openextensions_internal_test.go @@ -0,0 +1,99 @@ +package opensearch + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestAddOpenExtensionSource(t *testing.T) { + const prefix = "http://opencloud.eu/ns/extensions/com.example.project/" + body := map[string]any{"Name": "a.txt"} + addOpenExtensionSource(body, map[string]string{ + prefix + "state": "s:Open", + prefix + "priority": "n:3", + prefix + "tags": `S:["A","b"]`, + prefix + "due": "d:2026-10-01T00:00:00Z", + prefix + "site": "g:52.5,13.4", + }) + + got, err := json.Marshal(body) + require.NoError(t, err) + require.JSONEq(t, `{ + "Name": "a.txt", + "OpenExtensions": { + "`+prefix+`state": "s:Open", + "`+prefix+`priority": "n:3", + "`+prefix+`tags": "S:[\"A\",\"b\"]", + "`+prefix+`due": "d:2026-10-01T00:00:00Z", + "`+prefix+`site": "g:52.5,13.4" + }, + "ext": {"com": {"example": {"project": { + "state": {"@keyword": "Open", "@lower": "open"}, + "priority": {"@number": 3}, + "tags": {"@keyword": ["A", "b"], "@lower": ["a", "b"]}, + "due": {"@date": "2026-10-01T00:00:00Z"}, + "site": {"@geo": {"lat": 52.5, "lon": 13.4}} + }}}} + }`, string(got)) +} + +func TestAddOpenExtensionSourceWithoutExtensions(t *testing.T) { + body := map[string]any{"Name": "a.txt"} + addOpenExtensionSource(body, nil) + require.Equal(t, map[string]any{"Name": "a.txt"}, body) +} + +func TestClassifyOpenExtensionMapping(t *testing.T) { + templates := func() string { + b, err := json.Marshal(openExtensionTemplates()) + require.NoError(t, err) + return string(b) + } + local := func() map[string]any { + return map[string]any{"Name": map[string]any{"type": "keyword"}, "ext": map[string]any{"type": "object", "dynamic": true}} + } + // what GET _mapping returns once documents were indexed: the flag as a + // string and the concrete fields OpenSearch added under ext + remote := func() map[string]any { + return map[string]any{ + "Name": map[string]any{"type": "keyword"}, + "ext": map[string]any{"dynamic": "true", "properties": map[string]any{ + "com": map[string]any{"properties": map[string]any{"example": map[string]any{"properties": map[string]any{ + "project": map[string]any{"properties": map[string]any{"priority": map[string]any{"properties": map[string]any{"@number": map[string]any{"type": "double"}}}}}, + }}}}, + }}, + } + } + + t.Run("concrete fields under ext are not a difference", func(t *testing.T) { + l, r := local(), remote() + reasons := classifyOpenExtensionMapping(l, r, templates(), templates()) + require.Empty(t, reasons) + require.NotContains(t, l, "ext") + require.NotContains(t, r, "ext") + require.Equal(t, l, r, "what is left is compared by Classify") + }) + + t.Run("a changed template is breaking", func(t *testing.T) { + reasons := classifyOpenExtensionMapping(local(), remote(), templates(), `[{"ext_number":{"path_match":"ext.*","match":"@number","mapping":{"type":"long"}}}]`) + require.Len(t, reasons, 1) + require.Contains(t, reasons[0], "dynamic_templates changed") + }) + + t.Run("a changed dynamic flag is breaking", func(t *testing.T) { + r := remote() + r["ext"].(map[string]any)["dynamic"] = "strict" + reasons := classifyOpenExtensionMapping(local(), r, templates(), templates()) + require.Len(t, reasons, 1) + require.Contains(t, reasons[0], "dynamic changed") + }) + + t.Run("an index without ext leaves the addition to Classify", func(t *testing.T) { + l, r := local(), map[string]any{"Name": map[string]any{"type": "keyword"}} + reasons := classifyOpenExtensionMapping(l, r, templates(), "") + require.Empty(t, reasons) + require.Contains(t, l, "ext") + }) +} diff --git a/services/search/pkg/opensearch/testdata/resource.golden.json b/services/search/pkg/opensearch/testdata/resource.golden.json index ab343eef11..2372dbd9ab 100644 --- a/services/search/pkg/opensearch/testdata/resource.golden.json +++ b/services/search/pkg/opensearch/testdata/resource.golden.json @@ -1,5 +1,62 @@ { "mappings": { + "dynamic_templates": [ + { + "ext_keyword": { + "mapping": { + "type": "keyword" + }, + "match": "@keyword", + "path_match": "ext.*" + } + }, + { + "ext_lower": { + "mapping": { + "doc_values": false, + "type": "keyword" + }, + "match": "@lower", + "path_match": "ext.*" + } + }, + { + "ext_number": { + "mapping": { + "type": "double" + }, + "match": "@number", + "path_match": "ext.*" + } + }, + { + "ext_bool": { + "mapping": { + "type": "boolean" + }, + "match": "@bool", + "path_match": "ext.*" + } + }, + { + "ext_date": { + "mapping": { + "type": "date" + }, + "match": "@date", + "path_match": "ext.*" + } + }, + { + "ext_geo": { + "mapping": { + "type": "geo_point" + }, + "match": "@geo", + "path_match": "ext.*" + } + } + ], "properties": { "Content": { "analyzer": "words", @@ -9,6 +66,10 @@ "Deleted": { "type": "boolean" }, + "OpenExtensions": { + "enabled": false, + "type": "object" + }, "Favorites": { "type": "keyword" }, @@ -178,6 +239,10 @@ } } }, + "ext": { + "dynamic": true, + "type": "object" + }, "image": { "properties": { "height": { diff --git a/services/search/pkg/service/event/openextensions_internal_test.go b/services/search/pkg/service/event/openextensions_internal_test.go new file mode 100644 index 0000000000..1e7bd0d72a --- /dev/null +++ b/services/search/pkg/service/event/openextensions_internal_test.go @@ -0,0 +1,18 @@ +package event + +import "testing" + +func TestTouchesOpenExtensions(t *testing.T) { + const state = "http://opencloud.eu/ns/extensions/com.example.project/state" + for keys, want := range map[*[]string]bool{ + {state}: true, + {"tags", state}: true, + {"tags"}: false, + {"http://owncloud.org/ns/favorite"}: false, + {}: false, + } { + if got := touchesOpenExtensions(*keys); got != want { + t.Errorf("touchesOpenExtensions(%v) = %v, want %v", *keys, got, want) + } + } +} diff --git a/services/search/pkg/service/event/service.go b/services/search/pkg/service/event/service.go index c4eff9bfb7..6868b17f1a 100644 --- a/services/search/pkg/service/event/service.go +++ b/services/search/pkg/service/event/service.go @@ -10,6 +10,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/opencloud-eu/reva/v2/pkg/events" "github.com/opencloud-eu/reva/v2/pkg/events/raw" + "github.com/opencloud-eu/reva/v2/pkg/openextension" "github.com/opencloud-eu/reva/v2/pkg/storagespace" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" @@ -62,6 +63,7 @@ func New(ctx context.Context, stream raw.Stream, logger log.Logger, tp trace.Tra events.FileVersionRestored{}, events.TagsAdded{}, events.TagsRemoved{}, + events.ArbitraryMetadataUpdated{}, events.SpaceRenamed{}, events.SpaceDeleted{}, events.LabelAdded{}, @@ -207,6 +209,12 @@ func (s Service) processEvent(e raw.Event) error { case events.TagsRemoved: s.index.UpsertItem(ev.Ref) debounce(getSpaceID(ev.Ref)) + case events.ArbitraryMetadataUpdated: + // tags and favorites come with their own events + if touchesOpenExtensions(ev.Keys) { + s.index.UpsertItem(ev.Ref) + debounce(getSpaceID(ev.Ref)) + } case events.FileUploaded: debounce(getSpaceID(ev.Ref)) case events.UploadReady: @@ -253,3 +261,12 @@ func monitorMetrics(ctx context.Context, stream raw.Stream, name string, m *metr } }() } + +func touchesOpenExtensions(keys []string) bool { + for _, key := range keys { + if _, _, ok := openextension.SplitKey(key); ok { + return true + } + } + return false +} diff --git a/services/search/pkg/service/event/service_test.go b/services/search/pkg/service/event/service_test.go index d0afa2fea9..249ee9c5c6 100644 --- a/services/search/pkg/service/event/service_test.go +++ b/services/search/pkg/service/event/service_test.go @@ -63,6 +63,7 @@ var _ = DescribeTable("event", Entry("FileVersionRestored", []string{"IndexSpace"}, events.FileVersionRestored{}, false), Entry("TagsAdded", []string{"UpsertItem", "IndexSpace"}, events.TagsAdded{}, false), Entry("TagsRemoved", []string{"UpsertItem", "IndexSpace"}, events.TagsRemoved{}, false), + Entry("ArbitraryMetadataUpdated on an extension", []string{"UpsertItem", "IndexSpace"}, events.ArbitraryMetadataUpdated{Keys: []string{"http://opencloud.eu/ns/extensions/com.example.project/state"}}, false), Entry("FileUploaded", []string{"IndexSpace"}, events.FileUploaded{}, false), Entry("UploadReady", []string{"IndexSpace"}, events.UploadReady{ExecutingUser: &userv1beta1.User{}}, true), )