feat(search): index open extensions as typed siblings

Every property of an open extension is indexed under the sibling of its
value's kind (ext.<name>.<property>.@keyword/@lower/@number/@bool/@date/
@geo), so a property can change its kind between writes without a
mapping change. bleve builds the fields directly and indexes them with
IndexAdvanced, the mapping stays as it is; OpenSearch types them through
dynamic templates under a dynamic ext object, which the reconciler
compares by rule instead of by concrete field. The stored values travel
with the document so Move, Delete and Restore keep the extensions.

The search service follows ArbitraryMetadataUpdated events that touch an
extension.
This commit is contained in:
Dominik Schmidt committed 2026-09-16 09:20:39 +00:00
1 parent 44b97637ba
commit 12f76239de
16 files changed
+814 -9

No files matched your search

+10 -2
View File
@@ -6,6 +6,7 @@ import (
"strings"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/document"
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/opencloud-eu/reva/v2/pkg/utils"
@@ -45,11 +46,18 @@ func (b *Batch) Upsert(id string, r search.Resource) error {
// type-specific adaptations via the mapping package) and appends it to the
// batch under id.
func (b *Batch) indexResource(id string, r search.Resource) error {
doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
data, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
if err != nil {
return err
}
return b.batch.Index(id, doc)
doc := document.NewDocument(id)
if err := b.index.Mapping().MapDocument(doc, data); err != nil {
return err
}
if err := addOpenExtensionFields(doc, b.index.Mapping(), r.OpenExtensions); err != nil {
return err
}
return b.batch.IndexAdvanced(doc)
}
func (b *Batch) Move(id, parentID, location string) error {
+3 -1
View File
@@ -78,5 +78,7 @@ func hitToFacet[T any](fields map[string]any, prefix string) *T {
// for per-field parse errors, so corrupted hit values surface as zero
// values on individual fields instead of dropping the whole record.
func matchToResource(match *bleveSearch.DocumentMatch) *search.Resource {
return mapping.Deserialize[search.Resource](match.Fields)
r := mapping.Deserialize[search.Resource](match.Fields)
r.OpenExtensions = openExtensionsFromHit(match.Fields)
return r
}
@@ -0,0 +1,80 @@
package bleve
import (
"strings"
"time"
"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
"github.com/blevesearch/bleve/v2/document"
bleveMapping "github.com/blevesearch/bleve/v2/mapping"
index "github.com/blevesearch/bleve_index_api"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
)
// Open extensions bypass the mapping: it is fixed when the index is created
// and can only infer a type from a Go value. The typed siblings are built as
// document fields and indexed with IndexAdvanced, so the stored mapping never
// changes. The stored values are stored, not indexed, so Move, Delete and
// Restore can re-index a hit without losing them.
const openExtensionIndexOptions = index.IndexField | index.DocValues
func addOpenExtensionFields(doc *document.Document, im bleveMapping.IndexMapping, values map[string]string) error {
analyzer := im.AnalyzerNamed(keyword.Name)
for key, raw := range values {
doc.AddField(document.NewTextFieldCustom(mapping.OpenExtensionsStoredField+"."+key, nil, []byte(raw), index.StoreField, nil))
}
for _, leaf := range mapping.OpenExtensionLeaves(values) {
positions := func(i, n int) []uint64 {
if n == 1 {
return nil
}
return []uint64{uint64(i)} //nolint:gosec // a loop index is never negative
}
switch leaf.Sibling {
case mapping.SiblingKeyword, mapping.SiblingLower:
for i, s := range leaf.Strings {
doc.AddField(document.NewTextFieldCustom(leaf.Field, positions(i, len(leaf.Strings)), []byte(s), openExtensionIndexOptions, analyzer))
}
case mapping.SiblingNumber:
for i, f := range leaf.Numbers {
doc.AddField(document.NewNumericFieldWithIndexingOptions(leaf.Field, positions(i, len(leaf.Numbers)), f, openExtensionIndexOptions))
}
case mapping.SiblingBool:
for i, b := range leaf.Bools {
doc.AddField(document.NewBooleanFieldWithIndexingOptions(leaf.Field, positions(i, len(leaf.Bools)), b, openExtensionIndexOptions))
}
case mapping.SiblingDate:
for i, t := range leaf.Times {
f, err := document.NewDateTimeFieldWithIndexingOptions(leaf.Field, positions(i, len(leaf.Times)), t, time.RFC3339Nano, openExtensionIndexOptions)
if err != nil {
return err
}
doc.AddField(f)
}
case mapping.SiblingGeo:
doc.AddField(document.NewGeoPointFieldWithIndexingOptions(leaf.Field, nil, leaf.Geo.Longitude, leaf.Geo.Latitude, openExtensionIndexOptions))
}
}
return nil
}
func openExtensionsFromHit(fields map[string]any) map[string]string {
var values map[string]string
prefix := mapping.OpenExtensionsStoredField + "."
for field, value := range fields {
if !strings.HasPrefix(field, prefix) {
continue
}
raw, ok := value.(string)
if !ok {
continue
}
if values == nil {
values = map[string]string{}
}
values[field[len(prefix):]] = raw
}
return values
}
@@ -0,0 +1,163 @@
package bleve_test
import (
"context"
"time"
bleveSearch "github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/search/query"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/pkg/log"
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
// Open extensions bypass the mapping (IndexAdvanced), so the invariants the
// rest of the engine relies on are pinned here: the typed siblings exist as
// fields, they answer typed queries, the mapping is untouched, and the stored
// values survive a Move, which re-indexes the resource from a hit.
var _ = Describe("Open extensions", func() {
var (
idx bleveSearch.Index
eng *bleve.Backend
)
key := func(property string) string {
return "http://opencloud.eu/ns/extensions/com.example.project/" + property
}
stored := map[string]string{
key("state"): "s:Open",
key("priority"): "n:3",
key("done"): "b:false",
key("due"): "d:2026-10-01T00:00:00Z",
key("site"): "g:52.5,13.4",
}
resource := func(id, path string, exts map[string]string) search.Resource {
return search.Resource{
ID: id,
RootID: "1$2!2",
ParentID: "1$2!2",
Path: path,
Type: 1,
Document: content.Document{Name: path[2:], OpenExtensions: exts},
}
}
hits := func(q query.Query) []string {
GinkgoHelper()
req := bleveSearch.NewSearchRequest(q)
req.Fields = []string{"*"}
res, err := idx.Search(req)
Expect(err).ToNot(HaveOccurred())
ids := make([]string, 0, len(res.Hits))
for _, h := range res.Hits {
ids = append(ids, h.ID)
}
return ids
}
term := func(field, value string) query.Query {
q := query.NewTermQuery(value)
q.SetField(field)
return q
}
BeforeEach(func() {
m, err := bleve.NewMapping()
Expect(err).ToNot(HaveOccurred())
idx, err = bleveSearch.NewMemOnly(m)
Expect(err).ToNot(HaveOccurred())
eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{})
Expect(eng.Upsert("1$2!a", resource("1$2!a", "./a.txt", stored))).To(Succeed())
Expect(eng.Upsert("1$2!b", resource("1$2!b", "./b.txt", map[string]string{key("state"): "s:closed", key("priority"): "s:high"}))).To(Succeed())
Expect(eng.Upsert("1$2!c", resource("1$2!c", "./c.txt", nil))).To(Succeed())
})
It("indexes every property under the sibling of its kind", func() {
fields, err := idx.Fields()
Expect(err).ToNot(HaveOccurred())
Expect(fields).To(ContainElements(
"ext.com.example.project.state.@keyword",
"ext.com.example.project.state.@lower",
"ext.com.example.project.priority.@number",
"ext.com.example.project.priority.@keyword",
"ext.com.example.project.done.@bool",
"ext.com.example.project.due.@date",
"ext.com.example.project.site.@geo",
))
})
It("answers typed queries on the siblings", func() {
Expect(hits(term("ext.com.example.project.state.@lower", "open"))).To(ConsistOf("1$2!a"))
Expect(hits(term("ext.com.example.project.state.@keyword", "Open"))).To(ConsistOf("1$2!a"))
Expect(hits(term("ext.com.example.project.state.@keyword", "open"))).To(BeEmpty(), "the keyword sibling is case-sensitive")
lo, hi := 2.0, 4.0
inclusive := true
nq := query.NewNumericRangeInclusiveQuery(&lo, &hi, &inclusive, &inclusive)
nq.SetField("ext.com.example.project.priority.@number")
Expect(hits(nq)).To(ConsistOf("1$2!a"), "b's priority is a string and lives in the keyword sibling")
Expect(hits(term("ext.com.example.project.priority.@lower", "high"))).To(ConsistOf("1$2!b"))
bq := query.NewBoolFieldQuery(false)
bq.SetField("ext.com.example.project.done.@bool")
Expect(hits(bq)).To(ConsistOf("1$2!a"))
start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
end := time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC)
dq := query.NewDateRangeQuery(start, end)
dq.SetField("ext.com.example.project.due.@date")
Expect(hits(dq)).To(ConsistOf("1$2!a"))
gq := query.NewGeoDistanceQuery(13.4, 52.5, "1km")
gq.SetField("ext.com.example.project.site.@geo")
Expect(hits(gq)).To(ConsistOf("1$2!a"))
})
It("stores the values with the hit and keeps them through a Move", func() {
req := bleveSearch.NewSearchRequest(bleveSearch.NewDocIDQuery([]string{"1$2!a"}))
req.Fields = []string{"*"}
res, err := idx.Search(req)
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits).To(HaveLen(1))
Expect(res.Hits[0].Fields).To(HaveKeyWithValue("OpenExtensions."+key("state"), "s:Open"))
Expect(res.Hits[0].Fields).To(HaveKeyWithValue("OpenExtensions."+key("site"), "g:52.5,13.4"))
Expect(eng.Move("1$2!a", "1$2!2", "./moved.txt")).To(Succeed())
Expect(hits(term("ext.com.example.project.state.@lower", "open"))).To(ConsistOf("1$2!a"), "the re-indexed resource keeps its extensions")
sr, err := eng.Search(context.Background(), &searchService.SearchIndexRequest{Query: `name:moved.txt`})
Expect(err).ToNot(HaveOccurred())
Expect(sr.Matches).To(HaveLen(1))
})
It("replaces the siblings on upsert instead of accumulating them", func() {
Expect(eng.Upsert("1$2!a", resource("1$2!a", "./a.txt", map[string]string{key("state"): "s:closed"}))).To(Succeed())
Expect(hits(term("ext.com.example.project.state.@lower", "open"))).To(BeEmpty())
Expect(hits(term("ext.com.example.project.state.@lower", "closed"))).To(ConsistOf("1$2!a", "1$2!b"))
lo, hi := 2.0, 4.0
inclusive := true
nq := query.NewNumericRangeInclusiveQuery(&lo, &hi, &inclusive, &inclusive)
nq.SetField("ext.com.example.project.priority.@number")
Expect(hits(nq)).To(BeEmpty(), "a property that is gone leaves no sibling behind")
})
It("leaves the stored mapping untouched", func() {
fields, err := idx.Fields()
Expect(err).ToNot(HaveOccurred())
Expect(fields).To(ContainElement("ext.com.example.project.state.@lower"))
// the mapping only knows the resource struct; an extension field never
// becomes part of it, so the reconciler never sees one
m := idx.Mapping()
Expect(m.AnalyzerNameForPath("ext.com.example.project.state.@lower")).To(Equal(m.AnalyzerNameForPath("does.not.exist")))
})
})
+10
View File
@@ -7,6 +7,7 @@ import (
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/opencloud-eu/opencloud/pkg/conversions"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/reva/v2/pkg/openextension"
"github.com/opencloud-eu/reva/v2/pkg/tags"
"github.com/opencloud-eu/reva/v2/pkg/utils"
)
@@ -33,6 +34,15 @@ func (b Basic) Extract(_ context.Context, ri *storageProvider.ResourceInfo) (Doc
if t, ok := m["tags"]; ok {
doc.Tags = tags.New(t).AsSlice()
}
for key, value := range m {
if _, _, ok := openextension.SplitKey(key); !ok {
continue
}
if doc.OpenExtensions == nil {
doc.OpenExtensions = map[string]string{}
}
doc.OpenExtensions[key] = value
}
}
if m := ri.Opaque.GetMap(); m != nil && m["favorites"] != nil {
+4
View File
@@ -30,6 +30,10 @@ type Document struct {
Video *libregraph.Video `json:"video,omitempty"`
MotionPhoto *libregraph.MotionPhoto `json:"motionPhoto,omitempty"`
LivePhoto *libregraph.LivePhoto `json:"livePhoto,omitempty"`
// OpenExtensions are the stored open extension properties by metadata key;
// the engines index them as typed siblings, see mapping.OpenExtensionLeaves.
OpenExtensions map[string]string `json:"-"`
}
func CleanString(content, langCode string) string {
@@ -0,0 +1,147 @@
package mapping
import (
"sort"
"strings"
"time"
"github.com/opencloud-eu/reva/v2/pkg/openextension"
)
// An open extension property is indexed under the sibling of its value's
// kind, ext.<extensionName>.<property>.@<sibling>, so it can change its kind
// between writes without a mapping change. The sibling segment starts with
// "@", which neither a name nor a property may, so a leaf never collides with
// an object of another extension in the dotted tree OpenSearch builds.
const (
OpenExtensionsRoot = "ext"
// OpenExtensionsStoredField holds the stored values by metadata key, so an
// engine can rebuild the document from a hit.
OpenExtensionsStoredField = "OpenExtensions"
// OpenExtensionsQueryPrefix: KQL addresses extensions.<extensionName>.<property>
OpenExtensionsQueryPrefix = "extensions."
SiblingKeyword = "@keyword"
SiblingLower = "@lower"
SiblingNumber = "@number"
SiblingBool = "@bool"
SiblingDate = "@date"
SiblingGeo = "@geo"
)
// OpenExtensionField is the indexed field name of one typed sibling.
func OpenExtensionField(name, property, sibling string) string {
return OpenExtensionsRoot + "." + name + "." + property + "." + sibling
}
// OpenExtensionFieldFromQuery turns a KQL key into the sibling's field name. The
// prefix is matched case-insensitively, name and property are taken as written.
func OpenExtensionFieldFromQuery(key, sibling string) (string, bool) {
rest, ok := openExtensionPath(key)
if !ok {
return "", false
}
return OpenExtensionsRoot + "." + rest + "." + sibling, true
}
// IsOpenExtensionQueryField reports whether a KQL key addresses an extension property.
func IsOpenExtensionQueryField(key string) bool {
_, ok := openExtensionPath(key)
return ok
}
func openExtensionPath(key string) (string, bool) {
if len(key) <= len(OpenExtensionsQueryPrefix) || !strings.EqualFold(key[:len(OpenExtensionsQueryPrefix)], OpenExtensionsQueryPrefix) {
return "", false
}
rest := key[len(OpenExtensionsQueryPrefix):]
// the property is the last segment, the name needs at least two more
if strings.Count(rest, ".") < 2 || strings.HasPrefix(rest, ".") || strings.HasSuffix(rest, ".") {
return "", false
}
return rest, true
}
// OpenExtensionLeaf is one typed sibling value, the unit both engines index.
type OpenExtensionLeaf struct {
Field string
Sibling string
Strings []string
Numbers []float64
Bools []bool
Times []time.Time
Geo *openextension.GeoPoint
}
// OpenExtensionLeaves flattens stored extension properties (metadata key to
// stored value) into the leaves to index, sorted by field. Unreadable values
// are skipped, the index must never fail on user data.
func OpenExtensionLeaves(values map[string]string) []OpenExtensionLeaf {
var leaves []OpenExtensionLeaf
for key, raw := range values {
name, property, ok := openextension.SplitKey(key)
if !ok {
continue
}
v, err := openextension.DecodeValue(raw)
if err != nil {
continue
}
leaves = append(leaves, propertyLeaves(name, property, v)...)
}
sort.Slice(leaves, func(i, j int) bool { return leaves[i].Field < leaves[j].Field })
return leaves
}
func propertyLeaves(name, property string, v openextension.Value) []OpenExtensionLeaf {
leaf := func(sibling string) OpenExtensionLeaf {
return OpenExtensionLeaf{Field: OpenExtensionField(name, property, sibling), Sibling: sibling}
}
switch v.Kind {
case openextension.KindString:
values := v.Strings()
if len(values) == 0 {
return nil
}
lower := make([]string, len(values))
for i, s := range values {
lower[i] = strings.ToLower(s)
}
keyword := leaf(SiblingKeyword)
keyword.Strings = values
lowered := leaf(SiblingLower)
lowered.Strings = lower
return []OpenExtensionLeaf{keyword, lowered}
case openextension.KindNumber:
l := leaf(SiblingNumber)
l.Numbers = v.Numbers()
if len(l.Numbers) == 0 {
return nil
}
return []OpenExtensionLeaf{l}
case openextension.KindBool:
l := leaf(SiblingBool)
l.Bools = v.Bools()
if len(l.Bools) == 0 {
return nil
}
return []OpenExtensionLeaf{l}
case openextension.KindDate:
l := leaf(SiblingDate)
l.Times = v.Times()
if len(l.Times) == 0 {
return nil
}
return []OpenExtensionLeaf{l}
case openextension.KindGeo:
p, ok := v.Geo()
if !ok {
return nil
}
l := leaf(SiblingGeo)
l.Geo = &p
return []OpenExtensionLeaf{l}
}
return nil
}
@@ -0,0 +1,74 @@
package mapping
import (
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("extension fields", func() {
key := func(name, property string) string {
return "http://opencloud.eu/ns/extensions/" + name + "/" + property
}
It("names a sibling and resolves the KQL spelling", func() {
Expect(OpenExtensionField("com.example.project", "priority", SiblingNumber)).To(Equal("ext.com.example.project.priority.@number"))
field, ok := OpenExtensionFieldFromQuery("extensions.com.example.project.priority", SiblingNumber)
Expect(ok).To(BeTrue())
Expect(field).To(Equal("ext.com.example.project.priority.@number"))
field, ok = OpenExtensionFieldFromQuery("Extensions.com.example.project.Priority", SiblingLower)
Expect(ok).To(BeTrue(), "the prefix is case-insensitive, the rest is taken as written")
Expect(field).To(Equal("ext.com.example.project.Priority.@lower"))
})
It("rejects keys that are not an extension property", func() {
for _, key := range []string{"Name", "extensions", "extensions.", "extensions.priority", "extensions.project.priority", "extensions.com.example.", "extension.com.example.x"} {
Expect(IsOpenExtensionQueryField(key)).To(BeFalse(), key)
}
Expect(IsOpenExtensionQueryField("extensions.com.example.priority")).To(BeTrue())
})
It("flattens stored extension properties into typed leaves", func() {
const project = "com.example.project"
leaves := OpenExtensionLeaves(map[string]string{
key(project, "state"): "s:Open",
key(project, "priority"): "n:3",
key(project, "done"): "b:false",
key(project, "tags"): `S:["A","b"]`,
key(project, "due"): "d:2026-10-01T00:00:00Z",
key(project, "site"): "g:52.5,13.4",
key(project, "broken"): "n:not a number",
"tags": "a,b",
})
byField := map[string]OpenExtensionLeaf{}
for _, l := range leaves {
byField[l.Field] = l
}
Expect(byField).To(HaveLen(8))
Expect(byField["ext.com.example.project.state.@keyword"].Strings).To(Equal([]string{"Open"}))
Expect(byField["ext.com.example.project.state.@lower"].Strings).To(Equal([]string{"open"}))
Expect(byField["ext.com.example.project.tags.@keyword"].Strings).To(Equal([]string{"A", "b"}))
Expect(byField["ext.com.example.project.tags.@lower"].Strings).To(Equal([]string{"a", "b"}))
Expect(byField["ext.com.example.project.priority.@number"].Numbers).To(Equal([]float64{3}))
Expect(byField["ext.com.example.project.done.@bool"].Bools).To(Equal([]bool{false}))
Expect(byField["ext.com.example.project.due.@date"].Times).To(Equal([]time.Time{time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)}))
Expect(byField["ext.com.example.project.site.@geo"].Geo.Latitude).To(Equal(52.5))
Expect(byField["ext.com.example.project.site.@geo"].Geo.Longitude).To(Equal(13.4))
Expect(leaves[0].Field < leaves[len(leaves)-1].Field).To(BeTrue(), "leaves are sorted by field")
})
It("indexes a property under the sibling of its current kind only", func() {
asNumber := OpenExtensionLeaves(map[string]string{key("x.y", "p"): "n:3"})
asString := OpenExtensionLeaves(map[string]string{key("x.y", "p"): "s:3"})
Expect(asNumber).To(HaveLen(1))
Expect(asNumber[0].Field).To(Equal("ext.x.y.p.@number"))
Expect(asString).To(HaveLen(2))
Expect(asString[0].Field).To(Equal("ext.x.y.p.@keyword"))
Expect(asString[1].Field).To(Equal("ext.x.y.p.@lower"))
})
})
+1
View File
@@ -48,6 +48,7 @@ func (b *Batch) Upsert(id string, r search.Resource) error {
if err != nil {
return fmt.Errorf("failed to marshal resource: %w", err)
}
addOpenExtensionSource(body, r.OpenExtensions)
op := func() []map[string]any {
return []map[string]any{
+31 -6
View File
@@ -74,6 +74,7 @@ func buildResourceMapping() ([]byte, error) {
if err != nil {
return nil, err
}
maps.Copy(props, openExtensionProperties())
index := map[string]any{
"settings": map[string]any{
@@ -108,7 +109,8 @@ func buildResourceMapping() ([]byte, error) {
},
},
"mappings": map[string]any{
"properties": props,
"dynamic_templates": openExtensionTemplates(),
"properties": props,
},
}
return json.Marshal(index)
@@ -198,15 +200,38 @@ func (r *osReconciler) Classify() (searchmapping.Classification, error) {
reasons = append(reasons, fmt.Sprintf("settings.analysis changed: index %s, code %s", rawOrUnset(rv), rawOrUnset(lv)))
}
classification := searchmapping.Classify(
propertiesMap(r.remote.Get("mappings.properties").Raw),
propertiesMap(r.local.Get("mappings.properties").Raw),
nil,
)
remoteProps := propertiesMap(r.remote.Get("mappings.properties").Raw)
localProps := propertiesMap(r.local.Get("mappings.properties").Raw)
reasons = append(reasons, classifyOpenExtensionMapping(localProps, remoteProps,
r.local.Get("mappings.dynamic_templates").Raw, r.remote.Get("mappings.dynamic_templates").Raw)...)
classification := searchmapping.Classify(remoteProps, localProps, nil)
classification.AddBreaking(reasons...)
return classification, nil
}
// classifyOpenExtensionMapping takes the ext tree out of the property comparison:
// the index grows concrete properties under it that the code never declares.
// Its dynamic flag and the templates are the rule, a changed rule is breaking.
func classifyOpenExtensionMapping(local, remote map[string]any, localTemplates, remoteTemplates string) []string {
var reasons []string
localExt, _ := local[searchmapping.OpenExtensionsRoot].(map[string]any)
remoteExt, _ := remote[searchmapping.OpenExtensionsRoot].(map[string]any)
if localExt == nil || remoteExt == nil {
return nil // an addition or a removal, Classify reports either
}
delete(local, searchmapping.OpenExtensionsRoot)
delete(remote, searchmapping.OpenExtensionsRoot)
// the flag comes back as a string
if lv, rv := fmt.Sprint(localExt["dynamic"]), fmt.Sprint(remoteExt["dynamic"]); lv != rv {
reasons = append(reasons, fmt.Sprintf("field %s: dynamic changed: index %s, code %s", searchmapping.OpenExtensionsRoot, rv, lv))
}
if !jsonEqual(localTemplates, remoteTemplates) {
reasons = append(reasons, fmt.Sprintf("mappings.dynamic_templates changed: index %s, code %s", rawOrUnset(remoteTemplates), rawOrUnset(localTemplates)))
}
return reasons
}
// ApplyAdditive puts the full code properties (only additions, per the
// classifier). The PUT is atomic, so persisted is true only on success.
func (r *osReconciler) ApplyAdditive() (bool, error) {
@@ -0,0 +1,91 @@
package opensearch
import (
"strings"
"time"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
)
// The ext object is dynamic and its siblings are typed by name through dynamic
// templates, so a new property needs no mapping change on our side.
func openExtensionTemplates() []map[string]any {
template := func(sibling string, m map[string]any) map[string]any {
return map[string]any{
"ext_" + strings.TrimPrefix(sibling, "@"): map[string]any{
"path_match": mapping.OpenExtensionsRoot + ".*",
"match": sibling,
"mapping": m,
},
}
}
return []map[string]any{
template(mapping.SiblingKeyword, map[string]any{"type": "keyword"}),
template(mapping.SiblingLower, map[string]any{"type": "keyword", "doc_values": false}),
template(mapping.SiblingNumber, map[string]any{"type": "double"}),
template(mapping.SiblingBool, map[string]any{"type": "boolean"}),
template(mapping.SiblingDate, map[string]any{"type": "date"}),
template(mapping.SiblingGeo, map[string]any{"type": "geo_point"}),
}
}
func openExtensionProperties() map[string]any {
return map[string]any{
mapping.OpenExtensionsStoredField: map[string]any{"type": "object", "enabled": false},
mapping.OpenExtensionsRoot: map[string]any{"type": "object", "dynamic": true},
}
}
func addOpenExtensionSource(body map[string]any, values map[string]string) {
if len(values) == 0 {
return
}
stored := make(map[string]any, len(values))
for key, raw := range values {
stored[key] = raw
}
body[mapping.OpenExtensionsStoredField] = stored
tree := map[string]any{}
for _, leaf := range mapping.OpenExtensionLeaves(values) {
var value any
switch leaf.Sibling {
case mapping.SiblingKeyword, mapping.SiblingLower:
value = single(leaf.Strings)
case mapping.SiblingNumber:
value = single(leaf.Numbers)
case mapping.SiblingBool:
value = single(leaf.Bools)
case mapping.SiblingDate:
formatted := make([]string, len(leaf.Times))
for i, t := range leaf.Times {
formatted[i] = t.UTC().Format(time.RFC3339Nano)
}
value = single(formatted)
case mapping.SiblingGeo:
value = map[string]any{"lat": leaf.Geo.Latitude, "lon": leaf.Geo.Longitude}
}
setPath(tree, strings.Split(strings.TrimPrefix(leaf.Field, mapping.OpenExtensionsRoot+"."), "."), value)
}
body[mapping.OpenExtensionsRoot] = tree
}
func single[T any](values []T) any {
if len(values) == 1 {
return values[0]
}
return values
}
func setPath(tree map[string]any, path []string, value any) {
for _, segment := range path[:len(path)-1] {
next, ok := tree[segment].(map[string]any)
if !ok {
next = map[string]any{}
tree[segment] = next
}
tree = next
}
tree[path[len(path)-1]] = value
}
@@ -0,0 +1,99 @@
package opensearch
import (
"encoding/json"
"testing"
"github.com/stretchr/testify/require"
)
func TestAddOpenExtensionSource(t *testing.T) {
const prefix = "http://opencloud.eu/ns/extensions/com.example.project/"
body := map[string]any{"Name": "a.txt"}
addOpenExtensionSource(body, map[string]string{
prefix + "state": "s:Open",
prefix + "priority": "n:3",
prefix + "tags": `S:["A","b"]`,
prefix + "due": "d:2026-10-01T00:00:00Z",
prefix + "site": "g:52.5,13.4",
})
got, err := json.Marshal(body)
require.NoError(t, err)
require.JSONEq(t, `{
"Name": "a.txt",
"OpenExtensions": {
"`+prefix+`state": "s:Open",
"`+prefix+`priority": "n:3",
"`+prefix+`tags": "S:[\"A\",\"b\"]",
"`+prefix+`due": "d:2026-10-01T00:00:00Z",
"`+prefix+`site": "g:52.5,13.4"
},
"ext": {"com": {"example": {"project": {
"state": {"@keyword": "Open", "@lower": "open"},
"priority": {"@number": 3},
"tags": {"@keyword": ["A", "b"], "@lower": ["a", "b"]},
"due": {"@date": "2026-10-01T00:00:00Z"},
"site": {"@geo": {"lat": 52.5, "lon": 13.4}}
}}}}
}`, string(got))
}
func TestAddOpenExtensionSourceWithoutExtensions(t *testing.T) {
body := map[string]any{"Name": "a.txt"}
addOpenExtensionSource(body, nil)
require.Equal(t, map[string]any{"Name": "a.txt"}, body)
}
func TestClassifyOpenExtensionMapping(t *testing.T) {
templates := func() string {
b, err := json.Marshal(openExtensionTemplates())
require.NoError(t, err)
return string(b)
}
local := func() map[string]any {
return map[string]any{"Name": map[string]any{"type": "keyword"}, "ext": map[string]any{"type": "object", "dynamic": true}}
}
// what GET _mapping returns once documents were indexed: the flag as a
// string and the concrete fields OpenSearch added under ext
remote := func() map[string]any {
return map[string]any{
"Name": map[string]any{"type": "keyword"},
"ext": map[string]any{"dynamic": "true", "properties": map[string]any{
"com": map[string]any{"properties": map[string]any{"example": map[string]any{"properties": map[string]any{
"project": map[string]any{"properties": map[string]any{"priority": map[string]any{"properties": map[string]any{"@number": map[string]any{"type": "double"}}}}},
}}}},
}},
}
}
t.Run("concrete fields under ext are not a difference", func(t *testing.T) {
l, r := local(), remote()
reasons := classifyOpenExtensionMapping(l, r, templates(), templates())
require.Empty(t, reasons)
require.NotContains(t, l, "ext")
require.NotContains(t, r, "ext")
require.Equal(t, l, r, "what is left is compared by Classify")
})
t.Run("a changed template is breaking", func(t *testing.T) {
reasons := classifyOpenExtensionMapping(local(), remote(), templates(), `[{"ext_number":{"path_match":"ext.*","match":"@number","mapping":{"type":"long"}}}]`)
require.Len(t, reasons, 1)
require.Contains(t, reasons[0], "dynamic_templates changed")
})
t.Run("a changed dynamic flag is breaking", func(t *testing.T) {
r := remote()
r["ext"].(map[string]any)["dynamic"] = "strict"
reasons := classifyOpenExtensionMapping(local(), r, templates(), templates())
require.Len(t, reasons, 1)
require.Contains(t, reasons[0], "dynamic changed")
})
t.Run("an index without ext leaves the addition to Classify", func(t *testing.T) {
l, r := local(), map[string]any{"Name": map[string]any{"type": "keyword"}}
reasons := classifyOpenExtensionMapping(l, r, templates(), "")
require.Empty(t, reasons)
require.Contains(t, l, "ext")
})
}
@@ -1,5 +1,62 @@
{
"mappings": {
"dynamic_templates": [
{
"ext_keyword": {
"mapping": {
"type": "keyword"
},
"match": "@keyword",
"path_match": "ext.*"
}
},
{
"ext_lower": {
"mapping": {
"doc_values": false,
"type": "keyword"
},
"match": "@lower",
"path_match": "ext.*"
}
},
{
"ext_number": {
"mapping": {
"type": "double"
},
"match": "@number",
"path_match": "ext.*"
}
},
{
"ext_bool": {
"mapping": {
"type": "boolean"
},
"match": "@bool",
"path_match": "ext.*"
}
},
{
"ext_date": {
"mapping": {
"type": "date"
},
"match": "@date",
"path_match": "ext.*"
}
},
{
"ext_geo": {
"mapping": {
"type": "geo_point"
},
"match": "@geo",
"path_match": "ext.*"
}
}
],
"properties": {
"Content": {
"analyzer": "words",
@@ -9,6 +66,10 @@
"Deleted": {
"type": "boolean"
},
"OpenExtensions": {
"enabled": false,
"type": "object"
},
"Favorites": {
"type": "keyword"
},
@@ -178,6 +239,10 @@
}
}
},
"ext": {
"dynamic": true,
"type": "object"
},
"image": {
"properties": {
"height": {
@@ -0,0 +1,18 @@
package event
import "testing"
func TestTouchesOpenExtensions(t *testing.T) {
const state = "http://opencloud.eu/ns/extensions/com.example.project/state"
for keys, want := range map[*[]string]bool{
{state}: true,
{"tags", state}: true,
{"tags"}: false,
{"http://owncloud.org/ns/favorite"}: false,
{}: false,
} {
if got := touchesOpenExtensions(*keys); got != want {
t.Errorf("touchesOpenExtensions(%v) = %v, want %v", *keys, got, want)
}
}
}
@@ -10,6 +10,7 @@ import (
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/opencloud-eu/reva/v2/pkg/events"
"github.com/opencloud-eu/reva/v2/pkg/events/raw"
"github.com/opencloud-eu/reva/v2/pkg/openextension"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
@@ -62,6 +63,7 @@ func New(ctx context.Context, stream raw.Stream, logger log.Logger, tp trace.Tra
events.FileVersionRestored{},
events.TagsAdded{},
events.TagsRemoved{},
events.ArbitraryMetadataUpdated{},
events.SpaceRenamed{},
events.SpaceDeleted{},
events.LabelAdded{},
@@ -207,6 +209,12 @@ func (s Service) processEvent(e raw.Event) error {
case events.TagsRemoved:
s.index.UpsertItem(ev.Ref)
debounce(getSpaceID(ev.Ref))
case events.ArbitraryMetadataUpdated:
// tags and favorites come with their own events
if touchesOpenExtensions(ev.Keys) {
s.index.UpsertItem(ev.Ref)
debounce(getSpaceID(ev.Ref))
}
case events.FileUploaded:
debounce(getSpaceID(ev.Ref))
case events.UploadReady:
@@ -253,3 +261,12 @@ func monitorMetrics(ctx context.Context, stream raw.Stream, name string, m *metr
}
}()
}
func touchesOpenExtensions(keys []string) bool {
for _, key := range keys {
if _, _, ok := openextension.SplitKey(key); ok {
return true
}
}
return false
}
@@ -63,6 +63,7 @@ var _ = DescribeTable("event",
Entry("FileVersionRestored", []string{"IndexSpace"}, events.FileVersionRestored{}, false),
Entry("TagsAdded", []string{"UpsertItem", "IndexSpace"}, events.TagsAdded{}, false),
Entry("TagsRemoved", []string{"UpsertItem", "IndexSpace"}, events.TagsRemoved{}, false),
Entry("ArbitraryMetadataUpdated on an extension", []string{"UpsertItem", "IndexSpace"}, events.ArbitraryMetadataUpdated{Keys: []string{"http://opencloud.eu/ns/extensions/com.example.project/state"}}, false),
Entry("FileUploaded", []string{"IndexSpace"}, events.FileUploaded{}, false),
Entry("UploadReady", []string{"IndexSpace"}, events.UploadReady{ExecutingUser: &userv1beta1.User{}}, true),
)