mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-09-25 13:37:09 -04:00
feat(search): index open extensions as typed siblings
Every property of an open extension is indexed under the sibling of its value's kind (ext.<name>.<property>.@keyword/@lower/@number/@bool/@date/ @geo), so a property can change its kind between writes without a mapping change. bleve builds the fields directly and indexes them with IndexAdvanced, the mapping stays as it is; OpenSearch types them through dynamic templates under a dynamic ext object, which the reconciler compares by rule instead of by concrete field. The stored values travel with the document so Move, Delete and Restore keep the extensions. The search service follows ArbitraryMetadataUpdated events that touch an extension.
This commit is contained in:
16 files changed
+814
-9
No files matched your search
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/document"
|
||||
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
|
||||
@@ -45,11 +46,18 @@ func (b *Batch) Upsert(id string, r search.Resource) error {
|
||||
// type-specific adaptations via the mapping package) and appends it to the
|
||||
// batch under id.
|
||||
func (b *Batch) indexResource(id string, r search.Resource) error {
|
||||
doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
|
||||
data, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return b.batch.Index(id, doc)
|
||||
doc := document.NewDocument(id)
|
||||
if err := b.index.Mapping().MapDocument(doc, data); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := addOpenExtensionFields(doc, b.index.Mapping(), r.OpenExtensions); err != nil {
|
||||
return err
|
||||
}
|
||||
return b.batch.IndexAdvanced(doc)
|
||||
}
|
||||
|
||||
func (b *Batch) Move(id, parentID, location string) error {
|
||||
|
||||
@@ -78,5 +78,7 @@ func hitToFacet[T any](fields map[string]any, prefix string) *T {
|
||||
// for per-field parse errors, so corrupted hit values surface as zero
|
||||
// values on individual fields instead of dropping the whole record.
|
||||
func matchToResource(match *bleveSearch.DocumentMatch) *search.Resource {
|
||||
return mapping.Deserialize[search.Resource](match.Fields)
|
||||
r := mapping.Deserialize[search.Resource](match.Fields)
|
||||
r.OpenExtensions = openExtensionsFromHit(match.Fields)
|
||||
return r
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package bleve
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
|
||||
"github.com/blevesearch/bleve/v2/document"
|
||||
bleveMapping "github.com/blevesearch/bleve/v2/mapping"
|
||||
index "github.com/blevesearch/bleve_index_api"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
)
|
||||
|
||||
// Open extensions bypass the mapping: it is fixed when the index is created
|
||||
// and can only infer a type from a Go value. The typed siblings are built as
|
||||
// document fields and indexed with IndexAdvanced, so the stored mapping never
|
||||
// changes. The stored values are stored, not indexed, so Move, Delete and
|
||||
// Restore can re-index a hit without losing them.
|
||||
|
||||
const openExtensionIndexOptions = index.IndexField | index.DocValues
|
||||
|
||||
func addOpenExtensionFields(doc *document.Document, im bleveMapping.IndexMapping, values map[string]string) error {
|
||||
analyzer := im.AnalyzerNamed(keyword.Name)
|
||||
for key, raw := range values {
|
||||
doc.AddField(document.NewTextFieldCustom(mapping.OpenExtensionsStoredField+"."+key, nil, []byte(raw), index.StoreField, nil))
|
||||
}
|
||||
for _, leaf := range mapping.OpenExtensionLeaves(values) {
|
||||
positions := func(i, n int) []uint64 {
|
||||
if n == 1 {
|
||||
return nil
|
||||
}
|
||||
return []uint64{uint64(i)} //nolint:gosec // a loop index is never negative
|
||||
}
|
||||
switch leaf.Sibling {
|
||||
case mapping.SiblingKeyword, mapping.SiblingLower:
|
||||
for i, s := range leaf.Strings {
|
||||
doc.AddField(document.NewTextFieldCustom(leaf.Field, positions(i, len(leaf.Strings)), []byte(s), openExtensionIndexOptions, analyzer))
|
||||
}
|
||||
case mapping.SiblingNumber:
|
||||
for i, f := range leaf.Numbers {
|
||||
doc.AddField(document.NewNumericFieldWithIndexingOptions(leaf.Field, positions(i, len(leaf.Numbers)), f, openExtensionIndexOptions))
|
||||
}
|
||||
case mapping.SiblingBool:
|
||||
for i, b := range leaf.Bools {
|
||||
doc.AddField(document.NewBooleanFieldWithIndexingOptions(leaf.Field, positions(i, len(leaf.Bools)), b, openExtensionIndexOptions))
|
||||
}
|
||||
case mapping.SiblingDate:
|
||||
for i, t := range leaf.Times {
|
||||
f, err := document.NewDateTimeFieldWithIndexingOptions(leaf.Field, positions(i, len(leaf.Times)), t, time.RFC3339Nano, openExtensionIndexOptions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
doc.AddField(f)
|
||||
}
|
||||
case mapping.SiblingGeo:
|
||||
doc.AddField(document.NewGeoPointFieldWithIndexingOptions(leaf.Field, nil, leaf.Geo.Longitude, leaf.Geo.Latitude, openExtensionIndexOptions))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func openExtensionsFromHit(fields map[string]any) map[string]string {
|
||||
var values map[string]string
|
||||
prefix := mapping.OpenExtensionsStoredField + "."
|
||||
for field, value := range fields {
|
||||
if !strings.HasPrefix(field, prefix) {
|
||||
continue
|
||||
}
|
||||
raw, ok := value.(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if values == nil {
|
||||
values = map[string]string{}
|
||||
}
|
||||
values[field[len(prefix):]] = raw
|
||||
}
|
||||
return values
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package bleve_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
bleveSearch "github.com/blevesearch/bleve/v2"
|
||||
"github.com/blevesearch/bleve/v2/search/query"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
// Open extensions bypass the mapping (IndexAdvanced), so the invariants the
|
||||
// rest of the engine relies on are pinned here: the typed siblings exist as
|
||||
// fields, they answer typed queries, the mapping is untouched, and the stored
|
||||
// values survive a Move, which re-indexes the resource from a hit.
|
||||
var _ = Describe("Open extensions", func() {
|
||||
var (
|
||||
idx bleveSearch.Index
|
||||
eng *bleve.Backend
|
||||
)
|
||||
|
||||
key := func(property string) string {
|
||||
return "http://opencloud.eu/ns/extensions/com.example.project/" + property
|
||||
}
|
||||
stored := map[string]string{
|
||||
key("state"): "s:Open",
|
||||
key("priority"): "n:3",
|
||||
key("done"): "b:false",
|
||||
key("due"): "d:2026-10-01T00:00:00Z",
|
||||
key("site"): "g:52.5,13.4",
|
||||
}
|
||||
|
||||
resource := func(id, path string, exts map[string]string) search.Resource {
|
||||
return search.Resource{
|
||||
ID: id,
|
||||
RootID: "1$2!2",
|
||||
ParentID: "1$2!2",
|
||||
Path: path,
|
||||
Type: 1,
|
||||
Document: content.Document{Name: path[2:], OpenExtensions: exts},
|
||||
}
|
||||
}
|
||||
|
||||
hits := func(q query.Query) []string {
|
||||
GinkgoHelper()
|
||||
req := bleveSearch.NewSearchRequest(q)
|
||||
req.Fields = []string{"*"}
|
||||
res, err := idx.Search(req)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
ids := make([]string, 0, len(res.Hits))
|
||||
for _, h := range res.Hits {
|
||||
ids = append(ids, h.ID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
term := func(field, value string) query.Query {
|
||||
q := query.NewTermQuery(value)
|
||||
q.SetField(field)
|
||||
return q
|
||||
}
|
||||
|
||||
BeforeEach(func() {
|
||||
m, err := bleve.NewMapping()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
idx, err = bleveSearch.NewMemOnly(m)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{})
|
||||
|
||||
Expect(eng.Upsert("1$2!a", resource("1$2!a", "./a.txt", stored))).To(Succeed())
|
||||
Expect(eng.Upsert("1$2!b", resource("1$2!b", "./b.txt", map[string]string{key("state"): "s:closed", key("priority"): "s:high"}))).To(Succeed())
|
||||
Expect(eng.Upsert("1$2!c", resource("1$2!c", "./c.txt", nil))).To(Succeed())
|
||||
})
|
||||
|
||||
It("indexes every property under the sibling of its kind", func() {
|
||||
fields, err := idx.Fields()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(fields).To(ContainElements(
|
||||
"ext.com.example.project.state.@keyword",
|
||||
"ext.com.example.project.state.@lower",
|
||||
"ext.com.example.project.priority.@number",
|
||||
"ext.com.example.project.priority.@keyword",
|
||||
"ext.com.example.project.done.@bool",
|
||||
"ext.com.example.project.due.@date",
|
||||
"ext.com.example.project.site.@geo",
|
||||
))
|
||||
})
|
||||
|
||||
It("answers typed queries on the siblings", func() {
|
||||
Expect(hits(term("ext.com.example.project.state.@lower", "open"))).To(ConsistOf("1$2!a"))
|
||||
Expect(hits(term("ext.com.example.project.state.@keyword", "Open"))).To(ConsistOf("1$2!a"))
|
||||
Expect(hits(term("ext.com.example.project.state.@keyword", "open"))).To(BeEmpty(), "the keyword sibling is case-sensitive")
|
||||
|
||||
lo, hi := 2.0, 4.0
|
||||
inclusive := true
|
||||
nq := query.NewNumericRangeInclusiveQuery(&lo, &hi, &inclusive, &inclusive)
|
||||
nq.SetField("ext.com.example.project.priority.@number")
|
||||
Expect(hits(nq)).To(ConsistOf("1$2!a"), "b's priority is a string and lives in the keyword sibling")
|
||||
Expect(hits(term("ext.com.example.project.priority.@lower", "high"))).To(ConsistOf("1$2!b"))
|
||||
|
||||
bq := query.NewBoolFieldQuery(false)
|
||||
bq.SetField("ext.com.example.project.done.@bool")
|
||||
Expect(hits(bq)).To(ConsistOf("1$2!a"))
|
||||
|
||||
start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
|
||||
end := time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC)
|
||||
dq := query.NewDateRangeQuery(start, end)
|
||||
dq.SetField("ext.com.example.project.due.@date")
|
||||
Expect(hits(dq)).To(ConsistOf("1$2!a"))
|
||||
|
||||
gq := query.NewGeoDistanceQuery(13.4, 52.5, "1km")
|
||||
gq.SetField("ext.com.example.project.site.@geo")
|
||||
Expect(hits(gq)).To(ConsistOf("1$2!a"))
|
||||
})
|
||||
|
||||
It("stores the values with the hit and keeps them through a Move", func() {
|
||||
req := bleveSearch.NewSearchRequest(bleveSearch.NewDocIDQuery([]string{"1$2!a"}))
|
||||
req.Fields = []string{"*"}
|
||||
res, err := idx.Search(req)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.Hits).To(HaveLen(1))
|
||||
Expect(res.Hits[0].Fields).To(HaveKeyWithValue("OpenExtensions."+key("state"), "s:Open"))
|
||||
Expect(res.Hits[0].Fields).To(HaveKeyWithValue("OpenExtensions."+key("site"), "g:52.5,13.4"))
|
||||
|
||||
Expect(eng.Move("1$2!a", "1$2!2", "./moved.txt")).To(Succeed())
|
||||
Expect(hits(term("ext.com.example.project.state.@lower", "open"))).To(ConsistOf("1$2!a"), "the re-indexed resource keeps its extensions")
|
||||
|
||||
sr, err := eng.Search(context.Background(), &searchService.SearchIndexRequest{Query: `name:moved.txt`})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(sr.Matches).To(HaveLen(1))
|
||||
})
|
||||
|
||||
It("replaces the siblings on upsert instead of accumulating them", func() {
|
||||
Expect(eng.Upsert("1$2!a", resource("1$2!a", "./a.txt", map[string]string{key("state"): "s:closed"}))).To(Succeed())
|
||||
Expect(hits(term("ext.com.example.project.state.@lower", "open"))).To(BeEmpty())
|
||||
Expect(hits(term("ext.com.example.project.state.@lower", "closed"))).To(ConsistOf("1$2!a", "1$2!b"))
|
||||
|
||||
lo, hi := 2.0, 4.0
|
||||
inclusive := true
|
||||
nq := query.NewNumericRangeInclusiveQuery(&lo, &hi, &inclusive, &inclusive)
|
||||
nq.SetField("ext.com.example.project.priority.@number")
|
||||
Expect(hits(nq)).To(BeEmpty(), "a property that is gone leaves no sibling behind")
|
||||
})
|
||||
|
||||
It("leaves the stored mapping untouched", func() {
|
||||
fields, err := idx.Fields()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(fields).To(ContainElement("ext.com.example.project.state.@lower"))
|
||||
|
||||
// the mapping only knows the resource struct; an extension field never
|
||||
// becomes part of it, so the reconciler never sees one
|
||||
m := idx.Mapping()
|
||||
Expect(m.AnalyzerNameForPath("ext.com.example.project.state.@lower")).To(Equal(m.AnalyzerNameForPath("does.not.exist")))
|
||||
})
|
||||
})
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/opencloud/pkg/conversions"
|
||||
"github.com/opencloud-eu/opencloud/pkg/log"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/openextension"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/tags"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
@@ -33,6 +34,15 @@ func (b Basic) Extract(_ context.Context, ri *storageProvider.ResourceInfo) (Doc
|
||||
if t, ok := m["tags"]; ok {
|
||||
doc.Tags = tags.New(t).AsSlice()
|
||||
}
|
||||
for key, value := range m {
|
||||
if _, _, ok := openextension.SplitKey(key); !ok {
|
||||
continue
|
||||
}
|
||||
if doc.OpenExtensions == nil {
|
||||
doc.OpenExtensions = map[string]string{}
|
||||
}
|
||||
doc.OpenExtensions[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
if m := ri.Opaque.GetMap(); m != nil && m["favorites"] != nil {
|
||||
|
||||
@@ -30,6 +30,10 @@ type Document struct {
|
||||
Video *libregraph.Video `json:"video,omitempty"`
|
||||
MotionPhoto *libregraph.MotionPhoto `json:"motionPhoto,omitempty"`
|
||||
LivePhoto *libregraph.LivePhoto `json:"livePhoto,omitempty"`
|
||||
|
||||
// OpenExtensions are the stored open extension properties by metadata key;
|
||||
// the engines index them as typed siblings, see mapping.OpenExtensionLeaves.
|
||||
OpenExtensions map[string]string `json:"-"`
|
||||
}
|
||||
|
||||
func CleanString(content, langCode string) string {
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/openextension"
|
||||
)
|
||||
|
||||
// An open extension property is indexed under the sibling of its value's
|
||||
// kind, ext.<extensionName>.<property>.@<sibling>, so it can change its kind
|
||||
// between writes without a mapping change. The sibling segment starts with
|
||||
// "@", which neither a name nor a property may, so a leaf never collides with
|
||||
// an object of another extension in the dotted tree OpenSearch builds.
|
||||
const (
|
||||
OpenExtensionsRoot = "ext"
|
||||
// OpenExtensionsStoredField holds the stored values by metadata key, so an
|
||||
// engine can rebuild the document from a hit.
|
||||
OpenExtensionsStoredField = "OpenExtensions"
|
||||
// OpenExtensionsQueryPrefix: KQL addresses extensions.<extensionName>.<property>
|
||||
OpenExtensionsQueryPrefix = "extensions."
|
||||
|
||||
SiblingKeyword = "@keyword"
|
||||
SiblingLower = "@lower"
|
||||
SiblingNumber = "@number"
|
||||
SiblingBool = "@bool"
|
||||
SiblingDate = "@date"
|
||||
SiblingGeo = "@geo"
|
||||
)
|
||||
|
||||
// OpenExtensionField is the indexed field name of one typed sibling.
|
||||
func OpenExtensionField(name, property, sibling string) string {
|
||||
return OpenExtensionsRoot + "." + name + "." + property + "." + sibling
|
||||
}
|
||||
|
||||
// OpenExtensionFieldFromQuery turns a KQL key into the sibling's field name. The
|
||||
// prefix is matched case-insensitively, name and property are taken as written.
|
||||
func OpenExtensionFieldFromQuery(key, sibling string) (string, bool) {
|
||||
rest, ok := openExtensionPath(key)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
return OpenExtensionsRoot + "." + rest + "." + sibling, true
|
||||
}
|
||||
|
||||
// IsOpenExtensionQueryField reports whether a KQL key addresses an extension property.
|
||||
func IsOpenExtensionQueryField(key string) bool {
|
||||
_, ok := openExtensionPath(key)
|
||||
return ok
|
||||
}
|
||||
|
||||
func openExtensionPath(key string) (string, bool) {
|
||||
if len(key) <= len(OpenExtensionsQueryPrefix) || !strings.EqualFold(key[:len(OpenExtensionsQueryPrefix)], OpenExtensionsQueryPrefix) {
|
||||
return "", false
|
||||
}
|
||||
rest := key[len(OpenExtensionsQueryPrefix):]
|
||||
// the property is the last segment, the name needs at least two more
|
||||
if strings.Count(rest, ".") < 2 || strings.HasPrefix(rest, ".") || strings.HasSuffix(rest, ".") {
|
||||
return "", false
|
||||
}
|
||||
return rest, true
|
||||
}
|
||||
|
||||
// OpenExtensionLeaf is one typed sibling value, the unit both engines index.
|
||||
type OpenExtensionLeaf struct {
|
||||
Field string
|
||||
Sibling string
|
||||
|
||||
Strings []string
|
||||
Numbers []float64
|
||||
Bools []bool
|
||||
Times []time.Time
|
||||
Geo *openextension.GeoPoint
|
||||
}
|
||||
|
||||
// OpenExtensionLeaves flattens stored extension properties (metadata key to
|
||||
// stored value) into the leaves to index, sorted by field. Unreadable values
|
||||
// are skipped, the index must never fail on user data.
|
||||
func OpenExtensionLeaves(values map[string]string) []OpenExtensionLeaf {
|
||||
var leaves []OpenExtensionLeaf
|
||||
for key, raw := range values {
|
||||
name, property, ok := openextension.SplitKey(key)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
v, err := openextension.DecodeValue(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
leaves = append(leaves, propertyLeaves(name, property, v)...)
|
||||
}
|
||||
sort.Slice(leaves, func(i, j int) bool { return leaves[i].Field < leaves[j].Field })
|
||||
return leaves
|
||||
}
|
||||
|
||||
func propertyLeaves(name, property string, v openextension.Value) []OpenExtensionLeaf {
|
||||
leaf := func(sibling string) OpenExtensionLeaf {
|
||||
return OpenExtensionLeaf{Field: OpenExtensionField(name, property, sibling), Sibling: sibling}
|
||||
}
|
||||
switch v.Kind {
|
||||
case openextension.KindString:
|
||||
values := v.Strings()
|
||||
if len(values) == 0 {
|
||||
return nil
|
||||
}
|
||||
lower := make([]string, len(values))
|
||||
for i, s := range values {
|
||||
lower[i] = strings.ToLower(s)
|
||||
}
|
||||
keyword := leaf(SiblingKeyword)
|
||||
keyword.Strings = values
|
||||
lowered := leaf(SiblingLower)
|
||||
lowered.Strings = lower
|
||||
return []OpenExtensionLeaf{keyword, lowered}
|
||||
case openextension.KindNumber:
|
||||
l := leaf(SiblingNumber)
|
||||
l.Numbers = v.Numbers()
|
||||
if len(l.Numbers) == 0 {
|
||||
return nil
|
||||
}
|
||||
return []OpenExtensionLeaf{l}
|
||||
case openextension.KindBool:
|
||||
l := leaf(SiblingBool)
|
||||
l.Bools = v.Bools()
|
||||
if len(l.Bools) == 0 {
|
||||
return nil
|
||||
}
|
||||
return []OpenExtensionLeaf{l}
|
||||
case openextension.KindDate:
|
||||
l := leaf(SiblingDate)
|
||||
l.Times = v.Times()
|
||||
if len(l.Times) == 0 {
|
||||
return nil
|
||||
}
|
||||
return []OpenExtensionLeaf{l}
|
||||
case openextension.KindGeo:
|
||||
p, ok := v.Geo()
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
l := leaf(SiblingGeo)
|
||||
l.Geo = &p
|
||||
return []OpenExtensionLeaf{l}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package mapping
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("extension fields", func() {
|
||||
key := func(name, property string) string {
|
||||
return "http://opencloud.eu/ns/extensions/" + name + "/" + property
|
||||
}
|
||||
|
||||
It("names a sibling and resolves the KQL spelling", func() {
|
||||
Expect(OpenExtensionField("com.example.project", "priority", SiblingNumber)).To(Equal("ext.com.example.project.priority.@number"))
|
||||
|
||||
field, ok := OpenExtensionFieldFromQuery("extensions.com.example.project.priority", SiblingNumber)
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(field).To(Equal("ext.com.example.project.priority.@number"))
|
||||
|
||||
field, ok = OpenExtensionFieldFromQuery("Extensions.com.example.project.Priority", SiblingLower)
|
||||
Expect(ok).To(BeTrue(), "the prefix is case-insensitive, the rest is taken as written")
|
||||
Expect(field).To(Equal("ext.com.example.project.Priority.@lower"))
|
||||
})
|
||||
|
||||
It("rejects keys that are not an extension property", func() {
|
||||
for _, key := range []string{"Name", "extensions", "extensions.", "extensions.priority", "extensions.project.priority", "extensions.com.example.", "extension.com.example.x"} {
|
||||
Expect(IsOpenExtensionQueryField(key)).To(BeFalse(), key)
|
||||
}
|
||||
Expect(IsOpenExtensionQueryField("extensions.com.example.priority")).To(BeTrue())
|
||||
})
|
||||
|
||||
It("flattens stored extension properties into typed leaves", func() {
|
||||
const project = "com.example.project"
|
||||
leaves := OpenExtensionLeaves(map[string]string{
|
||||
key(project, "state"): "s:Open",
|
||||
key(project, "priority"): "n:3",
|
||||
key(project, "done"): "b:false",
|
||||
key(project, "tags"): `S:["A","b"]`,
|
||||
key(project, "due"): "d:2026-10-01T00:00:00Z",
|
||||
key(project, "site"): "g:52.5,13.4",
|
||||
key(project, "broken"): "n:not a number",
|
||||
"tags": "a,b",
|
||||
})
|
||||
|
||||
byField := map[string]OpenExtensionLeaf{}
|
||||
for _, l := range leaves {
|
||||
byField[l.Field] = l
|
||||
}
|
||||
Expect(byField).To(HaveLen(8))
|
||||
Expect(byField["ext.com.example.project.state.@keyword"].Strings).To(Equal([]string{"Open"}))
|
||||
Expect(byField["ext.com.example.project.state.@lower"].Strings).To(Equal([]string{"open"}))
|
||||
Expect(byField["ext.com.example.project.tags.@keyword"].Strings).To(Equal([]string{"A", "b"}))
|
||||
Expect(byField["ext.com.example.project.tags.@lower"].Strings).To(Equal([]string{"a", "b"}))
|
||||
Expect(byField["ext.com.example.project.priority.@number"].Numbers).To(Equal([]float64{3}))
|
||||
Expect(byField["ext.com.example.project.done.@bool"].Bools).To(Equal([]bool{false}))
|
||||
Expect(byField["ext.com.example.project.due.@date"].Times).To(Equal([]time.Time{time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)}))
|
||||
Expect(byField["ext.com.example.project.site.@geo"].Geo.Latitude).To(Equal(52.5))
|
||||
Expect(byField["ext.com.example.project.site.@geo"].Geo.Longitude).To(Equal(13.4))
|
||||
|
||||
Expect(leaves[0].Field < leaves[len(leaves)-1].Field).To(BeTrue(), "leaves are sorted by field")
|
||||
})
|
||||
|
||||
It("indexes a property under the sibling of its current kind only", func() {
|
||||
asNumber := OpenExtensionLeaves(map[string]string{key("x.y", "p"): "n:3"})
|
||||
asString := OpenExtensionLeaves(map[string]string{key("x.y", "p"): "s:3"})
|
||||
Expect(asNumber).To(HaveLen(1))
|
||||
Expect(asNumber[0].Field).To(Equal("ext.x.y.p.@number"))
|
||||
Expect(asString).To(HaveLen(2))
|
||||
Expect(asString[0].Field).To(Equal("ext.x.y.p.@keyword"))
|
||||
Expect(asString[1].Field).To(Equal("ext.x.y.p.@lower"))
|
||||
})
|
||||
})
|
||||
@@ -48,6 +48,7 @@ func (b *Batch) Upsert(id string, r search.Resource) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal resource: %w", err)
|
||||
}
|
||||
addOpenExtensionSource(body, r.OpenExtensions)
|
||||
|
||||
op := func() []map[string]any {
|
||||
return []map[string]any{
|
||||
|
||||
@@ -74,6 +74,7 @@ func buildResourceMapping() ([]byte, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
maps.Copy(props, openExtensionProperties())
|
||||
|
||||
index := map[string]any{
|
||||
"settings": map[string]any{
|
||||
@@ -108,7 +109,8 @@ func buildResourceMapping() ([]byte, error) {
|
||||
},
|
||||
},
|
||||
"mappings": map[string]any{
|
||||
"properties": props,
|
||||
"dynamic_templates": openExtensionTemplates(),
|
||||
"properties": props,
|
||||
},
|
||||
}
|
||||
return json.Marshal(index)
|
||||
@@ -198,15 +200,38 @@ func (r *osReconciler) Classify() (searchmapping.Classification, error) {
|
||||
reasons = append(reasons, fmt.Sprintf("settings.analysis changed: index %s, code %s", rawOrUnset(rv), rawOrUnset(lv)))
|
||||
}
|
||||
|
||||
classification := searchmapping.Classify(
|
||||
propertiesMap(r.remote.Get("mappings.properties").Raw),
|
||||
propertiesMap(r.local.Get("mappings.properties").Raw),
|
||||
nil,
|
||||
)
|
||||
remoteProps := propertiesMap(r.remote.Get("mappings.properties").Raw)
|
||||
localProps := propertiesMap(r.local.Get("mappings.properties").Raw)
|
||||
reasons = append(reasons, classifyOpenExtensionMapping(localProps, remoteProps,
|
||||
r.local.Get("mappings.dynamic_templates").Raw, r.remote.Get("mappings.dynamic_templates").Raw)...)
|
||||
|
||||
classification := searchmapping.Classify(remoteProps, localProps, nil)
|
||||
classification.AddBreaking(reasons...)
|
||||
return classification, nil
|
||||
}
|
||||
|
||||
// classifyOpenExtensionMapping takes the ext tree out of the property comparison:
|
||||
// the index grows concrete properties under it that the code never declares.
|
||||
// Its dynamic flag and the templates are the rule, a changed rule is breaking.
|
||||
func classifyOpenExtensionMapping(local, remote map[string]any, localTemplates, remoteTemplates string) []string {
|
||||
var reasons []string
|
||||
localExt, _ := local[searchmapping.OpenExtensionsRoot].(map[string]any)
|
||||
remoteExt, _ := remote[searchmapping.OpenExtensionsRoot].(map[string]any)
|
||||
if localExt == nil || remoteExt == nil {
|
||||
return nil // an addition or a removal, Classify reports either
|
||||
}
|
||||
delete(local, searchmapping.OpenExtensionsRoot)
|
||||
delete(remote, searchmapping.OpenExtensionsRoot)
|
||||
// the flag comes back as a string
|
||||
if lv, rv := fmt.Sprint(localExt["dynamic"]), fmt.Sprint(remoteExt["dynamic"]); lv != rv {
|
||||
reasons = append(reasons, fmt.Sprintf("field %s: dynamic changed: index %s, code %s", searchmapping.OpenExtensionsRoot, rv, lv))
|
||||
}
|
||||
if !jsonEqual(localTemplates, remoteTemplates) {
|
||||
reasons = append(reasons, fmt.Sprintf("mappings.dynamic_templates changed: index %s, code %s", rawOrUnset(remoteTemplates), rawOrUnset(localTemplates)))
|
||||
}
|
||||
return reasons
|
||||
}
|
||||
|
||||
// ApplyAdditive puts the full code properties (only additions, per the
|
||||
// classifier). The PUT is atomic, so persisted is true only on success.
|
||||
func (r *osReconciler) ApplyAdditive() (bool, error) {
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package opensearch
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
)
|
||||
|
||||
// The ext object is dynamic and its siblings are typed by name through dynamic
|
||||
// templates, so a new property needs no mapping change on our side.
|
||||
|
||||
func openExtensionTemplates() []map[string]any {
|
||||
template := func(sibling string, m map[string]any) map[string]any {
|
||||
return map[string]any{
|
||||
"ext_" + strings.TrimPrefix(sibling, "@"): map[string]any{
|
||||
"path_match": mapping.OpenExtensionsRoot + ".*",
|
||||
"match": sibling,
|
||||
"mapping": m,
|
||||
},
|
||||
}
|
||||
}
|
||||
return []map[string]any{
|
||||
template(mapping.SiblingKeyword, map[string]any{"type": "keyword"}),
|
||||
template(mapping.SiblingLower, map[string]any{"type": "keyword", "doc_values": false}),
|
||||
template(mapping.SiblingNumber, map[string]any{"type": "double"}),
|
||||
template(mapping.SiblingBool, map[string]any{"type": "boolean"}),
|
||||
template(mapping.SiblingDate, map[string]any{"type": "date"}),
|
||||
template(mapping.SiblingGeo, map[string]any{"type": "geo_point"}),
|
||||
}
|
||||
}
|
||||
|
||||
func openExtensionProperties() map[string]any {
|
||||
return map[string]any{
|
||||
mapping.OpenExtensionsStoredField: map[string]any{"type": "object", "enabled": false},
|
||||
mapping.OpenExtensionsRoot: map[string]any{"type": "object", "dynamic": true},
|
||||
}
|
||||
}
|
||||
|
||||
func addOpenExtensionSource(body map[string]any, values map[string]string) {
|
||||
if len(values) == 0 {
|
||||
return
|
||||
}
|
||||
stored := make(map[string]any, len(values))
|
||||
for key, raw := range values {
|
||||
stored[key] = raw
|
||||
}
|
||||
body[mapping.OpenExtensionsStoredField] = stored
|
||||
|
||||
tree := map[string]any{}
|
||||
for _, leaf := range mapping.OpenExtensionLeaves(values) {
|
||||
var value any
|
||||
switch leaf.Sibling {
|
||||
case mapping.SiblingKeyword, mapping.SiblingLower:
|
||||
value = single(leaf.Strings)
|
||||
case mapping.SiblingNumber:
|
||||
value = single(leaf.Numbers)
|
||||
case mapping.SiblingBool:
|
||||
value = single(leaf.Bools)
|
||||
case mapping.SiblingDate:
|
||||
formatted := make([]string, len(leaf.Times))
|
||||
for i, t := range leaf.Times {
|
||||
formatted[i] = t.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
value = single(formatted)
|
||||
case mapping.SiblingGeo:
|
||||
value = map[string]any{"lat": leaf.Geo.Latitude, "lon": leaf.Geo.Longitude}
|
||||
}
|
||||
setPath(tree, strings.Split(strings.TrimPrefix(leaf.Field, mapping.OpenExtensionsRoot+"."), "."), value)
|
||||
}
|
||||
body[mapping.OpenExtensionsRoot] = tree
|
||||
}
|
||||
|
||||
func single[T any](values []T) any {
|
||||
if len(values) == 1 {
|
||||
return values[0]
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func setPath(tree map[string]any, path []string, value any) {
|
||||
for _, segment := range path[:len(path)-1] {
|
||||
next, ok := tree[segment].(map[string]any)
|
||||
if !ok {
|
||||
next = map[string]any{}
|
||||
tree[segment] = next
|
||||
}
|
||||
tree = next
|
||||
}
|
||||
tree[path[len(path)-1]] = value
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package opensearch
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAddOpenExtensionSource(t *testing.T) {
|
||||
const prefix = "http://opencloud.eu/ns/extensions/com.example.project/"
|
||||
body := map[string]any{"Name": "a.txt"}
|
||||
addOpenExtensionSource(body, map[string]string{
|
||||
prefix + "state": "s:Open",
|
||||
prefix + "priority": "n:3",
|
||||
prefix + "tags": `S:["A","b"]`,
|
||||
prefix + "due": "d:2026-10-01T00:00:00Z",
|
||||
prefix + "site": "g:52.5,13.4",
|
||||
})
|
||||
|
||||
got, err := json.Marshal(body)
|
||||
require.NoError(t, err)
|
||||
require.JSONEq(t, `{
|
||||
"Name": "a.txt",
|
||||
"OpenExtensions": {
|
||||
"`+prefix+`state": "s:Open",
|
||||
"`+prefix+`priority": "n:3",
|
||||
"`+prefix+`tags": "S:[\"A\",\"b\"]",
|
||||
"`+prefix+`due": "d:2026-10-01T00:00:00Z",
|
||||
"`+prefix+`site": "g:52.5,13.4"
|
||||
},
|
||||
"ext": {"com": {"example": {"project": {
|
||||
"state": {"@keyword": "Open", "@lower": "open"},
|
||||
"priority": {"@number": 3},
|
||||
"tags": {"@keyword": ["A", "b"], "@lower": ["a", "b"]},
|
||||
"due": {"@date": "2026-10-01T00:00:00Z"},
|
||||
"site": {"@geo": {"lat": 52.5, "lon": 13.4}}
|
||||
}}}}
|
||||
}`, string(got))
|
||||
}
|
||||
|
||||
func TestAddOpenExtensionSourceWithoutExtensions(t *testing.T) {
|
||||
body := map[string]any{"Name": "a.txt"}
|
||||
addOpenExtensionSource(body, nil)
|
||||
require.Equal(t, map[string]any{"Name": "a.txt"}, body)
|
||||
}
|
||||
|
||||
func TestClassifyOpenExtensionMapping(t *testing.T) {
|
||||
templates := func() string {
|
||||
b, err := json.Marshal(openExtensionTemplates())
|
||||
require.NoError(t, err)
|
||||
return string(b)
|
||||
}
|
||||
local := func() map[string]any {
|
||||
return map[string]any{"Name": map[string]any{"type": "keyword"}, "ext": map[string]any{"type": "object", "dynamic": true}}
|
||||
}
|
||||
// what GET _mapping returns once documents were indexed: the flag as a
|
||||
// string and the concrete fields OpenSearch added under ext
|
||||
remote := func() map[string]any {
|
||||
return map[string]any{
|
||||
"Name": map[string]any{"type": "keyword"},
|
||||
"ext": map[string]any{"dynamic": "true", "properties": map[string]any{
|
||||
"com": map[string]any{"properties": map[string]any{"example": map[string]any{"properties": map[string]any{
|
||||
"project": map[string]any{"properties": map[string]any{"priority": map[string]any{"properties": map[string]any{"@number": map[string]any{"type": "double"}}}}},
|
||||
}}}},
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("concrete fields under ext are not a difference", func(t *testing.T) {
|
||||
l, r := local(), remote()
|
||||
reasons := classifyOpenExtensionMapping(l, r, templates(), templates())
|
||||
require.Empty(t, reasons)
|
||||
require.NotContains(t, l, "ext")
|
||||
require.NotContains(t, r, "ext")
|
||||
require.Equal(t, l, r, "what is left is compared by Classify")
|
||||
})
|
||||
|
||||
t.Run("a changed template is breaking", func(t *testing.T) {
|
||||
reasons := classifyOpenExtensionMapping(local(), remote(), templates(), `[{"ext_number":{"path_match":"ext.*","match":"@number","mapping":{"type":"long"}}}]`)
|
||||
require.Len(t, reasons, 1)
|
||||
require.Contains(t, reasons[0], "dynamic_templates changed")
|
||||
})
|
||||
|
||||
t.Run("a changed dynamic flag is breaking", func(t *testing.T) {
|
||||
r := remote()
|
||||
r["ext"].(map[string]any)["dynamic"] = "strict"
|
||||
reasons := classifyOpenExtensionMapping(local(), r, templates(), templates())
|
||||
require.Len(t, reasons, 1)
|
||||
require.Contains(t, reasons[0], "dynamic changed")
|
||||
})
|
||||
|
||||
t.Run("an index without ext leaves the addition to Classify", func(t *testing.T) {
|
||||
l, r := local(), map[string]any{"Name": map[string]any{"type": "keyword"}}
|
||||
reasons := classifyOpenExtensionMapping(l, r, templates(), "")
|
||||
require.Empty(t, reasons)
|
||||
require.Contains(t, l, "ext")
|
||||
})
|
||||
}
|
||||
@@ -1,5 +1,62 @@
|
||||
{
|
||||
"mappings": {
|
||||
"dynamic_templates": [
|
||||
{
|
||||
"ext_keyword": {
|
||||
"mapping": {
|
||||
"type": "keyword"
|
||||
},
|
||||
"match": "@keyword",
|
||||
"path_match": "ext.*"
|
||||
}
|
||||
},
|
||||
{
|
||||
"ext_lower": {
|
||||
"mapping": {
|
||||
"doc_values": false,
|
||||
"type": "keyword"
|
||||
},
|
||||
"match": "@lower",
|
||||
"path_match": "ext.*"
|
||||
}
|
||||
},
|
||||
{
|
||||
"ext_number": {
|
||||
"mapping": {
|
||||
"type": "double"
|
||||
},
|
||||
"match": "@number",
|
||||
"path_match": "ext.*"
|
||||
}
|
||||
},
|
||||
{
|
||||
"ext_bool": {
|
||||
"mapping": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"match": "@bool",
|
||||
"path_match": "ext.*"
|
||||
}
|
||||
},
|
||||
{
|
||||
"ext_date": {
|
||||
"mapping": {
|
||||
"type": "date"
|
||||
},
|
||||
"match": "@date",
|
||||
"path_match": "ext.*"
|
||||
}
|
||||
},
|
||||
{
|
||||
"ext_geo": {
|
||||
"mapping": {
|
||||
"type": "geo_point"
|
||||
},
|
||||
"match": "@geo",
|
||||
"path_match": "ext.*"
|
||||
}
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"Content": {
|
||||
"analyzer": "words",
|
||||
@@ -9,6 +66,10 @@
|
||||
"Deleted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"OpenExtensions": {
|
||||
"enabled": false,
|
||||
"type": "object"
|
||||
},
|
||||
"Favorites": {
|
||||
"type": "keyword"
|
||||
},
|
||||
@@ -178,6 +239,10 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"ext": {
|
||||
"dynamic": true,
|
||||
"type": "object"
|
||||
},
|
||||
"image": {
|
||||
"properties": {
|
||||
"height": {
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
package event
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestTouchesOpenExtensions(t *testing.T) {
|
||||
const state = "http://opencloud.eu/ns/extensions/com.example.project/state"
|
||||
for keys, want := range map[*[]string]bool{
|
||||
{state}: true,
|
||||
{"tags", state}: true,
|
||||
{"tags"}: false,
|
||||
{"http://owncloud.org/ns/favorite"}: false,
|
||||
{}: false,
|
||||
} {
|
||||
if got := touchesOpenExtensions(*keys); got != want {
|
||||
t.Errorf("touchesOpenExtensions(%v) = %v, want %v", *keys, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events/raw"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/openextension"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
@@ -62,6 +63,7 @@ func New(ctx context.Context, stream raw.Stream, logger log.Logger, tp trace.Tra
|
||||
events.FileVersionRestored{},
|
||||
events.TagsAdded{},
|
||||
events.TagsRemoved{},
|
||||
events.ArbitraryMetadataUpdated{},
|
||||
events.SpaceRenamed{},
|
||||
events.SpaceDeleted{},
|
||||
events.LabelAdded{},
|
||||
@@ -207,6 +209,12 @@ func (s Service) processEvent(e raw.Event) error {
|
||||
case events.TagsRemoved:
|
||||
s.index.UpsertItem(ev.Ref)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.ArbitraryMetadataUpdated:
|
||||
// tags and favorites come with their own events
|
||||
if touchesOpenExtensions(ev.Keys) {
|
||||
s.index.UpsertItem(ev.Ref)
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
}
|
||||
case events.FileUploaded:
|
||||
debounce(getSpaceID(ev.Ref))
|
||||
case events.UploadReady:
|
||||
@@ -253,3 +261,12 @@ func monitorMetrics(ctx context.Context, stream raw.Stream, name string, m *metr
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func touchesOpenExtensions(keys []string) bool {
|
||||
for _, key := range keys {
|
||||
if _, _, ok := openextension.SplitKey(key); ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -63,6 +63,7 @@ var _ = DescribeTable("event",
|
||||
Entry("FileVersionRestored", []string{"IndexSpace"}, events.FileVersionRestored{}, false),
|
||||
Entry("TagsAdded", []string{"UpsertItem", "IndexSpace"}, events.TagsAdded{}, false),
|
||||
Entry("TagsRemoved", []string{"UpsertItem", "IndexSpace"}, events.TagsRemoved{}, false),
|
||||
Entry("ArbitraryMetadataUpdated on an extension", []string{"UpsertItem", "IndexSpace"}, events.ArbitraryMetadataUpdated{Keys: []string{"http://opencloud.eu/ns/extensions/com.example.project/state"}}, false),
|
||||
Entry("FileUploaded", []string{"IndexSpace"}, events.FileUploaded{}, false),
|
||||
Entry("UploadReady", []string{"IndexSpace"}, events.UploadReady{ExecutingUser: &userv1beta1.User{}}, true),
|
||||
)
|
||||
Reference in new issue
Block a user