From cafd791fdea6327f829c80400f734f45f7eb93cc Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Wed, 16 Sep 2026 07:17:43 +0000 Subject: [PATCH] feat(search): query open extension properties KQL addresses an open extension property as extensions... The literal picks the typed sibling: a number or date-time range asks the number or date sibling, an equality asks every sibling the literal fits (the lowercased string, and the number, boolean or date it also reads as), = asks the string as written. Both compilers build the same plan. The parity suite covers the queries and the lifecycle of extensions through re-typing, removal, move and trash. --- .../internal/convert/kql_transpile.go | 18 ++- .../internal/convert/openextensions.go | 62 +++++++++ .../internal/convert/openextensions_test.go | 87 +++++++++++++ services/search/pkg/parity/README.md | 51 ++++++++ services/search/pkg/parity/fixtures_test.go | 19 +++ .../pkg/parity/lifecycle_openext_test.go | 65 ++++++++++ services/search/pkg/parity/matrix_test.go | 7 + services/search/pkg/parity/parity_test.go | 2 + .../search/pkg/parity/query_openext_test.go | 50 ++++++++ services/search/pkg/query/bleve/compiler.go | 32 ++++- .../search/pkg/query/bleve/openextensions.go | 72 +++++++++++ .../pkg/query/bleve/openextensions_test.go | 121 ++++++++++++++++++ services/search/pkg/query/openextensions.go | 68 ++++++++++ services/search/pkg/query/resolver.go | 3 + 14 files changed, 651 insertions(+), 6 deletions(-) create mode 100644 services/search/pkg/opensearch/internal/convert/openextensions.go create mode 100644 services/search/pkg/opensearch/internal/convert/openextensions_test.go create mode 100644 services/search/pkg/parity/lifecycle_openext_test.go create mode 100644 services/search/pkg/parity/query_openext_test.go create mode 100644 services/search/pkg/query/bleve/openextensions.go create mode 100644 services/search/pkg/query/bleve/openextensions_test.go create mode 100644 services/search/pkg/query/openextensions.go diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index 9747c7e2b3..61d51d6ef3 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -107,8 +107,15 @@ func (t kqlOpensearchTranspiler) getOperatorValueAt(nodes []ast.Node, i int) str func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { switch node := node.(type) { case *ast.BooleanNode: + if query.IsOpenExtensionField(node.Key) { + return osu.NewTermQuery[bool](query.OpenExtensionField(node.Key, mapping.SiblingBool)).Value(node.Value), nil + } return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil case *ast.StringNode: + if query.IsOpenExtensionField(node.Key) { + return openExtensionStringQuery(node), nil + } + // hidden takes bool words only; anything else matches nothing if node.Key == "Hidden" { b, err := strconv.ParseBool(node.Value) @@ -176,6 +183,9 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { return nil, fmt.Errorf("unsupported string node value: %s", value) case *ast.DateTimeNode: + if query.IsOpenExtensionField(node.Key) { + return openExtensionDateQuery(node) + } return dateTimeNodeQuery(node) case *ast.NumberNode: return numberNodeQuery(node) @@ -218,11 +228,15 @@ func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) { return nil, fmt.Errorf("number node without operator: %w", ErrUnsupportedNodeType) } - if !slices.Contains([]string{"Size", "Type"}, node.Key) { + field := node.Key + switch { + case query.IsOpenExtensionField(node.Key): + field = query.OpenExtensionField(node.Key, mapping.SiblingNumber) + case !slices.Contains([]string{"Size", "Type"}, node.Key): return osu.NewMatchNoneQuery(), nil } - query := osu.NewRangeQuery[float64](node.Key) + query := osu.NewRangeQuery[float64](field) switch node.Operator.Value { case ">": diff --git a/services/search/pkg/opensearch/internal/convert/openextensions.go b/services/search/pkg/opensearch/internal/convert/openextensions.go new file mode 100644 index 0000000000..0004e21d30 --- /dev/null +++ b/services/search/pkg/opensearch/internal/convert/openextensions.go @@ -0,0 +1,62 @@ +package convert + +import ( + "fmt" + "strings" + "time" + + "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" + "github.com/opencloud-eu/opencloud/services/search/pkg/query" +) + +// mirrors the bleve compiler, see query.OpenExtensionEquality + +func openExtensionStringQuery(n *ast.StringNode) osu.Builder { + if strings.ContainsAny(n.Value, "*?") { + sibling, value := mapping.SiblingLower, strings.ToLower(n.Value) + if n.Exact { + sibling, value = mapping.SiblingKeyword, n.Value + } + return osu.NewWildcardQuery(query.OpenExtensionField(n.Key, sibling)).Value(value) + } + + var alternatives []osu.Builder + for _, term := range query.OpenExtensionEquality(n.Key, n.Value, n.Exact) { + switch term.Sibling { + case mapping.SiblingNumber: + alternatives = append(alternatives, osu.NewRangeQuery[float64](term.Field).Gte(term.Number).Lte(term.Number)) + case mapping.SiblingBool: + alternatives = append(alternatives, osu.NewTermQuery[bool](term.Field).Value(term.Bool)) + case mapping.SiblingDate: + alternatives = append(alternatives, osu.NewRangeQuery[time.Time](term.Field).Gte(term.Time).Lte(term.Time)) + default: + alternatives = append(alternatives, osu.NewTermQuery[string](term.Field).Value(term.String)) + } + } + if len(alternatives) == 1 { + return alternatives[0] + } + return osu.NewBoolQuery().Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).Should(alternatives...) +} + +// openExtensionDateQuery: an equality is the one-instant range. +func openExtensionDateQuery(n *ast.DateTimeNode) (osu.Builder, error) { + if n.Operator == nil { + return nil, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType) + } + q := osu.NewRangeQuery[time.Time](query.OpenExtensionField(n.Key, mapping.SiblingDate)) + switch n.Operator.Value { + case ">": + return q.Gt(n.Value), nil + case ">=": + return q.Gte(n.Value), nil + case "<": + return q.Lt(n.Value), nil + case "<=": + return q.Lte(n.Value), nil + default: + return q.Gte(n.Value).Lte(n.Value), nil + } +} diff --git a/services/search/pkg/opensearch/internal/convert/openextensions_test.go b/services/search/pkg/opensearch/internal/convert/openextensions_test.go new file mode 100644 index 0000000000..b2d47897e9 --- /dev/null +++ b/services/search/pkg/opensearch/internal/convert/openextensions_test.go @@ -0,0 +1,87 @@ +package convert_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" + "github.com/opencloud-eu/opencloud/services/search/pkg/query" +) + +func TestTranspileOpenExtensions(t *testing.T) { + oct := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + either := func(alternatives ...osu.Builder) osu.Builder { + return osu.NewBoolQuery().Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).Should(alternatives...) + } + + tests := []opensearchtest.TableTest[ast.Node, osu.Builder]{ + { + Name: "a plain string asks the lowercased sibling", + Got: &ast.StringNode{Key: "extensions.com.example.project.state", Value: "Open"}, + Want: osu.NewTermQuery[string]("ext.com.example.project.state.@lower").Value("open"), + }, + { + Name: "= asks the keyword sibling as written", + Got: &ast.StringNode{Key: "extensions.com.example.project.state", Value: "Open", Exact: true}, + Want: osu.NewTermQuery[string]("ext.com.example.project.state.@keyword").Value("Open"), + }, + { + Name: "a numeric literal also asks the number sibling", + Got: &ast.StringNode{Key: "extensions.com.example.project.priority", Value: "3"}, + Want: either( + osu.NewTermQuery[string]("ext.com.example.project.priority.@lower").Value("3"), + osu.NewRangeQuery[float64]("ext.com.example.project.priority.@number").Gte(3).Lte(3), + ), + }, + { + Name: "a boolean literal also asks the bool sibling", + Got: &ast.StringNode{Key: "extensions.com.example.project.done", Value: "false"}, + Want: either( + osu.NewTermQuery[string]("ext.com.example.project.done.@lower").Value("false"), + osu.NewTermQuery[bool]("ext.com.example.project.done.@bool").Value(false), + ), + }, + { + Name: "a date-time literal also asks the date sibling", + Got: &ast.StringNode{Key: "extensions.com.example.project.due", Value: "2026-10-01T00:00:00Z"}, + Want: either( + osu.NewTermQuery[string]("ext.com.example.project.due.@lower").Value("2026-10-01t00:00:00z"), + osu.NewRangeQuery[time.Time]("ext.com.example.project.due.@date").Gte(oct).Lte(oct), + ), + }, + { + Name: "a wildcard runs on the lowercased sibling", + Got: &ast.StringNode{Key: "extensions.com.example.project.state", Value: "Op*"}, + Want: osu.NewWildcardQuery("ext.com.example.project.state.@lower").Value("op*"), + }, + { + Name: "a number range asks the number sibling", + Got: &ast.NumberNode{Key: "extensions.com.example.project.priority", Operator: &ast.OperatorNode{Value: ">"}, Value: 3}, + Want: osu.NewRangeQuery[float64]("ext.com.example.project.priority.@number").Gt(3), + }, + { + Name: "a date range asks the date sibling", + Got: &ast.DateTimeNode{Key: "extensions.com.example.project.due", Operator: &ast.OperatorNode{Value: "<="}, Value: oct}, + Want: osu.NewRangeQuery[time.Time]("ext.com.example.project.due.@date").Lte(oct), + }, + { + Name: "a boolean node asks the bool sibling", + Got: &ast.BooleanNode{Key: "extensions.com.example.project.done", Value: true}, + Want: osu.NewTermQuery[bool]("ext.com.example.project.done.@bool").Value(true), + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + normalized := query.Normalize(&ast.Ast{Nodes: []ast.Node{test.Got}}, query.ResolveField) + dsl, err := convert.TranspileKQLToOpenSearch(normalized.Nodes) + assert.NoError(t, err) + assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, dsl)) + }) + } +} diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index 6558e96847..5125586534 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -450,6 +450,39 @@ Fixtures: | INVALID-01 | `AND mediatype:document` | bad request | bad request | bad request | ✅ | | INVALID-02 | `mediatype:document AND` | alpha.txt | alpha.txt | alpha.txt | ✅ | +### openext + +Fixtures: + +- `plan.txt`, com.example.project/done = `b:false`, com.example.project/due = `d:2026-10-01T00:00:00Z`, com.example.project/priority = `n:3`, com.example.project/site = `g:52.5,13.4`, com.example.project/state = `s:Open`, com.example.project/tags = `S:["Urgent","customer"]` +- `draft.txt`, com.example.project/done = `b:true`, com.example.project/due = `d:2026-12-24T00:00:00Z`, com.example.project/priority = `n:1.5`, com.example.project/state = `s:open` +- `legacy.txt`, com.example.project/due = `s:2026-10-01T00:00:00Z`, com.example.project/priority = `s:3`, com.example.project/state = `s:closed` +- `other.txt`, com.example.other/state = `s:open` +- `plain.txt` + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| OPENEXT-01 | `extensions.com.example.project.state:open` | draft.txt, plan.txt | draft.txt, plan.txt | draft.txt, plan.txt | ✅ | +| OPENEXT-02 | `extensions.com.example.project.state=Open` | plan.txt | plan.txt | plan.txt | ✅ | +| OPENEXT-03 | `extensions.com.example.project.state=open` | draft.txt | draft.txt | draft.txt | ✅ | +| OPENEXT-04 | `extensions.com.example.project.state:op*` | draft.txt, plan.txt | draft.txt, plan.txt | draft.txt, plan.txt | ✅ | +| OPENEXT-05 | `Extensions.com.example.project.state:closed` | legacy.txt | legacy.txt | legacy.txt | ✅ | +| OPENEXT-06 | `extensions.com.example.project.priority>2` | plan.txt | plan.txt | plan.txt | ✅ | +| OPENEXT-07 | `extensions.com.example.project.priority<2` | draft.txt | draft.txt | draft.txt | ✅ | +| OPENEXT-08 | `extensions.com.example.project.priority:3` | legacy.txt, plan.txt | legacy.txt, plan.txt | legacy.txt, plan.txt | ✅ | +| OPENEXT-09 | `extensions.com.example.project.priority:"3"` | legacy.txt, plan.txt | legacy.txt, plan.txt | legacy.txt, plan.txt | ✅ | +| OPENEXT-10 | `extensions.com.example.project.done:true` | draft.txt | draft.txt | draft.txt | ✅ | +| OPENEXT-11 | `extensions.com.example.project.done:false` | plan.txt | plan.txt | plan.txt | ✅ | +| OPENEXT-12 | `extensions.com.example.project.due>2026-11-01T00:00:00Z` | draft.txt | draft.txt | draft.txt | ✅ | +| OPENEXT-13 | `extensions.com.example.project.due<2026-11-01T00:00:00Z` | plan.txt | plan.txt | plan.txt | ✅ | +| OPENEXT-14 | `extensions.com.example.project.due:2026-10-01T00:00:00Z` | plan.txt | plan.txt | plan.txt | ✅ | +| OPENEXT-15 | `extensions.com.example.project.tags:urgent` | plan.txt | plan.txt | plan.txt | ✅ | +| OPENEXT-16 | `extensions.com.example.project.t...ample.project.state:open` | plan.txt | plan.txt | plan.txt | ✅ | +| OPENEXT-17 | `extensions.com.example.project.s...example.other.state:open` | draft.txt, other.txt, plan.txt | draft.txt, other.txt, plan.txt | draft.txt, other.txt, plan.txt | ✅ | +| OPENEXT-18 | `NOT extensions.com.example.project.state:open` | legacy.txt, other.txt, plain.txt | legacy.txt, other.txt, plain.txt | legacy.txt, other.txt, plain.txt | ✅ | +| OPENEXT-19 | `extensions.com.example.project.missing:x` | no match | no match | no match | ✅ | +| OPENEXT-20 | `extensions.com.example.unknown.state:open` | no match | no match | no match | ✅ | + ## Operations ### delete @@ -662,6 +695,24 @@ Fixtures: | BATCH-05 | keeps what another batch holds out of its push, then `name:"*added*"` | added.pdf | added.pdf | added.pdf | ✅ | | BATCH-05 | keeps what another batch holds out of its push, then `name:"*other*"` | no match | no match | no match | ✅ | +### openextops + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf, com.example.project/priority = `n:3`, com.example.project/state = `s:open` + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| OPENEXTOPS-01 | re-types a property on upsert, then `extensions.com.example.project.priority:high` | child.pdf | child.pdf | child.pdf | ✅ | +| OPENEXTOPS-01 | re-types a property on upsert, then `extensions.com.example.project.priority>2` | no match | no match | no match | ✅ | +| OPENEXTOPS-01 | re-types a property on upsert, then `extensions.com.example.project.state:open` | child.pdf | child.pdf | child.pdf | ✅ | +| OPENEXTOPS-02 | forgets a removed extension on upsert, then `extensions.com.example.project.state:open` | no match | no match | no match | ✅ | +| OPENEXTOPS-02 | forgets a removed extension on upsert, then `name:child.pdf` | child.pdf | child.pdf | child.pdf | ✅ | +| OPENEXTOPS-03 | keeps the extensions through a move, then `extensions.com.example.project.priority>2` | child.pdf | child.pdf | child.pdf | ✅ | +| OPENEXTOPS-03 | keeps the extensions through a move, then `path:"./renamed/child.pdf"` | child.pdf | child.pdf | child.pdf | ✅ | +| OPENEXTOPS-04 | keeps the extensions through the trash and back, then `extensions.com.example.project.state:open` | child.pdf | child.pdf | child.pdf | ✅ | + ## Response ### entity diff --git a/services/search/pkg/parity/fixtures_test.go b/services/search/pkg/parity/fixtures_test.go index 330fddc3dd..2457c1887e 100644 --- a/services/search/pkg/parity/fixtures_test.go +++ b/services/search/pkg/parity/fixtures_test.go @@ -8,6 +8,7 @@ import ( sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" libregraph "github.com/opencloud-eu/libre-graph-api-go" + "github.com/opencloud-eu/reva/v2/pkg/openextension" "github.com/opencloud-eu/opencloud/services/search/pkg/content" "github.com/opencloud-eu/opencloud/services/search/pkg/search" @@ -54,6 +55,24 @@ func withLocation(location *libregraph.GeoCoordinates) fixtureOption { return func(r *search.Resource) { r.Location = location } } +// withOpenExtension stores the properties of a libregraph openTypeExtension +// body as the codec would, one typed metadata value per property. +func withOpenExtension(name, body string) fixtureOption { + patch, err := openextension.Parse([]byte(body)) + if err != nil { + panic(err) + } + set, _ := patch.Metadata(name) + return func(r *search.Resource) { + if r.OpenExtensions == nil { + r.OpenExtensions = map[string]string{} + } + for key, value := range set { + r.OpenExtensions[key] = value + } + } +} + func fixtureDoc(name string, opts ...fixtureOption) search.Resource { mtime := fixtureNow r := search.Resource{ diff --git a/services/search/pkg/parity/lifecycle_openext_test.go b/services/search/pkg/parity/lifecycle_openext_test.go new file mode 100644 index 0000000000..2f3a9240b8 --- /dev/null +++ b/services/search/pkg/parity/lifecycle_openext_test.go @@ -0,0 +1,65 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +// The extension siblings follow the stored value: a re-typed property leaves +// its old sibling, and the operations that re-index a resource from the index +// (Move) keep the extensions. +func openextLifecycle() lifecycleGroup { + const project = "com.example.project" + parent, child := fixtureTree() + withOpenExtension(project, `{"state":"open","priority":3}`)(&child) + + retyped := child + retyped.OpenExtensions = nil + withOpenExtension(project, `{"state":"open","priority":"high"}`)(&retyped) + + dropped := child + dropped.OpenExtensions = nil + + return lifecycleGroup{ + name: "openextops", + fixtures: []search.Resource{parent, child}, + cases: []lifecycleCase{ + { + id: 1, title: "re-types a property on upsert", + do: func(e search.Engine) error { return e.Upsert(retyped.ID, retyped) }, + expect: []expectation{ + {`extensions.com.example.project.priority:high`, []string{"child.pdf"}}, + {`extensions.com.example.project.priority>2`, nil}, + {`extensions.com.example.project.state:open`, []string{"child.pdf"}}, + }, + }, + { + id: 2, title: "forgets a removed extension on upsert", + do: func(e search.Engine) error { return e.Upsert(dropped.ID, dropped) }, + expect: []expectation{ + {`extensions.com.example.project.state:open`, nil}, + {`name:child.pdf`, []string{"child.pdf"}}, + }, + }, + { + id: 3, title: "keeps the extensions through a move", + do: func(e search.Engine) error { return e.Move(parent.ID, parent.ParentID, "./renamed") }, + expect: []expectation{ + {`extensions.com.example.project.priority>2`, []string{"child.pdf"}}, + {`path:"./renamed/child.pdf"`, []string{"child.pdf"}}, + }, + }, + { + id: 4, title: "keeps the extensions through the trash and back", + do: func(e search.Engine) error { + if err := e.Delete(child.ID); err != nil { + return err + } + return e.Restore(child.ID) + }, + expect: []expectation{ + {`extensions.com.example.project.state:open`, []string{"child.pdf"}}, + }, + }, + }, + } +} diff --git a/services/search/pkg/parity/matrix_test.go b/services/search/pkg/parity/matrix_test.go index a1d51394d7..333129c294 100644 --- a/services/search/pkg/parity/matrix_test.go +++ b/services/search/pkg/parity/matrix_test.go @@ -3,7 +3,10 @@ package parity import ( "encoding/json" "fmt" + "github.com/opencloud-eu/reva/v2/pkg/openextension" + "maps" "os" + "slices" "sort" "strings" "time" @@ -342,6 +345,10 @@ func fixtureFields(f search.Resource, withID bool) string { add("Favorites = %s", strings.Join(f.Favorites, ", ")) } + for _, key := range slices.Sorted(maps.Keys(f.OpenExtensions)) { + add("%s = `%s`", strings.TrimPrefix(key, openextension.NamespacePrefix), f.OpenExtensions[key]) + } + if f.Size != 1000 { add("Size = %d", f.Size) } diff --git a/services/search/pkg/parity/parity_test.go b/services/search/pkg/parity/parity_test.go index 41695d18b0..c0a8e5d733 100644 --- a/services/search/pkg/parity/parity_test.go +++ b/services/search/pkg/parity/parity_test.go @@ -111,6 +111,7 @@ func queryGroups() []queryGroup { rangeGroup(), scopeGroup(), invalidGroup(), + openextGroup(), } } @@ -233,6 +234,7 @@ func lifecycleGroups() []lifecycleGroup { upsertLifecycle(), idempotencyLifecycle(), batchLifecycle(), + openextLifecycle(), } } diff --git a/services/search/pkg/parity/query_openext_test.go b/services/search/pkg/parity/query_openext_test.go new file mode 100644 index 0000000000..ab7e26c753 --- /dev/null +++ b/services/search/pkg/parity/query_openext_test.go @@ -0,0 +1,50 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +// Open extensions are addressed as extensions... Every +// property is indexed in the sibling of its value's kind, and the literal of a +// restriction picks the sibling: a range needs a typed literal, an equality +// asks every sibling the literal fits. +func openextGroup() queryGroup { + const project = "com.example.project" + + return queryGroup{ + name: "openext", + fixtures: []search.Resource{ + fixtureDoc("plan.txt", withOpenExtension(project, + `{"state":"Open","priority":3,"done":false,"due":"2026-10-01T00:00:00Z","due@odata.type":"#DateTimeOffset","tags":["Urgent","customer"],"site":{"latitude":52.5,"longitude":13.4},"site@odata.type":"#microsoft.graph.geoCoordinates"}`)), + fixtureDoc("draft.txt", withOpenExtension(project, + `{"state":"open","priority":1.5,"done":true,"due":"2026-12-24T00:00:00Z","due@odata.type":"#DateTimeOffset"}`)), + fixtureDoc("legacy.txt", withOpenExtension(project, `{"state":"closed","priority":"3","due":"2026-10-01T00:00:00Z"}`)), + fixtureDoc("other.txt", withOpenExtension("com.example.other", `{"state":"open"}`)), + fixtureDoc("plain.txt"), + }, + cases: []queryCase{ + {id: 1, query: `extensions.com.example.project.state:open`, want: []string{"plan.txt", "draft.txt"}}, + {id: 2, query: `extensions.com.example.project.state=Open`, want: []string{"plan.txt"}}, + {id: 3, query: `extensions.com.example.project.state=open`, want: []string{"draft.txt"}}, + {id: 4, query: `extensions.com.example.project.state:op*`, want: []string{"plan.txt", "draft.txt"}}, + {id: 5, query: `Extensions.com.example.project.state:closed`, want: []string{"legacy.txt"}}, + {id: 6, query: `extensions.com.example.project.priority>2`, want: []string{"plan.txt"}}, + {id: 7, query: `extensions.com.example.project.priority<2`, want: []string{"draft.txt"}}, + {id: 8, query: `extensions.com.example.project.priority:3`, want: []string{"plan.txt", "legacy.txt"}}, + {id: 9, query: `extensions.com.example.project.priority:"3"`, want: []string{"plan.txt", "legacy.txt"}}, + {id: 10, query: `extensions.com.example.project.done:true`, want: []string{"draft.txt"}}, + {id: 11, query: `extensions.com.example.project.done:false`, want: []string{"plan.txt"}}, + {id: 12, query: `extensions.com.example.project.due>2026-11-01T00:00:00Z`, want: []string{"draft.txt"}}, + {id: 13, query: `extensions.com.example.project.due<2026-11-01T00:00:00Z`, want: []string{"plan.txt"}}, + // a date-time literal is typed by the parser and asks the date sibling + // only; legacy.txt holds the same text as a plain string + {id: 14, query: `extensions.com.example.project.due:2026-10-01T00:00:00Z`, want: []string{"plan.txt"}}, + {id: 15, query: `extensions.com.example.project.tags:urgent`, want: []string{"plan.txt"}}, + {id: 16, query: `extensions.com.example.project.tags:customer AND extensions.com.example.project.state:open`, want: []string{"plan.txt"}}, + {id: 17, query: `extensions.com.example.project.state:open OR extensions.com.example.other.state:open`, want: []string{"plan.txt", "draft.txt", "other.txt"}}, + {id: 18, query: `NOT extensions.com.example.project.state:open`, want: []string{"legacy.txt", "other.txt", "plain.txt"}}, + {id: 19, query: `extensions.com.example.project.missing:x`}, + {id: 20, query: `extensions.com.example.unknown.state:open`}, + }, + } +} diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index a1dba9a65a..e0172318e9 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -72,6 +72,16 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { for i := offset; i < len(nodes); i++ { switch n := nodes[i].(type) { case *ast.StringNode: + if searchQuery.IsOpenExtensionField(n.Key) { + q := openExtensionStringQuery(n) + if prev == nil { + prev = q + } else { + next = q + } + break + } + // hidden takes bool words only; anything else matches nothing if n.Key == "Hidden" { var q bleveQuery.Query @@ -153,6 +163,19 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { next = q } case *ast.DateTimeNode: + if n.Operator == nil { + continue + } + if searchQuery.IsOpenExtensionField(n.Key) { + q := openExtensionDateQuery(n) + if prev == nil { + prev = q + } else { + next = q + } + break + } + q := &bleveQuery.DateRangeQuery{ Start: bleveQuery.BleveQueryTime{}, End: bleveQuery.BleveQueryTime{}, @@ -161,10 +184,6 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { FieldVal: n.Key, } - if n.Operator == nil { - continue - } - switch n.Operator.Value { case ">": q.Start.Time = n.Value @@ -191,6 +210,8 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { var q bleveQuery.Query if field := n.Key; slices.Contains([]string{"Size", "Type"}, field) { q = numberRange(field, n.Operator, n.Value) + } else if searchQuery.IsOpenExtensionField(n.Key) { + q = numberRange(searchQuery.OpenExtensionField(n.Key, mapping.SiblingNumber), n.Operator, n.Value) } else { // same answer as the OpenSearch backend: unknown numeric keys // match nothing instead of querying an arbitrary field @@ -208,6 +229,9 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { case *ast.BooleanNode: q := bleveQuery.NewBoolFieldQuery(n.Value) q.SetField(n.Key) + if searchQuery.IsOpenExtensionField(n.Key) { + q.SetField(searchQuery.OpenExtensionField(n.Key, mapping.SiblingBool)) + } if prev == nil { prev = q } else { diff --git a/services/search/pkg/query/bleve/openextensions.go b/services/search/pkg/query/bleve/openextensions.go new file mode 100644 index 0000000000..461b75e2d4 --- /dev/null +++ b/services/search/pkg/query/bleve/openextensions.go @@ -0,0 +1,72 @@ +package bleve + +import ( + "strings" + "time" + + "github.com/blevesearch/bleve/v2" + bleveQuery "github.com/blevesearch/bleve/v2/search/query" + + "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" + searchQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query" +) + +var time0 time.Time // open end of a date range + +func openExtensionStringQuery(n *ast.StringNode) bleveQuery.Query { + if strings.ContainsAny(n.Value, "*?") { + sibling, value := mapping.SiblingLower, strings.ToLower(n.Value) + if n.Exact { + sibling, value = mapping.SiblingKeyword, n.Value + } + wq := bleveQuery.NewWildcardQuery(value) + wq.SetField(searchQuery.OpenExtensionField(n.Key, sibling)) + return wq + } + + inclusive := true + plan := searchQuery.OpenExtensionEquality(n.Key, n.Value, n.Exact) + alternatives := make([]bleveQuery.Query, 0, len(plan)) + for _, term := range plan { + var q bleveQuery.FieldableQuery + switch term.Sibling { + case mapping.SiblingNumber: + f := term.Number + q = bleveQuery.NewNumericRangeInclusiveQuery(&f, &f, &inclusive, &inclusive) + case mapping.SiblingBool: + q = bleveQuery.NewBoolFieldQuery(term.Bool) + case mapping.SiblingDate: + q = bleveQuery.NewDateRangeInclusiveQuery(term.Time, term.Time, &inclusive, &inclusive) + default: + q = bleveQuery.NewTermQuery(term.String) + } + q.SetField(term.Field) + alternatives = append(alternatives, q) + } + if len(alternatives) == 1 { + return alternatives[0] + } + return bleve.NewDisjunctionQuery(alternatives...) +} + +// openExtensionDateQuery: an equality is the one-instant range. +func openExtensionDateQuery(n *ast.DateTimeNode) bleveQuery.Query { + inclusive, exclusive := true, false + field := searchQuery.OpenExtensionField(n.Key, mapping.SiblingDate) + var q *bleveQuery.DateRangeQuery + switch n.Operator.Value { + case ">": + q = bleveQuery.NewDateRangeInclusiveQuery(n.Value, time0, &exclusive, nil) + case ">=": + q = bleveQuery.NewDateRangeInclusiveQuery(n.Value, time0, &inclusive, nil) + case "<": + q = bleveQuery.NewDateRangeInclusiveQuery(time0, n.Value, nil, &exclusive) + case "<=": + q = bleveQuery.NewDateRangeInclusiveQuery(time0, n.Value, nil, &inclusive) + default: + q = bleveQuery.NewDateRangeInclusiveQuery(n.Value, n.Value, &inclusive, &inclusive) + } + q.SetField(field) + return q +} diff --git a/services/search/pkg/query/bleve/openextensions_test.go b/services/search/pkg/query/bleve/openextensions_test.go new file mode 100644 index 0000000000..8167c2b373 --- /dev/null +++ b/services/search/pkg/query/bleve/openextensions_test.go @@ -0,0 +1,121 @@ +package bleve + +import ( + "testing" + "time" + + "github.com/blevesearch/bleve/v2/search/query" + "github.com/stretchr/testify/assert" + + "github.com/opencloud-eu/opencloud/pkg/ast" + searchquery "github.com/opencloud-eu/opencloud/services/search/pkg/query" +) + +func Test_compileOpenExtensions(t *testing.T) { + inclusive, exclusive := true, false + term := func(field, value string) query.Query { + q := query.NewTermQuery(value) + q.SetField(field) + return q + } + number := func(field string, lo, hi *float64, loIn, hiIn *bool) query.Query { + q := query.NewNumericRangeInclusiveQuery(lo, hi, loIn, hiIn) + q.SetField(field) + return q + } + date := func(field string, start, end time.Time, startIn, endIn *bool) query.Query { + q := query.NewDateRangeInclusiveQuery(start, end, startIn, endIn) + q.SetField(field) + return q + } + boolean := func(field string, v bool) query.Query { + q := query.NewBoolFieldQuery(v) + q.SetField(field) + return q + } + three := 3.0 + oct := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) + + tests := []struct { + name string + node ast.Node + want query.Query + }{ + { + name: "a plain string asks the lowercased sibling", + node: &ast.StringNode{Key: "extensions.com.example.project.state", Value: "Open"}, + want: term("ext.com.example.project.state.@lower", "open"), + }, + { + name: "= asks the keyword sibling as written", + node: &ast.StringNode{Key: "extensions.com.example.project.state", Value: "Open", Exact: true}, + want: term("ext.com.example.project.state.@keyword", "Open"), + }, + { + name: "the prefix is case-insensitive", + node: &ast.StringNode{Key: "Extensions.com.example.project.State", Value: "x"}, + want: term("ext.com.example.project.State.@lower", "x"), + }, + { + name: "a numeric literal also asks the number sibling", + node: &ast.StringNode{Key: "extensions.com.example.project.priority", Value: "3"}, + want: query.NewDisjunctionQuery([]query.Query{ + term("ext.com.example.project.priority.@lower", "3"), + number("ext.com.example.project.priority.@number", &three, &three, &inclusive, &inclusive), + }), + }, + { + name: "a boolean literal also asks the bool sibling", + node: &ast.StringNode{Key: "extensions.com.example.project.done", Value: "true"}, + want: query.NewDisjunctionQuery([]query.Query{ + term("ext.com.example.project.done.@lower", "true"), + boolean("ext.com.example.project.done.@bool", true), + }), + }, + { + name: "a date-time literal also asks the date sibling", + node: &ast.StringNode{Key: "extensions.com.example.project.due", Value: "2026-10-01T00:00:00Z"}, + want: query.NewDisjunctionQuery([]query.Query{ + term("ext.com.example.project.due.@lower", "2026-10-01t00:00:00z"), + date("ext.com.example.project.due.@date", oct, oct, &inclusive, &inclusive), + }), + }, + { + name: "a wildcard runs on the lowercased sibling", + node: &ast.StringNode{Key: "extensions.com.example.project.state", Value: "Op*"}, + want: func() query.Query { + q := query.NewWildcardQuery("op*") + q.SetField("ext.com.example.project.state.@lower") + return q + }(), + }, + { + name: "a number range asks the number sibling", + node: &ast.NumberNode{Key: "extensions.com.example.project.priority", Operator: &ast.OperatorNode{Value: ">"}, Value: 3}, + want: number("ext.com.example.project.priority.@number", &three, nil, &exclusive, nil), + }, + { + name: "a date range asks the date sibling", + node: &ast.DateTimeNode{Key: "extensions.com.example.project.due", Operator: &ast.OperatorNode{Value: "<="}, Value: oct}, + want: date("ext.com.example.project.due.@date", time.Time{}, oct, nil, &inclusive), + }, + { + name: "a boolean node asks the bool sibling", + node: &ast.BooleanNode{Key: "extensions.com.example.project.done", Value: false}, + want: boolean("ext.com.example.project.done.@bool", false), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := compile(searchquery.Normalize(&ast.Ast{Nodes: []ast.Node{tt.node}}, searchquery.ResolveField)) + assert.NoError(t, err) + // compile wraps a single leaf in a conjunction and hands a disjunction through + want := tt.want + if _, ok := want.(*query.DisjunctionQuery); !ok { + want = query.NewConjunctionQuery([]query.Query{want}) + } + assert.Equal(t, want, got) + }) + } +} diff --git a/services/search/pkg/query/openextensions.go b/services/search/pkg/query/openextensions.go new file mode 100644 index 0000000000..a8fd6dea67 --- /dev/null +++ b/services/search/pkg/query/openextensions.go @@ -0,0 +1,68 @@ +package query + +import ( + "strconv" + "strings" + "time" + + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" +) + +// Nothing in a request says which kind an extension property has, so the +// literal decides: a range targets the sibling of its typed literal, an +// equality asks every sibling the literal fits. Both compilers follow this plan. + +// OpenExtensionTerm is one sibling an equality is asked on. +type OpenExtensionTerm struct { + Field string + Sibling string + + String string + Number float64 + Bool bool + Time time.Time +} + +// IsOpenExtensionField reports whether a KQL key addresses an extension property. +func IsOpenExtensionField(key string) bool { + return mapping.IsOpenExtensionQueryField(key) +} + +// OpenExtensionField is the indexed field name of the sibling for a KQL key. +func OpenExtensionField(key, sibling string) string { + field, _ := mapping.OpenExtensionFieldFromQuery(key, sibling) + return field +} + +// OpenExtensionEquality asks the string sibling, case-insensitively unless exact, +// and every other sibling the value reads as. +func OpenExtensionEquality(key, value string, exact bool) []OpenExtensionTerm { + term := func(sibling string) OpenExtensionTerm { + return OpenExtensionTerm{Field: OpenExtensionField(key, sibling), Sibling: sibling} + } + + str := term(mapping.SiblingLower) + str.String = strings.ToLower(value) + if exact { + str = term(mapping.SiblingKeyword) + str.String = value + } + plan := []OpenExtensionTerm{str} + + if f, err := strconv.ParseFloat(value, 64); err == nil { + num := term(mapping.SiblingNumber) + num.Number = f + plan = append(plan, num) + } + if b, err := strconv.ParseBool(value); err == nil && (value == "true" || value == "false") { + bl := term(mapping.SiblingBool) + bl.Bool = b + plan = append(plan, bl) + } + if t, err := time.Parse(time.RFC3339Nano, value); err == nil { + dt := term(mapping.SiblingDate) + dt.Time = t + plan = append(plan, dt) + } + return plan +} diff --git a/services/search/pkg/query/resolver.go b/services/search/pkg/query/resolver.go index 5628dd886f..5818af5e2b 100644 --- a/services/search/pkg/query/resolver.go +++ b/services/search/pkg/query/resolver.go @@ -61,6 +61,9 @@ func ResolveField(name string) string { if v, ok := fieldIndex()[strings.ToLower(name)]; ok { return v } + if mapping.IsOpenExtensionQueryField(name) { + return mapping.OpenExtensionsQueryPrefix + name[len(mapping.OpenExtensionsQueryPrefix):] + } return name }