mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-09-25 05:26:00 -04:00
feat(search): query open extension properties
KQL addresses an open extension property as extensions.<name>.<property>. The literal picks the typed sibling: a number or date-time range asks the number or date sibling, an equality asks every sibling the literal fits (the lowercased string, and the number, boolean or date it also reads as), = asks the string as written. Both compilers build the same plan. The parity suite covers the queries and the lifecycle of extensions through re-typing, removal, move and trash.
This commit is contained in:
14 files changed
+651
-6
No files matched your search
@@ -107,8 +107,15 @@ func (t kqlOpensearchTranspiler) getOperatorValueAt(nodes []ast.Node, i int) str
|
||||
func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
|
||||
switch node := node.(type) {
|
||||
case *ast.BooleanNode:
|
||||
if query.IsOpenExtensionField(node.Key) {
|
||||
return osu.NewTermQuery[bool](query.OpenExtensionField(node.Key, mapping.SiblingBool)).Value(node.Value), nil
|
||||
}
|
||||
return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil
|
||||
case *ast.StringNode:
|
||||
if query.IsOpenExtensionField(node.Key) {
|
||||
return openExtensionStringQuery(node), nil
|
||||
}
|
||||
|
||||
// hidden takes bool words only; anything else matches nothing
|
||||
if node.Key == "Hidden" {
|
||||
b, err := strconv.ParseBool(node.Value)
|
||||
@@ -176,6 +183,9 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
|
||||
|
||||
return nil, fmt.Errorf("unsupported string node value: %s", value)
|
||||
case *ast.DateTimeNode:
|
||||
if query.IsOpenExtensionField(node.Key) {
|
||||
return openExtensionDateQuery(node)
|
||||
}
|
||||
return dateTimeNodeQuery(node)
|
||||
case *ast.NumberNode:
|
||||
return numberNodeQuery(node)
|
||||
@@ -218,11 +228,15 @@ func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) {
|
||||
return nil, fmt.Errorf("number node without operator: %w", ErrUnsupportedNodeType)
|
||||
}
|
||||
|
||||
if !slices.Contains([]string{"Size", "Type"}, node.Key) {
|
||||
field := node.Key
|
||||
switch {
|
||||
case query.IsOpenExtensionField(node.Key):
|
||||
field = query.OpenExtensionField(node.Key, mapping.SiblingNumber)
|
||||
case !slices.Contains([]string{"Size", "Type"}, node.Key):
|
||||
return osu.NewMatchNoneQuery(), nil
|
||||
}
|
||||
|
||||
query := osu.NewRangeQuery[float64](node.Key)
|
||||
query := osu.NewRangeQuery[float64](field)
|
||||
|
||||
switch node.Operator.Value {
|
||||
case ">":
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
package convert
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
|
||||
)
|
||||
|
||||
// mirrors the bleve compiler, see query.OpenExtensionEquality
|
||||
|
||||
func openExtensionStringQuery(n *ast.StringNode) osu.Builder {
|
||||
if strings.ContainsAny(n.Value, "*?") {
|
||||
sibling, value := mapping.SiblingLower, strings.ToLower(n.Value)
|
||||
if n.Exact {
|
||||
sibling, value = mapping.SiblingKeyword, n.Value
|
||||
}
|
||||
return osu.NewWildcardQuery(query.OpenExtensionField(n.Key, sibling)).Value(value)
|
||||
}
|
||||
|
||||
var alternatives []osu.Builder
|
||||
for _, term := range query.OpenExtensionEquality(n.Key, n.Value, n.Exact) {
|
||||
switch term.Sibling {
|
||||
case mapping.SiblingNumber:
|
||||
alternatives = append(alternatives, osu.NewRangeQuery[float64](term.Field).Gte(term.Number).Lte(term.Number))
|
||||
case mapping.SiblingBool:
|
||||
alternatives = append(alternatives, osu.NewTermQuery[bool](term.Field).Value(term.Bool))
|
||||
case mapping.SiblingDate:
|
||||
alternatives = append(alternatives, osu.NewRangeQuery[time.Time](term.Field).Gte(term.Time).Lte(term.Time))
|
||||
default:
|
||||
alternatives = append(alternatives, osu.NewTermQuery[string](term.Field).Value(term.String))
|
||||
}
|
||||
}
|
||||
if len(alternatives) == 1 {
|
||||
return alternatives[0]
|
||||
}
|
||||
return osu.NewBoolQuery().Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).Should(alternatives...)
|
||||
}
|
||||
|
||||
// openExtensionDateQuery: an equality is the one-instant range.
|
||||
func openExtensionDateQuery(n *ast.DateTimeNode) (osu.Builder, error) {
|
||||
if n.Operator == nil {
|
||||
return nil, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
|
||||
}
|
||||
q := osu.NewRangeQuery[time.Time](query.OpenExtensionField(n.Key, mapping.SiblingDate))
|
||||
switch n.Operator.Value {
|
||||
case ">":
|
||||
return q.Gt(n.Value), nil
|
||||
case ">=":
|
||||
return q.Gte(n.Value), nil
|
||||
case "<":
|
||||
return q.Lt(n.Value), nil
|
||||
case "<=":
|
||||
return q.Lte(n.Value), nil
|
||||
default:
|
||||
return q.Gte(n.Value).Lte(n.Value), nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package convert_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
|
||||
)
|
||||
|
||||
func TestTranspileOpenExtensions(t *testing.T) {
|
||||
oct := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
|
||||
either := func(alternatives ...osu.Builder) osu.Builder {
|
||||
return osu.NewBoolQuery().Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).Should(alternatives...)
|
||||
}
|
||||
|
||||
tests := []opensearchtest.TableTest[ast.Node, osu.Builder]{
|
||||
{
|
||||
Name: "a plain string asks the lowercased sibling",
|
||||
Got: &ast.StringNode{Key: "extensions.com.example.project.state", Value: "Open"},
|
||||
Want: osu.NewTermQuery[string]("ext.com.example.project.state.@lower").Value("open"),
|
||||
},
|
||||
{
|
||||
Name: "= asks the keyword sibling as written",
|
||||
Got: &ast.StringNode{Key: "extensions.com.example.project.state", Value: "Open", Exact: true},
|
||||
Want: osu.NewTermQuery[string]("ext.com.example.project.state.@keyword").Value("Open"),
|
||||
},
|
||||
{
|
||||
Name: "a numeric literal also asks the number sibling",
|
||||
Got: &ast.StringNode{Key: "extensions.com.example.project.priority", Value: "3"},
|
||||
Want: either(
|
||||
osu.NewTermQuery[string]("ext.com.example.project.priority.@lower").Value("3"),
|
||||
osu.NewRangeQuery[float64]("ext.com.example.project.priority.@number").Gte(3).Lte(3),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "a boolean literal also asks the bool sibling",
|
||||
Got: &ast.StringNode{Key: "extensions.com.example.project.done", Value: "false"},
|
||||
Want: either(
|
||||
osu.NewTermQuery[string]("ext.com.example.project.done.@lower").Value("false"),
|
||||
osu.NewTermQuery[bool]("ext.com.example.project.done.@bool").Value(false),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "a date-time literal also asks the date sibling",
|
||||
Got: &ast.StringNode{Key: "extensions.com.example.project.due", Value: "2026-10-01T00:00:00Z"},
|
||||
Want: either(
|
||||
osu.NewTermQuery[string]("ext.com.example.project.due.@lower").Value("2026-10-01t00:00:00z"),
|
||||
osu.NewRangeQuery[time.Time]("ext.com.example.project.due.@date").Gte(oct).Lte(oct),
|
||||
),
|
||||
},
|
||||
{
|
||||
Name: "a wildcard runs on the lowercased sibling",
|
||||
Got: &ast.StringNode{Key: "extensions.com.example.project.state", Value: "Op*"},
|
||||
Want: osu.NewWildcardQuery("ext.com.example.project.state.@lower").Value("op*"),
|
||||
},
|
||||
{
|
||||
Name: "a number range asks the number sibling",
|
||||
Got: &ast.NumberNode{Key: "extensions.com.example.project.priority", Operator: &ast.OperatorNode{Value: ">"}, Value: 3},
|
||||
Want: osu.NewRangeQuery[float64]("ext.com.example.project.priority.@number").Gt(3),
|
||||
},
|
||||
{
|
||||
Name: "a date range asks the date sibling",
|
||||
Got: &ast.DateTimeNode{Key: "extensions.com.example.project.due", Operator: &ast.OperatorNode{Value: "<="}, Value: oct},
|
||||
Want: osu.NewRangeQuery[time.Time]("ext.com.example.project.due.@date").Lte(oct),
|
||||
},
|
||||
{
|
||||
Name: "a boolean node asks the bool sibling",
|
||||
Got: &ast.BooleanNode{Key: "extensions.com.example.project.done", Value: true},
|
||||
Want: osu.NewTermQuery[bool]("ext.com.example.project.done.@bool").Value(true),
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
normalized := query.Normalize(&ast.Ast{Nodes: []ast.Node{test.Got}}, query.ResolveField)
|
||||
dsl, err := convert.TranspileKQLToOpenSearch(normalized.Nodes)
|
||||
assert.NoError(t, err)
|
||||
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, dsl))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -450,6 +450,39 @@ Fixtures:
|
||||
| INVALID-01 | `AND mediatype:document` | bad request | bad request | bad request | ✅ |
|
||||
| INVALID-02 | `mediatype:document AND` | alpha.txt | alpha.txt | alpha.txt | ✅ |
|
||||
|
||||
### openext
|
||||
|
||||
Fixtures:
|
||||
|
||||
- `plan.txt`, com.example.project/done = `b:false`, com.example.project/due = `d:2026-10-01T00:00:00Z`, com.example.project/priority = `n:3`, com.example.project/site = `g:52.5,13.4`, com.example.project/state = `s:Open`, com.example.project/tags = `S:["Urgent","customer"]`
|
||||
- `draft.txt`, com.example.project/done = `b:true`, com.example.project/due = `d:2026-12-24T00:00:00Z`, com.example.project/priority = `n:1.5`, com.example.project/state = `s:open`
|
||||
- `legacy.txt`, com.example.project/due = `s:2026-10-01T00:00:00Z`, com.example.project/priority = `s:3`, com.example.project/state = `s:closed`
|
||||
- `other.txt`, com.example.other/state = `s:open`
|
||||
- `plain.txt`
|
||||
|
||||
| Case | Query | expected | bleve | OpenSearch | same? |
|
||||
|---|---|---|---|---|---|
|
||||
| OPENEXT-01 | `extensions.com.example.project.state:open` | draft.txt, plan.txt | draft.txt, plan.txt | draft.txt, plan.txt | ✅ |
|
||||
| OPENEXT-02 | `extensions.com.example.project.state=Open` | plan.txt | plan.txt | plan.txt | ✅ |
|
||||
| OPENEXT-03 | `extensions.com.example.project.state=open` | draft.txt | draft.txt | draft.txt | ✅ |
|
||||
| OPENEXT-04 | `extensions.com.example.project.state:op*` | draft.txt, plan.txt | draft.txt, plan.txt | draft.txt, plan.txt | ✅ |
|
||||
| OPENEXT-05 | `Extensions.com.example.project.state:closed` | legacy.txt | legacy.txt | legacy.txt | ✅ |
|
||||
| OPENEXT-06 | `extensions.com.example.project.priority>2` | plan.txt | plan.txt | plan.txt | ✅ |
|
||||
| OPENEXT-07 | `extensions.com.example.project.priority<2` | draft.txt | draft.txt | draft.txt | ✅ |
|
||||
| OPENEXT-08 | `extensions.com.example.project.priority:3` | legacy.txt, plan.txt | legacy.txt, plan.txt | legacy.txt, plan.txt | ✅ |
|
||||
| OPENEXT-09 | `extensions.com.example.project.priority:"3"` | legacy.txt, plan.txt | legacy.txt, plan.txt | legacy.txt, plan.txt | ✅ |
|
||||
| OPENEXT-10 | `extensions.com.example.project.done:true` | draft.txt | draft.txt | draft.txt | ✅ |
|
||||
| OPENEXT-11 | `extensions.com.example.project.done:false` | plan.txt | plan.txt | plan.txt | ✅ |
|
||||
| OPENEXT-12 | `extensions.com.example.project.due>2026-11-01T00:00:00Z` | draft.txt | draft.txt | draft.txt | ✅ |
|
||||
| OPENEXT-13 | `extensions.com.example.project.due<2026-11-01T00:00:00Z` | plan.txt | plan.txt | plan.txt | ✅ |
|
||||
| OPENEXT-14 | `extensions.com.example.project.due:2026-10-01T00:00:00Z` | plan.txt | plan.txt | plan.txt | ✅ |
|
||||
| OPENEXT-15 | `extensions.com.example.project.tags:urgent` | plan.txt | plan.txt | plan.txt | ✅ |
|
||||
| OPENEXT-16 | `extensions.com.example.project.t...ample.project.state:open` | plan.txt | plan.txt | plan.txt | ✅ |
|
||||
| OPENEXT-17 | `extensions.com.example.project.s...example.other.state:open` | draft.txt, other.txt, plan.txt | draft.txt, other.txt, plan.txt | draft.txt, other.txt, plan.txt | ✅ |
|
||||
| OPENEXT-18 | `NOT extensions.com.example.project.state:open` | legacy.txt, other.txt, plain.txt | legacy.txt, other.txt, plain.txt | legacy.txt, other.txt, plain.txt | ✅ |
|
||||
| OPENEXT-19 | `extensions.com.example.project.missing:x` | no match | no match | no match | ✅ |
|
||||
| OPENEXT-20 | `extensions.com.example.unknown.state:open` | no match | no match | no match | ✅ |
|
||||
|
||||
## Operations
|
||||
|
||||
### delete
|
||||
@@ -662,6 +695,24 @@ Fixtures:
|
||||
| BATCH-05 | keeps what another batch holds out of its push, then `name:"*added*"` | added.pdf | added.pdf | added.pdf | ✅ |
|
||||
| BATCH-05 | keeps what another batch holds out of its push, then `name:"*other*"` | no match | no match | no match | ✅ |
|
||||
|
||||
### openextops
|
||||
|
||||
Fixtures:
|
||||
|
||||
- `parent`, ID = 1$1!2, folder
|
||||
- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf, com.example.project/priority = `n:3`, com.example.project/state = `s:open`
|
||||
|
||||
| Case | Query | expected | bleve | OpenSearch | same? |
|
||||
|---|---|---|---|---|---|
|
||||
| OPENEXTOPS-01 | re-types a property on upsert, then `extensions.com.example.project.priority:high` | child.pdf | child.pdf | child.pdf | ✅ |
|
||||
| OPENEXTOPS-01 | re-types a property on upsert, then `extensions.com.example.project.priority>2` | no match | no match | no match | ✅ |
|
||||
| OPENEXTOPS-01 | re-types a property on upsert, then `extensions.com.example.project.state:open` | child.pdf | child.pdf | child.pdf | ✅ |
|
||||
| OPENEXTOPS-02 | forgets a removed extension on upsert, then `extensions.com.example.project.state:open` | no match | no match | no match | ✅ |
|
||||
| OPENEXTOPS-02 | forgets a removed extension on upsert, then `name:child.pdf` | child.pdf | child.pdf | child.pdf | ✅ |
|
||||
| OPENEXTOPS-03 | keeps the extensions through a move, then `extensions.com.example.project.priority>2` | child.pdf | child.pdf | child.pdf | ✅ |
|
||||
| OPENEXTOPS-03 | keeps the extensions through a move, then `path:"./renamed/child.pdf"` | child.pdf | child.pdf | child.pdf | ✅ |
|
||||
| OPENEXTOPS-04 | keeps the extensions through the trash and back, then `extensions.com.example.project.state:open` | child.pdf | child.pdf | child.pdf | ✅ |
|
||||
|
||||
## Response
|
||||
|
||||
### entity
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
libregraph "github.com/opencloud-eu/libre-graph-api-go"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/openextension"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
@@ -54,6 +55,24 @@ func withLocation(location *libregraph.GeoCoordinates) fixtureOption {
|
||||
return func(r *search.Resource) { r.Location = location }
|
||||
}
|
||||
|
||||
// withOpenExtension stores the properties of a libregraph openTypeExtension
|
||||
// body as the codec would, one typed metadata value per property.
|
||||
func withOpenExtension(name, body string) fixtureOption {
|
||||
patch, err := openextension.Parse([]byte(body))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
set, _ := patch.Metadata(name)
|
||||
return func(r *search.Resource) {
|
||||
if r.OpenExtensions == nil {
|
||||
r.OpenExtensions = map[string]string{}
|
||||
}
|
||||
for key, value := range set {
|
||||
r.OpenExtensions[key] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func fixtureDoc(name string, opts ...fixtureOption) search.Resource {
|
||||
mtime := fixtureNow
|
||||
r := search.Resource{
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package parity
|
||||
|
||||
import (
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
// The extension siblings follow the stored value: a re-typed property leaves
|
||||
// its old sibling, and the operations that re-index a resource from the index
|
||||
// (Move) keep the extensions.
|
||||
func openextLifecycle() lifecycleGroup {
|
||||
const project = "com.example.project"
|
||||
parent, child := fixtureTree()
|
||||
withOpenExtension(project, `{"state":"open","priority":3}`)(&child)
|
||||
|
||||
retyped := child
|
||||
retyped.OpenExtensions = nil
|
||||
withOpenExtension(project, `{"state":"open","priority":"high"}`)(&retyped)
|
||||
|
||||
dropped := child
|
||||
dropped.OpenExtensions = nil
|
||||
|
||||
return lifecycleGroup{
|
||||
name: "openextops",
|
||||
fixtures: []search.Resource{parent, child},
|
||||
cases: []lifecycleCase{
|
||||
{
|
||||
id: 1, title: "re-types a property on upsert",
|
||||
do: func(e search.Engine) error { return e.Upsert(retyped.ID, retyped) },
|
||||
expect: []expectation{
|
||||
{`extensions.com.example.project.priority:high`, []string{"child.pdf"}},
|
||||
{`extensions.com.example.project.priority>2`, nil},
|
||||
{`extensions.com.example.project.state:open`, []string{"child.pdf"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 2, title: "forgets a removed extension on upsert",
|
||||
do: func(e search.Engine) error { return e.Upsert(dropped.ID, dropped) },
|
||||
expect: []expectation{
|
||||
{`extensions.com.example.project.state:open`, nil},
|
||||
{`name:child.pdf`, []string{"child.pdf"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 3, title: "keeps the extensions through a move",
|
||||
do: func(e search.Engine) error { return e.Move(parent.ID, parent.ParentID, "./renamed") },
|
||||
expect: []expectation{
|
||||
{`extensions.com.example.project.priority>2`, []string{"child.pdf"}},
|
||||
{`path:"./renamed/child.pdf"`, []string{"child.pdf"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 4, title: "keeps the extensions through the trash and back",
|
||||
do: func(e search.Engine) error {
|
||||
if err := e.Delete(child.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
return e.Restore(child.ID)
|
||||
},
|
||||
expect: []expectation{
|
||||
{`extensions.com.example.project.state:open`, []string{"child.pdf"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,10 @@ package parity
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/openextension"
|
||||
"maps"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -342,6 +345,10 @@ func fixtureFields(f search.Resource, withID bool) string {
|
||||
add("Favorites = %s", strings.Join(f.Favorites, ", "))
|
||||
}
|
||||
|
||||
for _, key := range slices.Sorted(maps.Keys(f.OpenExtensions)) {
|
||||
add("%s = `%s`", strings.TrimPrefix(key, openextension.NamespacePrefix), f.OpenExtensions[key])
|
||||
}
|
||||
|
||||
if f.Size != 1000 {
|
||||
add("Size = %d", f.Size)
|
||||
}
|
||||
|
||||
@@ -111,6 +111,7 @@ func queryGroups() []queryGroup {
|
||||
rangeGroup(),
|
||||
scopeGroup(),
|
||||
invalidGroup(),
|
||||
openextGroup(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -233,6 +234,7 @@ func lifecycleGroups() []lifecycleGroup {
|
||||
upsertLifecycle(),
|
||||
idempotencyLifecycle(),
|
||||
batchLifecycle(),
|
||||
openextLifecycle(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
package parity
|
||||
|
||||
import (
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
// Open extensions are addressed as extensions.<name>.<property>. Every
|
||||
// property is indexed in the sibling of its value's kind, and the literal of a
|
||||
// restriction picks the sibling: a range needs a typed literal, an equality
|
||||
// asks every sibling the literal fits.
|
||||
func openextGroup() queryGroup {
|
||||
const project = "com.example.project"
|
||||
|
||||
return queryGroup{
|
||||
name: "openext",
|
||||
fixtures: []search.Resource{
|
||||
fixtureDoc("plan.txt", withOpenExtension(project,
|
||||
`{"state":"Open","priority":3,"done":false,"due":"2026-10-01T00:00:00Z","due@odata.type":"#DateTimeOffset","tags":["Urgent","customer"],"site":{"latitude":52.5,"longitude":13.4},"site@odata.type":"#microsoft.graph.geoCoordinates"}`)),
|
||||
fixtureDoc("draft.txt", withOpenExtension(project,
|
||||
`{"state":"open","priority":1.5,"done":true,"due":"2026-12-24T00:00:00Z","due@odata.type":"#DateTimeOffset"}`)),
|
||||
fixtureDoc("legacy.txt", withOpenExtension(project, `{"state":"closed","priority":"3","due":"2026-10-01T00:00:00Z"}`)),
|
||||
fixtureDoc("other.txt", withOpenExtension("com.example.other", `{"state":"open"}`)),
|
||||
fixtureDoc("plain.txt"),
|
||||
},
|
||||
cases: []queryCase{
|
||||
{id: 1, query: `extensions.com.example.project.state:open`, want: []string{"plan.txt", "draft.txt"}},
|
||||
{id: 2, query: `extensions.com.example.project.state=Open`, want: []string{"plan.txt"}},
|
||||
{id: 3, query: `extensions.com.example.project.state=open`, want: []string{"draft.txt"}},
|
||||
{id: 4, query: `extensions.com.example.project.state:op*`, want: []string{"plan.txt", "draft.txt"}},
|
||||
{id: 5, query: `Extensions.com.example.project.state:closed`, want: []string{"legacy.txt"}},
|
||||
{id: 6, query: `extensions.com.example.project.priority>2`, want: []string{"plan.txt"}},
|
||||
{id: 7, query: `extensions.com.example.project.priority<2`, want: []string{"draft.txt"}},
|
||||
{id: 8, query: `extensions.com.example.project.priority:3`, want: []string{"plan.txt", "legacy.txt"}},
|
||||
{id: 9, query: `extensions.com.example.project.priority:"3"`, want: []string{"plan.txt", "legacy.txt"}},
|
||||
{id: 10, query: `extensions.com.example.project.done:true`, want: []string{"draft.txt"}},
|
||||
{id: 11, query: `extensions.com.example.project.done:false`, want: []string{"plan.txt"}},
|
||||
{id: 12, query: `extensions.com.example.project.due>2026-11-01T00:00:00Z`, want: []string{"draft.txt"}},
|
||||
{id: 13, query: `extensions.com.example.project.due<2026-11-01T00:00:00Z`, want: []string{"plan.txt"}},
|
||||
// a date-time literal is typed by the parser and asks the date sibling
|
||||
// only; legacy.txt holds the same text as a plain string
|
||||
{id: 14, query: `extensions.com.example.project.due:2026-10-01T00:00:00Z`, want: []string{"plan.txt"}},
|
||||
{id: 15, query: `extensions.com.example.project.tags:urgent`, want: []string{"plan.txt"}},
|
||||
{id: 16, query: `extensions.com.example.project.tags:customer AND extensions.com.example.project.state:open`, want: []string{"plan.txt"}},
|
||||
{id: 17, query: `extensions.com.example.project.state:open OR extensions.com.example.other.state:open`, want: []string{"plan.txt", "draft.txt", "other.txt"}},
|
||||
{id: 18, query: `NOT extensions.com.example.project.state:open`, want: []string{"legacy.txt", "other.txt", "plain.txt"}},
|
||||
{id: 19, query: `extensions.com.example.project.missing:x`},
|
||||
{id: 20, query: `extensions.com.example.unknown.state:open`},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -72,6 +72,16 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
|
||||
for i := offset; i < len(nodes); i++ {
|
||||
switch n := nodes[i].(type) {
|
||||
case *ast.StringNode:
|
||||
if searchQuery.IsOpenExtensionField(n.Key) {
|
||||
q := openExtensionStringQuery(n)
|
||||
if prev == nil {
|
||||
prev = q
|
||||
} else {
|
||||
next = q
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
// hidden takes bool words only; anything else matches nothing
|
||||
if n.Key == "Hidden" {
|
||||
var q bleveQuery.Query
|
||||
@@ -153,6 +163,19 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
|
||||
next = q
|
||||
}
|
||||
case *ast.DateTimeNode:
|
||||
if n.Operator == nil {
|
||||
continue
|
||||
}
|
||||
if searchQuery.IsOpenExtensionField(n.Key) {
|
||||
q := openExtensionDateQuery(n)
|
||||
if prev == nil {
|
||||
prev = q
|
||||
} else {
|
||||
next = q
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
q := &bleveQuery.DateRangeQuery{
|
||||
Start: bleveQuery.BleveQueryTime{},
|
||||
End: bleveQuery.BleveQueryTime{},
|
||||
@@ -161,10 +184,6 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
|
||||
FieldVal: n.Key,
|
||||
}
|
||||
|
||||
if n.Operator == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
switch n.Operator.Value {
|
||||
case ">":
|
||||
q.Start.Time = n.Value
|
||||
@@ -191,6 +210,8 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
|
||||
var q bleveQuery.Query
|
||||
if field := n.Key; slices.Contains([]string{"Size", "Type"}, field) {
|
||||
q = numberRange(field, n.Operator, n.Value)
|
||||
} else if searchQuery.IsOpenExtensionField(n.Key) {
|
||||
q = numberRange(searchQuery.OpenExtensionField(n.Key, mapping.SiblingNumber), n.Operator, n.Value)
|
||||
} else {
|
||||
// same answer as the OpenSearch backend: unknown numeric keys
|
||||
// match nothing instead of querying an arbitrary field
|
||||
@@ -208,6 +229,9 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
|
||||
case *ast.BooleanNode:
|
||||
q := bleveQuery.NewBoolFieldQuery(n.Value)
|
||||
q.SetField(n.Key)
|
||||
if searchQuery.IsOpenExtensionField(n.Key) {
|
||||
q.SetField(searchQuery.OpenExtensionField(n.Key, mapping.SiblingBool))
|
||||
}
|
||||
if prev == nil {
|
||||
prev = q
|
||||
} else {
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package bleve
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/blevesearch/bleve/v2"
|
||||
bleveQuery "github.com/blevesearch/bleve/v2/search/query"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
searchQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query"
|
||||
)
|
||||
|
||||
var time0 time.Time // open end of a date range
|
||||
|
||||
func openExtensionStringQuery(n *ast.StringNode) bleveQuery.Query {
|
||||
if strings.ContainsAny(n.Value, "*?") {
|
||||
sibling, value := mapping.SiblingLower, strings.ToLower(n.Value)
|
||||
if n.Exact {
|
||||
sibling, value = mapping.SiblingKeyword, n.Value
|
||||
}
|
||||
wq := bleveQuery.NewWildcardQuery(value)
|
||||
wq.SetField(searchQuery.OpenExtensionField(n.Key, sibling))
|
||||
return wq
|
||||
}
|
||||
|
||||
inclusive := true
|
||||
plan := searchQuery.OpenExtensionEquality(n.Key, n.Value, n.Exact)
|
||||
alternatives := make([]bleveQuery.Query, 0, len(plan))
|
||||
for _, term := range plan {
|
||||
var q bleveQuery.FieldableQuery
|
||||
switch term.Sibling {
|
||||
case mapping.SiblingNumber:
|
||||
f := term.Number
|
||||
q = bleveQuery.NewNumericRangeInclusiveQuery(&f, &f, &inclusive, &inclusive)
|
||||
case mapping.SiblingBool:
|
||||
q = bleveQuery.NewBoolFieldQuery(term.Bool)
|
||||
case mapping.SiblingDate:
|
||||
q = bleveQuery.NewDateRangeInclusiveQuery(term.Time, term.Time, &inclusive, &inclusive)
|
||||
default:
|
||||
q = bleveQuery.NewTermQuery(term.String)
|
||||
}
|
||||
q.SetField(term.Field)
|
||||
alternatives = append(alternatives, q)
|
||||
}
|
||||
if len(alternatives) == 1 {
|
||||
return alternatives[0]
|
||||
}
|
||||
return bleve.NewDisjunctionQuery(alternatives...)
|
||||
}
|
||||
|
||||
// openExtensionDateQuery: an equality is the one-instant range.
|
||||
func openExtensionDateQuery(n *ast.DateTimeNode) bleveQuery.Query {
|
||||
inclusive, exclusive := true, false
|
||||
field := searchQuery.OpenExtensionField(n.Key, mapping.SiblingDate)
|
||||
var q *bleveQuery.DateRangeQuery
|
||||
switch n.Operator.Value {
|
||||
case ">":
|
||||
q = bleveQuery.NewDateRangeInclusiveQuery(n.Value, time0, &exclusive, nil)
|
||||
case ">=":
|
||||
q = bleveQuery.NewDateRangeInclusiveQuery(n.Value, time0, &inclusive, nil)
|
||||
case "<":
|
||||
q = bleveQuery.NewDateRangeInclusiveQuery(time0, n.Value, nil, &exclusive)
|
||||
case "<=":
|
||||
q = bleveQuery.NewDateRangeInclusiveQuery(time0, n.Value, nil, &inclusive)
|
||||
default:
|
||||
q = bleveQuery.NewDateRangeInclusiveQuery(n.Value, n.Value, &inclusive, &inclusive)
|
||||
}
|
||||
q.SetField(field)
|
||||
return q
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package bleve
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/blevesearch/bleve/v2/search/query"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
searchquery "github.com/opencloud-eu/opencloud/services/search/pkg/query"
|
||||
)
|
||||
|
||||
func Test_compileOpenExtensions(t *testing.T) {
|
||||
inclusive, exclusive := true, false
|
||||
term := func(field, value string) query.Query {
|
||||
q := query.NewTermQuery(value)
|
||||
q.SetField(field)
|
||||
return q
|
||||
}
|
||||
number := func(field string, lo, hi *float64, loIn, hiIn *bool) query.Query {
|
||||
q := query.NewNumericRangeInclusiveQuery(lo, hi, loIn, hiIn)
|
||||
q.SetField(field)
|
||||
return q
|
||||
}
|
||||
date := func(field string, start, end time.Time, startIn, endIn *bool) query.Query {
|
||||
q := query.NewDateRangeInclusiveQuery(start, end, startIn, endIn)
|
||||
q.SetField(field)
|
||||
return q
|
||||
}
|
||||
boolean := func(field string, v bool) query.Query {
|
||||
q := query.NewBoolFieldQuery(v)
|
||||
q.SetField(field)
|
||||
return q
|
||||
}
|
||||
three := 3.0
|
||||
oct := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
node ast.Node
|
||||
want query.Query
|
||||
}{
|
||||
{
|
||||
name: "a plain string asks the lowercased sibling",
|
||||
node: &ast.StringNode{Key: "extensions.com.example.project.state", Value: "Open"},
|
||||
want: term("ext.com.example.project.state.@lower", "open"),
|
||||
},
|
||||
{
|
||||
name: "= asks the keyword sibling as written",
|
||||
node: &ast.StringNode{Key: "extensions.com.example.project.state", Value: "Open", Exact: true},
|
||||
want: term("ext.com.example.project.state.@keyword", "Open"),
|
||||
},
|
||||
{
|
||||
name: "the prefix is case-insensitive",
|
||||
node: &ast.StringNode{Key: "Extensions.com.example.project.State", Value: "x"},
|
||||
want: term("ext.com.example.project.State.@lower", "x"),
|
||||
},
|
||||
{
|
||||
name: "a numeric literal also asks the number sibling",
|
||||
node: &ast.StringNode{Key: "extensions.com.example.project.priority", Value: "3"},
|
||||
want: query.NewDisjunctionQuery([]query.Query{
|
||||
term("ext.com.example.project.priority.@lower", "3"),
|
||||
number("ext.com.example.project.priority.@number", &three, &three, &inclusive, &inclusive),
|
||||
}),
|
||||
},
|
||||
{
|
||||
name: "a boolean literal also asks the bool sibling",
|
||||
node: &ast.StringNode{Key: "extensions.com.example.project.done", Value: "true"},
|
||||
want: query.NewDisjunctionQuery([]query.Query{
|
||||
term("ext.com.example.project.done.@lower", "true"),
|
||||
boolean("ext.com.example.project.done.@bool", true),
|
||||
}),
|
||||
},
|
||||
{
|
||||
name: "a date-time literal also asks the date sibling",
|
||||
node: &ast.StringNode{Key: "extensions.com.example.project.due", Value: "2026-10-01T00:00:00Z"},
|
||||
want: query.NewDisjunctionQuery([]query.Query{
|
||||
term("ext.com.example.project.due.@lower", "2026-10-01t00:00:00z"),
|
||||
date("ext.com.example.project.due.@date", oct, oct, &inclusive, &inclusive),
|
||||
}),
|
||||
},
|
||||
{
|
||||
name: "a wildcard runs on the lowercased sibling",
|
||||
node: &ast.StringNode{Key: "extensions.com.example.project.state", Value: "Op*"},
|
||||
want: func() query.Query {
|
||||
q := query.NewWildcardQuery("op*")
|
||||
q.SetField("ext.com.example.project.state.@lower")
|
||||
return q
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "a number range asks the number sibling",
|
||||
node: &ast.NumberNode{Key: "extensions.com.example.project.priority", Operator: &ast.OperatorNode{Value: ">"}, Value: 3},
|
||||
want: number("ext.com.example.project.priority.@number", &three, nil, &exclusive, nil),
|
||||
},
|
||||
{
|
||||
name: "a date range asks the date sibling",
|
||||
node: &ast.DateTimeNode{Key: "extensions.com.example.project.due", Operator: &ast.OperatorNode{Value: "<="}, Value: oct},
|
||||
want: date("ext.com.example.project.due.@date", time.Time{}, oct, nil, &inclusive),
|
||||
},
|
||||
{
|
||||
name: "a boolean node asks the bool sibling",
|
||||
node: &ast.BooleanNode{Key: "extensions.com.example.project.done", Value: false},
|
||||
want: boolean("ext.com.example.project.done.@bool", false),
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := compile(searchquery.Normalize(&ast.Ast{Nodes: []ast.Node{tt.node}}, searchquery.ResolveField))
|
||||
assert.NoError(t, err)
|
||||
// compile wraps a single leaf in a conjunction and hands a disjunction through
|
||||
want := tt.want
|
||||
if _, ok := want.(*query.DisjunctionQuery); !ok {
|
||||
want = query.NewConjunctionQuery([]query.Query{want})
|
||||
}
|
||||
assert.Equal(t, want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
)
|
||||
|
||||
// Nothing in a request says which kind an extension property has, so the
|
||||
// literal decides: a range targets the sibling of its typed literal, an
|
||||
// equality asks every sibling the literal fits. Both compilers follow this plan.
|
||||
|
||||
// OpenExtensionTerm is one sibling an equality is asked on.
|
||||
type OpenExtensionTerm struct {
|
||||
Field string
|
||||
Sibling string
|
||||
|
||||
String string
|
||||
Number float64
|
||||
Bool bool
|
||||
Time time.Time
|
||||
}
|
||||
|
||||
// IsOpenExtensionField reports whether a KQL key addresses an extension property.
|
||||
func IsOpenExtensionField(key string) bool {
|
||||
return mapping.IsOpenExtensionQueryField(key)
|
||||
}
|
||||
|
||||
// OpenExtensionField is the indexed field name of the sibling for a KQL key.
|
||||
func OpenExtensionField(key, sibling string) string {
|
||||
field, _ := mapping.OpenExtensionFieldFromQuery(key, sibling)
|
||||
return field
|
||||
}
|
||||
|
||||
// OpenExtensionEquality asks the string sibling, case-insensitively unless exact,
|
||||
// and every other sibling the value reads as.
|
||||
func OpenExtensionEquality(key, value string, exact bool) []OpenExtensionTerm {
|
||||
term := func(sibling string) OpenExtensionTerm {
|
||||
return OpenExtensionTerm{Field: OpenExtensionField(key, sibling), Sibling: sibling}
|
||||
}
|
||||
|
||||
str := term(mapping.SiblingLower)
|
||||
str.String = strings.ToLower(value)
|
||||
if exact {
|
||||
str = term(mapping.SiblingKeyword)
|
||||
str.String = value
|
||||
}
|
||||
plan := []OpenExtensionTerm{str}
|
||||
|
||||
if f, err := strconv.ParseFloat(value, 64); err == nil {
|
||||
num := term(mapping.SiblingNumber)
|
||||
num.Number = f
|
||||
plan = append(plan, num)
|
||||
}
|
||||
if b, err := strconv.ParseBool(value); err == nil && (value == "true" || value == "false") {
|
||||
bl := term(mapping.SiblingBool)
|
||||
bl.Bool = b
|
||||
plan = append(plan, bl)
|
||||
}
|
||||
if t, err := time.Parse(time.RFC3339Nano, value); err == nil {
|
||||
dt := term(mapping.SiblingDate)
|
||||
dt.Time = t
|
||||
plan = append(plan, dt)
|
||||
}
|
||||
return plan
|
||||
}
|
||||
@@ -61,6 +61,9 @@ func ResolveField(name string) string {
|
||||
if v, ok := fieldIndex()[strings.ToLower(name)]; ok {
|
||||
return v
|
||||
}
|
||||
if mapping.IsOpenExtensionQueryField(name) {
|
||||
return mapping.OpenExtensionsQueryPrefix + name[len(mapping.OpenExtensionsQueryPrefix):]
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user