mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-07-31 17:59:49 -04:00
feat(search): add shared KQL lowering pass
query.Normalize resolves field names (query.ResolveField, from the derived index + a small alias overlay) and expands media-type restrictions (mimetype.Expand) once, between parse and backend compilation.
This commit is contained in:
96
services/search/pkg/query/mimetype/mimetype.go
Normal file
96
services/search/pkg/query/mimetype/mimetype.go
Normal file
@@ -0,0 +1,96 @@
|
||||
// Package mimetype maps the "mediatype" KQL restriction (field name and value)
|
||||
// to a concrete MimeType query.
|
||||
package mimetype
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/pkg/kql"
|
||||
)
|
||||
|
||||
// field is the real index field a mediatype restriction targets.
|
||||
const field = "MimeType"
|
||||
|
||||
// Expand turns mediatype:<value> into the MimeType query it stands for: category
|
||||
// values (file/document/image/...) expand to their MIME set, anything else is a
|
||||
// literal MimeType:<value>. Returns nil for non-mediatype keys.
|
||||
func Expand(key, value string) []ast.Node {
|
||||
if strings.ToLower(key) != "mediatype" {
|
||||
return nil
|
||||
}
|
||||
switch value {
|
||||
case "file":
|
||||
return []ast.Node{
|
||||
&ast.OperatorNode{Value: kql.BoolNOT},
|
||||
&ast.StringNode{Key: field, Value: "httpd/unix-directory"},
|
||||
}
|
||||
case "folder":
|
||||
return term("httpd/unix-directory")
|
||||
case "document":
|
||||
return group(
|
||||
"application/msword",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.form",
|
||||
"application/vnd.oasis.opendocument.text",
|
||||
"text/plain",
|
||||
"text/markdown",
|
||||
"application/rtf",
|
||||
"application/vnd.apple.pages",
|
||||
)
|
||||
case "spreadsheet":
|
||||
return group(
|
||||
"application/vnd.ms-excel",
|
||||
"application/vnd.oasis.opendocument.spreadsheet",
|
||||
"text/csv",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"application/vnd.apple.numbers",
|
||||
)
|
||||
case "presentation":
|
||||
return group(
|
||||
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
|
||||
"application/vnd.oasis.opendocument.presentation",
|
||||
"application/vnd.ms-powerpoint",
|
||||
"application/vnd.apple.keynote",
|
||||
)
|
||||
case "pdf":
|
||||
return term("application/pdf")
|
||||
case "image":
|
||||
return term("image/*")
|
||||
case "video":
|
||||
return term("video/*")
|
||||
case "audio":
|
||||
return term("audio/*")
|
||||
case "archive":
|
||||
return group(
|
||||
"application/zip",
|
||||
"application/gzip",
|
||||
"application/x-gzip",
|
||||
"application/x-7z-compressed",
|
||||
"application/x-rar-compressed",
|
||||
"application/x-tar",
|
||||
"application/x-bzip2",
|
||||
"application/x-bzip",
|
||||
"application/x-tgz",
|
||||
)
|
||||
}
|
||||
// not a category: treat the value as a literal MIME type.
|
||||
return term(value)
|
||||
}
|
||||
|
||||
// term is a single MimeType:value restriction.
|
||||
func term(value string) []ast.Node {
|
||||
return []ast.Node{&ast.StringNode{Key: field, Value: value}}
|
||||
}
|
||||
|
||||
// group is a single OR group of MimeType:value restrictions.
|
||||
func group(values ...string) []ast.Node {
|
||||
nodes := make([]ast.Node, 0, len(values)*2-1)
|
||||
for i, v := range values {
|
||||
if i > 0 {
|
||||
nodes = append(nodes, &ast.OperatorNode{Value: kql.BoolOR})
|
||||
}
|
||||
nodes = append(nodes, &ast.StringNode{Key: field, Value: v})
|
||||
}
|
||||
return []ast.Node{&ast.GroupNode{Nodes: nodes}}
|
||||
}
|
||||
97
services/search/pkg/query/mimetype/mimetype_test.go
Normal file
97
services/search/pkg/query/mimetype/mimetype_test.go
Normal file
@@ -0,0 +1,97 @@
|
||||
package mimetype_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/query/mimetype"
|
||||
)
|
||||
|
||||
// This is the single place the mediatype -> MimeType mapping is tested. The
|
||||
// query pipeline consumes Expand via query.Normalize and must NOT re-test it.
|
||||
|
||||
func TestExpand_onlyTriggersOnMediatype(t *testing.T) {
|
||||
require.Nil(t, mimetype.Expand("Name", "document"))
|
||||
require.Nil(t, mimetype.Expand("MimeType", "file")) // the real field name is not the trigger
|
||||
require.Nil(t, mimetype.Expand("Tags", "file"))
|
||||
}
|
||||
|
||||
func TestExpand_keyIsCaseInsensitive(t *testing.T) {
|
||||
require.NotNil(t, mimetype.Expand("MediaType", "file"))
|
||||
}
|
||||
|
||||
// A non-category value is a literal MIME type and targets the MimeType field.
|
||||
func TestExpand_literalValuePassesThroughToMimeType(t *testing.T) {
|
||||
require.Equal(t, []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "application/pdf"},
|
||||
}, mimetype.Expand("mediatype", "application/pdf"))
|
||||
|
||||
require.Equal(t, []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "image/jpeg"},
|
||||
}, mimetype.Expand("mediatype", "image/jpeg"))
|
||||
}
|
||||
|
||||
func TestExpand_fileIsNotAFolder(t *testing.T) {
|
||||
require.Equal(t, []ast.Node{
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
|
||||
}, mimetype.Expand("mediatype", "file"))
|
||||
}
|
||||
|
||||
func TestExpand_folderIsASingleTerm(t *testing.T) {
|
||||
require.Equal(t, []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
|
||||
}, mimetype.Expand("mediatype", "folder"))
|
||||
}
|
||||
|
||||
func TestExpand_wildcardCategories(t *testing.T) {
|
||||
for value, mime := range map[string]string{
|
||||
"image": "image/*", "video": "video/*", "audio": "audio/*", "pdf": "application/pdf",
|
||||
} {
|
||||
require.Equal(t, []ast.Node{
|
||||
&ast.StringNode{Key: "MimeType", Value: mime},
|
||||
}, mimetype.Expand("mediatype", value), value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpand_documentGroup(t *testing.T) {
|
||||
got := mimetype.Expand("mediatype", "document")
|
||||
require.Len(t, got, 1)
|
||||
group, ok := got[0].(*ast.GroupNode)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, mimeValues(group), []string{
|
||||
"application/msword",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.form",
|
||||
"application/vnd.oasis.opendocument.text",
|
||||
"text/plain",
|
||||
"text/markdown",
|
||||
"application/rtf",
|
||||
"application/vnd.apple.pages",
|
||||
})
|
||||
}
|
||||
|
||||
// spreadsheet asserts the exact MIME set, in order, with no duplicate entry.
|
||||
func TestExpand_spreadsheet(t *testing.T) {
|
||||
group := mimetype.Expand("mediatype", "spreadsheet")[0].(*ast.GroupNode)
|
||||
require.Equal(t, mimeValues(group), []string{
|
||||
"application/vnd.ms-excel",
|
||||
"application/vnd.oasis.opendocument.spreadsheet",
|
||||
"text/csv",
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"application/vnd.apple.numbers",
|
||||
})
|
||||
}
|
||||
|
||||
// mimeValues extracts the StringNode values from an OR group, dropping operators.
|
||||
func mimeValues(group *ast.GroupNode) []string {
|
||||
var out []string
|
||||
for _, n := range group.Nodes {
|
||||
if s, ok := n.(*ast.StringNode); ok {
|
||||
out = append(out, s.Value)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
73
services/search/pkg/query/normalize.go
Normal file
73
services/search/pkg/query/normalize.go
Normal file
@@ -0,0 +1,73 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/query/mimetype"
|
||||
)
|
||||
|
||||
// Normalize is the shared KQL lowering pass between parse and compile: it
|
||||
// resolves keys to real field names (via resolve) and expands media-type
|
||||
// restrictions, so the backends compile a plain field:value AST.
|
||||
func Normalize(a *ast.Ast, resolve func(string) string) *ast.Ast {
|
||||
a.Nodes = normalizeNodes(a.Nodes, resolve, "")
|
||||
return a
|
||||
}
|
||||
|
||||
// normalizeNodes rewrites nodes in place. defaultKey is what a bare restriction
|
||||
// inherits: its enclosing group's key, or "" at the top level.
|
||||
func normalizeNodes(nodes []ast.Node, resolve func(string) string, defaultKey string) []ast.Node {
|
||||
resolveKey := func(key string) string {
|
||||
if key == "" && defaultKey != "" {
|
||||
return defaultKey // bare child inherits the group key
|
||||
}
|
||||
return resolve(key)
|
||||
}
|
||||
|
||||
out := make([]ast.Node, 0, len(nodes))
|
||||
for _, n := range nodes {
|
||||
n = toPointer(n) // ensure a pointer so in-place key rewrites persist
|
||||
switch node := n.(type) {
|
||||
case *ast.StringNode:
|
||||
node.Key = resolveKey(node.Key)
|
||||
if exp := mimetype.Expand(node.Key, node.Value); exp != nil {
|
||||
out = append(out, normalizeNodes(exp, resolve, defaultKey)...)
|
||||
continue
|
||||
}
|
||||
out = append(out, node)
|
||||
case *ast.DateTimeNode:
|
||||
node.Key = resolveKey(node.Key)
|
||||
out = append(out, node)
|
||||
case *ast.BooleanNode:
|
||||
node.Key = resolveKey(node.Key)
|
||||
out = append(out, node)
|
||||
case *ast.GroupNode:
|
||||
groupKey := defaultKey
|
||||
if node.Key != "" {
|
||||
node.Key = resolve(node.Key)
|
||||
groupKey = node.Key
|
||||
}
|
||||
node.Nodes = normalizeNodes(node.Nodes, resolve, groupKey)
|
||||
out = append(out, node)
|
||||
default:
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// toPointer returns n as a pointer; the parser emits some nodes by value and the
|
||||
// in-place key rewrites would be lost on those.
|
||||
func toPointer(n ast.Node) ast.Node {
|
||||
rv := reflect.ValueOf(n)
|
||||
if rv.Kind() == reflect.Ptr {
|
||||
return n
|
||||
}
|
||||
ptr := reflect.New(rv.Type())
|
||||
ptr.Elem().Set(rv)
|
||||
if pn, ok := ptr.Interface().(ast.Node); ok {
|
||||
return pn
|
||||
}
|
||||
return n
|
||||
}
|
||||
90
services/search/pkg/query/normalize_test.go
Normal file
90
services/search/pkg/query/normalize_test.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package query_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/pkg/ast"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
|
||||
)
|
||||
|
||||
// This is the single place the shared KQL lowering pass is tested (field
|
||||
// resolution, media-type expansion, group-key defaulting, pointer conversion).
|
||||
// The backend query compilers consume its canonical output and must not re-test
|
||||
// it.
|
||||
|
||||
func norm(nodes ...ast.Node) []ast.Node {
|
||||
return query.Normalize(&ast.Ast{Nodes: nodes}, query.ResolveField).Nodes
|
||||
}
|
||||
|
||||
func TestResolveField(t *testing.T) {
|
||||
require.Equal(t, "Name", query.ResolveField("")) // empty -> free-text default
|
||||
require.Equal(t, "Name", query.ResolveField("NAME")) // case-insensitive
|
||||
require.Equal(t, "Tags", query.ResolveField("tag")) // singular alias
|
||||
require.Equal(t, "MimeType", query.ResolveField("mimetype")) // real field, case-insensitive
|
||||
require.Equal(t, "photo.cameraMake", query.ResolveField("photo.CAMERAMAKE")) // facet, case-insensitive
|
||||
require.Equal(t, "unknown.field", query.ResolveField("unknown.field")) // unknown key: unchanged, becomes a dead query
|
||||
}
|
||||
|
||||
func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) {
|
||||
got := norm(
|
||||
&ast.StringNode{Key: "", Value: "free"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "TAG", Value: "x"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "photo.cameramake", Value: "canon"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "mediatype", Value: "file"},
|
||||
)
|
||||
require.Equal(t, []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "free"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "Tags", Value: "x"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.StringNode{Key: "photo.cameraMake", Value: "canon"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.OperatorNode{Value: "NOT"},
|
||||
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
|
||||
}, got)
|
||||
}
|
||||
|
||||
// A bare restriction inside a named group inherits the group key; a keyed child
|
||||
// keeps its own key; a bare restriction in an unnamed group falls back to Name.
|
||||
func TestNormalize_GroupKeyDefaulting(t *testing.T) {
|
||||
got := norm(
|
||||
&ast.GroupNode{Key: "author", Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "name", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Value: "e"},
|
||||
}},
|
||||
)
|
||||
require.Equal(t, []ast.Node{
|
||||
&ast.GroupNode{Key: "author", Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "author", Value: "b"},
|
||||
&ast.OperatorNode{Value: "OR"},
|
||||
&ast.StringNode{Key: "Name", Value: "d"},
|
||||
}},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.GroupNode{Nodes: []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "e"},
|
||||
}},
|
||||
}, got)
|
||||
}
|
||||
|
||||
func TestNormalize_ConvertsValueNodesToPointers(t *testing.T) {
|
||||
got := norm(
|
||||
ast.StringNode{Key: "name", Value: "x"},
|
||||
ast.OperatorNode{Value: "AND"},
|
||||
ast.DateTimeNode{Key: "mtime"},
|
||||
)
|
||||
require.Equal(t, []ast.Node{
|
||||
&ast.StringNode{Key: "Name", Value: "x"},
|
||||
&ast.OperatorNode{Value: "AND"},
|
||||
&ast.DateTimeNode{Key: "Mtime"},
|
||||
}, got)
|
||||
}
|
||||
41
services/search/pkg/query/resolver.go
Normal file
41
services/search/pkg/query/resolver.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package query
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
|
||||
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
|
||||
)
|
||||
|
||||
// aliases are KQL spellings the derived index can't produce (fields are plural).
|
||||
var aliases = map[string]string{
|
||||
"tag": "Tags",
|
||||
"favorite": "Favorites",
|
||||
}
|
||||
|
||||
// fieldIndex maps a lowercased KQL key to the real field name: derived once from
|
||||
// the resource struct, overlaid with the explicit aliases.
|
||||
var fieldIndex = sync.OnceValue(func() map[string]string {
|
||||
idx := mapping.FieldNameIndex(
|
||||
reflect.TypeFor[search.Resource](),
|
||||
search.Resource{}.SearchFieldOverrides(),
|
||||
)
|
||||
for k, v := range aliases {
|
||||
idx[k] = v
|
||||
}
|
||||
return idx
|
||||
})
|
||||
|
||||
// ResolveField maps a KQL key to the index field name: empty -> Name, a known
|
||||
// key (case-insensitive) -> its field, anything else unchanged.
|
||||
func ResolveField(name string) string {
|
||||
if name == "" {
|
||||
return "Name"
|
||||
}
|
||||
if v, ok := fieldIndex()[strings.ToLower(name)]; ok {
|
||||
return v
|
||||
}
|
||||
return name
|
||||
}
|
||||
Reference in New Issue
Block a user