feat(search): add shared KQL lowering pass

query.Normalize resolves field names (query.ResolveField, from the derived
index + a small alias overlay) and expands media-type restrictions
(mimetype.Expand) once, between parse and backend compilation.
This commit is contained in:
Dominik Schmidt
2026-07-29 17:39:23 +02:00
parent 1a4daf38bf
commit e1c634e546
5 changed files with 397 additions and 0 deletions

View File

@@ -0,0 +1,96 @@
// Package mimetype maps the "mediatype" KQL restriction (field name and value)
// to a concrete MimeType query.
package mimetype
import (
"strings"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/pkg/kql"
)
// field is the real index field a mediatype restriction targets.
const field = "MimeType"
// Expand turns mediatype:<value> into the MimeType query it stands for: category
// values (file/document/image/...) expand to their MIME set, anything else is a
// literal MimeType:<value>. Returns nil for non-mediatype keys.
func Expand(key, value string) []ast.Node {
if strings.ToLower(key) != "mediatype" {
return nil
}
switch value {
case "file":
return []ast.Node{
&ast.OperatorNode{Value: kql.BoolNOT},
&ast.StringNode{Key: field, Value: "httpd/unix-directory"},
}
case "folder":
return term("httpd/unix-directory")
case "document":
return group(
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.openxmlformats-officedocument.wordprocessingml.form",
"application/vnd.oasis.opendocument.text",
"text/plain",
"text/markdown",
"application/rtf",
"application/vnd.apple.pages",
)
case "spreadsheet":
return group(
"application/vnd.ms-excel",
"application/vnd.oasis.opendocument.spreadsheet",
"text/csv",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.apple.numbers",
)
case "presentation":
return group(
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"application/vnd.oasis.opendocument.presentation",
"application/vnd.ms-powerpoint",
"application/vnd.apple.keynote",
)
case "pdf":
return term("application/pdf")
case "image":
return term("image/*")
case "video":
return term("video/*")
case "audio":
return term("audio/*")
case "archive":
return group(
"application/zip",
"application/gzip",
"application/x-gzip",
"application/x-7z-compressed",
"application/x-rar-compressed",
"application/x-tar",
"application/x-bzip2",
"application/x-bzip",
"application/x-tgz",
)
}
// not a category: treat the value as a literal MIME type.
return term(value)
}
// term is a single MimeType:value restriction.
func term(value string) []ast.Node {
return []ast.Node{&ast.StringNode{Key: field, Value: value}}
}
// group is a single OR group of MimeType:value restrictions.
func group(values ...string) []ast.Node {
nodes := make([]ast.Node, 0, len(values)*2-1)
for i, v := range values {
if i > 0 {
nodes = append(nodes, &ast.OperatorNode{Value: kql.BoolOR})
}
nodes = append(nodes, &ast.StringNode{Key: field, Value: v})
}
return []ast.Node{&ast.GroupNode{Nodes: nodes}}
}

View File

@@ -0,0 +1,97 @@
package mimetype_test
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/pkg/query/mimetype"
)
// This is the single place the mediatype -> MimeType mapping is tested. The
// query pipeline consumes Expand via query.Normalize and must NOT re-test it.
func TestExpand_onlyTriggersOnMediatype(t *testing.T) {
require.Nil(t, mimetype.Expand("Name", "document"))
require.Nil(t, mimetype.Expand("MimeType", "file")) // the real field name is not the trigger
require.Nil(t, mimetype.Expand("Tags", "file"))
}
func TestExpand_keyIsCaseInsensitive(t *testing.T) {
require.NotNil(t, mimetype.Expand("MediaType", "file"))
}
// A non-category value is a literal MIME type and targets the MimeType field.
func TestExpand_literalValuePassesThroughToMimeType(t *testing.T) {
require.Equal(t, []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/pdf"},
}, mimetype.Expand("mediatype", "application/pdf"))
require.Equal(t, []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "image/jpeg"},
}, mimetype.Expand("mediatype", "image/jpeg"))
}
func TestExpand_fileIsNotAFolder(t *testing.T) {
require.Equal(t, []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
}, mimetype.Expand("mediatype", "file"))
}
func TestExpand_folderIsASingleTerm(t *testing.T) {
require.Equal(t, []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
}, mimetype.Expand("mediatype", "folder"))
}
func TestExpand_wildcardCategories(t *testing.T) {
for value, mime := range map[string]string{
"image": "image/*", "video": "video/*", "audio": "audio/*", "pdf": "application/pdf",
} {
require.Equal(t, []ast.Node{
&ast.StringNode{Key: "MimeType", Value: mime},
}, mimetype.Expand("mediatype", value), value)
}
}
func TestExpand_documentGroup(t *testing.T) {
got := mimetype.Expand("mediatype", "document")
require.Len(t, got, 1)
group, ok := got[0].(*ast.GroupNode)
require.True(t, ok)
require.Equal(t, mimeValues(group), []string{
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.openxmlformats-officedocument.wordprocessingml.form",
"application/vnd.oasis.opendocument.text",
"text/plain",
"text/markdown",
"application/rtf",
"application/vnd.apple.pages",
})
}
// spreadsheet asserts the exact MIME set, in order, with no duplicate entry.
func TestExpand_spreadsheet(t *testing.T) {
group := mimetype.Expand("mediatype", "spreadsheet")[0].(*ast.GroupNode)
require.Equal(t, mimeValues(group), []string{
"application/vnd.ms-excel",
"application/vnd.oasis.opendocument.spreadsheet",
"text/csv",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.apple.numbers",
})
}
// mimeValues extracts the StringNode values from an OR group, dropping operators.
func mimeValues(group *ast.GroupNode) []string {
var out []string
for _, n := range group.Nodes {
if s, ok := n.(*ast.StringNode); ok {
out = append(out, s.Value)
}
}
return out
}

View File

@@ -0,0 +1,73 @@
package query
import (
"reflect"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/pkg/query/mimetype"
)
// Normalize is the shared KQL lowering pass between parse and compile: it
// resolves keys to real field names (via resolve) and expands media-type
// restrictions, so the backends compile a plain field:value AST.
func Normalize(a *ast.Ast, resolve func(string) string) *ast.Ast {
a.Nodes = normalizeNodes(a.Nodes, resolve, "")
return a
}
// normalizeNodes rewrites nodes in place. defaultKey is what a bare restriction
// inherits: its enclosing group's key, or "" at the top level.
func normalizeNodes(nodes []ast.Node, resolve func(string) string, defaultKey string) []ast.Node {
resolveKey := func(key string) string {
if key == "" && defaultKey != "" {
return defaultKey // bare child inherits the group key
}
return resolve(key)
}
out := make([]ast.Node, 0, len(nodes))
for _, n := range nodes {
n = toPointer(n) // ensure a pointer so in-place key rewrites persist
switch node := n.(type) {
case *ast.StringNode:
node.Key = resolveKey(node.Key)
if exp := mimetype.Expand(node.Key, node.Value); exp != nil {
out = append(out, normalizeNodes(exp, resolve, defaultKey)...)
continue
}
out = append(out, node)
case *ast.DateTimeNode:
node.Key = resolveKey(node.Key)
out = append(out, node)
case *ast.BooleanNode:
node.Key = resolveKey(node.Key)
out = append(out, node)
case *ast.GroupNode:
groupKey := defaultKey
if node.Key != "" {
node.Key = resolve(node.Key)
groupKey = node.Key
}
node.Nodes = normalizeNodes(node.Nodes, resolve, groupKey)
out = append(out, node)
default:
out = append(out, n)
}
}
return out
}
// toPointer returns n as a pointer; the parser emits some nodes by value and the
// in-place key rewrites would be lost on those.
func toPointer(n ast.Node) ast.Node {
rv := reflect.ValueOf(n)
if rv.Kind() == reflect.Ptr {
return n
}
ptr := reflect.New(rv.Type())
ptr.Elem().Set(rv)
if pn, ok := ptr.Interface().(ast.Node); ok {
return pn
}
return n
}

View File

@@ -0,0 +1,90 @@
package query_test
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
)
// This is the single place the shared KQL lowering pass is tested (field
// resolution, media-type expansion, group-key defaulting, pointer conversion).
// The backend query compilers consume its canonical output and must not re-test
// it.
func norm(nodes ...ast.Node) []ast.Node {
return query.Normalize(&ast.Ast{Nodes: nodes}, query.ResolveField).Nodes
}
func TestResolveField(t *testing.T) {
require.Equal(t, "Name", query.ResolveField("")) // empty -> free-text default
require.Equal(t, "Name", query.ResolveField("NAME")) // case-insensitive
require.Equal(t, "Tags", query.ResolveField("tag")) // singular alias
require.Equal(t, "MimeType", query.ResolveField("mimetype")) // real field, case-insensitive
require.Equal(t, "photo.cameraMake", query.ResolveField("photo.CAMERAMAKE")) // facet, case-insensitive
require.Equal(t, "unknown.field", query.ResolveField("unknown.field")) // unknown key: unchanged, becomes a dead query
}
func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) {
got := norm(
&ast.StringNode{Key: "", Value: "free"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "TAG", Value: "x"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "photo.cameramake", Value: "canon"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "mediatype", Value: "file"},
)
require.Equal(t, []ast.Node{
&ast.StringNode{Key: "Name", Value: "free"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Tags", Value: "x"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "photo.cameraMake", Value: "canon"},
&ast.OperatorNode{Value: "AND"},
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
}, got)
}
// A bare restriction inside a named group inherits the group key; a keyed child
// keeps its own key; a bare restriction in an unnamed group falls back to Name.
func TestNormalize_GroupKeyDefaulting(t *testing.T) {
got := norm(
&ast.GroupNode{Key: "author", Nodes: []ast.Node{
&ast.StringNode{Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "name", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Value: "e"},
}},
)
require.Equal(t, []ast.Node{
&ast.GroupNode{Key: "author", Nodes: []ast.Node{
&ast.StringNode{Key: "author", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "Name", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "e"},
}},
}, got)
}
func TestNormalize_ConvertsValueNodesToPointers(t *testing.T) {
got := norm(
ast.StringNode{Key: "name", Value: "x"},
ast.OperatorNode{Value: "AND"},
ast.DateTimeNode{Key: "mtime"},
)
require.Equal(t, []ast.Node{
&ast.StringNode{Key: "Name", Value: "x"},
&ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: "Mtime"},
}, got)
}

View File

@@ -0,0 +1,41 @@
package query
import (
"reflect"
"strings"
"sync"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
// aliases are KQL spellings the derived index can't produce (fields are plural).
var aliases = map[string]string{
"tag": "Tags",
"favorite": "Favorites",
}
// fieldIndex maps a lowercased KQL key to the real field name: derived once from
// the resource struct, overlaid with the explicit aliases.
var fieldIndex = sync.OnceValue(func() map[string]string {
idx := mapping.FieldNameIndex(
reflect.TypeFor[search.Resource](),
search.Resource{}.SearchFieldOverrides(),
)
for k, v := range aliases {
idx[k] = v
}
return idx
})
// ResolveField maps a KQL key to the index field name: empty -> Name, a known
// key (case-insensitive) -> its field, anything else unchanged.
func ResolveField(name string) string {
if name == "" {
return "Name"
}
if v, ok := fieldIndex()[strings.ToLower(name)]; ok {
return v
}
return name
}