From e1c634e546d690db4182e2d42c34ea136293fbb9 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Wed, 29 Jul 2026 17:39:23 +0200 Subject: [PATCH] feat(search): add shared KQL lowering pass query.Normalize resolves field names (query.ResolveField, from the derived index + a small alias overlay) and expands media-type restrictions (mimetype.Expand) once, between parse and backend compilation. --- .../search/pkg/query/mimetype/mimetype.go | 96 ++++++++++++++++++ .../pkg/query/mimetype/mimetype_test.go | 97 +++++++++++++++++++ services/search/pkg/query/normalize.go | 73 ++++++++++++++ services/search/pkg/query/normalize_test.go | 90 +++++++++++++++++ services/search/pkg/query/resolver.go | 41 ++++++++ 5 files changed, 397 insertions(+) create mode 100644 services/search/pkg/query/mimetype/mimetype.go create mode 100644 services/search/pkg/query/mimetype/mimetype_test.go create mode 100644 services/search/pkg/query/normalize.go create mode 100644 services/search/pkg/query/normalize_test.go create mode 100644 services/search/pkg/query/resolver.go diff --git a/services/search/pkg/query/mimetype/mimetype.go b/services/search/pkg/query/mimetype/mimetype.go new file mode 100644 index 0000000000..98692d9524 --- /dev/null +++ b/services/search/pkg/query/mimetype/mimetype.go @@ -0,0 +1,96 @@ +// Package mimetype maps the "mediatype" KQL restriction (field name and value) +// to a concrete MimeType query. +package mimetype + +import ( + "strings" + + "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/pkg/kql" +) + +// field is the real index field a mediatype restriction targets. +const field = "MimeType" + +// Expand turns mediatype: into the MimeType query it stands for: category +// values (file/document/image/...) expand to their MIME set, anything else is a +// literal MimeType:. Returns nil for non-mediatype keys. +func Expand(key, value string) []ast.Node { + if strings.ToLower(key) != "mediatype" { + return nil + } + switch value { + case "file": + return []ast.Node{ + &ast.OperatorNode{Value: kql.BoolNOT}, + &ast.StringNode{Key: field, Value: "httpd/unix-directory"}, + } + case "folder": + return term("httpd/unix-directory") + case "document": + return group( + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.wordprocessingml.form", + "application/vnd.oasis.opendocument.text", + "text/plain", + "text/markdown", + "application/rtf", + "application/vnd.apple.pages", + ) + case "spreadsheet": + return group( + "application/vnd.ms-excel", + "application/vnd.oasis.opendocument.spreadsheet", + "text/csv", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.apple.numbers", + ) + case "presentation": + return group( + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/vnd.oasis.opendocument.presentation", + "application/vnd.ms-powerpoint", + "application/vnd.apple.keynote", + ) + case "pdf": + return term("application/pdf") + case "image": + return term("image/*") + case "video": + return term("video/*") + case "audio": + return term("audio/*") + case "archive": + return group( + "application/zip", + "application/gzip", + "application/x-gzip", + "application/x-7z-compressed", + "application/x-rar-compressed", + "application/x-tar", + "application/x-bzip2", + "application/x-bzip", + "application/x-tgz", + ) + } + // not a category: treat the value as a literal MIME type. + return term(value) +} + +// term is a single MimeType:value restriction. +func term(value string) []ast.Node { + return []ast.Node{&ast.StringNode{Key: field, Value: value}} +} + +// group is a single OR group of MimeType:value restrictions. +func group(values ...string) []ast.Node { + nodes := make([]ast.Node, 0, len(values)*2-1) + for i, v := range values { + if i > 0 { + nodes = append(nodes, &ast.OperatorNode{Value: kql.BoolOR}) + } + nodes = append(nodes, &ast.StringNode{Key: field, Value: v}) + } + return []ast.Node{&ast.GroupNode{Nodes: nodes}} +} diff --git a/services/search/pkg/query/mimetype/mimetype_test.go b/services/search/pkg/query/mimetype/mimetype_test.go new file mode 100644 index 0000000000..0b25844e28 --- /dev/null +++ b/services/search/pkg/query/mimetype/mimetype_test.go @@ -0,0 +1,97 @@ +package mimetype_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/services/search/pkg/query/mimetype" +) + +// This is the single place the mediatype -> MimeType mapping is tested. The +// query pipeline consumes Expand via query.Normalize and must NOT re-test it. + +func TestExpand_onlyTriggersOnMediatype(t *testing.T) { + require.Nil(t, mimetype.Expand("Name", "document")) + require.Nil(t, mimetype.Expand("MimeType", "file")) // the real field name is not the trigger + require.Nil(t, mimetype.Expand("Tags", "file")) +} + +func TestExpand_keyIsCaseInsensitive(t *testing.T) { + require.NotNil(t, mimetype.Expand("MediaType", "file")) +} + +// A non-category value is a literal MIME type and targets the MimeType field. +func TestExpand_literalValuePassesThroughToMimeType(t *testing.T) { + require.Equal(t, []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "application/pdf"}, + }, mimetype.Expand("mediatype", "application/pdf")) + + require.Equal(t, []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "image/jpeg"}, + }, mimetype.Expand("mediatype", "image/jpeg")) +} + +func TestExpand_fileIsNotAFolder(t *testing.T) { + require.Equal(t, []ast.Node{ + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + }, mimetype.Expand("mediatype", "file")) +} + +func TestExpand_folderIsASingleTerm(t *testing.T) { + require.Equal(t, []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + }, mimetype.Expand("mediatype", "folder")) +} + +func TestExpand_wildcardCategories(t *testing.T) { + for value, mime := range map[string]string{ + "image": "image/*", "video": "video/*", "audio": "audio/*", "pdf": "application/pdf", + } { + require.Equal(t, []ast.Node{ + &ast.StringNode{Key: "MimeType", Value: mime}, + }, mimetype.Expand("mediatype", value), value) + } +} + +func TestExpand_documentGroup(t *testing.T) { + got := mimetype.Expand("mediatype", "document") + require.Len(t, got, 1) + group, ok := got[0].(*ast.GroupNode) + require.True(t, ok) + require.Equal(t, mimeValues(group), []string{ + "application/msword", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.wordprocessingml.form", + "application/vnd.oasis.opendocument.text", + "text/plain", + "text/markdown", + "application/rtf", + "application/vnd.apple.pages", + }) +} + +// spreadsheet asserts the exact MIME set, in order, with no duplicate entry. +func TestExpand_spreadsheet(t *testing.T) { + group := mimetype.Expand("mediatype", "spreadsheet")[0].(*ast.GroupNode) + require.Equal(t, mimeValues(group), []string{ + "application/vnd.ms-excel", + "application/vnd.oasis.opendocument.spreadsheet", + "text/csv", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.apple.numbers", + }) +} + +// mimeValues extracts the StringNode values from an OR group, dropping operators. +func mimeValues(group *ast.GroupNode) []string { + var out []string + for _, n := range group.Nodes { + if s, ok := n.(*ast.StringNode); ok { + out = append(out, s.Value) + } + } + return out +} diff --git a/services/search/pkg/query/normalize.go b/services/search/pkg/query/normalize.go new file mode 100644 index 0000000000..8c1269604b --- /dev/null +++ b/services/search/pkg/query/normalize.go @@ -0,0 +1,73 @@ +package query + +import ( + "reflect" + + "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/services/search/pkg/query/mimetype" +) + +// Normalize is the shared KQL lowering pass between parse and compile: it +// resolves keys to real field names (via resolve) and expands media-type +// restrictions, so the backends compile a plain field:value AST. +func Normalize(a *ast.Ast, resolve func(string) string) *ast.Ast { + a.Nodes = normalizeNodes(a.Nodes, resolve, "") + return a +} + +// normalizeNodes rewrites nodes in place. defaultKey is what a bare restriction +// inherits: its enclosing group's key, or "" at the top level. +func normalizeNodes(nodes []ast.Node, resolve func(string) string, defaultKey string) []ast.Node { + resolveKey := func(key string) string { + if key == "" && defaultKey != "" { + return defaultKey // bare child inherits the group key + } + return resolve(key) + } + + out := make([]ast.Node, 0, len(nodes)) + for _, n := range nodes { + n = toPointer(n) // ensure a pointer so in-place key rewrites persist + switch node := n.(type) { + case *ast.StringNode: + node.Key = resolveKey(node.Key) + if exp := mimetype.Expand(node.Key, node.Value); exp != nil { + out = append(out, normalizeNodes(exp, resolve, defaultKey)...) + continue + } + out = append(out, node) + case *ast.DateTimeNode: + node.Key = resolveKey(node.Key) + out = append(out, node) + case *ast.BooleanNode: + node.Key = resolveKey(node.Key) + out = append(out, node) + case *ast.GroupNode: + groupKey := defaultKey + if node.Key != "" { + node.Key = resolve(node.Key) + groupKey = node.Key + } + node.Nodes = normalizeNodes(node.Nodes, resolve, groupKey) + out = append(out, node) + default: + out = append(out, n) + } + } + return out +} + +// toPointer returns n as a pointer; the parser emits some nodes by value and the +// in-place key rewrites would be lost on those. +func toPointer(n ast.Node) ast.Node { + rv := reflect.ValueOf(n) + if rv.Kind() == reflect.Ptr { + return n + } + ptr := reflect.New(rv.Type()) + ptr.Elem().Set(rv) + if pn, ok := ptr.Interface().(ast.Node); ok { + return pn + } + return n +} diff --git a/services/search/pkg/query/normalize_test.go b/services/search/pkg/query/normalize_test.go new file mode 100644 index 0000000000..49cea2c153 --- /dev/null +++ b/services/search/pkg/query/normalize_test.go @@ -0,0 +1,90 @@ +package query_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/services/search/pkg/query" +) + +// This is the single place the shared KQL lowering pass is tested (field +// resolution, media-type expansion, group-key defaulting, pointer conversion). +// The backend query compilers consume its canonical output and must not re-test +// it. + +func norm(nodes ...ast.Node) []ast.Node { + return query.Normalize(&ast.Ast{Nodes: nodes}, query.ResolveField).Nodes +} + +func TestResolveField(t *testing.T) { + require.Equal(t, "Name", query.ResolveField("")) // empty -> free-text default + require.Equal(t, "Name", query.ResolveField("NAME")) // case-insensitive + require.Equal(t, "Tags", query.ResolveField("tag")) // singular alias + require.Equal(t, "MimeType", query.ResolveField("mimetype")) // real field, case-insensitive + require.Equal(t, "photo.cameraMake", query.ResolveField("photo.CAMERAMAKE")) // facet, case-insensitive + require.Equal(t, "unknown.field", query.ResolveField("unknown.field")) // unknown key: unchanged, becomes a dead query +} + +func TestNormalize_ResolvesFieldsAndExpandsMediatype(t *testing.T) { + got := norm( + &ast.StringNode{Key: "", Value: "free"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "TAG", Value: "x"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "photo.cameramake", Value: "canon"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "mediatype", Value: "file"}, + ) + require.Equal(t, []ast.Node{ + &ast.StringNode{Key: "Name", Value: "free"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "Tags", Value: "x"}, + &ast.OperatorNode{Value: "AND"}, + &ast.StringNode{Key: "photo.cameraMake", Value: "canon"}, + &ast.OperatorNode{Value: "AND"}, + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"}, + }, got) +} + +// A bare restriction inside a named group inherits the group key; a keyed child +// keeps its own key; a bare restriction in an unnamed group falls back to Name. +func TestNormalize_GroupKeyDefaulting(t *testing.T) { + got := norm( + &ast.GroupNode{Key: "author", Nodes: []ast.Node{ + &ast.StringNode{Value: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "name", Value: "d"}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Value: "e"}, + }}, + ) + require.Equal(t, []ast.Node{ + &ast.GroupNode{Key: "author", Nodes: []ast.Node{ + &ast.StringNode{Key: "author", Value: "b"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: "Name", Value: "d"}, + }}, + &ast.OperatorNode{Value: "AND"}, + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: "Name", Value: "e"}, + }}, + }, got) +} + +func TestNormalize_ConvertsValueNodesToPointers(t *testing.T) { + got := norm( + ast.StringNode{Key: "name", Value: "x"}, + ast.OperatorNode{Value: "AND"}, + ast.DateTimeNode{Key: "mtime"}, + ) + require.Equal(t, []ast.Node{ + &ast.StringNode{Key: "Name", Value: "x"}, + &ast.OperatorNode{Value: "AND"}, + &ast.DateTimeNode{Key: "Mtime"}, + }, got) +} diff --git a/services/search/pkg/query/resolver.go b/services/search/pkg/query/resolver.go new file mode 100644 index 0000000000..5a4127a817 --- /dev/null +++ b/services/search/pkg/query/resolver.go @@ -0,0 +1,41 @@ +package query + +import ( + "reflect" + "strings" + "sync" + + "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +// aliases are KQL spellings the derived index can't produce (fields are plural). +var aliases = map[string]string{ + "tag": "Tags", + "favorite": "Favorites", +} + +// fieldIndex maps a lowercased KQL key to the real field name: derived once from +// the resource struct, overlaid with the explicit aliases. +var fieldIndex = sync.OnceValue(func() map[string]string { + idx := mapping.FieldNameIndex( + reflect.TypeFor[search.Resource](), + search.Resource{}.SearchFieldOverrides(), + ) + for k, v := range aliases { + idx[k] = v + } + return idx +}) + +// ResolveField maps a KQL key to the index field name: empty -> Name, a known +// key (case-insensitive) -> its field, anything else unchanged. +func ResolveField(name string) string { + if name == "" { + return "Name" + } + if v, ok := fieldIndex()[strings.ToLower(name)]; ok { + return v + } + return name +}