mirror/opencloud
1
0 Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-09-12 21:58:58 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3345 from opencloud-eu/refactor/search-mapping

Browse Source 
refactor: reflection-based search mapping + location geopoint
This commit is contained in:
Dominik Schmidt authored and GitHub committed 2026-08-31 15:12:49 +02:00
2 parents 109e9555c8 1bf813068e
commit d38fbc8e52
84 files changed
+3922 -2096
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

pkg/ast/ast.go
+3
View File
@@ -44,6 +44,9 @@ type StringNode struct {
Key string
Value string
Exact bool
// CaseInsensitive marks a case-insensitive restriction; set by the search
// lowering pass, a backend routes it to the field's lowercased form.
CaseInsensitive bool
}
// BooleanNode represents a bool value
pkg/kql/cast.go
+2 -3
View File
@@ -7,7 +7,6 @@ import (
"github.com/jinzhu/now"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
)
func toNode[T ast.Node](in any) (T, error) {
@@ -85,7 +84,7 @@ func toTimeRange(in any) (*time.Time, *time.Time, error) {
value, err := toString(in)
if err != nil {
return &from, &to, &query.UnsupportedTimeRangeError{}
return &from, &to, &UnsupportedTimeRangeError{}
}
c := &now.Config{
@@ -132,7 +131,7 @@ func toTimeRange(in any) (*time.Time, *time.Time, error) {
}
if from.IsZero() || to.IsZero() {
return nil, nil, &query.UnsupportedTimeRangeError{}
return nil, nil, &UnsupportedTimeRangeError{}
}
return &from, &to, nil
pkg/kql/dictionary_test.go
+10 -11
View File
@@ -9,7 +9,6 @@ import (
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/pkg/ast/test"
"github.com/opencloud-eu/opencloud/pkg/kql"
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
tAssert "github.com/stretchr/testify/assert"
)
@@ -34,13 +33,13 @@ func TestParse_Spec(t *testing.T) {
},
{
name: `AND`,
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolAND},
},
},
{
name: `AND cat AND dog`,
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolAND},
},
},
@@ -80,13 +79,13 @@ func TestParse_Spec(t *testing.T) {
},
{
name: `OR`,
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolOR},
},
},
{
name: `OR cat AND dog`,
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolOR},
},
},
@@ -930,37 +929,37 @@ func TestParse_Errors(t *testing.T) {
tests := []testCase{
{
query: "animal:(mammal:cat mammal:dog reptile:turtle)",
error: query.NamedGroupInvalidNodesError{
error: kql.NamedGroupInvalidNodesError{
Node: &ast.StringNode{Key: "mammal", Value: "cat"},
},
},
{
query: "animal:(cat mammal:dog turtle)",
error: query.NamedGroupInvalidNodesError{
error: kql.NamedGroupInvalidNodesError{
Node: &ast.StringNode{Key: "mammal", Value: "dog"},
},
},
{
query: "animal:(AND cat)",
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolAND},
},
},
{
query: "animal:(OR cat)",
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolOR},
},
},
{
query: "(AND cat)",
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolAND},
},
},
{
query: "(OR cat)",
error: query.StartsWithBinaryOperatorError{
error: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolOR},
},
},
services/search/pkg/query/error.go → pkg/kql/error.go
+4 -3
View File
@@ -1,4 +1,4 @@
package query
package kql
import (
"errors"
@@ -39,8 +39,9 @@ func (e UnsupportedTimeRangeError) Error() string {
return fmt.Sprintf("unable to convert '%v' to a time range", e.Value)
}
// IsValidationError says whether the query itself is at fault, which makes it a
// bad request and not an error of ours.
// IsValidationError reports whether err is one of the KQL parse/validation
// errors produced by this package, i.e. the query itself is at fault and the
// caller should treat it as a bad request.
func IsValidationError(err error) bool {
var (
startsWithBinaryOperator *StartsWithBinaryOperatorError
pkg/kql/kql_test.go
+1 -2
View File
@@ -5,7 +5,6 @@ import (
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/pkg/kql"
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
tAssert "github.com/stretchr/testify/assert"
)
@@ -22,7 +21,7 @@ func TestNewAST(t *testing.T) {
{
name: "error",
givenQuery: kql.BoolAND,
expectedError: query.StartsWithBinaryOperatorError{
expectedError: kql.StartsWithBinaryOperatorError{
Node: &ast.OperatorNode{Value: kql.BoolAND},
},
},
pkg/kql/validate.go
+3 -4
View File
@@ -2,7 +2,6 @@ package kql
import (
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
)
func validateAst(a *ast.Ast) error {
@@ -10,7 +9,7 @@ func validateAst(a *ast.Ast) error {
case *ast.OperatorNode:
switch node.Value {
case BoolAND, BoolOR:
return &query.StartsWithBinaryOperatorError{Node: node}
return &StartsWithBinaryOperatorError{Node: node}
}
}
return nil
@@ -21,14 +20,14 @@ func validateGroupNode(n *ast.GroupNode) error {
case *ast.OperatorNode:
switch node.Value {
case BoolAND, BoolOR:
return &query.StartsWithBinaryOperatorError{Node: node}
return &StartsWithBinaryOperatorError{Node: node}
}
}
if n.Key != "" {
for _, node := range n.Nodes {
if ast.NodeKey(node) != "" {
return &query.NamedGroupInvalidNodesError{Node: node}
return &NamedGroupInvalidNodesError{Node: node}
}
}
}
services/graph/pkg/service/v0/driveitems.go
+10 -121
View File
@@ -9,9 +9,7 @@ import (
"net/http"
"net/url"
"path"
"reflect"
"strconv"
"strings"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
@@ -28,6 +26,7 @@ import (
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/graph/pkg/errorcode"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
)
// CreateUploadSession create an upload session to allow your app to upload files up to the maximum file size.
@@ -452,130 +451,20 @@ func cs3ResourceToDriveItem(logger *log.Logger, publicBaseURL *url.URL, res *sto
}
}
if res.GetArbitraryMetadata() != nil {
driveItem.Audio = cs3ResourceToDriveItemAudioFacet(logger, res)
driveItem.Image = cs3ResourceToDriveItemImageFacet(logger, res)
driveItem.Location = cs3ResourceToDriveItemLocationFacet(logger, res)
driveItem.Photo = cs3ResourceToDriveItemPhotoFacet(logger, res)
if metadata := res.GetArbitraryMetadata().GetMetadata(); metadata != nil {
driveItem.Audio = metadataToFacet[libregraph.Audio](metadata, "audio")
driveItem.Image = metadataToFacet[libregraph.Image](metadata, "image")
driveItem.Location = metadataToFacet[libregraph.GeoCoordinates](metadata, "location")
driveItem.Photo = metadataToFacet[libregraph.Photo](metadata, "photo")
}
return driveItem, nil
}
func cs3ResourceToDriveItemAudioFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Audio {
if !strings.HasPrefix(res.GetMimeType(), "audio/") {
return nil
}
k := res.GetArbitraryMetadata().GetMetadata()
if k == nil {
return nil
}
var audio = &libregraph.Audio{}
if ok := unmarshalStringMap(logger, audio, k, "libre.graph.audio."); ok {
return audio
}
return nil
}
func cs3ResourceToDriveItemImageFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Image {
k := res.GetArbitraryMetadata().GetMetadata()
if k == nil {
return nil
}
var image = &libregraph.Image{}
if ok := unmarshalStringMap(logger, image, k, "libre.graph.image."); ok {
return image
}
return nil
}
func cs3ResourceToDriveItemLocationFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.GeoCoordinates {
k := res.GetArbitraryMetadata().GetMetadata()
if k == nil {
return nil
}
var location = &libregraph.GeoCoordinates{}
if ok := unmarshalStringMap(logger, location, k, "libre.graph.location."); ok {
return location
}
return nil
}
func cs3ResourceToDriveItemPhotoFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Photo {
k := res.GetArbitraryMetadata().GetMetadata()
if k == nil {
return nil
}
var photo = &libregraph.Photo{}
if ok := unmarshalStringMap(logger, photo, k, "libre.graph.photo."); ok {
return photo
}
return nil
}
func getFieldName(structField reflect.StructField) string {
tag := structField.Tag.Get("json")
if tag == "" {
return structField.Name
}
return strings.Split(tag, ",")[0]
}
func unmarshalStringMap(logger *log.Logger, out any, flatMap map[string]string, prefix string) bool {
nonEmpty := false
obj := reflect.ValueOf(out).Elem()
timeKind := reflect.TypeOf(&time.Time{}).Elem().Kind()
for i := 0; i < obj.NumField(); i++ {
field := obj.Field(i)
structField := obj.Type().Field(i)
mapKey := prefix + getFieldName(structField)
if value, ok := flatMap[mapKey]; ok {
if field.Kind() == reflect.Ptr {
newValue := reflect.New(field.Type().Elem())
var tmp any
var err error
switch t := newValue.Type().Elem().Kind(); t {
case reflect.String:
tmp = value
case reflect.Int32:
tmp, err = strconv.ParseInt(value, 10, 32)
case reflect.Int64:
tmp, err = strconv.ParseInt(value, 10, 64)
case reflect.Float32:
tmp, err = strconv.ParseFloat(value, 32)
case reflect.Float64:
tmp, err = strconv.ParseFloat(value, 64)
case reflect.Bool:
tmp, err = strconv.ParseBool(value)
case timeKind:
tmp, err = time.Parse(time.RFC3339, value)
default:
err = errors.New("unsupported type")
logger.Error().Err(err).Str("type", t.String()).Str("mapKey", mapKey).Msg("target field type for value of mapKey is not supported")
}
if err != nil {
logger.Error().Err(err).Str("mapKey", mapKey).Msg("unmarshalling failed")
continue
}
newValue.Elem().Set(reflect.ValueOf(tmp).Convert(field.Type().Elem()))
field.Set(newValue)
nonEmpty = true
}
}
}
return nonEmpty
// metadataToFacet builds a DriveItem facet *T from CS3 arbitrary metadata under
// the "libre.graph.<facet>." key prefix. Nil when no such keys are present.
func metadataToFacet[T any](metadata map[string]string, facet string) *T {
return mapping.DeserializeStringsAt[T](metadata, "libre.graph."+facet+".")
}
func cs3ResourceToRemoteItem(res *storageprovider.ResourceInfo) (*libregraph.RemoteItem, error) {
services/search/MIGRATION.md
+9 -22
View File
@@ -9,38 +9,24 @@ until you remove it.
### OpenSearch
The new index is `opencloud-resource-v3`. Fill it in one of two ways:
- copy the old index, fast and keeps the extracted file contents, or
- index all spaces again, slower since every file is read once more, but drops
documents that no longer have a resource.
The address below is the one from `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_ADDRESSES`,
`opencloud-resource` the name from
`SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME`.
Fill the new index by indexing all spaces again:
```shell
# either copy the old index
curl -X POST "https://opensearch.example.com:9200/_reindex?wait_for_completion=false" \
-H 'Content-Type: application/json' -d '
{"source":{"index":"opencloud-resource"},"dest":{"index":"opencloud-resource-v3"}}'
# the answer carries a task id, watch it while it runs
curl "https://opensearch.example.com:9200/_tasks/<task-id>"
# or index all spaces again, the service keeps running while it happens
# the service keeps running while it happens
opencloud search index --all-spaces
```
Once the new index is filled, remove the old one:
Once the new index is filled, every index but the one with the highest
`-v<N>` suffix can go (indexes up to 7.4 have no suffix):
```shell
curl -X DELETE "https://opensearch.example.com:9200/opencloud-resource"
curl "https://os.example.com:9200/_cat/indices/opencloud-resource*"
curl -X DELETE "https://os.example.com:9200/opencloud-resource"
```
### bleve
The new index is the `bleve-v2` directory next to the old `bleve` one, both in
The new index is a directory next to the old `bleve` one, both in
`$OC_BASE_DATA_PATH/search` by default (`SEARCH_ENGINE_BLEVE_DATA_PATH`). A
bleve index cannot be copied, index all spaces again:
@@ -48,7 +34,8 @@ bleve index cannot be copied, index all spaces again:
opencloud search index --all-spaces
```
Once the new index is filled, remove the old one:
Once the new index is filled, every directory but the one with the highest
`bleve-v<N>` suffix can go (directories up to 7.4 have no suffix):
```shell
rm -r "$OC_BASE_DATA_PATH/search/bleve"
services/search/README.md
+4 -1
View File
@@ -39,7 +39,10 @@ To enable OpenSearch as a backend, the following settings must be set:
Additionally, the following optional settings can be set:
* `SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME=val` (default: `opencloud-resource`): Name of the OpenSearch index
* `SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME=val` (default:
`opencloud-resource`): base name of the OpenSearch index. The running
index is suffixed with the current schema version; a breaking schema
change targets a fresh index, the old one stays in place.
* `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_USERNAME=val`: Username for HTTP Basic Authentication.
* `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_PASSWORD=val`: Password for HTTP Basic Authentication.
* `SEARCH_ENGINE_OPEN_SEARCH_CLIENT_HEADER=val`: HTTP headers to include in requests.
services/search/internal/opensearchtest/os.go
+19
View File
@@ -137,6 +137,19 @@ func (tc *TestClient) IndicesCreate(ctx context.Context, index string, body io.R
}
}
// IndicesCount returns the number of documents in the given indices.
func (tc *TestClient) IndicesCount(ctx context.Context, indices []string, body io.Reader) (int, error) {
resp, err := tc.c.Indices.Count(ctx, &opensearchgoAPI.IndicesCountReq{
Indices: indices,
Body: body,
})
if err != nil {
return 0, fmt.Errorf("failed to count documents in %v: %w", indices, err)
}
return resp.Count, nil
}
type testRequireClient struct {
tc *TestClient
t testing.TB
@@ -157,3 +170,9 @@ func (trc *testRequireClient) IndicesCreate(index string, body io.Reader) {
func (trc *testRequireClient) IndicesDelete(indices []string) {
require.NoError(trc.t, trc.tc.IndicesDelete(trc.t.Context(), indices))
}
func (trc *testRequireClient) IndicesCount(indices []string, body io.Reader, want int) {
got, err := trc.tc.IndicesCount(trc.t.Context(), indices, body)
require.NoError(trc.t, err)
require.Equal(trc.t, want, got)
}
services/search/internal/opensearchtest/testdata/resource_file.json
Vendored
+1 -1
View File
@@ -8,7 +8,7 @@
"Name" : "dummy name",
"Content" : "dummy content",
"Size" : 42,
"Mtime" : "2025-07-24 15:15:01.324093 +0200 CEST m=+0.000056251",
"Mtime" : "2025-07-24T15:15:01.324093+02:00",
"MimeType" : "image/jpeg",
"Tags" : [ "dummy" ],
"Deleted" : false,
services/search/internal/opensearchtest/testdata/resource_folder.json
Vendored
+2 -1
View File
@@ -3,5 +3,6 @@
"RootID" : "1$1!1",
"ParentID" : "1$1!1",
"Path" : "./parent d!r",
"Type" : 2
"Type" : 2,
"Mtime" : "2025-07-24T15:15:01.324093+02:00"
}
services/search/internal/opensearchtest/testdata/resource_root.json
Vendored
+2 -1
View File
@@ -1,5 +1,6 @@
{
"ID" : "1$1!1",
"RootID" : "1$1!1",
"Path" : "."
"Path" : ".",
"Mtime" : "2025-07-24T15:15:01.324093+02:00"
}
services/search/pkg/bleve/backend.go
+16 -17
View File
@@ -3,7 +3,6 @@ package bleve
import (
"context"
"math"
"strings"
"time"
"github.com/blevesearch/bleve/v2"
@@ -14,6 +13,7 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/utils"
"google.golang.org/protobuf/types/known/timestamppb"
"github.com/opencloud-eu/opencloud/pkg/kql"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
@@ -45,7 +45,7 @@ func NewBackend(index bleve.Index, queryCreator searchQuery.Creator[query.Query]
func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) {
createdQuery, err := b.queryCreator.Create(sir.Query)
if err != nil {
if searchQuery.IsValidationError(err) {
if kql.IsValidationError(err) {
return nil, errtypes.BadRequest(err.Error())
}
return nil, err
@@ -74,6 +74,16 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
),
},
)
// Scope below the space root: restrict at query level so totals and
// paging respect the path too. Path is a case-preserving keyword
// (paths act as references, /Foo and /foo are distinct), so the exact
// folder or the folder prefix matches all of, and only, the scope.
if requestedPath := utils.MakeRelativePath(sir.Ref.Path); requestedPath != "." {
q.Conjuncts = append(q.Conjuncts, query.NewDisjunctionQuery([]query.Query{
&query.TermQuery{FieldVal: "Path", Term: requestedPath},
&query.PrefixQuery{FieldVal: "Path", Prefix: requestedPath + "/"},
}))
}
}
bleveReq := bleve.NewSearchRequest(q)
@@ -97,17 +107,6 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
matches := make([]*searchMessage.Match, 0, len(res.Hits))
totalMatches := res.Total
for _, hit := range res.Hits {
if sir.Ref != nil {
hitPath := strings.TrimSuffix(getFieldValue[string](hit.Fields, "Path"), "/")
requestedPath := utils.MakeRelativePath(sir.Ref.Path)
isRoot := hitPath == requestedPath
if !isRoot && requestedPath != "." && !strings.HasPrefix(hitPath, requestedPath+"/") {
totalMatches--
continue
}
}
rootID, err := storagespace.ParseID(getFieldValue[string](hit.Fields, "RootID"))
if err != nil {
return nil, err
@@ -136,10 +135,10 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
Tags: getFieldSliceValue[string](hit.Fields, "Tags"),
Favorites: getFieldSliceValue[string](hit.Fields, "Favorites"),
Highlights: getFragmentValue(hit.Fragments, "Content", 0),
Audio: getAudioValue[searchMessage.Audio](hit.Fields),
Image: getImageValue[searchMessage.Image](hit.Fields),
Location: getLocationValue[searchMessage.GeoCoordinates](hit.Fields),
Photo: getPhotoValue[searchMessage.Photo](hit.Fields),
Audio: hitToFacet[searchMessage.Audio](hit.Fields, "audio"),
Image: hitToFacet[searchMessage.Image](hit.Fields, "image"),
Location: hitToFacet[searchMessage.GeoCoordinates](hit.Fields, "location"),
Photo: hitToFacet[searchMessage.Photo](hit.Fields, "photo"),
},
}
services/search/pkg/bleve/backend_test.go
+118
View File
@@ -0,0 +1,118 @@
package bleve_test
import (
"fmt"
bleveSearch "github.com/blevesearch/bleve/v2"
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
var _ = Describe("Bleve", func() {
var (
eng *bleve.Backend
idx bleveSearch.Index
rootResource search.Resource
parentResource search.Resource
childResource search.Resource
)
BeforeEach(func() {
mapping, err := bleve.NewMapping()
Expect(err).ToNot(HaveOccurred())
idx, err = bleveSearch.NewMemOnly(mapping)
Expect(err).ToNot(HaveOccurred())
eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{})
Expect(err).ToNot(HaveOccurred())
rootResource = search.Resource{
ID: "1$2!2",
RootID: "1$2!2",
Path: ".",
Document: content.Document{},
}
parentResource = search.Resource{
ID: "1$2!3",
ParentID: rootResource.ID,
RootID: rootResource.ID,
Path: "./parent d!r",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER),
Document: content.Document{Name: "parent d!r"},
}
childResource = search.Resource{
ID: "1$2!4",
ParentID: parentResource.ID,
RootID: rootResource.ID,
Path: "./parent d!r/child.pdf",
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{Name: "child.pdf"},
}
})
Describe("PurgeSpace", func() {
It("takes every record of that space out of the index", func() {
otherSpace := search.Resource{
ID: "1$9!9",
RootID: "1$9!9",
Path: ".",
Document: content.Document{Name: "other"},
}
for _, resource := range []search.Resource{rootResource, parentResource, childResource, otherSpace} {
Expect(eng.Upsert(resource.ID, resource)).To(Succeed())
}
Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed())
count, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(1)), "only the records of that space are gone")
})
It("takes a space out that holds more records than one round", func() {
otherSpace := search.Resource{
ID: "1$9!9",
RootID: "1$9!9",
Path: ".",
Document: content.Document{Name: "other"},
}
Expect(eng.Upsert(otherSpace.ID, otherSpace)).To(Succeed())
for i := range 120 {
resource := search.Resource{
ID: fmt.Sprintf("%s!file-%d", rootResource.RootID, i),
RootID: rootResource.RootID,
Path: fmt.Sprintf("./file-%d", i),
Document: content.Document{Name: fmt.Sprintf("file-%d", i)},
}
Expect(eng.Upsert(resource.ID, resource)).To(Succeed())
}
Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed())
count, err := idx.DocCount()
Expect(err).ToNot(HaveOccurred())
Expect(count).To(Equal(uint64(1)), "only the record of the other space is left")
})
})
Describe("New", func() {
It("returns a new index instance", func() {
b := bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{})
Expect(b).ToNot(BeNil())
})
})
})
services/search/pkg/bleve/batch.go
+18 -6
View File
@@ -10,6 +10,7 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/utils"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
@@ -36,18 +37,29 @@ func NewBatch(index bleve.Index, size int) (*Batch, error) {
func (b *Batch) Upsert(id string, r search.Resource) error {
return b.withSizeLimit(func() error {
return b.batch.Index(id, r)
return b.indexResource(id, r)
})
}
func (b *Batch) Move(id string, parentID string, targetPath string) error {
// indexResource prepares r for bleve (resolving json tags and splicing in
// type-specific adaptations via the mapping package) and appends it to the
// batch under id.
func (b *Batch) indexResource(id string, r search.Resource) error {
doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
if err != nil {
return err
}
return b.batch.Index(id, doc)
}
func (b *Batch) Move(id, parentID, location string) error {
return b.withSizeLimit(func() error {
rootResource, err := searchResourceByID(id, b.index)
if err != nil {
return err
}
currentPath := rootResource.Path
nextPath := utils.MakeRelativePath(targetPath)
nextPath := utils.MakeRelativePath(location)
rootResource.Path = nextPath
rootResource.Name = path.Base(nextPath)
@@ -70,7 +82,7 @@ func (b *Batch) Move(id string, parentID string, targetPath string) error {
for _, resource := range resources {
resource.Hidden = search.IsHidden(resource.Path)
if err := b.batch.Index(resource.ID, resource); err != nil {
if err := b.indexResource(resource.ID, *resource); err != nil {
return err
}
if b.batch.Size() >= b.size {
@@ -92,7 +104,7 @@ func (b *Batch) Delete(id string) error {
}
for _, resource := range affectedResources {
if err := b.batch.Index(resource.ID, resource); err != nil {
if err := b.indexResource(resource.ID, *resource); err != nil {
return err
}
if b.batch.Size() >= b.size {
@@ -114,7 +126,7 @@ func (b *Batch) Restore(id string) error {
}
for _, resource := range affectedResources {
if err := b.batch.Index(resource.ID, resource); err != nil {
if err := b.indexResource(resource.ID, *resource); err != nil {
return err
}
if b.batch.Size() >= b.size {
services/search/pkg/bleve/bleve.go
+11 -129
View File
@@ -1,18 +1,13 @@
package bleve
import (
"reflect"
"regexp"
"strings"
"time"
bleveSearch "github.com/blevesearch/bleve/v2/search"
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"google.golang.org/protobuf/types/known/timestamppb"
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
@@ -75,132 +70,19 @@ func getFragmentValue(m bleveSearch.FieldFragmentMap, key string, idx int) strin
return val[idx]
}
func getAudioValue[T any](fields map[string]any) *T {
if !strings.HasPrefix(getFieldValue[string](fields, "MimeType"), "audio/") {
return nil
}
var audio = newPointerOfType[T]()
if ok := unmarshalInterfaceMap(audio, fields, "audio."); ok {
return audio
}
return nil
}
func getImageValue[T any](fields map[string]any) *T {
var image = newPointerOfType[T]()
if ok := unmarshalInterfaceMap(image, fields, "image."); ok {
return image
}
return nil
}
func getLocationValue[T any](fields map[string]any) *T {
var location = newPointerOfType[T]()
if ok := unmarshalInterfaceMap(location, fields, "location."); ok {
return location
}
return nil
}
func getPhotoValue[T any](fields map[string]any) *T {
var photo = newPointerOfType[T]()
if ok := unmarshalInterfaceMap(photo, fields, "photo."); ok {
return photo
}
return nil
}
func newPointerOfType[T any]() *T {
t := reflect.TypeOf((*T)(nil)).Elem()
ptr := reflect.New(t).Interface()
return ptr.(*T)
}
func unmarshalInterfaceMap(out any, flatMap map[string]any, prefix string) bool {
nonEmpty := false
obj := reflect.ValueOf(out).Elem()
for i := 0; i < obj.NumField(); i++ {
field := obj.Field(i)
structField := obj.Type().Field(i)
mapKey := prefix + getFieldName(structField)
if value, ok := flatMap[mapKey]; ok {
if field.Kind() == reflect.Ptr {
alloc := reflect.New(field.Type().Elem())
elemType := field.Type().Elem()
// convert time strings from index for search requests
if elemType == reflect.TypeOf(timestamppb.Timestamp{}) {
if strValue, ok := value.(string); ok {
if parsedTime, err := time.Parse(time.RFC3339, strValue); err == nil {
alloc.Elem().Set(reflect.ValueOf(*timestamppb.New(parsedTime)))
field.Set(alloc)
nonEmpty = true
}
}
continue
}
// convert time strings from index for libregraph structs when updating resources
if elemType == reflect.TypeOf(time.Time{}) {
if strValue, ok := value.(string); ok {
if parsedTime, err := time.Parse(time.RFC3339, strValue); err == nil {
alloc.Elem().Set(reflect.ValueOf(parsedTime))
field.Set(alloc)
nonEmpty = true
}
}
continue
}
alloc.Elem().Set(reflect.ValueOf(value).Convert(elemType))
field.Set(alloc)
nonEmpty = true
}
}
}
return nonEmpty
}
func getFieldName(structField reflect.StructField) string {
tag := structField.Tag.Get("json")
if tag == "" {
return structField.Name
}
return strings.Split(tag, ",")[0]
// hitToFacet builds a search Entity facet *T from a bleve hit's fields under the
// given key prefix. Nil when the hit has no such fields.
func hitToFacet[T any](fields map[string]any, prefix string) *T {
return mapping.DeserializeAt[T](fields, prefix)
}
// matchToResource reconstructs a search.Resource from a bleve hit. Used by
// the Move / Delete / Restore / Purge paths that round-trip a record through
// the index. Always returns a non-nil *Resource: Deserialize is fail-soft
// for per-field parse errors, so corrupted hit values surface as zero
// values on individual fields instead of dropping the whole record.
func matchToResource(match *bleveSearch.DocumentMatch) *search.Resource {
return &search.Resource{
ID: getFieldValue[string](match.Fields, "ID"),
RootID: getFieldValue[string](match.Fields, "RootID"),
Path: getFieldValue[string](match.Fields, "Path"),
ParentID: getFieldValue[string](match.Fields, "ParentID"),
Type: uint64(getFieldValue[float64](match.Fields, "Type")),
Deleted: getFieldValue[bool](match.Fields, "Deleted"),
Hidden: getFieldValue[bool](match.Fields, "Hidden"),
Document: content.Document{
Name: getFieldValue[string](match.Fields, "Name"),
Title: getFieldValue[string](match.Fields, "Title"),
Size: uint64(getFieldValue[float64](match.Fields, "Size")),
Mtime: getFieldValue[string](match.Fields, "Mtime"),
MimeType: getFieldValue[string](match.Fields, "MimeType"),
Content: getFieldValue[string](match.Fields, "Content"),
Tags: getFieldSliceValue[string](match.Fields, "Tags"),
Favorites: getFieldSliceValue[string](match.Fields, "Favorites"),
Audio: getAudioValue[libregraph.Audio](match.Fields),
Image: getImageValue[libregraph.Image](match.Fields),
Location: getLocationValue[libregraph.GeoCoordinates](match.Fields),
Photo: getPhotoValue[libregraph.Photo](match.Fields),
},
}
return mapping.Deserialize[search.Resource](match.Fields)
}
func escapeQuery(s string) string {
services/search/pkg/bleve/geo_verify_test.go
+127
View File
@@ -0,0 +1,127 @@
package bleve_test
import (
bleveSearch "github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/search/query"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
// geoFixture builds an in-memory bleve index with a single resource carrying
// the given lon/lat/alt, indexed through the full bleve pipeline.
func geoFixture(lon, lat, alt float64) bleveSearch.Index {
idxMapping, err := bleve.NewMapping()
Expect(err).ToNot(HaveOccurred())
idx, err := bleveSearch.NewMemOnly(idxMapping)
Expect(err).ToNot(HaveOccurred())
r := search.Resource{
ID: "x",
Document: content.Document{
Name: "team.jpg",
Location: &libregraph.GeoCoordinates{
Longitude: &lon,
Latitude: &lat,
Altitude: &alt,
},
},
}
doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
Expect(err).ToNot(HaveOccurred())
Expect(idx.Index(r.ID, doc)).To(Succeed())
return idx
}
var _ = Describe("Location geo queries", func() {
// Every Location subfield (including altitude) must end up in hit.Fields
// when a Resource is indexed through the full bleve pipeline. This is the
// invariant the Move / Delete / Restore round-trip depends on.
It("round-trips every Location subfield into hit.Fields", func() {
idx := geoFixture(11.103870357204285, 49.48675890884328, 1047.7)
req := bleveSearch.NewSearchRequest(bleveSearch.NewMatchAllQuery())
req.Fields = []string{"*"}
res, err := idx.Search(req)
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits).ToNot(BeEmpty())
for _, k := range []string{"location.longitude", "location.latitude", "location.altitude"} {
Expect(res.Hits[0].Fields).To(HaveKey(k))
}
})
It("matches a latitude numeric range and misses outside it", func() {
idx := geoFixture(11.1, 49.48, 1000)
min, max := 49.0, 50.0
incl := true
q := query.NewNumericRangeInclusiveQuery(&min, &max, &incl, &incl)
q.SetField("location.latitude")
res, err := idx.Search(bleveSearch.NewSearchRequest(q))
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits).To(HaveLen(1))
lowMin, lowMax := 0.0, 10.0
q2 := query.NewNumericRangeInclusiveQuery(&lowMin, &lowMax, &incl, &incl)
q2.SetField("location.latitude")
res, err = idx.Search(bleveSearch.NewSearchRequest(q2))
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits).To(BeEmpty())
})
It("matches a longitude numeric range", func() {
idx := geoFixture(11.1, 49.48, 1000)
min, max := 11.0, 12.0
incl := true
q := query.NewNumericRangeInclusiveQuery(&min, &max, &incl, &incl)
q.SetField("location.longitude")
res, err := idx.Search(bleveSearch.NewSearchRequest(q))
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits).To(HaveLen(1))
})
It("matches an altitude lower-bound range and misses above it", func() {
idx := geoFixture(11.1, 49.48, 1047.7)
min := 1000.0
incl := true
q := query.NewNumericRangeInclusiveQuery(&min, nil, &incl, nil)
q.SetField("location.altitude")
res, err := idx.Search(bleveSearch.NewSearchRequest(q))
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits).To(HaveLen(1))
highMin := 2000.0
q2 := query.NewNumericRangeInclusiveQuery(&highMin, nil, &incl, nil)
q2.SetField("location.altitude")
res, err = idx.Search(bleveSearch.NewSearchRequest(q2))
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits).To(BeEmpty())
})
It("matches a geo-distance query near the point and misses far away", func() {
// Nuremberg-ish coordinates.
idx := geoFixture(11.103870357204285, 49.48675890884328, 1047.7)
// 10 km radius around the indexed point should match.
near := query.NewGeoDistanceQuery(11.103870357204285, 49.48675890884328, "10km")
near.SetField("location" + mapping.GeopointSuffix)
res, err := idx.Search(bleveSearch.NewSearchRequest(near))
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits).To(HaveLen(1))
// Far away (Berlin, ~400 km) with a 10 km radius should miss.
far := query.NewGeoDistanceQuery(13.404954, 52.520008, "10km")
far.SetField("location" + mapping.GeopointSuffix)
res, err = idx.Search(bleveSearch.NewSearchRequest(far))
Expect(err).ToNot(HaveOccurred())
Expect(res.Hits).To(BeEmpty())
})
})
services/search/pkg/bleve/index.go
+24 -73
View File
@@ -2,33 +2,26 @@ package bleve
import (
"errors"
"fmt"
"math"
"path/filepath"
"reflect"
"github.com/blevesearch/bleve/v2"
"github.com/blevesearch/bleve/v2/analysis/analyzer/custom"
"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
regexpCharFilter "github.com/blevesearch/bleve/v2/analysis/char/regexp"
"github.com/blevesearch/bleve/v2/analysis/char/regexp"
"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/single"
"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
"github.com/blevesearch/bleve/v2/mapping"
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
const (
wildcardSuffix = ".wildcard"
indexVersion = "v2"
)
func indexPath(root string) string {
return filepath.Join(root, "bleve-"+indexVersion)
}
func NewIndex(root string) (bleve.Index, error) {
destination := indexPath(root)
destination := filepath.Join(root, fmt.Sprintf("bleve-v%d", search.SchemaVersion))
index, err := bleve.Open(destination)
if errors.Is(err, bleve.ErrorIndexPathDoesNotExist) {
indexMapping, err := NewMapping()
@@ -47,77 +40,35 @@ func NewIndex(root string) (bleve.Index, error) {
}
func NewMapping() (mapping.IndexMapping, error) {
words := func() *mapping.FieldMapping {
fm := bleve.NewTextFieldMapping()
fm.Analyzer = "lowercaseWords"
return fm
resourceType := reflect.TypeFor[search.Resource]()
overrides := search.Resource{}.SearchFieldOverrides()
if err := searchmapping.Validate(resourceType, overrides); err != nil {
return nil, err
}
whole := func(field string) *mapping.FieldMapping {
fm := bleve.NewTextFieldMapping()
fm.Analyzer = "lowercaseKeyword"
fm.IncludeInAll = false
fm.Name = field + wildcardSuffix
return fm
docMapping, err := searchmapping.BleveBuildMapping(resourceType, overrides)
if err != nil {
return nil, err
}
lowercaseMapping := bleve.NewTextFieldMapping()
lowercaseMapping.IncludeInAll = false
lowercaseMapping.Analyzer = "lowercaseKeyword"
contentMapping := words()
contentMapping.IncludeInAll = false
docMapping := bleve.NewDocumentMapping()
docMapping.AddFieldMappingsAt("Name",
words(),
whole("Name"),
)
docMapping.AddFieldMappingsAt("Title",
words(),
whole("Title"),
)
docMapping.AddFieldMappingsAt("Tags", lowercaseMapping)
docMapping.AddFieldMappingsAt("Favorites", lowercaseMapping)
docMapping.AddFieldMappingsAt("Content", contentMapping)
indexMapping := bleve.NewIndexMapping()
indexMapping.DefaultAnalyzer = keyword.Name
indexMapping.DefaultMapping = docMapping
err := indexMapping.AddCustomCharFilter("dotToSpace",
map[string]any{
"type": regexpCharFilter.Name,
"regexp": `\.`,
"replace": " ",
},
)
// words: split into lowercased words, a dot is a word boundary too so that
// "report" finds "Report.txt"; no stemming, a name is not prose
err = indexMapping.AddCustomCharFilter("dot_to_space", map[string]any{
"type": regexp.Name,
"regexp": `\.`,
"replace": " ",
})
if err != nil {
return nil, err
}
err = indexMapping.AddCustomAnalyzer("lowercaseWords",
err = indexMapping.AddCustomAnalyzer(searchmapping.WordsAnalyzer,
map[string]any{
"type": custom.Name,
"char_filters": []string{"dotToSpace"},
"tokenizer": unicode.Name,
"token_filters": []string{
lowercase.Name,
},
},
)
if err != nil {
return nil, err
}
err = indexMapping.AddCustomAnalyzer("lowercaseKeyword",
map[string]any{
"type": custom.Name,
"tokenizer": single.Name,
"token_filters": []string{
lowercase.Name,
},
"type": custom.Name,
"char_filters": []string{"dot_to_space"},
"tokenizer": unicode.Name,
"token_filters": []string{lowercase.Name},
},
)
if err != nil {
services/search/pkg/bleve/index_test.go
+5 -3
View File
@@ -1,12 +1,14 @@
package bleve_test
import (
"fmt"
"path/filepath"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
var _ = Describe("Index", func() {
@@ -18,8 +20,8 @@ var _ = Describe("Index", func() {
Expect(err).ToNot(HaveOccurred())
DeferCleanup(index.Close)
Expect(index.Name()).To(Equal(filepath.Join(root, "bleve-v2")))
Expect(filepath.Join(root, "bleve-v2")).To(BeADirectory())
Expect(index.Name()).To(Equal(filepath.Join(root, fmt.Sprintf("bleve-v%d", search.SchemaVersion))))
Expect(filepath.Join(root, fmt.Sprintf("bleve-v%d", search.SchemaVersion))).To(BeADirectory())
})
It("opens the index that is already there", func() {
@@ -33,7 +35,7 @@ var _ = Describe("Index", func() {
Expect(err).ToNot(HaveOccurred())
DeferCleanup(reopened.Close)
Expect(reopened.Name()).To(Equal(filepath.Join(root, "bleve-v2")))
Expect(reopened.Name()).To(Equal(filepath.Join(root, fmt.Sprintf("bleve-v%d", search.SchemaVersion))))
})
})
})
services/search/pkg/bleve/mtime_test.go
+41
View File
@@ -0,0 +1,41 @@
package bleve_test
import (
"time"
"github.com/opencloud-eu/opencloud/pkg/conversions"
bleveSearch "github.com/blevesearch/bleve/v2"
bquery "github.com/blevesearch/bleve/v2/search/query"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/services/search/pkg/bleve"
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
// Mtime is typed as a date, so range queries are chronological, not a
// lexicographic keyword compare.
var _ = Describe("Mtime date range", func() {
It("compares chronologically, not lexicographically", func() {
m, err := bleve.NewMapping()
Expect(err).ToNot(HaveOccurred())
idx, err := bleveSearch.NewMemOnly(m)
Expect(err).ToNot(HaveOccurred())
r := search.Resource{ID: "x", Document: content.Document{Name: "f", Mtime: conversions.ToPointer(time.Date(2026, 3, 15, 12, 0, 0, 123456789, time.UTC))}}
doc, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
Expect(err).ToNot(HaveOccurred())
Expect(idx.Index(r.ID, doc)).To(Succeed())
hits := func(qs string) uint64 {
res, err := idx.Search(bleveSearch.NewSearchRequest(bquery.NewQueryStringQuery(qs)))
Expect(err).ToNot(HaveOccurred(), qs)
return res.Total
}
Expect(hits(`Mtime:>"2026-01-01T00:00:00Z"`)).To(Equal(uint64(1)), "in-range")
Expect(hits(`Mtime:>"2026-06-01T00:00:00Z"`)).To(Equal(uint64(0)), "out-of-range")
})
})
services/search/pkg/command/server.go
+2 -1
View File
@@ -119,7 +119,8 @@ func Server(cfg *config.Config) *cobra.Command {
return fmt.Errorf("failed to create OpenSearch client: %w", err)
}
openSearchBackend, err := opensearch.NewBackend(cfg.Engine.OpenSearch.ResourceIndex.Name, client)
indexName := opensearch.VersionedIndexName(cfg.Engine.OpenSearch.ResourceIndex.Name)
openSearchBackend, err := opensearch.NewBackend(indexName, client)
if err != nil {
return fmt.Errorf("failed to create OpenSearch backend: %w", err)
}
services/search/pkg/config/engine.go
+1 -1
View File
@@ -25,7 +25,7 @@ type EngineOpenSearch struct {
// EngineOpenSearchResourceIndex defines the OpenSearch index for resources
type EngineOpenSearchResourceIndex struct {
Name string `yaml:"name" env:"SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME" desc:"The name of the OpenSearch index for resources." introductionVersion:"4.0.0"`
Name string `yaml:"name" env:"SEARCH_ENGINE_OPEN_SEARCH_RESOURCE_INDEX_NAME" desc:"The base name of the OpenSearch index for resources. The running index is suffixed with the current schema version." introductionVersion:"4.0.0"`
}
// EngineOpenSearchClient configures the OpenSearch client
services/search/pkg/content/basic.go
+2 -2
View File
@@ -3,9 +3,9 @@ package content
import (
"context"
"encoding/json"
"time"
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/opencloud-eu/opencloud/pkg/conversions"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/reva/v2/pkg/tags"
"github.com/opencloud-eu/reva/v2/pkg/utils"
@@ -54,7 +54,7 @@ func (b Basic) Extract(_ context.Context, ri *storageProvider.ResourceInfo) (Doc
}
if ri.Mtime != nil {
doc.Mtime = utils.TSToTime(ri.Mtime).UTC().Format(time.RFC3339Nano)
doc.Mtime = conversions.ToPointer(utils.TSToTime(ri.Mtime).UTC())
}
return doc, nil
services/search/pkg/content/basic_test.go
+8 -4
View File
@@ -1,6 +1,10 @@
package content_test
import (
"time"
"github.com/opencloud-eu/opencloud/pkg/conversions"
"context"
"encoding/json"
@@ -69,11 +73,11 @@ var _ = Describe("Basic", func() {
It("RFC3339 mtime", func() {
for _, data := range []struct {
second uint64
expect string
expect *time.Time
}{
{second: 4000, expect: "1970-01-01T01:06:40Z"},
{second: 3000, expect: "1970-01-01T00:50:00Z"},
{expect: ""},
{second: 4000, expect: conversions.ToPointer(time.Unix(4000, 0).UTC())},
{second: 3000, expect: conversions.ToPointer(time.Unix(3000, 0).UTC())},
{},
} {
ri := &storageProvider.ResourceInfo{}
services/search/pkg/content/content.go
+9 -8
View File
@@ -2,6 +2,7 @@ package content
import (
"strings"
"time"
"github.com/bbalet/stopwords"
libregraph "github.com/opencloud-eu/libre-graph-api-go"
@@ -14,14 +15,14 @@ func init() {
// Document wraps all resource meta fields,
// it is used as a content extraction result.
type Document struct {
Title string
Name string
Content string
Size uint64
Mtime string `json:"Mtime,omitempty"`
MimeType string
Tags []string
Favorites []string
Title string `json:"Title"`
Name string `json:"Name"`
Content string `json:"Content"`
Size uint64 `json:"Size"`
Mtime *time.Time `json:"Mtime,omitempty"`
MimeType string `json:"MimeType"`
Tags []string `json:"Tags"`
Favorites []string `json:"Favorites"`
Audio *libregraph.Audio `json:"audio,omitempty"`
Image *libregraph.Image `json:"image,omitempty"`
Location *libregraph.GeoCoordinates `json:"location,omitempty"`
services/search/pkg/mapping/bleve.go
+142
View File
@@ -0,0 +1,142 @@
package mapping
import (
"fmt"
"reflect"
"github.com/blevesearch/bleve/v2"
bleveMapping "github.com/blevesearch/bleve/v2/mapping"
)
// BleveBuildMapping builds a bleve DocumentMapping for t by walking the
// struct via reflection. Field names come from json tags; overrides are
// keyed by those names (or dotted paths for nested fields).
//
// The returned mapping references the words analyzer for Fulltext fields;
// the caller registers it on the enclosing IndexMapping.
func BleveBuildMapping(t reflect.Type, overrides map[string]FieldOpts) (*bleveMapping.DocumentMapping, error) {
return buildBleveDocMapping(t, overrides, "")
}
func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix string) (*bleveMapping.DocumentMapping, error) {
doc := bleve.NewDocumentMapping()
err := walkFields(t, func(fi fieldInfo) error {
key := fi.Name
if prefix != "" {
key = prefix + "." + fi.Name
}
opts := overrides[key]
fieldType := opts.Type
if fieldType == "" {
fieldType = inferType(fi.GoField.Type)
}
if fieldType == TypeObject {
sub := structType(fi.GoField.Type)
if sub == nil {
return fmt.Errorf("mapping: object type on non-struct field %q", key)
}
subDoc, err := buildBleveDocMapping(sub, overrides, key)
if err != nil {
return err
}
doc.AddSubDocumentMapping(fi.Name, subDoc)
return nil
}
if fieldType == TypeGeopoint {
// Keep the facet object, add a sibling _geopoint field (see GeopointSuffix).
sub := structType(fi.GoField.Type)
if sub == nil {
return fmt.Errorf("mapping: geopoint type on non-struct field %q", key)
}
subDoc, err := buildBleveDocMapping(sub, overrides, key)
if err != nil {
return err
}
doc.AddSubDocumentMapping(fi.Name, subDoc)
doc.AddFieldMappingsAt(fi.Name+GeopointSuffix, bleve.NewGeoPointFieldMapping())
return nil
}
if fieldType == TypeKeyword || fieldType == TypePath {
// bleve has no path tokenizer, so a path is a plain keyword here.
base := bleveKeywordMapping(fieldType, opts)
doc.AddFieldMappingsAt(fi.Name, base)
if opts.caseInsensitive() {
doc.AddFieldMappingsAt(fi.Name+LowercaseSuffix, searchSibling(base))
}
if fieldType == TypeKeyword && opts.wordBroken() {
words := searchSibling(base)
words.Analyzer = WordsAnalyzer
doc.AddFieldMappingsAt(fi.Name+WordsSuffix, words)
}
return nil
}
fm, err := bleveFieldMapping(fieldType, opts)
if err != nil {
return fmt.Errorf("mapping: field %q: %w", key, err)
}
doc.AddFieldMappingsAt(fi.Name, fm)
return nil
})
return doc, err
}
// bleveKeywordMapping is a case-preserving keyword field; path fields stay out
// of _all by default.
func bleveKeywordMapping(fieldType string, opts FieldOpts) *bleveMapping.FieldMapping {
fm := bleve.NewKeywordFieldMapping()
switch {
case opts.IncludeInAll != nil:
fm.IncludeInAll = *opts.IncludeInAll
case fieldType == TypePath:
fm.IncludeInAll = false
}
return fm
}
// searchSibling derives a search-only shadow of a keyword/path field from its
// base mapping (the _lowercase and _words siblings): indexed but never stored,
// kept out of _all, and without doc values, since the case-preserved base field
// is what we return and aggregate on.
func searchSibling(base *bleveMapping.FieldMapping) *bleveMapping.FieldMapping {
fm := *base
fm.Store = false
fm.IncludeInAll = false
fm.DocValues = false
return &fm
}
func bleveFieldMapping(fieldType string, opts FieldOpts) (*bleveMapping.FieldMapping, error) {
switch fieldType {
case TypeWildcard:
// bleve has no wildcard type; fall back to keyword-ish text.
fieldType = TypeKeyword
fallthrough
case TypeKeyword, TypeFulltext:
fm := bleve.NewTextFieldMapping()
if fieldType == TypeFulltext {
fm.Analyzer = WordsAnalyzer
}
switch {
case opts.IncludeInAll != nil:
fm.IncludeInAll = *opts.IncludeInAll
case fieldType == TypeFulltext:
fm.IncludeInAll = false
}
return fm, nil
case TypeNumeric:
return bleve.NewNumericFieldMapping(), nil
case TypeBool:
return bleve.NewBooleanFieldMapping(), nil
case TypeDatetime:
return bleve.NewDateTimeFieldMapping(), nil
case TypeGeopoint:
return bleve.NewGeoPointFieldMapping(), nil
case "":
return nil, fmt.Errorf("no type inferred and no override")
}
return nil, fmt.Errorf("unsupported type %q", fieldType)
}
services/search/pkg/mapping/bleve_test.go
+138
View File
@@ -0,0 +1,138 @@
package mapping
import (
"reflect"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
type bleveDoc struct {
Name string `json:"Name"`
Content string `json:"Content"`
Tags []string `json:"Tags"`
Size uint64 `json:"Size"`
Deleted bool `json:"Deleted"`
CreatedAt time.Time `json:"CreatedAt"`
Nested *nested `json:"nested,omitempty"`
}
type nested struct {
Artist string `json:"artist"`
Year int `json:"year"`
}
var _ = Describe("BleveBuildMapping", func() {
It("falls back to text for wildcard fields", func() {
// bleve wildcard falls back to keyword-ish text (bleve has no wildcard type).
type doc struct {
Mime string `json:"mime"`
}
dm, err := BleveBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"mime": {Type: TypeWildcard}})
Expect(err).ToNot(HaveOccurred())
fms := dm.Properties["mime"].Fields
Expect(fms).To(HaveLen(1))
Expect(fms[0].Type).To(Equal("text"), "wildcard should map to text")
})
DescribeTable("infers field types",
func(field, wantType string) {
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil)
Expect(err).ToNot(HaveOccurred())
prop := dm.Properties[field]
Expect(prop).ToNot(BeNil(), "missing property %q", field)
Expect(prop.Fields).ToNot(BeEmpty(), "%q: no field mappings", field)
Expect(prop.Fields[0].Type).To(Equal(wantType), "%q type", field)
},
Entry("Name", "Name", "text"),
Entry("Content", "Content", "text"),
Entry("Tags", "Tags", "text"),
Entry("Size", "Size", "number"),
Entry("Deleted", "Deleted", "boolean"),
Entry("CreatedAt", "CreatedAt", "datetime"),
)
It("maps a nested struct as a sub-document", func() {
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil)
Expect(err).ToNot(HaveOccurred())
sub := dm.Properties["nested"]
Expect(sub).ToNot(BeNil(), "missing nested sub-document")
Expect(sub.Properties["artist"]).ToNot(BeNil())
Expect(sub.Properties["year"]).ToNot(BeNil())
Expect(sub.Properties["artist"].Fields[0].Type).To(Equal("text"), "nested.artist")
Expect(sub.Properties["year"].Fields[0].Type).To(Equal("number"), "nested.year")
})
It("applies field overrides", func() {
True, False := true, false
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), map[string]FieldOpts{
"Name": {CaseInsensitive: &True},
"Content": {Type: TypeFulltext},
"Tags": {CaseInsensitive: &True, IncludeInAll: &False},
})
Expect(err).ToNot(HaveOccurred())
// Name: case-preserved base keyword + lowercased sibling.
Expect(dm.Properties["Name"]).ToNot(BeNil(), "Name base field")
Expect(dm.Properties["Name"].Fields[0].Analyzer).To(Equal("keyword"), "Name base is a keyword")
Expect(dm.Properties["Name"].Fields[0].Store).To(BeTrue(), "Name base is stored (returned)")
Expect(dm.Properties["Name_lowercase"]).ToNot(BeNil(), "Name_lowercase sibling")
// The sibling is a search-only shadow: indexed but never stored, kept out
// of _all, no doc values (the base is what we return).
sibling := dm.Properties["Name_lowercase"].Fields[0]
Expect(sibling.Index).To(BeTrue(), "Name_lowercase is indexed")
Expect(sibling.Store).To(BeFalse(), "Name_lowercase is not stored")
Expect(sibling.IncludeInAll).To(BeFalse(), "Name_lowercase is out of _all")
Expect(sibling.DocValues).To(BeFalse(), "Name_lowercase has no doc values")
contentField := dm.Properties["Content"].Fields[0]
Expect(contentField.Analyzer).To(Equal(WordsAnalyzer), "Content analyzer")
Expect(contentField.IncludeInAll).To(BeFalse(), "Content IncludeInAll should default to false for the fulltext type")
// Tags: base + lowercased sibling, both honoring the IncludeInAll override.
Expect(dm.Properties["Tags"].Fields[0].IncludeInAll).To(BeFalse(), "Tags base IncludeInAll honored")
Expect(dm.Properties["Tags_lowercase"].Fields[0].IncludeInAll).To(BeFalse(), "Tags sibling IncludeInAll honored")
})
It("gives a keyword its lowercase and words siblings by default", func() {
dm, err := BleveBuildMapping(reflect.TypeFor[bleveDoc](), nil)
Expect(err).ToNot(HaveOccurred())
// the base stays a keyword, the words go to a search-only sibling
Expect(dm.Properties["Name"].Fields[0].Analyzer).To(Equal("keyword"), "Name base stays a keyword")
Expect(dm.Properties["Name"].Fields[0].Store).To(BeTrue(), "Name base is stored (returned)")
Expect(dm.Properties["Name_lowercase"].Fields[0].Analyzer).To(Equal("keyword"), "Name_lowercase stays a keyword")
words := dm.Properties["Name_words"].Fields[0]
Expect(words.Analyzer).To(Equal(WordsAnalyzer), "Name_words is split into words")
Expect(words.Store).To(BeFalse(), "Name_words is not stored")
Expect(words.IncludeInAll).To(BeFalse(), "Name_words is out of _all")
})
It("builds an object sub-document plus a geopoint sibling", func() {
type geoDoc struct {
Location *struct {
Lon *float64 `json:"longitude,omitempty"`
Lat *float64 `json:"latitude,omitempty"`
Alt *float64 `json:"altitude,omitempty"`
} `json:"location,omitempty"`
}
dm, err := BleveBuildMapping(reflect.TypeFor[geoDoc](), map[string]FieldOpts{
"location": {Type: TypeGeopoint},
})
Expect(err).ToNot(HaveOccurred())
// Original facet stays as an object sub-document with numeric
// sub-properties - for data retrieval via hit.Fields and ordinary
// numeric queries.
loc := dm.Properties["location"]
Expect(loc).ToNot(BeNil(), "location sub-document missing")
Expect(loc.Fields).To(BeEmpty(), "location should not carry field mappings directly")
for _, sub := range []string{"longitude", "latitude", "altitude"} {
prop, ok := loc.Properties[sub]
Expect(ok).To(BeTrue(), "missing sub-field %q under location", sub)
Expect(prop.Fields).ToNot(BeEmpty(), "location.%s Fields", sub)
Expect(prop.Fields[0].Type).To(Equal("number"), "location.%s type", sub)
}
// Sibling geopoint at "<name>_geopoint" for geo-distance queries.
sibling := dm.Properties["location"+GeopointSuffix]
Expect(sibling).ToNot(BeNil(), "location%s missing", GeopointSuffix)
Expect(sibling.Fields).ToNot(BeEmpty(), "location%s Fields", GeopointSuffix)
Expect(sibling.Fields[0].Type).To(Equal("geopoint"), "location%s type", GeopointSuffix)
})
})
services/search/pkg/mapping/casing.go
+100
View File
@@ -0,0 +1,100 @@
package mapping
import (
"reflect"
"strings"
)
// addSearchSiblings writes the _lowercase and _words siblings next to their
// base values, for every keyword/path field of t that has them (see
// SearchSiblings).
func addSearchSiblings(m map[string]any, t reflect.Type, overrides map[string]FieldOpts) {
for key, siblings := range SearchSiblings(t, overrides) {
parent, leaf, ok := resolveLeaf(m, key)
if !ok {
continue
}
if siblings.Lowercase {
addLowercaseSibling(parent, leaf)
}
if siblings.Words {
addWordsSibling(parent, leaf)
}
}
}
// Siblings says which search-only siblings a field carries.
type Siblings struct {
Lowercase bool
Words bool
}
// SearchSiblings lists the fields of t (json names, nested as parent.child)
// that carry a _lowercase or _words sibling, from the effective field type and
// the overrides. It is the one place that decides, the renderers, the
// document writer and the query lowering all follow it.
func SearchSiblings(t reflect.Type, overrides map[string]FieldOpts) map[string]Siblings {
out := map[string]Siblings{}
for key, goType := range collectFields(t, "") {
opts := overrides[key]
eff := opts.Type
if eff == "" {
eff = inferType(goType)
}
if eff != TypeKeyword && eff != TypePath {
continue
}
siblings := Siblings{
Lowercase: opts.caseInsensitive(),
Words: eff == TypeKeyword && opts.wordBroken(),
}
if siblings.Lowercase || siblings.Words {
out[key] = siblings
}
}
return out
}
// addWordsSibling copies the value to a <leaf>_words sibling; the words
// analyzer does the splitting and lowercasing. No-op for non-strings.
func addWordsSibling(parent map[string]any, leaf string) {
switch v := parent[leaf].(type) {
case string, []any, []string:
parent[leaf+WordsSuffix] = v
}
}
func resolveLeaf(m map[string]any, dottedPath string) (map[string]any, string, bool) {
parts := strings.Split(dottedPath, ".")
parent := m
for _, p := range parts[:len(parts)-1] {
next, ok := parent[p].(map[string]any)
if !ok {
return nil, "", false
}
parent = next
}
return parent, parts[len(parts)-1], true
}
// addLowercaseSibling writes a <leaf>_lowercase sibling; no-op for non-strings.
func addLowercaseSibling(parent map[string]any, leaf string) {
switch v := parent[leaf].(type) {
case string:
parent[leaf+LowercaseSuffix] = strings.ToLower(v)
case []any:
out := make([]any, 0, len(v))
for _, item := range v {
if s, ok := item.(string); ok {
out = append(out, strings.ToLower(s))
}
}
parent[leaf+LowercaseSuffix] = out
case []string:
out := make([]string, len(v))
for i, s := range v {
out[i] = strings.ToLower(s)
}
parent[leaf+LowercaseSuffix] = out
}
}
services/search/pkg/mapping/casing_test.go
+76
View File
@@ -0,0 +1,76 @@
package mapping
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("PrepareForIndex casing", func() {
It("adds lowercased siblings for CaseInsensitive keyword and path fields", func() {
True := true
type doc struct {
Name string `json:"Name"`
Path string `json:"Path"`
Tags []string `json:"Tags"`
}
d := doc{Name: "Report FINAL", Path: "/Foo/Bar", Tags: []string{"Work", "Urgent"}}
m, err := PrepareForIndex(d, map[string]FieldOpts{
"Name": {CaseInsensitive: &True},
"Path": {Type: TypePath, CaseInsensitive: &True},
"Tags": {CaseInsensitive: &True},
})
Expect(err).ToNot(HaveOccurred())
// Originals stay for the case-preserved base fields and the cascade.
Expect(m["Name"]).To(Equal("Report FINAL"))
Expect(m["Path"]).To(Equal("/Foo/Bar"))
Expect(m["Name_lowercase"]).To(Equal("report final"))
Expect(m["Path_lowercase"]).To(Equal("/foo/bar"))
Expect(m["Tags_lowercase"]).To(Equal([]any{"work", "urgent"}))
})
It("copies the value to a words sibling by default", func() {
type doc struct {
Name string `json:"Name"`
}
m, err := PrepareForIndex(doc{Name: "Report FINAL"}, nil)
Expect(err).ToNot(HaveOccurred())
// the analyzer splits and lowercases, the value goes over as is
Expect(m["Name"]).To(Equal("Report FINAL"))
Expect(m["Name_lowercase"]).To(Equal("report final"))
Expect(m["Name_words"]).To(Equal("Report FINAL"))
})
It("writes the lowercased sibling by default", func() {
type doc struct {
ID string `json:"ID"`
}
m, err := PrepareForIndex(doc{ID: "ABC"}, nil)
Expect(err).ToNot(HaveOccurred())
Expect(m["ID"+LowercaseSuffix]).To(Equal("abc"))
})
It("writes no sibling with CaseInsensitive off", func() {
False := false
type doc struct {
ID string `json:"ID"`
}
m, err := PrepareForIndex(doc{ID: "ABC"}, map[string]FieldOpts{"ID": {CaseInsensitive: &False}})
Expect(err).ToNot(HaveOccurred())
Expect(m).ToNot(HaveKey("ID" + LowercaseSuffix))
})
It("writes an empty sibling for an empty array, like a non-empty one", func() {
True := true
type doc struct {
Tags []string `json:"Tags"`
}
m, err := PrepareForIndex(doc{Tags: []string{}}, map[string]FieldOpts{
"Tags": {CaseInsensitive: &True},
})
Expect(err).ToNot(HaveOccurred())
Expect(m).To(HaveKey("Tags" + LowercaseSuffix))
Expect(m["Tags"+LowercaseSuffix]).To(BeEmpty())
})
})
services/search/pkg/mapping/deserialize.go
+180
View File
@@ -0,0 +1,180 @@
package mapping
import (
"fmt"
"reflect"
"time"
"google.golang.org/protobuf/types/known/timestamppb"
)
// Deserialize builds a *T from bleve's flat hit.Fields map (json-tag keys,
// "parent.child" for nested pointers). Used to rebuild a search Resource from a
// hit. Fail-soft: an unparseable field stays at its zero value.
func Deserialize[T any](fields map[string]any) *T {
t := reflect.TypeFor[T]()
if t.Kind() != reflect.Struct {
panic(fmt.Sprintf("mapping: Deserialize requires a struct type, got %v", t))
}
out := reflect.New(t)
fillStruct(out.Elem(), fields, "", setValue)
return out.Interface().(*T)
}
// DeserializeAt is Deserialize scoped to a dotted prefix, used to rebuild one
// search-result facet (e.g. "audio"). Returns nil when nothing matched, so the
// caller can leave the enclosing pointer nil.
func DeserializeAt[T any](fields map[string]any, prefix string) *T {
t := reflect.TypeFor[T]()
if t.Kind() != reflect.Struct {
panic(fmt.Sprintf("mapping: DeserializeAt requires a struct type, got %v", t))
}
out := reflect.New(t)
if !fillStruct(out.Elem(), fields, prefix, setValue) {
return nil
}
return out.Interface().(*T)
}
// fillStruct walks v's exported fields, reading values from the flat fields map
// (json-tag keys, "parent.child" for nested pointers). setLeaf converts each raw
// value into a leaf, which is what lets the any-valued and string-valued
// deserializers share one walker. Returns true if any leaf was populated.
func fillStruct[V any](v reflect.Value, fields map[string]V, prefix string, setLeaf func(reflect.Value, V) error) bool {
t := v.Type()
touched := false
for i := 0; i < t.NumField(); i++ {
fi := resolveField(t.Field(i))
if fi.Skip {
continue
}
fv := v.Field(i)
if fi.Embedded {
// Embedded *struct: allocate, recurse, keep the pointer only if set.
if fv.Kind() == reflect.Ptr {
if fv.Type().Elem().Kind() != reflect.Struct || !fv.CanSet() {
continue
}
alloc := reflect.New(fv.Type().Elem())
if fillStruct(alloc.Elem(), fields, prefix, setLeaf) {
fv.Set(alloc)
touched = true
}
continue
}
if fv.Kind() == reflect.Struct && fillStruct(fv, fields, prefix, setLeaf) {
touched = true
}
continue
}
key := fi.Name
if prefix != "" {
key = prefix + "." + fi.Name
}
// Pointer to a nested (non-time) struct: recurse, keep the pointer only
// if a field was populated.
if fv.Kind() == reflect.Ptr {
if elem := fv.Type().Elem(); elem.Kind() == reflect.Struct && elem != timeType && elem != timestampType {
alloc := reflect.New(elem)
if fillStruct(alloc.Elem(), fields, key, setLeaf) {
fv.Set(alloc)
touched = true
}
continue
}
}
// Value nested struct (e.g. a tagged embedded struct): recurse under key.
if fv.Kind() == reflect.Struct && fv.Type() != timeType && fv.Type() != timestampType {
if fillStruct(fv, fields, key, setLeaf) {
touched = true
}
continue
}
if raw, ok := fields[key]; ok && setLeaf(fv, raw) == nil {
touched = true
}
}
return touched
}
// setValue writes raw (an any value from a bleve hit) into v, converting to the
// field's type. Returns an error on nil or a type mismatch.
func setValue(v reflect.Value, raw any) error {
if v.Kind() == reflect.Ptr {
alloc := reflect.New(v.Type().Elem())
if err := setValue(alloc.Elem(), raw); err != nil {
return err
}
v.Set(alloc)
return nil
}
if v.Type() == timeType || v.Type() == timestampType {
t, ok := parseTime(raw)
if !ok {
return fmt.Errorf("not an RFC3339 time: %v", raw)
}
setParsedTime(v, t)
return nil
}
if v.Kind() == reflect.Slice {
return setSlice(v, raw)
}
rv := reflect.ValueOf(raw)
if !rv.IsValid() {
return fmt.Errorf("nil value for %s", v.Type())
}
if !rv.Type().ConvertibleTo(v.Type()) {
return fmt.Errorf("cannot convert %s to %s", rv.Type(), v.Type())
}
v.Set(rv.Convert(v.Type()))
return nil
}
func setSlice(v reflect.Value, raw any) error {
items, ok := raw.([]any)
if !ok {
// bleve unwraps single-element slices; re-wrap here.
items = []any{raw}
}
// Compact in place with a single MakeSlice: unparseable elements are
// dropped, Slice(0, j) trims the tail.
out := reflect.MakeSlice(v.Type(), len(items), len(items))
j := 0
for _, item := range items {
if setValue(out.Index(j), item) == nil {
j++
}
}
if j == 0 {
return fmt.Errorf("no slice elements set from %T", raw)
}
v.Set(out.Slice(0, j))
return nil
}
func parseTime(raw any) (time.Time, bool) {
s, ok := raw.(string)
if !ok {
return time.Time{}, false
}
t, err := time.Parse(time.RFC3339, s)
if err != nil {
return time.Time{}, false
}
return t, true
}
// setParsedTime writes t into v, which must be a time.Time or
// timestamppb.Timestamp field.
func setParsedTime(v reflect.Value, t time.Time) {
if v.Type() == timestampType {
v.Set(reflect.ValueOf(*timestamppb.New(t)))
return
}
v.Set(reflect.ValueOf(t))
}
services/search/pkg/mapping/deserialize_string.go
+82
View File
@@ -0,0 +1,82 @@
package mapping
import (
"fmt"
"reflect"
"strconv"
"strings"
"time"
)
// DeserializeStringsAt is DeserializeAt for a string-valued map (e.g. CS3
// ArbitraryMetadata), parsing each string into the field's Go type via strconv/
// time.Parse. Used to build a graph DriveItem facet. Returns nil when nothing
// under the prefix matched.
func DeserializeStringsAt[T any](fields map[string]string, prefix string) *T {
t := reflect.TypeFor[T]()
if t.Kind() != reflect.Struct {
panic(fmt.Sprintf("mapping: DeserializeStringsAt requires a struct type, got %v", t))
}
out := reflect.New(t)
// Callers pass a flat-key prefix with a trailing dot (e.g.
// "libre.graph.audio."); fillStruct joins segments with ".", so drop it.
if !fillStruct(out.Elem(), fields, strings.TrimSuffix(prefix, "."), setValueFromString) {
return nil
}
return out.Interface().(*T)
}
// setValueFromString parses the string raw into v's Go type via strconv/
// time.Parse, returning a descriptive error on failure.
func setValueFromString(v reflect.Value, raw string) error {
if v.Kind() == reflect.Ptr {
alloc := reflect.New(v.Type().Elem())
if err := setValueFromString(alloc.Elem(), raw); err != nil {
return err
}
v.Set(alloc)
return nil
}
if v.Type() == timeType || v.Type() == timestampType {
t, err := time.Parse(time.RFC3339, raw)
if err != nil {
return fmt.Errorf("parse time %q: %w", raw, err)
}
setParsedTime(v, t)
return nil
}
switch v.Kind() {
case reflect.String:
v.SetString(raw)
return nil
case reflect.Bool:
b, err := strconv.ParseBool(raw)
if err != nil {
return fmt.Errorf("parse bool %q: %w", raw, err)
}
v.SetBool(b)
return nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
n, err := strconv.ParseInt(raw, 10, v.Type().Bits())
if err != nil {
return fmt.Errorf("parse int %q: %w", raw, err)
}
v.SetInt(n)
return nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
n, err := strconv.ParseUint(raw, 10, v.Type().Bits())
if err != nil {
return fmt.Errorf("parse uint %q: %w", raw, err)
}
v.SetUint(n)
return nil
case reflect.Float32, reflect.Float64:
f, err := strconv.ParseFloat(raw, v.Type().Bits())
if err != nil {
return fmt.Errorf("parse float %q: %w", raw, err)
}
v.SetFloat(f)
return nil
}
return fmt.Errorf("unsupported target kind %s", v.Kind())
}
services/search/pkg/mapping/deserialize_string_test.go
+98
View File
@@ -0,0 +1,98 @@
package mapping
import (
"reflect"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"google.golang.org/protobuf/types/known/timestamppb"
)
type stringFacet struct {
Artist *string `json:"artist,omitempty"`
Year *int32 `json:"year,omitempty"`
Duration *int64 `json:"duration,omitempty"`
Rating *float64 `json:"rating,omitempty"`
Explicit *bool `json:"explicit,omitempty"`
Taken *time.Time `json:"takenDateTime,omitempty"`
}
var _ = Describe("DeserializeStringsAt", func() {
It("errors on an unsupported target kind", func() {
v := reflect.New(reflect.TypeFor[[]int]()).Elem() // settable slice
Expect(setValueFromString(v, "x")).To(HaveOccurred(), "expected error for unsupported target kind (slice)")
})
It("parses basic types", func() {
r := DeserializeStringsAt[stringFacet](map[string]string{
"libre.graph.audio.artist": "Queen",
"libre.graph.audio.year": "1975",
"libre.graph.audio.duration": "354000",
"libre.graph.audio.rating": "4.9",
"libre.graph.audio.explicit": "true",
"libre.graph.audio.takenDateTime": "2024-01-02T03:04:05Z",
}, "libre.graph.audio.")
Expect(r).ToNot(BeNil())
Expect(r.Artist).ToNot(BeNil())
Expect(*r.Artist).To(Equal("Queen"))
Expect(r.Year).ToNot(BeNil())
Expect(*r.Year).To(Equal(int32(1975)))
Expect(r.Duration).ToNot(BeNil())
Expect(*r.Duration).To(Equal(int64(354000)))
Expect(r.Rating).ToNot(BeNil())
Expect(*r.Rating).To(Equal(4.9))
Expect(r.Explicit).ToNot(BeNil())
Expect(*r.Explicit).To(BeTrue())
Expect(r.Taken).ToNot(BeNil())
Expect(r.Taken.Equal(time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC))).To(BeTrue(), "Taken: %#v", r.Taken)
})
It("returns nil when nothing matches the prefix", func() {
r := DeserializeStringsAt[stringFacet](map[string]string{
"libre.graph.image.width": "1200",
}, "libre.graph.audio.")
Expect(r).To(BeNil())
})
It("parses into a timestamppb.Timestamp", func() {
type photoFacet struct {
Taken *timestamppb.Timestamp `json:"takenDateTime,omitempty"`
}
r := DeserializeStringsAt[photoFacet](map[string]string{
"libre.graph.photo.takenDateTime": "2024-05-06T07:08:09Z",
}, "libre.graph.photo.")
Expect(r).ToNot(BeNil())
Expect(r.Taken).ToNot(BeNil())
want := time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC)
Expect(r.Taken.AsTime().Equal(want)).To(BeTrue(), "Taken: got %v, want %v", r.Taken.AsTime(), want)
})
It("is fail-soft on malformed fields", func() {
// A single malformed field (year is unparseable as int) must not drop
// the whole facet. The bad field stays at zero value, the rest of the
// facet still populates. Mirrors the bleve-hit Deserialize behavior.
r := DeserializeStringsAt[stringFacet](map[string]string{
"libre.graph.audio.artist": "Iron Maiden",
"libre.graph.audio.year": "not-a-number",
"libre.graph.audio.duration": "354000",
"libre.graph.audio.explicit": "not-a-bool",
"libre.graph.audio.rating": "4.9",
}, "libre.graph.audio.")
Expect(r).ToNot(BeNil())
Expect(r.Artist).ToNot(BeNil())
Expect(*r.Artist).To(Equal("Iron Maiden"), "Artist should still be populated")
Expect(r.Duration).ToNot(BeNil())
Expect(*r.Duration).To(Equal(int64(354000)), "Duration should still be populated")
Expect(r.Rating).ToNot(BeNil())
Expect(*r.Rating).To(Equal(4.9), "Rating should still be populated")
Expect(r.Year).To(BeNil(), "Year should stay nil for bad int")
Expect(r.Explicit).To(BeNil(), "Explicit should stay nil for bad bool")
})
It("panics for a non-struct T", func() {
Expect(func() {
DeserializeStringsAt[int](nil, "")
}).To(Panic())
})
})
services/search/pkg/mapping/deserialize_test.go
+139
View File
@@ -0,0 +1,139 @@
package mapping
import (
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"google.golang.org/protobuf/types/known/timestamppb"
)
type Leaf struct {
Name string `json:"Name"`
Size uint64 `json:"Size"`
Deleted bool `json:"Deleted"`
Tags []string `json:"Tags"`
Favorites []string `json:"Favorites"`
}
type audio struct {
Artist *string `json:"artist,omitempty"`
Year *int32 `json:"year,omitempty"`
}
type photo struct {
Taken *timestamppb.Timestamp `json:"takenDateTime,omitempty"`
Mtime *time.Time `json:"mtime,omitempty"`
}
type embedded struct {
Leaf
Audio *audio `json:"audio,omitempty"`
Photo *photo `json:"photo,omitempty"`
}
// taggedEmbedded embeds Leaf with a json tag, so encoding/json nests it under
// "leaf" rather than flattening it onto the parent.
type taggedEmbedded struct {
Leaf `json:"leaf"`
Top string `json:"top"`
}
var _ = Describe("Deserialize", func() {
It("panics for a non-struct type at a prefix", func() {
Expect(func() {
_ = DeserializeAt[int](map[string]any{}, "")
}).To(Panic())
})
It("panics for a non-struct T", func() {
Expect(func() {
Deserialize[int](nil)
}).To(Panic())
})
It("deserializes leaf fields", func() {
r := Deserialize[Leaf](map[string]any{
"Name": "n",
"Size": float64(42),
"Deleted": true,
})
Expect(r.Name).To(Equal("n"))
Expect(r.Size).To(Equal(uint64(42)))
Expect(r.Deleted).To(BeTrue())
})
It("coerces a scalar into a slice field", func() {
r := Deserialize[Leaf](map[string]any{
"Tags": "single",
"Favorites": []any{"a", "b"},
})
Expect(r.Tags).To(Equal([]string{"single"}))
Expect(r.Favorites).To(Equal([]string{"a", "b"}))
})
It("parses timestamps", func() {
r := Deserialize[embedded](map[string]any{
"photo.takenDateTime": "2024-01-02T03:04:05Z",
"photo.mtime": "2024-05-06T07:08:09Z",
})
Expect(r.Photo).ToNot(BeNil())
Expect(r.Photo.Taken).ToNot(BeNil())
expected := time.Date(2024, 1, 2, 3, 4, 5, 0, time.UTC)
Expect(r.Photo.Taken.AsTime().Equal(expected)).To(BeTrue(), "Taken: got %v, want %v", r.Photo.Taken.AsTime(), expected)
Expect(r.Photo.Mtime).ToNot(BeNil())
Expect(r.Photo.Mtime.Equal(time.Date(2024, 5, 6, 7, 8, 9, 0, time.UTC))).To(BeTrue(), "Mtime: %v", r.Photo.Mtime)
})
It("is fail-soft on malformed values", func() {
// Malformed values (type mismatch, unparseable time) leave the
// affected field at its zero value instead of dropping the whole
// record. Matches the pre-refactor getFieldValue behavior so
// matchToResource never returns nil on a corrupted hit.
r := Deserialize[embedded](map[string]any{
"Name": "n",
"Size": "not-a-number", // wrong type
"Deleted": true,
"photo.takenDateTime": "not-an-rfc3339-time",
"photo.mtime": "2024-05-06T07:08:09Z",
})
Expect(r).ToNot(BeNil())
Expect(r.Name).To(Equal("n"))
Expect(r.Size).To(Equal(uint64(0)), "Size should stay zero on mismatch")
Expect(r.Deleted).To(BeTrue())
Expect(r.Photo).ToNot(BeNil(), "Photo should be populated because Mtime parsed ok")
Expect(r.Photo.Taken).To(BeNil(), "Taken should stay nil for unparseable time")
Expect(r.Photo.Mtime).ToNot(BeNil(), "Mtime should be parsed")
})
It("returns nil when nothing matches the prefix", func() {
r := DeserializeAt[audio](map[string]any{"Name": "n"}, "audio")
Expect(r).To(BeNil())
})
It("returns a value when the prefix matches", func() {
r := DeserializeAt[audio](map[string]any{
"audio.artist": "A",
"audio.year": float64(2024), // setValue: pointer + numeric convert
}, "audio")
Expect(r).ToNot(BeNil())
Expect(r.Artist).ToNot(BeNil())
Expect(*r.Artist).To(Equal("A"))
Expect(r.Year).ToNot(BeNil())
Expect(*r.Year).To(Equal(int32(2024)))
})
It("nests a tagged embedded struct instead of flattening it", func() {
// matches encoding/json: a tagged embedded struct is read under its tag.
r := Deserialize[taggedEmbedded](map[string]any{
"leaf.Name": "n",
"top": "t",
})
Expect(r.Leaf.Name).To(Equal("n"))
Expect(r.Top).To(Equal("t"))
// the flattened top-level key must NOT populate the nested field.
flat := Deserialize[taggedEmbedded](map[string]any{"Name": "flat"})
Expect(flat.Leaf.Name).To(BeEmpty())
})
})
services/search/pkg/mapping/fieldindex.go
+36
View File
@@ -0,0 +1,36 @@
package mapping
import (
"reflect"
"strings"
)
// FieldNameIndex maps a lowercased field path to the real field name for every
// field of t, recursing into nested facets (photo.cameraMake, ...). Names come
// from json tags, so it is backend-neutral; the query layer resolves KQL keys
// case-insensitively against it.
func FieldNameIndex(t reflect.Type, overrides map[string]FieldOpts) map[string]string {
out := map[string]string{}
var walk func(t reflect.Type, prefix string)
walk = func(t reflect.Type, prefix string) {
_ = walkFields(t, func(fi fieldInfo) error {
name := fi.Name
if prefix != "" {
name = prefix + "." + fi.Name
}
out[strings.ToLower(name)] = name
fieldType := overrides[name].Type
if fieldType == "" {
fieldType = inferType(fi.GoField.Type)
}
// recurse into nested facets; time.Time is a struct too but a leaf.
if sub := structType(fi.GoField.Type); sub != nil && fieldType != TypeDatetime {
walk(sub, name)
}
return nil
})
}
walk(t, "")
return out
}
services/search/pkg/mapping/fieldindex_test.go
+101
View File
@@ -0,0 +1,101 @@
package mapping_test
import (
"reflect"
"strings"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
func resourceFieldIndex() map[string]string {
return mapping.FieldNameIndex(
reflect.TypeFor[search.Resource](),
search.Resource{}.SearchFieldOverrides(),
)
}
// resolve looks up the lowercased key in the index, falling back to the key.
func resolve(idx map[string]string, key string) string {
if v, ok := idx[strings.ToLower(key)]; ok {
return v
}
return key
}
// NestInner is embedded with a json tag below, so it must nest, not flatten.
type NestInner struct {
A string `json:"A"`
}
type taggedOuter struct {
NestInner `json:"inner"`
Top string `json:"top"`
}
var _ = Describe("FieldNameIndex", func() {
It("resolves top-level fields case-insensitively", func() {
idx := resourceFieldIndex()
for in, want := range map[string]string{
"rootid": "RootID", "ROOTID": "RootID", "RootID": "RootID",
"name": "Name", "NAME": "Name",
"mimetype": "MimeType", "MimeType": "MimeType",
"tags": "Tags", "favorites": "Favorites",
"mtime": "Mtime", "parentid": "ParentID", "id": "ID",
} {
Expect(resolve(idx, in)).To(Equal(want), "resolve(%q)", in)
}
})
// Facet sub-fields are lowerCamelCase in the index (from libregraph json
// tags); the derived index resolves them case-insensitively.
It("resolves facet sub-fields case-insensitively", func() {
idx := resourceFieldIndex()
for in, want := range map[string]string{
// case-insensitive: same field, different casings
"photo.cameramake": "photo.cameraMake",
"photo.CAMERAMAKE": "photo.cameraMake",
// a representative sub-field across each facet
"photo.takendatetime": "photo.takenDateTime",
"audio.artist": "audio.artist",
"audio.albumartist": "audio.albumArtist",
"image.width": "image.width",
"location.latitude": "location.latitude",
} {
Expect(resolve(idx, in)).To(Equal(want), "resolve(%q)", in)
}
})
It("passes unknown keys through unchanged", func() {
idx := resourceFieldIndex()
Expect(resolve(idx, "nope.field")).To(Equal("nope.field"))
Expect(resolve(idx, "custom")).To(Equal("custom"))
})
// All top-level fields are covered from one derived source, so both
// backends resolve them the same way.
It("covers all top-level fields", func() {
idx := resourceFieldIndex()
for in, want := range map[string]string{
"rootid": "RootID", "path": "Path", "id": "ID", "name": "Name",
"size": "Size", "mtime": "Mtime", "type": "Type",
"content": "Content", "hidden": "Hidden", "tags": "Tags",
"favorites": "Favorites",
} {
Expect(resolve(idx, in)).To(Equal(want), "derived should cover %q", in)
}
})
// A json-tagged embedded struct nests under its tag in the derived index
// too (walkFields must match encoding/json), so its fields are "inner.A",
// not "A".
It("nests json-tagged embedded structs", func() {
idx := mapping.FieldNameIndex(reflect.TypeFor[taggedOuter](), nil)
Expect(resolve(idx, "inner.a")).To(Equal("inner.A")) // nested under the tag
Expect(resolve(idx, "top")).To(Equal("top"))
Expect(resolve(idx, "a")).To(Equal("a")) // not flattened: bare "a" is not a key
})
})
services/search/pkg/mapping/fillstruct_test.go
+106
View File
@@ -0,0 +1,106 @@
package mapping
import (
"errors"
"reflect"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
// The walker is shared by both deserializers, so its structural behavior
// (flattening embedded structs, recursing into nested pointers, joining the
// prefix, keeping a pointer only when a field was set, fail-soft skipping) is
// tested here once against a trivial string setter instead of twice through
// Deserialize and DeserializeStringsAt.
// FsEmbVal/FsEmbPtr are exported so their embedded field is exported; an
// unexported embedded type would be skipped by resolveField.
type FsEmbVal struct {
EV string `json:"ev"`
}
type FsEmbPtr struct {
EP string `json:"ep"`
}
type fsNested struct {
N string `json:"n"`
}
type fsRoot struct {
FsEmbVal // embedded value struct: fields promoted
*FsEmbPtr // embedded pointer struct: allocated on demand
Leaf string `json:"leaf"`
Nested *fsNested `json:"nested"` // nested pointer: recursed under "nested."
}
var errBadLeaf = errors.New("bad leaf")
// fsSet writes raw into a string field; the "BAD" sentinel simulates a parse
// failure so the fail-soft path can be exercised.
func fsSet(v reflect.Value, raw string) error {
if raw == "BAD" {
return errBadLeaf
}
v.SetString(raw)
return nil
}
var _ = Describe("fillStruct", func() {
fill := func(fields map[string]string, prefix string) (fsRoot, bool) {
var root fsRoot
touched := fillStruct(reflect.ValueOf(&root).Elem(), fields, prefix, fsSet)
return root, touched
}
It("flattens embedded and recurses nested", func() {
root, touched := fill(map[string]string{
"leaf": "L",
"ev": "EV",
"ep": "EP",
"nested.n": "N",
}, "")
Expect(touched).To(BeTrue())
Expect(root.Leaf).To(Equal("L"))
Expect(root.EV).To(Equal("EV"), "embedded value not promoted")
Expect(root.FsEmbPtr).ToNot(BeNil(), "embedded pointer not allocated")
Expect(root.EP).To(Equal("EP"))
Expect(root.Nested).ToNot(BeNil(), "nested pointer not populated")
Expect(root.Nested.N).To(Equal("N"))
})
It("leaves touched false and pointers nil when nothing matches", func() {
root, touched := fill(map[string]string{"other": "x"}, "")
Expect(touched).To(BeFalse())
Expect(root.Nested).To(BeNil(), "Nested should stay nil")
Expect(root.FsEmbPtr).To(BeNil(), "embedded pointer should stay nil")
})
It("joins the prefix arg with the field name", func() {
root, touched := fill(map[string]string{
"pre.leaf": "L",
"pre.nested.n": "N",
}, "pre")
Expect(touched).To(BeTrue())
Expect(root.Leaf).To(Equal("L"))
Expect(root.Nested).ToNot(BeNil())
Expect(root.Nested.N).To(Equal("N"))
})
It("is fail-soft: errored leaf stays zero, walk continues", func() {
root, touched := fill(map[string]string{
"leaf": "BAD",
"ev": "EV",
}, "")
Expect(touched).To(BeTrue(), "expected touched because ev was set")
Expect(root.Leaf).To(BeEmpty(), "errored leaf should stay zero")
Expect(root.EV).To(Equal("EV"), "walk should continue past the error")
})
It("drops the embedded pointer when its only field errors", func() {
root, touched := fill(map[string]string{"ep": "BAD"}, "")
Expect(touched).To(BeFalse())
Expect(root.FsEmbPtr).To(BeNil(), "embedded pointer should stay nil on error")
})
})
services/search/pkg/mapping/geo.go
+51
View File
@@ -0,0 +1,51 @@
package mapping
import "strings"
// GeopointSuffix is appended to a field's name to produce the sibling key
// that carries the geo_point / bleve-geopoint representation of the
// original facet. For example, a libregraph "location" object with
// longitude / latitude / altitude is preserved as-is under "location" (for
// data retrieval and numeric queries) while "location_geopoint" carries
// the {lat, lon} form the geo indices understand.
const GeopointSuffix = "_geopoint"
// addGeopointSiblings walks the overrides; for each TypeGeopoint entry at
// a dotted path (e.g. "location" or "journey.start") it writes a sibling
// under the suffixed key with the {lat, lon} form both bleve's
// ExtractGeoPoint and OpenSearch's geo_point parser accept. The original
// facet object stays untouched so downstream code still sees the full
// libregraph shape (including altitude).
func addGeopointSiblings(m map[string]any, overrides map[string]FieldOpts) {
for key, opts := range overrides {
if opts.Type == TypeGeopoint {
addGeopointSibling(m, key)
}
}
}
// addGeopointSibling resolves dottedPath within m and, if the target is a
// libregraph-shaped geo object (with numeric "longitude" and "latitude"),
// writes the `{lat, lon}` sibling at the same level under the suffixed key.
func addGeopointSibling(m map[string]any, dottedPath string) {
parts := strings.Split(dottedPath, ".")
parent := m
for _, p := range parts[:len(parts)-1] {
next, ok := parent[p].(map[string]any)
if !ok {
return
}
parent = next
}
leaf := parts[len(parts)-1]
obj, ok := parent[leaf].(map[string]any)
if !ok {
return
}
lon, hasLon := obj["longitude"].(float64)
lat, hasLat := obj["latitude"].(float64)
if !hasLon || !hasLat {
return
}
parent[leaf+GeopointSuffix] = map[string]any{"lat": lat, "lon": lon}
}
services/search/pkg/mapping/geo_test.go
+143
View File
@@ -0,0 +1,143 @@
package mapping
import (
"reflect"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("BuildMapping geopoint errors", func() {
// build-mapping error paths: an override that doesn't fit the Go field.
It("errors for geopoint on a non-struct field on both backends", func() {
type doc struct {
Name string `json:"name"`
}
// Geopoint on a non-struct field must error on both backends.
_, err := BleveBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"name": {Type: TypeGeopoint}})
Expect(err).To(HaveOccurred(), "bleve: expected error for geopoint on string field")
_, err = OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{"name": {Type: TypeGeopoint}})
Expect(err).To(HaveOccurred(), "opensearch: expected error for geopoint on string field")
})
})
var _ = Describe("addGeopointSibling", func() {
It("bails without panicking when the intermediate is missing", func() {
m := map[string]any{"journey": "not-a-map"}
addGeopointSibling(m, "journey.start") // must bail, not panic
Expect(m).ToNot(HaveKey("journey.start" + GeopointSuffix))
})
})
var _ = Describe("PrepareForIndex geopoint", func() {
It("adds a geopoint sibling", func() {
type geoDoc struct {
Location *struct {
Longitude *float64 `json:"longitude,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
Altitude *float64 `json:"altitude,omitempty"`
} `json:"location,omitempty"`
}
lon, lat, alt := 11.1, 49.4, 1047.7
doc := geoDoc{Location: &struct {
Longitude *float64 `json:"longitude,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
Altitude *float64 `json:"altitude,omitempty"`
}{Longitude: &lon, Latitude: &lat, Altitude: &alt}}
m, err := PrepareForIndex(doc, map[string]FieldOpts{
"location": {Type: TypeGeopoint},
})
Expect(err).ToNot(HaveOccurred())
// Original location object stays untouched (full libregraph shape).
orig, ok := m["location"].(map[string]any)
Expect(ok).To(BeTrue(), "expected location object preserved, got %T", m["location"])
Expect(orig["longitude"]).To(Equal(lon))
Expect(orig["latitude"]).To(Equal(lat))
Expect(orig["altitude"]).To(Equal(alt))
// Sibling location_geopoint has {lat, lon} for the geo indices.
gp, ok := m["location"+GeopointSuffix].(map[string]any)
Expect(ok).To(BeTrue(), "expected location_geopoint sibling, got %T", m["location"+GeopointSuffix])
Expect(gp["lat"]).To(Equal(lat))
Expect(gp["lon"]).To(Equal(lon))
})
It("skips incomplete geopoints", func() {
type geoDoc struct {
Location *struct {
Altitude *float64 `json:"altitude,omitempty"`
} `json:"location,omitempty"`
}
alt := 100.0
doc := geoDoc{Location: &struct {
Altitude *float64 `json:"altitude,omitempty"`
}{Altitude: &alt}}
m, err := PrepareForIndex(doc, map[string]FieldOpts{
"location": {Type: TypeGeopoint},
})
Expect(err).ToNot(HaveOccurred())
// Original stays (altitude alone is still useful metadata).
Expect(m).To(HaveKey("location"), "location should still be present when only altitude is set")
// No sibling without both lon and lat.
Expect(m).ToNot(HaveKey("location"+GeopointSuffix), "no sibling expected")
})
It("writes no sibling without the geopoint override", func() {
type geoDoc struct {
Location *struct {
Longitude *float64 `json:"longitude,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
} `json:"location,omitempty"`
}
lon, lat := 11.1, 49.4
doc := geoDoc{Location: &struct {
Longitude *float64 `json:"longitude,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
}{Longitude: &lon, Latitude: &lat}}
m, err := PrepareForIndex(doc, nil)
Expect(err).ToNot(HaveOccurred())
Expect(m).ToNot(HaveKey("location"+GeopointSuffix), "no sibling expected without override")
})
It("handles nested geopoints via the dotted-path walker", func() {
// journey.start and journey.end - two geopoints in the same facet,
// demonstrating the dotted-path walker.
type geo struct {
Longitude *float64 `json:"longitude,omitempty"`
Latitude *float64 `json:"latitude,omitempty"`
}
type journey struct {
Start *geo `json:"start,omitempty"`
End *geo `json:"end,omitempty"`
}
type doc struct {
Journey *journey `json:"journey,omitempty"`
}
slon, slat := 11.0, 49.0
elon, elat := 13.4, 52.5
d := doc{Journey: &journey{
Start: &geo{Longitude: &slon, Latitude: &slat},
End: &geo{Longitude: &elon, Latitude: &elat},
}}
m, err := PrepareForIndex(d, map[string]FieldOpts{
"journey.start": {Type: TypeGeopoint},
"journey.end": {Type: TypeGeopoint},
})
Expect(err).ToNot(HaveOccurred())
j, ok := m["journey"].(map[string]any)
Expect(ok).To(BeTrue(), "journey not an object: %T", m["journey"])
startGp, ok := j["start"+GeopointSuffix].(map[string]any)
Expect(ok).To(BeTrue(), "journey.start sibling: %#v", j["start"+GeopointSuffix])
Expect(startGp["lat"]).To(Equal(slat))
Expect(startGp["lon"]).To(Equal(slon))
endGp, ok := j["end"+GeopointSuffix].(map[string]any)
Expect(ok).To(BeTrue(), "journey.end sibling: %#v", j["end"+GeopointSuffix])
Expect(endGp["lat"]).To(Equal(elat))
Expect(endGp["lon"]).To(Equal(elon))
})
})
services/search/pkg/mapping/infer.go
+121
View File
@@ -0,0 +1,121 @@
package mapping
import (
"reflect"
"strings"
"time"
"google.golang.org/protobuf/types/known/timestamppb"
)
var (
timeType = reflect.TypeFor[time.Time]()
timestampType = reflect.TypeFor[timestamppb.Timestamp]()
)
// deref unwraps pointer and slice types to their element type.
func deref(t reflect.Type) reflect.Type {
for t.Kind() == reflect.Ptr || t.Kind() == reflect.Slice {
t = t.Elem()
}
return t
}
// inferType returns the mapping type for a Go type. Pointers and slices are
// unwrapped to their element type. time.Time and timestamppb.Timestamp become
// datetime; other structs become object.
func inferType(t reflect.Type) string {
t = deref(t)
switch t.Kind() {
case reflect.String:
return TypeKeyword
case reflect.Bool:
return TypeBool
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
reflect.Float32, reflect.Float64:
return TypeNumeric
case reflect.Struct:
if t == timeType || t == timestampType {
return TypeDatetime
}
return TypeObject
}
return ""
}
// fieldInfo is the resolved metadata for one struct field.
type fieldInfo struct {
Name string
GoField reflect.StructField
Skip bool
Embedded bool
}
// resolveField resolves a struct field's json-tag name and skip/embed state.
func resolveField(sf reflect.StructField) fieldInfo {
if !sf.IsExported() {
return fieldInfo{Skip: true}
}
name := sf.Name
named := false
tag := sf.Tag.Get("json")
if tag != "" {
first, _, _ := strings.Cut(tag, ",")
if first == "-" {
return fieldInfo{Skip: true}
}
if first != "" {
name = first
named = true
}
}
return fieldInfo{
Name: name,
GoField: sf,
// An anonymous field is embedded (flattened onto the parent) only when it
// has no json tag name, matching encoding/json: a tag name nests it as a
// regular field instead.
Embedded: sf.Anonymous && !named,
}
}
// walkFields visits exported leaf fields of t, flattening embedded structs
// onto the enclosing level. It returns the first error returned by fn.
func walkFields(t reflect.Type, fn func(fi fieldInfo) error) error {
for t.Kind() == reflect.Ptr {
t = t.Elem()
}
if t.Kind() != reflect.Struct {
return nil
}
for i := 0; i < t.NumField(); i++ {
fi := resolveField(t.Field(i))
if fi.Skip {
continue
}
if fi.Embedded {
if err := walkFields(fi.GoField.Type, fn); err != nil {
return err
}
continue
}
if err := fn(fi); err != nil {
return err
}
}
return nil
}
// structType returns the underlying struct type, unwrapping pointers and
// slices. Returns nil when t is not a walkable struct (e.g. time.Time).
func structType(t reflect.Type) reflect.Type {
t = deref(t)
if t.Kind() != reflect.Struct {
return nil
}
if t == timeType || t == timestampType {
return nil
}
return t
}
services/search/pkg/mapping/infer_test.go
+100
View File
@@ -0,0 +1,100 @@
package mapping
import (
"reflect"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"google.golang.org/protobuf/types/known/timestamppb"
)
var _ = Describe("inferType", func() {
It("returns empty for unsupported kinds", func() {
Expect(inferType(reflect.TypeFor[map[string]int]())).To(BeEmpty(), "map")
Expect(inferType(reflect.TypeFor[chan int]())).To(BeEmpty(), "chan")
})
DescribeTable("infers the mapping type",
func(in any, want string) {
Expect(inferType(reflect.TypeOf(in))).To(Equal(want))
},
Entry("string", "", TypeKeyword),
Entry("*string", (*string)(nil), TypeKeyword),
Entry("[]string", []string(nil), TypeKeyword),
Entry("bool", false, TypeBool),
Entry("int", int(0), TypeNumeric),
Entry("int64", int64(0), TypeNumeric),
Entry("uint64", uint64(0), TypeNumeric),
Entry("float64", float64(0), TypeNumeric),
Entry("time.Time", time.Time{}, TypeDatetime),
Entry("*time.Time", (*time.Time)(nil), TypeDatetime),
Entry("*timestamppb.Timestamp", (*timestamppb.Timestamp)(nil), TypeDatetime),
Entry("struct", struct{ X int }{}, TypeObject),
Entry("*struct", (*struct{ X int })(nil), TypeObject),
)
})
var _ = Describe("resolveField", func() {
type S struct {
Exported string `json:"exp"`
Renamed string `json:"renamed,omitempty"`
NoTag string
OmitOnly string `json:",omitempty"`
Skipped string `json:"-"`
unexported string //nolint:unused
}
st := reflect.TypeFor[S]()
DescribeTable("resolves the field name and skip flag",
func(fieldIdx int, wantName string, wantSkip bool) {
fi := resolveField(st.Field(fieldIdx))
Expect(fi.Skip).To(Equal(wantSkip), "field %d skip", fieldIdx)
if !wantSkip {
Expect(fi.Name).To(Equal(wantName), "field %d name", fieldIdx)
}
},
Entry("exported json tag", 0, "exp", false),
Entry("renamed with omitempty", 1, "renamed", false),
Entry("no tag", 2, "NoTag", false),
Entry("omitempty only", 3, "OmitOnly", false),
Entry("json:- skipped", 4, "", true),
Entry("unexported skipped", 5, "", true),
)
})
var _ = Describe("walkFields", func() {
It("flattens embedded structs", func() {
type Inner struct {
A string `json:"a"`
B int `json:"b"`
}
type Outer struct {
Inner
C bool `json:"c"`
}
var names []string
err := walkFields(reflect.TypeFor[Outer](), func(fi fieldInfo) error {
names = append(names, fi.Name)
return nil
})
Expect(err).ToNot(HaveOccurred())
Expect(names).To(Equal([]string{"a", "b", "c"}))
})
})
var _ = Describe("structType", func() {
type S struct{ X int }
DescribeTable("resolves struct-ish types",
func(in reflect.Type, wantNil bool) {
got := structType(in)
Expect(got == nil).To(Equal(wantNil))
},
Entry("struct", reflect.TypeFor[S](), false),
Entry("*struct", reflect.TypeFor[*S](), false),
Entry("[]struct", reflect.TypeFor[[]S](), false),
Entry("time.Time", reflect.TypeFor[time.Time](), true),
Entry("string", reflect.TypeFor[string](), true),
)
})
services/search/pkg/mapping/mapping_suite_test.go
+13
View File
@@ -0,0 +1,13 @@
package mapping
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestMapping(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Mapping Suite")
}
services/search/pkg/mapping/opensearch.go
+136
View File
@@ -0,0 +1,136 @@
package mapping
import (
"fmt"
"maps"
"reflect"
)
// OpenSearchBuildMapping builds the OpenSearch "properties" map (the value
// of mappings.properties) for type t by walking the struct via reflection.
// Field names come from json tags; overrides are keyed by those names.
//
// The returned map contains plain JSON-friendly values (strings, bools,
// nested maps) and can be marshalled directly.
func OpenSearchBuildMapping(t reflect.Type, overrides map[string]FieldOpts) (map[string]any, error) {
return buildOpenSearchProperties(t, overrides, "")
}
func buildOpenSearchProperties(t reflect.Type, overrides map[string]FieldOpts, prefix string) (map[string]any, error) {
props := map[string]any{}
err := walkFields(t, func(fi fieldInfo) error {
key := fi.Name
if prefix != "" {
key = prefix + "." + fi.Name
}
opts := overrides[key]
fieldType := opts.Type
if fieldType == "" {
fieldType = inferType(fi.GoField.Type)
}
if fieldType == TypeObject {
sub := structType(fi.GoField.Type)
if sub == nil {
return fmt.Errorf("mapping: object type on non-struct field %q", key)
}
subProps, err := buildOpenSearchProperties(sub, overrides, key)
if err != nil {
return err
}
props[fi.Name] = map[string]any{"properties": subProps}
return nil
}
if fieldType == TypeGeopoint {
// Keep the facet object, add a sibling _geopoint field (see GeopointSuffix).
sub := structType(fi.GoField.Type)
if sub == nil {
return fmt.Errorf("mapping: geopoint type on non-struct field %q", key)
}
subProps, err := buildOpenSearchProperties(sub, overrides, key)
if err != nil {
return err
}
props[fi.Name] = map[string]any{"properties": subProps}
props[fi.Name+GeopointSuffix] = map[string]any{"type": "geo_point"}
return nil
}
if fieldType == TypeKeyword || fieldType == TypePath {
// path_hierarchy is case-preserving here; casing lives in the value.
m := map[string]any{"type": "keyword"}
if fieldType == TypePath {
m = map[string]any{"type": "text", "analyzer": "path_hierarchy"}
}
props[fi.Name] = m
if opts.caseInsensitive() {
// search-only, like the bleve sibling: never sorted or
// aggregated on, so no doc_values
sibling := maps.Clone(m)
if sibling["type"] == "keyword" {
sibling["doc_values"] = false
}
props[fi.Name+LowercaseSuffix] = sibling
}
if fieldType == TypeKeyword && opts.wordBroken() {
props[fi.Name+WordsSuffix] = map[string]any{"type": "text", "analyzer": WordsAnalyzer}
}
return nil
}
fm, err := openSearchFieldMapping(fieldType, fi.GoField.Type)
if err != nil {
return fmt.Errorf("mapping: field %q: %w", key, err)
}
props[fi.Name] = fm
return nil
})
return props, err
}
// openSearchFieldMapping handles the non-keyword/path types; keyword and path
// are emitted (with their cased forms) by buildOpenSearchProperties directly.
func openSearchFieldMapping(fieldType string, goType reflect.Type) (map[string]any, error) {
switch fieldType {
case TypeFulltext:
return map[string]any{
"type": "text",
"term_vector": "with_positions_offsets",
"analyzer": WordsAnalyzer,
}, nil
case TypeWildcard:
// OpenSearch stores wildcard fields with doc_values=false by
// default, so emit it explicitly to keep local and remote
// mappings in sync for the Apply comparison.
return map[string]any{"type": "wildcard", "doc_values": false}, nil
case TypeNumeric:
return map[string]any{"type": openSearchNumericType(goType)}, nil
case TypeBool:
return map[string]any{"type": "boolean"}, nil
case TypeDatetime:
return map[string]any{"type": "date"}, nil
case TypeGeopoint:
return map[string]any{"type": "geo_point"}, nil
case "":
return nil, fmt.Errorf("no type inferred and no override")
}
return nil, fmt.Errorf("unsupported type %q", fieldType)
}
// openSearchNumericType maps a Go numeric type to an OpenSearch numeric
// field type.
func openSearchNumericType(t reflect.Type) string {
t = deref(t)
switch t.Kind() {
case reflect.Float32:
return "float"
case reflect.Float64:
return "double"
case reflect.Int8, reflect.Uint8, reflect.Int16, reflect.Uint16:
return "short"
case reflect.Int32, reflect.Uint32:
return "integer"
}
return "long"
}
services/search/pkg/mapping/opensearch_test.go
+161
View File
@@ -0,0 +1,161 @@
package mapping
import (
"reflect"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
type osDoc struct {
ID string `json:"ID"`
Size uint64 `json:"Size"`
Deleted bool `json:"Deleted"`
CreatedAt time.Time `json:"CreatedAt"`
Rating float64 `json:"Rating"`
Nested *struct {
Artist string `json:"artist"`
Year int32 `json:"year"`
} `json:"nested,omitempty"`
}
var _ = Describe("OpenSearchBuildMapping", func() {
DescribeTable("maps numeric Go types to OpenSearch types",
func(field, wantType string) {
type doc struct {
A int8 `json:"a"`
B int16 `json:"b"`
C int32 `json:"c"`
D int64 `json:"d"`
E uint8 `json:"e"`
F uint64 `json:"f"`
G float32 `json:"g"`
H float64 `json:"h"`
}
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), nil)
Expect(err).ToNot(HaveOccurred())
Expect(props[field].(map[string]any)["type"]).To(Equal(wantType), "%s type", field)
},
Entry("int8 -> short", "a", "short"),
Entry("int16 -> short", "b", "short"),
Entry("int32 -> integer", "c", "integer"),
Entry("int64 -> long", "d", "long"),
Entry("uint8 -> short", "e", "short"),
Entry("uint64 -> long", "f", "long"),
Entry("float32 -> float", "g", "float"),
Entry("float64 -> double", "h", "double"),
)
DescribeTable("infers field types",
func(field, wantType string) {
props, err := OpenSearchBuildMapping(reflect.TypeFor[osDoc](), nil)
Expect(err).ToNot(HaveOccurred())
m, ok := props[field].(map[string]any)
Expect(ok).To(BeTrue(), "%s: missing or not a map: %#v", field, props[field])
Expect(m["type"]).To(Equal(wantType), "%s type", field)
},
Entry("ID -> keyword", "ID", "keyword"),
Entry("Size -> long", "Size", "long"),
Entry("Deleted -> boolean", "Deleted", "boolean"),
Entry("CreatedAt -> date", "CreatedAt", "date"),
Entry("Rating -> double", "Rating", "double"),
)
It("maps nested structs with their sub-properties", func() {
props, err := OpenSearchBuildMapping(reflect.TypeFor[osDoc](), nil)
Expect(err).ToNot(HaveOccurred())
nested, ok := props["nested"].(map[string]any)
Expect(ok).To(BeTrue(), "nested: not a map: %#v", props["nested"])
sub, ok := nested["properties"].(map[string]any)
Expect(ok).To(BeTrue(), "nested.properties: missing: %#v", nested)
artist, ok := sub["artist"].(map[string]any)
Expect(ok).To(BeTrue(), "nested.artist: %#v", sub)
Expect(artist["type"]).To(Equal("keyword"), "nested.artist.type")
year, ok := sub["year"].(map[string]any)
Expect(ok).To(BeTrue(), "nested.year: %#v", sub)
Expect(year["type"]).To(Equal("integer"), "nested.year.type (int32 -> integer expected)")
})
It("applies field overrides", func() {
True := true
type doc struct {
Name string `json:"Name"`
Content string `json:"Content"`
Path string `json:"Path"`
MimeType string `json:"MimeType"`
}
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{
"Name": {CaseInsensitive: &True},
"Content": {Type: TypeFulltext},
"Path": {Type: TypePath, CaseInsensitive: &True},
"MimeType": {Type: TypeWildcard},
})
Expect(err).ToNot(HaveOccurred())
// Name: case-preserved keyword base + lowercased search-only sibling.
Expect(props["Name"]).To(Equal(map[string]any{"type": "keyword"}))
Expect(props["Name_lowercase"]).To(Equal(map[string]any{"type": "keyword", "doc_values": false}))
content := props["Content"].(map[string]any)
Expect(content["type"]).To(Equal("text"), "Content: %#v", content)
Expect(content["term_vector"]).To(Equal("with_positions_offsets"), "Content: %#v", content)
Expect(content["analyzer"]).To(Equal(WordsAnalyzer), "Content uses the words analyzer, like bleve")
// Path: path_hierarchy base + lowercased sibling, both case-preserving.
Expect(props["Path"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"}))
Expect(props["Path_lowercase"]).To(Equal(map[string]any{"type": "text", "analyzer": "path_hierarchy"}))
mime := props["MimeType"].(map[string]any)
Expect(mime["type"]).To(Equal("wildcard"), "MimeType: %#v", mime)
})
It("gives a keyword its lowercase and words siblings by default", func() {
type doc struct {
Name string `json:"Name"`
}
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), nil)
Expect(err).ToNot(HaveOccurred())
// the base stays a keyword, the words go to their own sibling
Expect(props["Name"]).To(Equal(map[string]any{"type": "keyword"}))
Expect(props["Name_lowercase"]).To(Equal(map[string]any{"type": "keyword", "doc_values": false}))
Expect(props["Name_words"]).To(Equal(map[string]any{"type": "text", "analyzer": WordsAnalyzer}))
})
It("leaves a keyword one whole value with NoWordBreaker", func() {
True := true
type doc struct {
Tag string `json:"Tag"`
}
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{
"Tag": {NoWordBreaker: &True},
})
Expect(err).ToNot(HaveOccurred())
Expect(props).To(HaveKey("Tag_lowercase"))
Expect(props).ToNot(HaveKey("Tag_words"))
})
It("builds an object plus a geo_point sibling for geopoints", func() {
type doc struct {
Location *struct {
Lon float64 `json:"longitude"`
Lat float64 `json:"latitude"`
Alt float64 `json:"altitude"`
} `json:"location,omitempty"`
}
props, err := OpenSearchBuildMapping(reflect.TypeFor[doc](), map[string]FieldOpts{
"location": {Type: TypeGeopoint},
})
Expect(err).ToNot(HaveOccurred())
// Object for libregraph-shape data retrieval.
loc, ok := props["location"].(map[string]any)
Expect(ok).To(BeTrue(), "location: %#v", props["location"])
sub, ok := loc["properties"].(map[string]any)
Expect(ok).To(BeTrue(), "location should have numeric sub-properties, got %#v", loc)
for _, k := range []string{"longitude", "latitude", "altitude"} {
prop, ok := sub[k].(map[string]any)
Expect(ok).To(BeTrue(), "location.%s: %#v", k, sub[k])
Expect(prop["type"]).To(Equal("double"), "location.%s: %#v", k, sub[k])
}
// Sibling geo_point for spatial queries.
gp, ok := props["location"+GeopointSuffix].(map[string]any)
Expect(ok).To(BeTrue(), "location%s: %#v", GeopointSuffix, props["location"+GeopointSuffix])
Expect(gp["type"]).To(Equal("geo_point"), "location%s.type", GeopointSuffix)
})
})
services/search/pkg/mapping/opts.go
+57
View File
@@ -0,0 +1,57 @@
// Package mapping builds search index mappings for bleve and OpenSearch from
// a Go struct via reflection. Field names come from json tags; the caller
// provides overrides for fields that need a specific type or analyzer.
package mapping
// Field type constants used in FieldOpts.Type. An empty Type means the type
// is inferred from the Go field via reflection.
const (
TypeKeyword = "keyword"
TypeFulltext = "fulltext"
TypePath = "path"
TypeWildcard = "wildcard"
TypeNumeric = "numeric"
TypeDatetime = "datetime"
TypeBool = "bool"
TypeObject = "object"
TypeGeopoint = "geopoint"
)
// LowercaseSuffix names the lowercased sibling of a keyword/path field.
const LowercaseSuffix = "_lowercase"
// WordsSuffix names the word-broken sibling of a keyword field.
const WordsSuffix = "_words"
// WordsAnalyzer names the analyzer both engines register for the words sibling.
const WordsAnalyzer = "words"
// FieldOpts overrides the default type inference for a struct field. Keys in
// the override map are json-tag names (e.g. "Name", "location", "audio.artist"),
// not Go field names.
type FieldOpts struct {
// Type is one of the Type* constants. Empty means "infer from Go type".
Type string
// CaseInsensitive additionally indexes a lowercased <name>_lowercase sibling
// for case-insensitive search; the case-preserved base is always indexed.
// On by default for keyword/path fields, KQL searches case-insensitively;
// false opts a field out (ids, paths).
CaseInsensitive *bool
// NoWordBreaker is SharePoint's switch, with its default: a keyword field
// additionally indexes a <name>_words sibling split into lowercased words
// (no stemming), so a single word matches a value that contains it,
// "report" finds "Report.txt"; true opts a field out and leaves it one
// whole value (tags, ids, paths). The base stays the whole value for
// returning and aggregating; wildcards and whole-value matches use the
// _lowercase sibling. Keyword only.
NoWordBreaker *bool
// IncludeInAll controls bleve's _all field inclusion. Nil means "use the
// bleve default for this field type". Has no effect on OpenSearch.
IncludeInAll *bool
}
func (o FieldOpts) caseInsensitive() bool { return o.CaseInsensitive == nil || *o.CaseInsensitive }
func (o FieldOpts) wordBroken() bool { return o.NoWordBreaker == nil || !*o.NoWordBreaker }
services/search/pkg/mapping/serialize.go
+25
View File
@@ -0,0 +1,25 @@
package mapping
import (
"fmt"
"reflect"
"github.com/opencloud-eu/opencloud/pkg/conversions"
)
// PrepareForIndex converts v to the flat map[string]any the backend index
// clients expect: a json round-trip (conversions.To) plus type-specific
// adaptations (currently geopoint siblings). Pass the same overrides as the
// *BuildMapping calls so the document and the mapping stay in sync.
func PrepareForIndex(v any, overrides map[string]FieldOpts) (map[string]any, error) {
out, err := conversions.To[map[string]any](v)
if err != nil {
return nil, fmt.Errorf("mapping: prepare %T: %w", v, err)
}
if out == nil {
return out, nil
}
addGeopointSiblings(out, overrides)
addSearchSiblings(out, deref(reflect.TypeOf(v)), overrides)
return out, nil
}
services/search/pkg/mapping/serialize_test.go
+72
View File
@@ -0,0 +1,72 @@
package mapping
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("PrepareForIndex serialization", func() {
It("errors for a non-marshallable value", func() {
// a func field can't be json-marshalled -> conversions.To errors
type bad struct {
F func() `json:"f"`
}
_, err := PrepareForIndex(bad{}, nil)
Expect(err).To(HaveOccurred())
})
It("returns nil map for a typed nil pointer", func() {
// a typed nil pointer marshals to null -> nil map, no error, no panic
out, err := PrepareForIndex((*struct{})(nil), nil)
Expect(err).ToNot(HaveOccurred())
Expect(out).To(BeNil())
})
It("flattens embedded structs", func() {
type Inner struct {
Name string `json:"Name"`
Size uint64 `json:"Size"`
}
type outer struct {
Inner
ID string `json:"ID"`
}
m, err := PrepareForIndex(outer{Inner: Inner{Name: "a", Size: 7}, ID: "x"}, nil)
Expect(err).ToNot(HaveOccurred())
// keywords carry their lowercase and words search siblings by default
want := map[string]any{
"Name": "a", "Name_lowercase": "a", "Name_words": "a",
"Size": float64(7),
"ID": "x", "ID_lowercase": "x", "ID_words": "x",
}
Expect(m).To(Equal(want))
})
It("omits nil fields tagged omitempty", func() {
type facet struct {
Artist string `json:"artist"`
}
type doc struct {
Name string `json:"Name"`
Audio *facet `json:"audio,omitempty"`
}
m, err := PrepareForIndex(doc{Name: "n"}, nil)
Expect(err).ToNot(HaveOccurred())
Expect(m).ToNot(HaveKey("audio"))
Expect(m["Name"]).To(Equal("n"))
})
It("includes nested facets when set", func() {
type facet struct {
Artist string `json:"artist"`
}
type doc struct {
Audio *facet `json:"audio,omitempty"`
}
m, err := PrepareForIndex(doc{Audio: &facet{Artist: "A"}}, nil)
Expect(err).ToNot(HaveOccurred())
nested, ok := m["audio"].(map[string]any)
Expect(ok).To(BeTrue(), "audio should be a nested map: %#v", m["audio"])
Expect(nested["artist"]).To(Equal("A"))
})
})
services/search/pkg/mapping/validate.go
+92
View File
@@ -0,0 +1,92 @@
package mapping
import (
"fmt"
"reflect"
"sort"
"strings"
)
// Validate returns an error if any override key does not match a known field
// name in t. Top-level fields are identified by their json-tag name; nested
// named struct fields are reachable as "parent.child". Embedded (anonymous)
// structs are flattened, so their fields sit at the parent level (as with
// encoding/json).
func Validate(t reflect.Type, overrides map[string]FieldOpts) error {
if len(overrides) == 0 {
return nil
}
fields := collectFields(t, "")
var unknown, miscased, unbroken []string
for k, opts := range overrides {
goType, ok := fields[k]
if !ok {
unknown = append(unknown, k)
continue
}
if opts.NoWordBreaker != nil && !*opts.NoWordBreaker && !effectivelyKeyword(opts, goType) {
unbroken = append(unbroken, k)
}
// CaseInsensitive routes queries to a <field>_lowercase sibling, which is
// only generated for keyword/path fields; on any other type the query
// would target a non-existent field and silently match nothing. Use the
// effective type (override, else the inferred Go type), since an override
// with no explicit Type still infers keyword/numeric/... from the field.
if opts.CaseInsensitive != nil && *opts.CaseInsensitive && !effectivelyCased(opts, goType) {
miscased = append(miscased, k)
}
}
if len(unknown) > 0 {
sort.Strings(unknown)
return fmt.Errorf("mapping: unknown override keys: %s", strings.Join(unknown, ", "))
}
if len(miscased) > 0 {
sort.Strings(miscased)
return fmt.Errorf("mapping: CaseInsensitive is only valid on keyword/path fields: %s", strings.Join(miscased, ", "))
}
if len(unbroken) > 0 {
sort.Strings(unbroken)
return fmt.Errorf("mapping: NoWordBreaker is only valid on keyword fields: %s", strings.Join(unbroken, ", "))
}
return nil
}
// effectivelyCased reports whether a field is keyword/path (the only types that
// get a _lowercase sibling), from the override type or the inferred Go type.
func effectivelyCased(opts FieldOpts, goType reflect.Type) bool {
eff := opts.Type
if eff == "" && goType != nil {
eff = inferType(goType)
}
return eff == TypeKeyword || eff == TypePath
}
// effectivelyKeyword reports whether a field is a keyword, the only type
// NoWordBreaker applies to.
func effectivelyKeyword(opts FieldOpts, goType reflect.Type) bool {
eff := opts.Type
if eff == "" && goType != nil {
eff = inferType(goType)
}
return eff == TypeKeyword
}
// collectFields maps every known field name (nested as "parent.child") to its Go
// type. Embedded structs are flattened, matching encoding/json.
func collectFields(t reflect.Type, prefix string) map[string]reflect.Type {
out := map[string]reflect.Type{}
_ = walkFields(t, func(fi fieldInfo) error {
key := fi.Name
if prefix != "" {
key = prefix + "." + fi.Name
}
out[key] = fi.GoField.Type
if sub := structType(fi.GoField.Type); sub != nil {
for k, v := range collectFields(sub, key) {
out[k] = v
}
}
return nil
})
return out
}
services/search/pkg/mapping/validate_test.go
+91
View File
@@ -0,0 +1,91 @@
package mapping
import (
"reflect"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
type inner struct {
Artist string `json:"artist"`
}
type sample struct {
Name string `json:"Name"`
Audio *inner `json:"audio,omitempty"`
Location *struct { //nolint:unused
Lon float64 `json:"longitude"`
Lat float64 `json:"latitude"`
} `json:"location,omitempty"`
}
var _ = Describe("Validate", func() {
It("accepts known override keys", func() {
err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{
"Name": {},
"audio": {Type: TypeObject},
"audio.artist": {},
"location": {Type: TypeGeopoint},
})
Expect(err).ToNot(HaveOccurred())
})
It("rejects unknown override keys", func() {
err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{
"nope": {},
"audio.zzz": {},
})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("nope"))
Expect(err.Error()).To(ContainSubstring("audio.zzz"))
})
It("accepts empty overrides", func() {
Expect(Validate(reflect.TypeFor[sample](), nil)).To(Succeed())
})
It("rejects CaseInsensitive on a non-keyword/path field", func() {
True := true
err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{
"Name": {Type: TypeFulltext, CaseInsensitive: &True},
})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("Name"))
Expect(err.Error()).To(ContainSubstring("CaseInsensitive"))
})
It("rejects switching NoWordBreaker off on a non-keyword field", func() {
False := false
err := Validate(reflect.TypeFor[sample](), map[string]FieldOpts{
"Name": {Type: TypeFulltext, NoWordBreaker: &False},
})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("Name"))
Expect(err.Error()).To(ContainSubstring("NoWordBreaker"))
})
It("rejects CaseInsensitive on an inferred non-keyword field (empty Type)", func() {
True := true
type doc struct {
Size uint64 `json:"Size"`
}
err := Validate(reflect.TypeFor[doc](), map[string]FieldOpts{
"Size": {CaseInsensitive: &True}, // no explicit Type -> inferred numeric
})
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("Size"))
})
It("accepts CaseInsensitive on an inferred keyword field (empty Type)", func() {
True := true
type doc struct {
Name string `json:"Name"`
Tags []string `json:"Tags"`
}
Expect(Validate(reflect.TypeFor[doc](), map[string]FieldOpts{
"Name": {CaseInsensitive: &True},
"Tags": {CaseInsensitive: &True},
})).To(Succeed())
})
})
services/search/pkg/opensearch/backend.go
+12 -15
View File
@@ -3,7 +3,6 @@ package opensearch
import (
"context"
"fmt"
"strings"
"time"
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
@@ -14,11 +13,11 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/utils"
"github.com/opencloud-eu/opencloud/pkg/conversions"
"github.com/opencloud-eu/opencloud/pkg/kql"
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
searchQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
@@ -35,7 +34,7 @@ type Backend struct {
// NewBackend creates a backend on the versioned generation of the named index.
func NewBackend(name string, client *opensearchgoAPI.Client) (*Backend, error) {
index := IndexName(name)
index := VersionedIndexName(name)
pingResp, err := client.Ping(context.TODO(), &opensearchgoAPI.PingReq{})
switch {
@@ -74,7 +73,7 @@ func NewBackend(name string, client *opensearchgoAPI.Client) (*Backend, error) {
func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) {
boolQuery, err := convert.KQLToOpenSearchBoolQuery(sir.Query)
switch {
case searchQuery.IsValidationError(err):
case kql.IsValidationError(err):
return nil, errtypes.BadRequest(err.Error())
case err != nil:
return nil, fmt.Errorf("failed to convert KQL query to OpenSearch bool query: %w", err)
@@ -98,6 +97,15 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
),
),
)
// Scope below the space root: restrict at query level so totals and
// paging respect the path too. Path uses the case-preserving
// path_hierarchy analyzer, so the folder path is an indexed token of
// the folder itself and every descendant.
if requestedPath := utils.MakeRelativePath(sir.Ref.Path); requestedPath != "." {
boolQuery.Filter(
osu.NewTermQuery[string]("Path").Value(requestedPath),
)
}
}
searchParams := opensearchgoAPI.SearchParams{
@@ -150,17 +158,6 @@ func (b *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequ
return nil, fmt.Errorf("failed to convert hit to match: %w", err)
}
if sir.Ref != nil {
hitPath := strings.TrimSuffix(match.GetEntity().GetRef().GetPath(), "/")
requestedPath := utils.MakeRelativePath(sir.Ref.Path)
isRoot := hitPath == requestedPath
if !isRoot && requestedPath != "." && !strings.HasPrefix(hitPath, requestedPath+"/") {
totalMatches--
continue
}
}
matches = append(matches, match)
}
services/search/pkg/opensearch/backend_test.go
+48
View File
@@ -1,6 +1,7 @@
package opensearch_test
import (
"context"
"testing"
. "github.com/onsi/ginkgo/v2"
@@ -8,6 +9,7 @@ import (
opensearchgo "github.com/opensearch-project/opensearch-go/v4"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
)
@@ -16,6 +18,12 @@ func TestOpenSearchBackend(t *testing.T) {
RunSpecs(t, "OpenSearch Backend Suite")
}
func deleteIndexOnCleanup(tc *opensearchtest.TestClient, indexName string) {
DeferCleanup(func() {
Expect(tc.IndicesDelete(context.Background(), []string{indexName})).To(Succeed())
})
}
var _ = Describe("Backend", func() {
Describe("NewBackend", func() {
It("fails to create if the cluster is not healthy", func() {
@@ -31,4 +39,44 @@ var _ = Describe("Backend", func() {
Expect(err).To(MatchError(opensearch.ErrUnhealthyCluster))
})
})
Describe("Upsert", func() {
const indexName = "opencloud-test-engine-upsert"
var (
tc *opensearchtest.TestClient
backend *opensearch.Backend
)
BeforeEach(func() {
// the backend versions the physical index by schema generation
physical := opensearch.VersionedIndexName(indexName)
tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client)
tc.Require.IndicesReset([]string{physical})
deleteIndexOnCleanup(tc, physical)
var err error
backend, err = opensearch.NewBackend(indexName, tc.Client())
Expect(err).ToNot(HaveOccurred())
})
It("upserts a full document", func() {
document := opensearchtest.Testdata.Resources.File
Expect(backend.Upsert(document.ID, document)).To(Succeed())
tc.Require.IndicesCount([]string{opensearch.VersionedIndexName(indexName)}, nil, 1)
})
It("upserts a document without an mtime", func() {
// content.Extract leaves Mtime nil when the resource info carries none
document := opensearchtest.Testdata.Resources.File
document.ID = "1$1!4"
document.Mtime = nil
Expect(backend.Upsert(document.ID, document)).To(Succeed())
tc.Require.IndicesCount([]string{opensearch.VersionedIndexName(indexName)}, nil, 1)
})
})
})
services/search/pkg/opensearch/batch.go
+41 -17
View File
@@ -14,6 +14,7 @@ import (
"github.com/opencloud-eu/opencloud/pkg/conversions"
"github.com/opencloud-eu/opencloud/pkg/log"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
@@ -43,7 +44,7 @@ func NewBatch(client *opensearchgoAPI.Client, index string, size int) (*Batch, e
func (b *Batch) Upsert(id string, r search.Resource) error {
return b.withSizeLimit(func() error {
body, err := conversions.To[map[string]any](r)
body, err := mapping.PrepareForIndex(r, r.SearchFieldOverrides())
if err != nil {
return fmt.Errorf("failed to marshal resource: %w", err)
}
@@ -63,27 +64,46 @@ func (b *Batch) Upsert(id string, r search.Resource) error {
})
}
func (b *Batch) Move(id string, parentID string, targetPath string) error {
func (b *Batch) Move(id, parentID, location string) error {
return b.withSizeLimit(func() error {
op := func() error {
return updateSelfAndDescendants(context.Background(), b.client, b.index, id, func(rootResource search.Resource) *osu.BodyParamScript {
newPath := utils.MakeRelativePath(location)
newName := path.Base(newPath)
return &osu.BodyParamScript{
Source: `
if (ctx._source.ID == params.id ) { ctx._source.Name = params.newName; ctx._source.ParentID = params.parentID; }
ctx._source.Path = ctx._source.Path.replace(params.oldPath, params.newPath);
boolean hidden = false;
for (String name : ctx._source.Path.splitOnToken('/')) {
if (!name.equals('.') && !name.equals('..') && name.startsWith('.')) { hidden = true; break; }
}
ctx._source.Hidden = hidden;
`,
// Keep Name and its search siblings in sync; Path has
// no sibling (case-sensitive by design). Only the leading
// oldPath is replaced (startsWith + substring, not
// String.replace, which would also rewrite a repeated segment
// deeper in a descendant's path, e.g. /Music/Music.m3u). The
// lowercased new name comes from Go's strings.ToLower via
// params, so the sibling stays byte-identical to what
// PrepareForIndex writes on upsert (painless toLowerCase would
// lowercase differently than Go).
Source: fmt.Sprintf(`
if (ctx._source.ID == params.id) {
ctx._source.Name = params.newName;
ctx._source.ParentID = params.parentID;
if (ctx._source.Name%[1]s != null) { ctx._source.Name%[1]s = params.newNameLower; }
if (ctx._source.Name%[2]s != null) { ctx._source.Name%[2]s = params.newName; }
}
if (ctx._source.Path != null && ctx._source.Path.startsWith(params.oldPath)) {
ctx._source.Path = params.newPath + ctx._source.Path.substring(params.oldPath.length());
}
boolean hidden = false;
for (String name : ctx._source.Path.splitOnToken('/')) {
if (!name.equals('.') && !name.equals('..') && name.startsWith('.')) { hidden = true; break; }
}
ctx._source.Hidden = hidden;
`, mapping.LowercaseSuffix, mapping.WordsSuffix),
Lang: "painless",
Params: map[string]any{
"id": id,
"parentID": parentID,
"oldPath": rootResource.Path,
"newPath": utils.MakeRelativePath(targetPath),
"newName": path.Base(utils.MakeRelativePath(targetPath)),
"id": id,
"parentID": parentID,
"oldPath": rootResource.Path,
"newPath": newPath,
"newName": newName,
"newNameLower": strings.ToLower(newName),
},
}
})
@@ -148,7 +168,11 @@ func (b *Batch) Purge(id string, onlyDeleted bool) error {
return fmt.Errorf("failed to get resource: %w", err)
}
query := osu.NewBoolQuery().Must(osu.NewTermQuery[string]("Path").Value(resource.Path))
// scope to the resource's space: the same path exists in other spaces
query := osu.NewBoolQuery().Must(
osu.NewTermQuery[string]("RootID").Value(resource.RootID),
osu.NewTermQuery[string]("Path").Value(resource.Path),
)
if onlyDeleted {
query.Must(osu.NewTermQuery[bool]("Deleted").Value(true))
}
services/search/pkg/opensearch/index.go
+81 -40
View File
@@ -3,51 +3,38 @@ package opensearch
import (
"bytes"
"context"
"embed"
"errors"
"fmt"
"path"
"maps"
"reflect"
"strings"
"github.com/go-jose/go-jose/v3/json"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/tidwall/gjson"
searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
var (
ErrManualActionRequired = errors.New("manual action required")
IndexManagerLatest = IndexIndexManagerResourceV3
IndexIndexManagerResourceV3 IndexManager = "resource_v3.json"
ErrManualActionRequired = errors.New("manual action required")
// IndexManagerLatest identifies the current resource mapping; its version is
// derived from search.SchemaVersion so it never drifts from the index name.
IndexManagerLatest = IndexManager(fmt.Sprintf("resource_v%d", search.SchemaVersion))
)
//go:embed internal/indexes/*.json
var indexes embed.FS
// VersionedIndexName suffixes the base index name with the schema version, e.g.
// "opencloud-resource" -> the name suffixed with the current schema version.
func VersionedIndexName(base string) string {
return fmt.Sprintf("%s-v%d", base, search.SchemaVersion)
}
type IndexManager string
// Version is the part of the definition file name that says which generation it
// is, resource_v3.json carries v3.
func (m IndexManager) Version() string {
name := strings.TrimSuffix(string(m), path.Ext(string(m)))
_, version, found := strings.Cut(name, "_")
if !found {
return ""
}
return version
}
// IndexName puts the generation of the definition behind the configured name,
// so a new one starts on an index of its own instead of refusing to work with
// the one that is there.
func IndexName(name string) string {
version := IndexManagerLatest.Version()
if version == "" {
return name
}
return name + "-" + version
// indexGenerators dispatches each IndexManager variant to its builder.
var indexGenerators = map[IndexManager]func() ([]byte, error){
IndexManagerLatest: buildResourceMapping,
}
func (m IndexManager) String() string {
@@ -60,16 +47,67 @@ func (m IndexManager) String() string {
}
func (m IndexManager) MarshalJSON() ([]byte, error) {
filePath := string(m)
body, err := indexes.ReadFile(path.Join("./internal/indexes", filePath))
switch {
case err != nil:
return nil, fmt.Errorf("failed to read index file %s: %w", filePath, err)
case len(body) <= 0:
return nil, fmt.Errorf("index file %s is empty", filePath)
gen, ok := indexGenerators[m]
if !ok {
return nil, fmt.Errorf("unknown index manager %q", string(m))
}
return gen()
}
// buildResourceMapping renders the OpenSearch index template for a
// search.Resource from the shared SearchFieldOverrides. OpenSearch-specific
// tweaks (wildcard MimeType, path_hierarchy Path) are applied on top.
func buildResourceMapping() ([]byte, error) {
resourceType := reflect.TypeFor[search.Resource]()
overrides := maps.Clone(search.Resource{}.SearchFieldOverrides())
mimeType := overrides["MimeType"]
mimeType.Type = searchmapping.TypeWildcard
overrides["MimeType"] = mimeType
if err := searchmapping.Validate(resourceType, overrides); err != nil {
return nil, err
}
props, err := searchmapping.OpenSearchBuildMapping(resourceType, overrides)
if err != nil {
return nil, err
}
return body, nil
index := map[string]any{
"settings": map[string]any{
"number_of_shards": "1",
"number_of_replicas": "1",
"analysis": map[string]any{
// path_hierarchy is case-preserving; casing lives in the value.
"analyzer": map[string]any{
"path_hierarchy": map[string]any{
"type": "custom",
"tokenizer": "path_hierarchy",
},
// words: split into lowercased words, a dot is a word boundary
// too so that "report" finds "Report.txt"; no stemming, a name
// is not prose
searchmapping.WordsAnalyzer: map[string]any{
"type": "custom",
"char_filter": []string{"dot_to_space"},
"tokenizer": "standard",
"filter": []string{"lowercase"},
},
},
"char_filter": map[string]any{
"dot_to_space": map[string]any{
"type": "mapping",
"mappings": []string{`. => \u0020`},
},
},
"tokenizer": map[string]any{
"path_hierarchy": map[string]any{"type": "path_hierarchy"},
},
},
},
"mappings": map[string]any{
"properties": props,
},
}
return json.Marshal(index)
}
func coveredAt(declared, index gjson.Result, declaredPath, indexPath string) (string, string, bool) {
@@ -163,8 +201,11 @@ func (m IndexManager) Apply(ctx context.Context, name string, client *opensearch
if errs != nil {
return fmt.Errorf(
"index %s already exists and is different from the requested version, %w: %w",
name,
"index %s already exists with a different mapping than the requested version. "+
"There is no in-place migration today: drop the index in OpenSearch (DELETE /%s) "+
"and restart the search service. The index will be recreated with the new mapping. "+
"%w: %w",
name, name,
ErrManualActionRequired,
errors.Join(errs...),
)
services/search/pkg/opensearch/index_test.go
+15
View File
@@ -1,6 +1,7 @@
package opensearch_test
import (
"fmt"
"strings"
"testing"
@@ -9,8 +10,22 @@ import (
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
// TestVersionedIndexName guards that the index name and the generator identity
// carry the same schema version.
func TestVersionedIndexName(t *testing.T) {
require.Equal(t,
fmt.Sprintf("opencloud-resource-v%d", search.SchemaVersion),
opensearch.VersionedIndexName("opencloud-resource"),
)
require.Equal(t,
fmt.Sprintf("resource_v%d", search.SchemaVersion),
string(opensearch.IndexManagerLatest),
)
}
func TestIndexManager(t *testing.T) {
t.Run("index plausibility", func(t *testing.T) {
tests := []opensearchtest.TableTest[opensearch.IndexManager, struct{}]{
services/search/pkg/opensearch/internal/convert/kql_expand.go
-208
View File
@@ -1,208 +0,0 @@
package convert
import (
"fmt"
"reflect"
"slices"
"strconv"
"strings"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/opencloud-eu/opencloud/pkg/ast"
)
func ExpandKQL(nodes []ast.Node) ([]ast.Node, error) {
return kqlExpander{}.expand(nodes, "")
}
type kqlExpander struct{}
func (e kqlExpander) expand(nodes []ast.Node, defaultKey string) ([]ast.Node, error) {
for i, node := range nodes {
rnode := reflect.ValueOf(node)
// we need to ensure that the node is a pointer to an ast.Node in every case
if rnode.Kind() != reflect.Ptr {
ptr := reflect.New(rnode.Type())
ptr.Elem().Set(rnode)
rnode = ptr
cnode, ok := rnode.Interface().(ast.Node)
if !ok {
return nil, fmt.Errorf("expected node to be of type ast.Node, got %T", rnode.Interface())
}
node = cnode // Update the original node to the pointer
nodes[i] = node // Update the original slice with the pointer
}
var unfoldedNodes []ast.Node
switch cnode := node.(type) {
case *ast.GroupNode:
if cnode.Key != "" { // group nodes should not get a default key
cnode.Key = e.remapKey(cnode.Key, defaultKey)
}
groupNodes, err := e.expand(cnode.Nodes, cnode.Key)
if err != nil {
return nil, err
}
cnode.Nodes = groupNodes
case *ast.StringNode:
cnode.Key = e.remapKey(cnode.Key, defaultKey)
cnode.Value = e.lowerValue(cnode.Key, cnode.Value)
unfoldedNodes = e.unfoldValue(cnode.Key, cnode.Value)
case *ast.DateTimeNode:
cnode.Key = e.remapKey(cnode.Key, defaultKey)
case *ast.BooleanNode:
cnode.Key = e.remapKey(cnode.Key, defaultKey)
case *ast.NumberNode:
cnode.Key = e.remapKey(cnode.Key, defaultKey)
}
if unfoldedNodes != nil {
// Insert unfolded nodes at the current index
nodes = append(nodes[:i], append(unfoldedNodes, nodes[i+1:]...)...)
// Adjust index to account for new nodes
i += len(unfoldedNodes) - 1
}
}
return nodes, nil
}
func (_ kqlExpander) remapKey(current string, defaultKey string) string {
if defaultKey == "" {
defaultKey = "Name" // Set a default key if none is provided
}
key, ok := map[string]string{
"": defaultKey, // Default case if current is empty
"title": "Title",
"rootid": "RootID",
"path": "Path",
"id": "ID",
"name": "Name",
"size": "Size",
"mtime": "Mtime",
"mediatype": "MimeType",
"type": "Type",
"tag": "Tags",
"tags": "Tags",
"content": "Content",
"hidden": "Hidden",
"favorite": "Favorites",
}[strings.ToLower(current)]
if !ok {
return current // Return the original key if not found
}
return key
}
func (_ kqlExpander) lowerValue(key, value string) string {
if slices.Contains([]string{"Name", "Title", "Tags", "Content", "MimeType", "Type", "Hidden"}, key) {
return strings.ToLower(value)
}
return value
}
func (_ kqlExpander) unfoldValue(key, value string) []ast.Node {
result, ok := map[string][]ast.Node{
"Type:file": {
&ast.StringNode{Key: key, Value: strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10)},
},
"Type:folder": {
&ast.StringNode{Key: key, Value: strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10)},
},
"MimeType:file": {
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: key, Value: "httpd/unix-directory"},
},
"MimeType:folder": {
&ast.StringNode{Key: key, Value: "httpd/unix-directory"},
},
"MimeType:document": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/msword"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.text"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "text/plain"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "text/markdown"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/rtf"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.apple.pages"},
}},
},
"MimeType:spreadsheet": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/vnd.ms-excel"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadsheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "text/csv"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"},
}},
},
"MimeType:presentation": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.ms-powerpoint"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/vnd.apple.keynote"},
}},
},
"MimeType:pdf": {
&ast.StringNode{Key: key, Value: "application/pdf"},
},
"MimeType:image": {
&ast.StringNode{Key: key, Value: "image/*"},
},
"MimeType:video": {
&ast.StringNode{Key: key, Value: "video/*"},
},
"MimeType:audio": {
&ast.StringNode{Key: key, Value: "audio/*"},
},
"MimeType:archive": {
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: key, Value: "application/zip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-7z-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-rar-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-tar"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-bzip2"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-bzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: key, Value: "application/x-tgz"},
}},
},
}[fmt.Sprintf("%s:%s", key, value)]
if !ok {
return nil
}
return result
}
services/search/pkg/opensearch/internal/convert/kql_expand_test.go
-643
View File
@@ -1,643 +0,0 @@
package convert_test
import (
"fmt"
"testing"
"github.com/stretchr/testify/require"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest"
)
func TestExpandKQLAST(t *testing.T) {
t.Run("always converts a value node to a pointer node", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "ast.node.V -> ast.node.PTR",
Got: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: "c"},
&ast.OperatorNode{Value: "OR"},
ast.DateTimeNode{Key: "d"},
ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: "f"},
&ast.OperatorNode{Value: "NOT"},
ast.BooleanNode{Key: "g"},
ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
}},
}},
}},
ast.GroupNode{Key: "i", Nodes: []ast.Node{
ast.StringNode{Key: "a"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
ast.OperatorNode{Value: "OR"},
ast.GroupNode{Key: "h", Nodes: []ast.Node{
ast.StringNode{Key: "a"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
ast.OperatorNode{Value: "OR"},
ast.GroupNode{Key: "h", Nodes: []ast.Node{
ast.StringNode{Key: "a"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "b"},
}},
}},
}},
},
Want: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: "c"},
&ast.OperatorNode{Value: "OR"},
&ast.DateTimeNode{Key: "d"},
&ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: "f"},
&ast.OperatorNode{Value: "NOT"},
&ast.BooleanNode{Key: "g"},
&ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
}},
}},
}},
&ast.GroupNode{Key: "i", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: "h", Nodes: []ast.Node{
&ast.StringNode{Key: "a"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "b"},
}},
}},
}},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.Equal(t, test.Want, result)
})
}
})
t.Run("remaps some keys", func(t *testing.T) {
var tests []opensearchtest.TableTest[[]ast.Node, []ast.Node]
for k, v := range map[string]string{
"": "Name", // Default to "Name" if no key is provided
"rootid": "RootID",
"path": "Path",
"id": "ID",
"name": "Name",
"size": "Size",
"mtime": "Mtime",
"mediatype": "MimeType",
"type": "Type",
"tag": "Tags",
"tags": "Tags",
"content": "Content",
"hidden": "Hidden",
"favorite": "Favorites",
"any": "any", // Example of an unknown key that should remain unchanged
} {
tests = append(tests, opensearchtest.TableTest[[]ast.Node, []ast.Node]{
Name: fmt.Sprintf("%s -> %s", k, v),
Got: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: k},
&ast.OperatorNode{Value: "OR"},
ast.DateTimeNode{Key: k},
ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: k},
&ast.OperatorNode{Value: "NOT"},
ast.BooleanNode{Key: k},
ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: k, Nodes: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: k, Nodes: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: k, Nodes: []ast.Node{
&ast.StringNode{Key: k},
&ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: k},
}},
}},
}},
},
Want: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.DateTimeNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.BooleanNode{Key: v},
&ast.OperatorNode{Value: "NOT"},
&ast.BooleanNode{Key: v},
&ast.OperatorNode{Value: "NOT"},
&ast.GroupNode{Key: func() string {
switch {
case k == "":
return k
default:
return v
}
}(), Nodes: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: func() string {
switch {
case k == "":
return k
default:
return v
}
}(), Nodes: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "OR"},
&ast.GroupNode{Key: func() string {
switch {
case k == "":
return k
default:
return v
}
}(), Nodes: []ast.Node{
&ast.StringNode{Key: v},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: v},
}},
}},
}},
},
})
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.Equal(t, test.Want, result)
})
}
})
t.Run("lowercases some values", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "Name: StringNode -> stringnode",
Got: []ast.Node{
ast.StringNode{Key: "Name", Value: "StringNode"},
ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
ast.StringNode{Key: "Name", Value: "StringNode"},
}},
},
Want: []ast.Node{
&ast.StringNode{Key: "Name", Value: "stringnode"},
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "stringnode"},
}},
},
},
{
Name: "aBc: StringNode -> StringNode",
Got: []ast.Node{
ast.StringNode{Key: "aBc", Value: "StringNode"},
},
Want: []ast.Node{
&ast.StringNode{Key: "aBc", Value: "StringNode"},
},
},
{
Name: "Path: ./Documents -> ./Documents",
Got: []ast.Node{
ast.StringNode{Key: "Path", Value: "./Documents"},
ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
ast.StringNode{Key: "Path", Value: "./Documents"},
}},
},
Want: []ast.Node{
&ast.StringNode{Key: "Path", Value: "./Documents"},
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
&ast.StringNode{Key: "Path", Value: "./Documents"},
}},
},
},
{
Name: "Hidden: TRUE -> true",
Got: []ast.Node{
ast.StringNode{Key: "Hidden", Value: "TRUE"},
ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
ast.StringNode{Key: "Hidden", Value: "TRUE"},
}},
},
Want: []ast.Node{
&ast.StringNode{Key: "Hidden", Value: "true"},
&ast.GroupNode{Key: "GroupNode", Nodes: []ast.Node{
&ast.StringNode{Key: "Hidden", Value: "true"},
}},
},
},
{
Name: "ID: 1$1!AB23 -> 1$1!AB23",
Got: []ast.Node{
ast.StringNode{Key: "ID", Value: "1$1!AB23"},
ast.StringNode{Key: "RootID", Value: "1$1!AB23"},
ast.StringNode{Key: "ParentID", Value: "1$1!AB23"},
},
Want: []ast.Node{
&ast.StringNode{Key: "ID", Value: "1$1!AB23"},
&ast.StringNode{Key: "RootID", Value: "1$1!AB23"},
&ast.StringNode{Key: "ParentID", Value: "1$1!AB23"},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.Equal(t, test.Want, result)
})
}
})
t.Run("unfolds some values", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "MimeType:unknown",
Got: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "unknown"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: "some-name"},
},
Want: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "unknown"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:file",
Got: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "file"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: "some-name"},
},
Want: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:folder",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "folder"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:document",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "document"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/msword"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.text"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "text/plain"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "text/markdown"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/rtf"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.pages"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:spreadsheet",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "spreadsheet"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-excel"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.spreadsheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "text/csv"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.numbers"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:presentation",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "presentation"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.oasis.opendocument.presentation"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.ms-powerpoint"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/vnd.apple.keynote"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:pdf",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "pdf"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "application/pdf"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:image",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "image"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "image/*"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:video",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "video"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "video/*"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:audio",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "audio"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "MimeType", Value: "audio/*"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
{
Name: "MimeType:archive",
Got: []ast.Node{
ast.BooleanNode{Key: "Deleted", Value: false},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Key: "MimeType", Value: "archive"},
ast.OperatorNode{Value: "AND"},
ast.StringNode{Value: "some-name"},
},
Want: []ast.Node{
&ast.BooleanNode{Key: "Deleted", Value: false},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/zip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-gzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-7z-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-rar-compressed"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-tar"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-bzip2"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-bzip"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "MimeType", Value: "application/x-tgz"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: `some-name`},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
if test.Skip {
t.Skip("Skipping test due to known issue")
}
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.EqualValues(t, test.Want, result)
})
}
})
t.Run("different cases", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
Name: "use the group node key as default key",
Got: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "a", Nodes: []ast.Node{
&ast.StringNode{Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "mediatype", Nodes: []ast.Node{
&ast.StringNode{Value: "file"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "mediatype", Value: "file"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
},
Want: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "a", Nodes: []ast.Node{
&ast.StringNode{Key: "a", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Key: "MimeType", Nodes: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "c", Value: "d"},
}},
},
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
if test.Skip {
t.Skip("Skipping test due to known issue")
}
result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.EqualValues(t, test.Want, result)
})
}
})
}
services/search/pkg/opensearch/internal/convert/kql_query.go
+4 -5
View File
@@ -5,6 +5,7 @@ import (
"github.com/opencloud-eu/opencloud/pkg/kql"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
)
var (
@@ -17,12 +18,10 @@ func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) {
return nil, err
}
kqlNodes, err := ExpandKQL(kqlAst.Nodes)
if err != nil {
return nil, fmt.Errorf("failed to expand KQL AST nodes: %w", err)
}
// shared lowering: field resolution, media-type expansion, value lowercasing.
kqlAst = query.Normalize(kqlAst, query.ResolveField)
builder, err := TranspileKQLToOpenSearch(kqlNodes)
builder, err := TranspileKQLToOpenSearch(kqlAst.Nodes)
if err != nil {
return nil, fmt.Errorf("failed to compile query: %w", err)
}
services/search/pkg/opensearch/internal/convert/kql_transpile.go
+98 -92
View File
@@ -10,7 +10,9 @@ import (
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/pkg/kql"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
)
func TranspileKQLToOpenSearch(nodes []ast.Node) (osu.Builder, error) {
@@ -48,13 +50,21 @@ func (t kqlOpensearchTranspiler) transpile(nodes []ast.Node) (osu.Builder, error
nextOp := t.getOperatorValueAt(nodes, i+1)
prevOp := t.getOperatorValueAt(nodes, i-1)
// A preceding NOT negates this node regardless of what follows (NOT x AND y
// is (NOT x) AND y), so it must win over nextOp. The prevOp AND/OR cases
// give the right operand its own bucket instead of inheriting the previous
// one, which matters right after a NOT (its MustNot must not carry over).
switch {
case prevOp == kql.BoolNOT:
boolQueryAdd = boolQuery.MustNot
case nextOp == kql.BoolOR:
boolQueryAdd = boolQuery.Should
case nextOp == kql.BoolAND:
boolQueryAdd = boolQuery.Must
case prevOp == kql.BoolNOT:
boolQueryAdd = boolQuery.MustNot
case prevOp == kql.BoolOR:
boolQueryAdd = boolQuery.Should
case prevOp == kql.BoolAND:
boolQueryAdd = boolQuery.Must
}
builder, err := t.toBuilder(node)
@@ -71,7 +81,7 @@ func (t kqlOpensearchTranspiler) transpile(nodes []ast.Node) (osu.Builder, error
continue
}
if nextOp == kql.BoolOR {
if nextOp == kql.BoolOR || prevOp == kql.BoolOR {
// if there are should clauses, we set the minimum should match to 1
boolQueryParams.MinimumShouldMatch = 1
}
@@ -99,7 +109,72 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
case *ast.BooleanNode:
return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil
case *ast.StringNode:
return stringNodeQuery(node), nil
// hidden takes bool words only; anything else matches nothing
if node.Key == "Hidden" {
b, err := strconv.ParseBool(node.Value)
if err != nil {
return osu.NewMatchNoneQuery(), nil
}
return osu.NewTermQuery[bool](node.Key).Value(b), nil
}
field, value := node.Key, node.Value
if query.FieldIsPath(node.Key) {
value = strings.TrimSuffix(value, "/")
}
if node.CaseInsensitive {
field += mapping.LowercaseSuffix
value = strings.ToLower(value)
}
if isWildcard := strings.ContainsAny(value, "*?"); isWildcard {
// a wildcard on a word-broken field forgives a missing extension:
// *report also matches Report.txt
if query.FieldIsWordBroken(node.Key) && !strings.HasSuffix(value, "*") {
return osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewWildcardQuery(field).Value(value),
osu.NewWildcardQuery(field).Value(value+".*"),
), nil
}
return osu.NewWildcardQuery(field).Value(value), nil
}
// = matches the whole value, on the lowercased sibling for
// case-insensitive fields
if node.Exact {
return osu.NewTermQuery[string](field).Value(value), nil
}
// a word-broken field matches the value as a phrase of its words on the
// _words sibling, whose analyzer lowercases; wildcards stay on _lowercase
if query.FieldIsWordBroken(node.Key) {
return osu.NewMatchPhraseQuery(node.Key + mapping.WordsSuffix).Query(node.Value), nil
}
if query.FieldIsFulltext(node.Key) {
return osu.NewMatchPhraseQuery(field).Query(value), nil
}
// a path value is a single term in the path_hierarchy token stream; a
// phrase match would analyze the query into its path prefixes and match
// everything under the root, so paths with spaces must stay term queries.
if query.FieldIsPath(node.Key) {
return osu.NewTermQuery[string](field).Value(value), nil
}
totalTerms := strings.Split(value, " ")
isSingleTerm := len(totalTerms) == 1
isMultiTerm := len(totalTerms) >= 1
switch {
case isSingleTerm:
return osu.NewTermQuery[string](field).Value(value), nil
case isMultiTerm:
return osu.NewMatchPhraseQuery(field).Query(value), nil
}
return nil, fmt.Errorf("unsupported string node value: %s", value)
case *ast.DateTimeNode:
return dateTimeNodeQuery(node)
case *ast.NumberNode:
@@ -116,67 +191,26 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
return nil, fmt.Errorf("%w: %T", ErrUnsupportedNodeType, node)
}
// stringNodeQuery picks the query a string node turns into.
func stringNodeQuery(node *ast.StringNode) osu.Builder {
isWildcard := strings.ContainsAny(node.Value, "*?")
switch {
// Name: "*oo-bar", "*oo ba*", "*OO*"
// Title: "*rterly rep*"
// Tags: "*spaced tag*"
case isWildcard && slices.Contains([]string{"Name", "Title"}, node.Key):
patterns := []osu.Builder{wildcardOn(node.Key+".wildcard", node.Value)}
if !strings.HasSuffix(node.Value, "*") {
patterns = append(patterns, wildcardOn(node.Key+".wildcard", node.Value+".*"))
}
return osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(patterns...)
// Tags: "*foo*", "*paced ta*"
case isWildcard && node.Key == "Tags":
return wildcardOn(node.Key+".wildcard", node.Value)
// Path: "./foo*", MimeType: "*plain"
case isWildcard:
return osu.NewWildcardQuery(node.Key).Value(node.Value)
// Name: =new, Title: ="quarterly report"
case node.Exact && slices.Contains([]string{"Name", "Title"}, node.Key):
return osu.NewTermQuery[string](node.Key + ".wildcard").
Value(node.Value).
Params(&osu.TermQueryParams{CaseInsensitive: true})
// Tags: "foo-bar", "spaced tag", "FOO-BAR"
case node.Key == "Tags":
return osu.NewTermQuery[string](node.Key + ".wildcard").
Value(node.Value).
Params(&osu.TermQueryParams{CaseInsensitive: true})
// Name: "foo-bar", "foo bar"
// Title: "quarterly report"
// Content: "foo bar"
case slices.Contains([]string{"Name", "Title", "Content"}, node.Key):
return osu.NewMatchPhraseQuery(node.Key).Query(node.Value)
// Size: "42", Type: "1"
case slices.Contains([]string{"Size", "Type"}, node.Key):
number, err := strconv.ParseInt(node.Value, 10, 64)
if err != nil {
return osu.NewMatchNoneQuery()
}
return osu.NewTermQuery[int64](node.Key).Value(number)
// Path: "./foo bar/", the hierarchy tokens carry no trailing slash
case node.Key == "Path":
return osu.NewTermQuery[string](node.Key).Value(strings.TrimSuffix(node.Value, "/"))
// Hidden: "TRUE" arrives lowered, anything that is no bool matches nothing
case node.Key == "Hidden":
value, err := strconv.ParseBool(node.Value)
if err != nil {
return osu.NewMatchNoneQuery()
}
return osu.NewTermQuery[bool](node.Key).Value(value)
// MimeType: "text/plain"
default:
return osu.NewTermQuery[string](node.Key).Value(node.Value)
// dateTimeNodeQuery turns a date time node into a range query.
func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) {
if node.Operator == nil {
return nil, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
}
query := osu.NewRangeQuery[time.Time](node.Key)
switch node.Operator.Value {
case ">":
return query.Gt(node.Value), nil
case ">=":
return query.Gte(node.Value), nil
case "<":
return query.Lt(node.Value), nil
case "<=":
return query.Lte(node.Value), nil
}
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
}
func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) {
@@ -203,31 +237,3 @@ func numberNodeQuery(node *ast.NumberNode) (osu.Builder, error) {
return nil, fmt.Errorf("unsupported operator %s for number node: %w", node.Operator.Value, ErrUnsupportedNodeType)
}
func wildcardOn(field, value string) osu.Builder {
return osu.NewWildcardQuery(field).
Value(value).
Params(&osu.WildcardQueryParams{CaseInsensitive: true})
}
// dateTimeNodeQuery turns a date time node into a range query.
func dateTimeNodeQuery(node *ast.DateTimeNode) (osu.Builder, error) {
if node.Operator == nil {
return nil, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
}
query := osu.NewRangeQuery[time.Time](node.Key)
switch node.Operator.Value {
case ">":
return query.Gt(node.Value), nil
case ">=":
return query.Gte(node.Value), nil
case "<":
return query.Lt(node.Value), nil
case "<=":
return query.Lte(node.Value), nil
}
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
}
services/search/pkg/opensearch/internal/convert/kql_transpile_test.go
+69 -64
View File
@@ -16,13 +16,40 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{
// kql to os dsl - type tests
{
Name: "match phrase query - string node on an analyzed field",
Name: "word-broken field matches the value as a phrase on its words sibling",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "openCloud"},
},
},
Want: osu.NewMatchPhraseQuery("Name").Query("openCloud"),
Want: osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
},
{
Name: "case-insensitive term routes to the lowercased sibling",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Tags", Value: "openCloud", CaseInsensitive: true},
},
},
Want: osu.NewTermQuery[string]("Tags_lowercase").Value("opencloud"),
},
{
Name: "case-insensitive wildcard routes to the lowercased sibling",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "Open*", CaseInsensitive: true},
},
},
Want: osu.NewWildcardQuery("Name_lowercase").Value("open*"),
},
{
Name: "full-text field uses an analyzed match query, not an unanalyzed term",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Content", Value: "Running"},
},
},
Want: osu.NewMatchPhraseQuery("Content").Query("Running"),
},
{
Name: "term query - boolean node - true",
@@ -49,7 +76,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
&ast.StringNode{Key: "Name", Value: "open cloud"},
},
},
Want: osu.NewMatchPhraseQuery("Name").Query(`open cloud`),
Want: osu.NewMatchPhraseQuery("Name_words").Query(`open cloud`),
},
{
Name: "wildcard query - string node",
@@ -58,16 +85,10 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
&ast.StringNode{Key: "Name", Value: "open*"},
},
},
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewWildcardQuery("Name.wildcard").
Value("open*").
Params(&osu.WildcardQueryParams{CaseInsensitive: true}),
),
Want: osu.NewWildcardQuery("Name").Value("open*"),
},
{
Name: "wildcard query - string node without an unanalyzed sub field",
Name: "wildcard query - fulltext field",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Content", Value: "open*"},
@@ -76,60 +97,24 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
Want: osu.NewWildcardQuery("Content").Value("open*"),
},
{
Name: "wildcard query - a question mark counts as a wildcard",
// a phrase match would analyze the query with path_hierarchy and match
// everything under the root
Name: "path with spaces stays an unanalyzed term query",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "fo?o"},
&ast.StringNode{Key: "Path", Value: "./parent d!r/child.pdf"},
},
},
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewWildcardQuery("Name.wildcard").
Value("fo?o").
Params(&osu.WildcardQueryParams{CaseInsensitive: true}),
osu.NewWildcardQuery("Name.wildcard").
Value("fo?o.*").
Params(&osu.WildcardQueryParams{CaseInsensitive: true}),
),
Want: osu.NewTermQuery[string]("Path").Value("./parent d!r/child.pdf"),
},
{
Name: "term query - an equals restriction matches the whole name",
Name: "case-insensitive path with spaces routes to the lowercased sibling as a term query",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "foo bar.txt", Exact: true},
&ast.StringNode{Key: "Path", Value: "./Parent Dir", CaseInsensitive: true},
},
},
Want: osu.NewTermQuery[string]("Name.wildcard").
Value("foo bar.txt").
Params(&osu.TermQueryParams{CaseInsensitive: true}),
},
{
Name: "term query - a path loses its trailing slash",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Path", Value: "./Documents/"},
},
},
Want: osu.NewTermQuery[string]("Path").Value("./Documents"),
},
{
Name: "term query - a hidden string turns into a bool",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Hidden", Value: "true"},
},
},
Want: osu.NewTermQuery[bool]("Hidden").Value(true),
},
{
Name: "match-none query - a hidden string that is no bool",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "Hidden", Value: "banana"},
},
},
Want: osu.NewMatchNoneQuery(),
Want: osu.NewTermQuery[string]("Path_lowercase").Value("./parent dir"),
},
{
Name: "bool query",
@@ -142,8 +127,8 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
},
},
Want: osu.NewBoolQuery().Must(
osu.NewMatchPhraseQuery("Name").Query("a"),
osu.NewMatchPhraseQuery("Name").Query("b"),
osu.NewMatchPhraseQuery("Name_words").Query("a"),
osu.NewMatchPhraseQuery("Name_words").Query("b"),
),
},
{
@@ -155,7 +140,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
}},
},
},
Want: osu.NewMatchPhraseQuery("Name").Query("any"),
Want: osu.NewMatchPhraseQuery("Name_words").Query("any"),
},
{
Name: "range query >",
@@ -217,7 +202,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
&ast.StringNode{Key: "Name", Value: "openCloud"},
},
},
Want: osu.NewMatchPhraseQuery("Name").Query("openCloud"),
Want: osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
},
{
Name: "[* *]",
@@ -229,7 +214,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
},
Want: osu.NewBoolQuery().
Must(
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
osu.NewTermQuery[string]("age").Value("32"),
),
},
@@ -244,7 +229,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
},
Want: osu.NewBoolQuery().
Must(
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
osu.NewTermQuery[string]("age").Value("32"),
),
},
@@ -260,7 +245,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
osu.NewTermQuery[string]("age").Value("32"),
),
},
@@ -288,12 +273,32 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
},
Want: osu.NewBoolQuery().
Must(
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
).
MustNot(
osu.NewTermQuery[string]("age").Value("32"),
),
},
{
// NOT binds to the node directly after it, not to whatever operator
// follows that node: NOT x AND y is (NOT x) AND y.
Name: "[NOT * AND *]",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "age", Value: "32"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Name", Value: "openCloud"},
},
},
Want: osu.NewBoolQuery().
MustNot(
osu.NewTermQuery[string]("age").Value("32"),
).
Must(
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
),
},
{
Name: "[* OR * OR *]",
Got: &ast.Ast{
@@ -308,7 +313,7 @@ func TestTranspileKQLToOpenSearch(t *testing.T) {
Want: osu.NewBoolQuery().
Params(&osu.BoolQueryParams{MinimumShouldMatch: 1}).
Should(
osu.NewMatchPhraseQuery("Name").Query("openCloud"),
osu.NewMatchPhraseQuery("Name_words").Query("openCloud"),
osu.NewTermQuery[string]("age").Value("32"),
osu.NewTermQuery[string]("age").Value("44"),
),
services/search/pkg/opensearch/internal/convert/opensearch.go
+17 -23
View File
@@ -3,7 +3,6 @@ package convert
import (
"fmt"
"strings"
"time"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"google.golang.org/protobuf/types/known/timestamppb"
@@ -15,6 +14,17 @@ import (
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
// copyFacet converts a typed pointer from the indexed shape (libregraph) to
// the protobuf shape via conversions.To. Returns nil when src is nil so the
// enclosing Match.Entity field stays nil.
func copyFacet[Dst, Src any](src *Src) *Dst {
if src == nil {
return nil
}
dst, _ := conversions.To[*Dst](src)
return dst
}
func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match, error) {
resource, err := conversions.To[search.Resource](hit.Source)
if err != nil {
@@ -69,31 +79,15 @@ func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match,
return strings.Join(contentHighlights[:], "; ")
}(),
Audio: func() *searchMessage.Audio {
if !strings.HasPrefix(resource.MimeType, "audio/") {
return nil
}
audio, _ := conversions.To[*searchMessage.Audio](resource.Audio)
return audio
}(),
Image: func() *searchMessage.Image {
image, _ := conversions.To[*searchMessage.Image](resource.Image)
return image
}(),
Location: func() *searchMessage.GeoCoordinates {
geoCoordinates, _ := conversions.To[*searchMessage.GeoCoordinates](resource.Location)
return geoCoordinates
}(),
Photo: func() *searchMessage.Photo {
photo, _ := conversions.To[*searchMessage.Photo](resource.Photo)
return photo
}(),
Audio: copyFacet[searchMessage.Audio](resource.Audio),
Image: copyFacet[searchMessage.Image](resource.Image),
Location: copyFacet[searchMessage.GeoCoordinates](resource.Location),
Photo: copyFacet[searchMessage.Photo](resource.Photo),
},
}
if mtime, err := time.Parse(time.RFC3339, resource.Mtime); err == nil {
match.Entity.LastModifiedTime = &timestamppb.Timestamp{Seconds: mtime.Unix(), Nanos: int32(mtime.Nanosecond())}
if resource.Mtime != nil {
match.Entity.LastModifiedTime = timestamppb.New(*resource.Mtime)
}
return match, nil
services/search/pkg/opensearch/internal/convert/opensearch_test.go
+1 -1
View File
@@ -36,7 +36,7 @@ var _ = Describe("OpenSearchHitToMatch", func() {
resource = opensearchtest.Testdata.Resources.File
resource.MimeType = "audio/mpeg"
mtime = time.Date(2025, 7, 24, 15, 15, 1, 0, time.UTC)
resource.Mtime = mtime.Format(time.RFC3339)
resource.Mtime = &mtime
resource.Favorites = []string{"cbf24bce-3e6e-4d9e-a2a2-cbf24bce3e6e"}
hit = opensearchgoAPI.SearchHit{
services/search/pkg/opensearch/internal/indexes/resource_v1.json
-49
View File
@@ -1,49 +0,0 @@
{
"settings": {
"number_of_shards": "1",
"number_of_replicas": "1",
"analysis": {
"analyzer": {
"path_hierarchy": {
"filter": [
"lowercase"
],
"tokenizer": "path_hierarchy",
"type": "custom"
}
},
"tokenizer": {
"path_hierarchy": {
"type": "path_hierarchy"
}
}
}
},
"mappings": {
"properties": {
"ID": {
"type": "keyword"
},
"ParentID": {
"type": "keyword"
},
"RootID": {
"type": "keyword"
},
"MimeType": {
"type": "wildcard",
"doc_values": false
},
"Path": {
"type": "text",
"analyzer": "path_hierarchy"
},
"Deleted": {
"type": "boolean"
},
"Hidden": {
"type": "boolean"
}
}
}
}
services/search/pkg/opensearch/internal/indexes/resource_v2.json
-56
View File
@@ -1,56 +0,0 @@
{
"settings": {
"number_of_shards": "1",
"number_of_replicas": "1",
"analysis": {
"analyzer": {
"path_hierarchy": {
"filter": [
"lowercase"
],
"tokenizer": "path_hierarchy",
"type": "custom"
}
},
"tokenizer": {
"path_hierarchy": {
"type": "path_hierarchy"
}
}
}
},
"mappings": {
"properties": {
"Content": {
"type": "text",
"term_vector": "with_positions_offsets"
},
"ID": {
"type": "keyword"
},
"ParentID": {
"type": "keyword"
},
"RootID": {
"type": "keyword"
},
"MimeType": {
"type": "wildcard",
"doc_values": false
},
"Path": {
"type": "text",
"analyzer": "path_hierarchy"
},
"Deleted": {
"type": "boolean"
},
"Hidden": {
"type": "boolean"
},
"Favorites": {
"type": "keyword"
}
}
}
}
services/search/pkg/opensearch/internal/indexes/resource_v3.json
-122
View File
@@ -1,122 +0,0 @@
{
"settings": {
"number_of_shards": "1",
"number_of_replicas": "1",
"analysis": {
"analyzer": {
"path_hierarchy": {
"tokenizer": "path_hierarchy",
"type": "custom"
},
"name_words": {
"type": "custom",
"char_filter": [
"dot_to_space"
],
"tokenizer": "standard",
"filter": [
"lowercase"
]
}
},
"tokenizer": {
"path_hierarchy": {
"type": "path_hierarchy"
}
},
"normalizer": {
"lowercase": {
"type": "custom",
"filter": [
"lowercase"
]
}
},
"char_filter": {
"dot_to_space": {
"type": "pattern_replace",
"pattern": "\\.",
"replacement": " "
}
}
}
},
"mappings": {
"properties": {
"Content": {
"type": "text",
"analyzer": "name_words",
"term_vector": "with_positions_offsets"
},
"ID": {
"type": "keyword"
},
"ParentID": {
"type": "keyword"
},
"RootID": {
"type": "keyword"
},
"MimeType": {
"type": "wildcard"
},
"Path": {
"type": "text",
"analyzer": "path_hierarchy"
},
"Deleted": {
"type": "boolean"
},
"Hidden": {
"type": "boolean"
},
"Favorites": {
"type": "keyword"
},
"Tags": {
"type": "text",
"fields": {
"wildcard": {
"type": "wildcard",
"normalizer": "lowercase"
}
}
},
"Name": {
"type": "text",
"analyzer": "name_words",
"fields": {
"wildcard": {
"type": "wildcard",
"normalizer": "lowercase"
}
}
},
"Title": {
"type": "text",
"analyzer": "name_words",
"fields": {
"wildcard": {
"type": "wildcard",
"normalizer": "lowercase"
}
}
},
"Mtime": {
"type": "date",
"ignore_malformed": true
}
},
"dynamic_templates": [
{
"audio_facets": {
"path_match": "audio.*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
}
]
}
}
services/search/pkg/parity/README.md
+8 -1
View File
@@ -181,6 +181,10 @@ Fixtures:
| MEDIATYPE-04 | `mediatype:*jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ |
| MEDIATYPE-05 | `mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ |
| MEDIATYPE-06 | `mediatype:folder` | albums, drafts | albums, drafts | albums, drafts | ✅ |
| MEDIATYPE-07 | `mediatype:file` | notes.md, photo.jpg | notes.md, photo.jpg | notes.md, photo.jpg | ✅ |
| MEDIATYPE-08 | `NOT mediatype:file` | albums, drafts | albums, drafts | albums, drafts | ✅ |
| MEDIATYPE-09 | `mediatype:file OR mediatype:image` | notes.md, photo.jpg | notes.md, photo.jpg | notes.md, photo.jpg | ✅ |
| MEDIATYPE-10 | `NOT mediatype:(image OR folder)` | notes.md | notes.md | notes.md | ✅ |
### path
@@ -232,7 +236,8 @@ Fixtures:
| FIELDS-11 | `id:"1$1!AB-23"` | cased.txt | cased.txt | cased.txt | ✅ |
| FIELDS-12 | `id:"1$1!ab-23"` | no match | no match | no match | ✅ |
| FIELDS-13 | `audio.artist:"Some Artist"` | song.mp3 | song.mp3 | song.mp3 | ✅ |
| FIELDS-14 | `audio.artist:"some artist"` | no match | no match | no match | ✅ |
| FIELDS-14 | `audio.artist:"some artist"` | song.mp3 | song.mp3 | song.mp3 | ✅ |
| FIELDS-15 | `audio.duration>100` | no match | no match | no match | ✅ |
### deleted
@@ -499,6 +504,8 @@ Fixtures:
| ROOTSCOPE-01 | deletes only the one in the target root, then `name:"*twin*"` | twin.txt | twin.txt | twin.txt | ✅ |
| ROOTSCOPE-02 | restores only the one in the target root, then `name:"*target*"` | target.txt | target.txt | target.txt | ✅ |
| ROOTSCOPE-02 | restores only the one in the target root, then `name:"*twin*"` | no match | no match | no match | ✅ |
| ROOTSCOPE-04 | purges only the one in the target root, then `name:"*target*"` | no match | no match | no match | ✅ |
| ROOTSCOPE-04 | purges only the one in the target root, then `name:"*twin*"` | twin.txt | twin.txt | twin.txt | ✅ |
| ROOTSCOPE-03 | moves only the one in the target root, then `path:"./moved.txt"` | moved.txt | moved.txt | moved.txt | ✅ |
| ROOTSCOPE-03 | moves only the one in the target root, then `path:"./same/path.txt"` | twin.txt | twin.txt | twin.txt | ✅ |
services/search/pkg/parity/engines_test.go
+1 -1
View File
@@ -92,7 +92,7 @@ func newOpenSearch(name string, fixtures []search.Resource) testEngine {
GinkgoHelper()
tc := opensearchtest.NewDefaultTestClient(GinkgoTB(), openSearchClient)
index := opensearch.IndexName(name)
index := opensearch.VersionedIndexName(name)
if err := tc.IndicesReset(context.Background(), []string{index}); err != nil {
return testEngine{name: "opensearch", unavailable: err.Error()}
services/search/pkg/parity/fixtures_test.go
+11 -2
View File
@@ -24,7 +24,15 @@ func withMime(mime string) fixtureOption { return func(r *search.Resource) { r.M
func withTitle(t string) fixtureOption { return func(r *search.Resource) { r.Title = t } }
func withContent(c string) fixtureOption { return func(r *search.Resource) { r.Content = c } }
func withSize(s uint64) fixtureOption { return func(r *search.Resource) { r.Size = s } }
func withMtime(m string) fixtureOption { return func(r *search.Resource) { r.Mtime = m } }
func withMtime(m string) fixtureOption {
return func(r *search.Resource) {
t, err := time.Parse(time.RFC3339Nano, m)
if err != nil {
panic(err)
}
r.Mtime = &t
}
}
func withID(id string) fixtureOption { return func(r *search.Resource) { r.ID = id } }
func withParent(id string) fixtureOption { return func(r *search.Resource) { r.ParentID = id } }
func withRoot(id string) fixtureOption { return func(r *search.Resource) { r.RootID = id } }
@@ -47,6 +55,7 @@ func withLocation(location *libregraph.GeoCoordinates) fixtureOption {
}
func fixtureDoc(name string, opts ...fixtureOption) search.Resource {
mtime := fixtureNow
r := search.Resource{
ID: "1$1!" + name,
RootID: "1$1!1",
@@ -56,7 +65,7 @@ func fixtureDoc(name string, opts ...fixtureOption) search.Resource {
Document: content.Document{
Name: name,
MimeType: "text/plain",
Mtime: fixtureNow.Format(time.RFC3339Nano),
Mtime: &mtime,
Size: 1000,
},
}
services/search/pkg/parity/lifecycle_rootscope_test.go
+8
View File
@@ -37,6 +37,14 @@ func rootScopeLifecycle() lifecycleGroup {
{`name:"*twin*"`, nil},
},
},
{
id: 4, title: "purges only the one in the target root",
do: func(e search.Engine) error { return e.Purge(target.ID, false) },
expect: []expectation{
{`name:"*target*"`, nil},
{`name:"*twin*"`, []string{"twin.txt"}},
},
},
{
id: 3, title: "moves only the one in the target root",
do: func(e search.Engine) error { return e.Move(target.ID, target.ParentID, "./moved.txt") },
services/search/pkg/parity/matrix_test.go
+3 -2
View File
@@ -6,6 +6,7 @@ import (
"os"
"sort"
"strings"
"time"
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
. "github.com/onsi/ginkgo/v2"
@@ -336,8 +337,8 @@ func fixtureFields(f search.Resource, withID bool) string {
add("Size = %d", f.Size)
}
if !strings.HasPrefix(f.Mtime, fixtureNow.Format("2006-01-02")) {
add("Mtime = %s", f.Mtime)
if f.Mtime == nil || f.Mtime.Format("2006-01-02") != fixtureNow.Format("2006-01-02") {
add("Mtime = %s", f.Mtime.Format(time.RFC3339))
}
if f.Hidden {
services/search/pkg/parity/query_fields_test.go
+3 -2
View File
@@ -18,7 +18,7 @@ func fieldsGroup() queryGroup {
fixtureDoc("plain.txt"),
fixtureFolder("box"),
fixtureDoc("boxed.txt", withParent("1$1!box"), withPath("./box/boxed.txt")),
fixtureDoc("song.mp3", withMime("audio/mpeg"), withAudio(&libregraph.Audio{Artist: libregraph.PtrString("Some Artist")})),
fixtureDoc("song.mp3", withMime("audio/mpeg"), withAudio(&libregraph.Audio{Artist: libregraph.PtrString("Some Artist"), Duration: libregraph.PtrInt64(200)})),
},
cases: []queryCase{
{id: 1, query: `size:42`, want: []string{"small.txt"}},
@@ -35,7 +35,8 @@ func fieldsGroup() queryGroup {
{id: 12, query: `id:"1$1!ab-23"`},
// a facet value keeps its case, the field is not marked lowercase
{id: 13, query: `audio.artist:"Some Artist"`, want: []string{"song.mp3"}},
{id: 14, query: `audio.artist:"some artist"`},
{id: 14, query: `audio.artist:"some artist"`, want: []string{"song.mp3"}}, // facets search case-insensitively
{id: 15, query: `audio.duration>100`}, // number queries are gated to Size and Type on both engines
},
}
}
services/search/pkg/parity/query_mediatype_test.go
+4
View File
@@ -20,6 +20,10 @@ func mediatypeGroup() queryGroup {
{id: 4, query: `mediatype:*jpeg`, want: []string{"photo.jpg"}},
{id: 5, query: `mediatype:image`, want: []string{"photo.jpg"}},
{id: 6, query: `mediatype:folder`, want: []string{"albums", "drafts"}},
{id: 7, query: `mediatype:file`, want: []string{"notes.md", "photo.jpg"}},
{id: 8, query: `NOT mediatype:file`, want: []string{"albums", "drafts"}},
{id: 9, query: `mediatype:file OR mediatype:image`, want: []string{"notes.md", "photo.jpg"}},
{id: 10, query: `NOT mediatype:(image OR folder)`, want: []string{"notes.md"}},
},
}
}
services/search/pkg/query/bleve/bleve.go
+4
View File
@@ -22,6 +22,10 @@ func (c Creator[T]) Create(qs string) (T, error) {
return t, err
}
// shared KQL lowering pass: resolve field names + expand media-type aliases
// once, so the compiler below sees only canonical field:value nodes.
builderAst = query.Normalize(builderAst, query.ResolveField)
t, err = c.compiler.Compile(builderAst)
if err != nil {
return t, err
services/search/pkg/query/bleve/compiler.go
+90 -206
View File
@@ -6,41 +6,14 @@ import (
"strconv"
"strings"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/blevesearch/bleve/v2"
bleveQuery "github.com/blevesearch/bleve/v2/search/query"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/pkg/kql"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
searchQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query"
)
var lowercaseFields = map[string]struct{}{
"Name": {},
"Title": {},
"Tags": {},
"Favorites": {},
"Content": {},
"MimeType": {},
"Hidden": {},
}
var _fields = map[string]string{
"rootid": "RootID",
"path": "Path",
"id": "ID",
"name": "Name",
"size": "Size",
"mtime": "Mtime",
"mediatype": "MimeType",
"type": "Type",
"tag": "Tags",
"tags": "Tags",
"content": "Content",
"title": "Title",
"hidden": "Hidden",
"favorite": "Favorites",
}
// The following quoted string enumerates the characters which may be escaped: "+-=&|><!(){}[]^\"~*?:\\/ "
// based on bleve docs https://blevesearch.com/docs/Query-String-Query/
// Wildcards * and ? are excluded
@@ -99,55 +72,83 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
for i := offset; i < len(nodes); i++ {
switch n := nodes[i].(type) {
case *ast.StringNode:
k := getField(n.Key)
v := n.Value
if k != "ID" && k != "Size" {
v = bleveEscaper.Replace(n.Value)
}
if _, ok := lowercaseFields[k]; ok {
v = strings.ToLower(v)
}
if k == "Type" {
v = resourceType(v)
}
var q bleveQuery.Query
var group bool
switch {
case k == "Hidden":
value, err := strconv.ParseBool(v)
if err != nil {
// hidden takes bool words only; anything else matches nothing
if n.Key == "Hidden" {
var q bleveQuery.Query
if b, err := strconv.ParseBool(n.Value); err == nil {
bq := bleveQuery.NewBoolFieldQuery(b)
bq.SetField(n.Key)
q = bq
} else {
q = bleveQuery.NewMatchNoneQuery()
break
}
bq := bleveQuery.NewBoolFieldQuery(value)
bq.SetField(k)
q = bq
case k == "MimeType":
q, group = mimeType(k, v)
if prev == nil {
isGroup = group
}
case slices.Contains([]string{"Name", "Title"}, k) && strings.ContainsAny(n.Value, "*?"):
patterns := []bleveQuery.Query{bleveQuery.NewQueryStringQuery(k + ".wildcard:" + v)}
if !strings.HasSuffix(v, "*") {
patterns = append(patterns, bleveQuery.NewQueryStringQuery(k+".wildcard:"+v+".*"))
prev = q
} else {
next = q
}
break
}
q = closed(bleveQuery.NewDisjunctionQuery(patterns))
case n.Exact && !strings.ContainsAny(n.Value, "*?") && slices.Contains([]string{"Name", "Title"}, k):
q = bleveQuery.NewQueryStringQuery(k + ".wildcard:" + v)
case k == "Path" && !strings.ContainsAny(n.Value, "*?"):
q = pathAndBelow(k, n.Value)
case slices.Contains([]string{"Name", "Title", "Content"}, k) && !strings.ContainsAny(n.Value, "*?"):
q = phrase(k, n.Value)
case strings.Contains(n.Value, " ") && !strings.ContainsAny(n.Value, "*?"):
q = phrase(k, n.Value)
default:
q = bleveQuery.NewQueryStringQuery(k + ":" + v)
// keys are resolved and media-type expanded by normalize. MimeType
// skips the escaper so the category wildcards (image/*) keep their `*`;
// bleve treats `/` and `+` as literals mid-term, so a literal MIME like
// image/svg+xml still matches exactly.
val := n.Value
if searchQuery.FieldIsPath(n.Key) {
val = strings.TrimSuffix(val, "/")
}
k := n.Key
v := val
if k != "ID" && k != "Size" && k != "MimeType" {
v = bleveEscaper.Replace(val)
}
if n.CaseInsensitive {
k += mapping.LowercaseSuffix
v = strings.ToLower(v)
val = strings.ToLower(val)
}
isWildcard := strings.ContainsAny(val, "*?")
// a word-broken field matches the value as a phrase of its words on the
// _words sibling (a quoted query string term is a match phrase query
// run through the field's analyzer); wildcards stay on _lowercase.
// A fulltext field is its own words field, the phrase runs on it.
if searchQuery.FieldIsWordBroken(n.Key) && !isWildcard && !n.Exact {
k, v = n.Key+mapping.WordsSuffix, `"`+strings.ReplaceAll(val, `"`, `\"`)+`"`
} else if searchQuery.FieldIsFulltext(n.Key) && !isWildcard && !n.Exact {
v = `"` + strings.ReplaceAll(val, `"`, `\"`) + `"`
}
var q bleveQuery.Query = bleveQuery.NewQueryStringQuery(k + ":" + v)
switch {
case n.Exact && !isWildcard:
// = matches the whole value, on the lowercased sibling for
// case-insensitive fields
tq := bleveQuery.NewTermQuery(val)
tq.SetField(k)
q = tq
case isWildcard && searchQuery.FieldIsWordBroken(n.Key) && !strings.HasSuffix(val, "*"):
// a wildcard on a word-broken field forgives a missing extension:
// *report also matches Report.txt
bq := bleve.NewBooleanQuery()
bq.AddShould(
bleveQuery.NewQueryStringQuery(k+":"+v),
bleveQuery.NewQueryStringQuery(k+":"+v+".*"),
)
bq.SetMinShould(1)
q = bq
}
if searchQuery.FieldIsPath(n.Key) {
// bleve has no path hierarchy analyzer, unlike OpenSearch: match the
// folder itself and its descendants (`\/*`). A BooleanQuery keeps
// this atomic; a DisjunctionQuery would be redistributed by an
// enclosing AND (mapBinary treats a left disjunction as an OR-chain).
bq := bleve.NewBooleanQuery()
bq.AddShould(q, bleveQuery.NewQueryStringQuery(k+":"+v+`\/*`))
bq.SetMinShould(1)
q = bq
}
if prev == nil {
@@ -161,7 +162,7 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
End: bleveQuery.BleveQueryTime{},
InclusiveStart: nil,
InclusiveEnd: nil,
FieldVal: getField(n.Key),
FieldVal: n.Key,
}
if n.Operator == nil {
@@ -191,7 +192,14 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
next = q
}
case *ast.NumberNode:
q := numberRange(getField(n.Key), n.Operator, n.Value)
var q bleveQuery.Query
if field := n.Key; slices.Contains([]string{"Size", "Type"}, field) {
q = numberRange(field, n.Operator, n.Value)
} else {
// same answer as the OpenSearch backend: unknown numeric keys
// match nothing instead of querying an arbitrary field
q = bleveQuery.NewMatchNoneQuery()
}
if q == nil {
continue
}
@@ -203,16 +211,14 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
}
case *ast.BooleanNode:
q := bleveQuery.NewBoolFieldQuery(n.Value)
q.SetField(getField(n.Key))
q.SetField(n.Key)
if prev == nil {
prev = q
} else {
next = q
}
case *ast.GroupNode:
if n.Key != "" {
n = normalizeGroupingProperty(n)
}
// keys resolved and grouping property propagated in normalize
q, _, err := walk(0, n.Nodes)
if err != nil {
return nil, 0, err
@@ -235,8 +241,11 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
q := bleve.NewBooleanQuery()
q.AddMustNot(next)
if prev == nil {
// unary in the beginning
// unary at the beginning: the term was consumed into the
// MustNot via nextNode, so clear next, otherwise a following
// operator would bind the stale term (NOT x AND y drops y).
prev = q
next = nil
} else {
next = q
}
@@ -260,10 +269,7 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
func nextNode(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) {
if n, ok := nodes[offset].(*ast.GroupNode); ok {
if n.Key != "" {
n = normalizeGroupingProperty(n)
}
// keys are resolved and group keys propagated by normalize
gq, _, err := walk(0, n.Nodes)
if err != nil {
return nil, 0, err
@@ -350,125 +356,3 @@ func numberRange(field string, operator *ast.OperatorNode, value float64) bleveQ
return q
}
func pathAndBelow(field, path string) bleveQuery.Query {
path = strings.TrimSuffix(path, "/")
self := bleveQuery.NewTermQuery(path)
self.SetField(field)
below := bleveQuery.NewPrefixQuery(path + "/")
below.SetField(field)
return closed(bleveQuery.NewDisjunctionQuery([]bleveQuery.Query{self, below}))
}
func closed(q bleveQuery.Query) bleveQuery.Query {
// a bare disjunction reads as an open OR chain to mapBinary, a later OR
// would merge into it and widen the group
return bleveQuery.NewConjunctionQuery([]bleveQuery.Query{q})
}
func phrase(field, value string) bleveQuery.Query {
q := bleveQuery.NewMatchPhraseQuery(value)
q.SetField(field)
return q
}
func getField(name string) string {
if name == "" {
return "Name"
}
if _, ok := _fields[strings.ToLower(name)]; ok {
return _fields[strings.ToLower(name)]
}
return name
}
func normalizeGroupingProperty(group *ast.GroupNode) *ast.GroupNode {
for _, n := range group.Nodes {
if onode, ok := n.(*ast.StringNode); ok {
onode.Key = group.Key
}
}
return group
}
func resourceType(value string) string {
switch strings.ToLower(value) {
case "file":
return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10)
case "folder":
return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10)
default:
return value
}
}
func mimeType(k, v string) (bleveQuery.Query, bool) {
switch v {
case "file":
q := bleve.NewBooleanQuery()
q.AddMustNot(bleveQuery.NewQueryStringQuery(k + ":httpd/unix-directory"))
return q, false
case "folder":
return bleveQuery.NewQueryStringQuery(k + ":httpd/unix-directory"), false
case "document":
return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k,
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.openxmlformats-officedocument.wordprocessingml.form",
"application/vnd.oasis.opendocument.text",
"text/plain",
"text/markdown",
"application/rtf",
"application/vnd.apple.pages",
)), true
case "spreadsheet":
return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k,
"application/vnd.ms-excel",
"application/vnd.oasis.opendocument.spreadsheet",
"text/csv",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.apple.numbers",
)), true
case "presentation":
return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k,
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"application/vnd.oasis.opendocument.presentation",
"application/vnd.ms-powerpoint",
"application/vnd.apple.keynote",
)), true
case "pdf":
return bleveQuery.NewQueryStringQuery(k + ":application/pdf"), false
case "image":
return bleveQuery.NewQueryStringQuery(k + ":image/*"), false
case "video":
return bleveQuery.NewQueryStringQuery(k + ":video/*"), false
case "audio":
return bleveQuery.NewQueryStringQuery(k + ":audio/*"), false
case "archive":
return bleveQuery.NewDisjunctionQuery(newQueryStringQueryList(k,
"application/zip",
"application/gzip",
"application/x-gzip",
"application/x-7z-compressed",
"application/x-rar-compressed",
"application/x-tar",
"application/x-bzip2",
"application/x-bzip",
"application/x-tgz",
)), true
default:
return bleveQuery.NewQueryStringQuery(k + ":" + v), false
}
}
func newQueryStringQueryList(k string, v ...string) []bleveQuery.Query {
list := make([]bleveQuery.Query, len(v))
for i := 0; i < len(v); i++ {
list[i] = bleveQuery.NewQueryStringQuery(k + ":" + v[i])
}
return list
}
services/search/pkg/query/bleve/compiler_test.go
+64 -89
View File
@@ -1,12 +1,12 @@
package bleve
import (
"strings"
"testing"
"time"
"github.com/blevesearch/bleve/v2/search/query"
"github.com/opencloud-eu/opencloud/pkg/ast"
searchquery "github.com/opencloud-eu/opencloud/services/search/pkg/query"
tAssert "github.com/stretchr/testify/assert"
)
@@ -19,22 +19,11 @@ var timeMustParse = func(t *testing.T, ts string) time.Time {
return tp
}
func wildcardQuery(field, value string) query.Query {
patterns := []query.Query{query.NewQueryStringQuery(field + ".wildcard:" + value)}
if !strings.HasSuffix(value, "*") {
patterns = append(patterns, query.NewQueryStringQuery(field+".wildcard:"+value+".*"))
}
return query.NewConjunctionQuery([]query.Query{query.NewDisjunctionQuery(patterns)})
}
func phraseQuery(field, value string) query.Query {
q := query.NewMatchPhraseQuery(value)
q.SetField(field)
return q
}
// TODO(followup): make this a pure compiler test. Field resolution and
// media-type expansion live in query.Normalize, so this test could feed
// canonical ASTs (real field names, media-type already expanded) and call
// compile() directly, dropping the query.Normalize wrapper and the mediatype
// cases.
func boolFieldQuery(field string, value bool) query.Query {
q := query.NewBoolFieldQuery(value)
q.SetField(field)
@@ -57,10 +46,31 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
phraseQuery("Name", "federated"),
query.NewQueryStringQuery(`Name_words:"federated"`),
}),
wantErr: false,
},
{
// path fields expand to match the folder itself and its descendants,
// since bleve has no path hierarchy analyzer.
name: `path:/Foo`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "path", Value: "/Foo"},
},
},
// a BooleanQuery (should: exact OR descendants), not a DisjunctionQuery,
// so an enclosing AND does not redistribute the folder-itself clause.
want: func() query.Query {
bq := query.NewBooleanQuery(nil, []query.Query{
query.NewQueryStringQuery(`Path:\/Foo`),
query.NewQueryStringQuery(`Path:\/Foo\/*`),
}, nil)
bq.SetMinShould(1)
return query.NewConjunctionQuery([]query.Query{bq})
}(),
wantErr: false,
},
{
name: `"John Smith"`,
args: &ast.Ast{
@@ -69,7 +79,7 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
phraseQuery("Name", "John Smith"),
query.NewQueryStringQuery(`Name_words:"john smith"`),
}),
wantErr: false,
},
@@ -83,8 +93,8 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
phraseQuery("Name", "John Smith"),
phraseQuery("Name", "Jane"),
query.NewQueryStringQuery(`Name_words:"john smith"`),
query.NewQueryStringQuery(`Name_words:"jane"`),
}),
wantErr: false,
},
@@ -98,8 +108,8 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Tags:bestseller`),
query.NewQueryStringQuery(`Tags:book`),
query.NewQueryStringQuery(`Tags_lowercase:bestseller`),
query.NewQueryStringQuery(`Tags_lowercase:book`),
}),
wantErr: false,
},
@@ -115,10 +125,10 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewDisjunctionQuery([]query.Query{
wildcardQuery("Name", `moby\ di*`),
query.NewQueryStringQuery(`Name_lowercase:moby\ di*`),
query.NewConjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Tags:bestseller`),
query.NewQueryStringQuery(`Tags:book`),
query.NewQueryStringQuery(`Tags_lowercase:bestseller`),
query.NewQueryStringQuery(`Tags_lowercase:book`),
}),
}),
wantErr: false,
@@ -136,10 +146,10 @@ func Test_compile(t *testing.T) {
},
want: query.NewDisjunctionQuery([]query.Query{
query.NewConjunctionQuery([]query.Query{
phraseQuery("Name", "a"),
phraseQuery("Name", "b"),
query.NewQueryStringQuery(`Name_words:"a"`),
query.NewQueryStringQuery(`Name_words:"b"`),
}),
phraseQuery("Name", "c"),
query.NewQueryStringQuery(`Name_words:"c"`),
}),
wantErr: false,
},
@@ -155,10 +165,10 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewDisjunctionQuery([]query.Query{
phraseQuery("Name", "a"),
query.NewQueryStringQuery(`Name_words:"a"`),
query.NewConjunctionQuery([]query.Query{
phraseQuery("Name", "b"),
phraseQuery("Name", "c"),
query.NewQueryStringQuery(`Name_words:"b"`),
query.NewQueryStringQuery(`Name_words:"c"`),
}),
}),
wantErr: false,
@@ -180,11 +190,11 @@ func Test_compile(t *testing.T) {
},
want: query.NewConjunctionQuery([]query.Query{
query.NewDisjunctionQuery([]query.Query{
phraseQuery("Name", "a"),
phraseQuery("Name", "b"),
phraseQuery("Name", "c"),
query.NewQueryStringQuery(`Name_words:"a"`),
query.NewQueryStringQuery(`Name_words:"b"`),
query.NewQueryStringQuery(`Name_words:"c"`),
}),
phraseQuery("Name", "d"),
query.NewQueryStringQuery(`Name_words:"d"`),
}),
wantErr: false,
},
@@ -203,10 +213,10 @@ func Test_compile(t *testing.T) {
},
want: query.NewConjunctionQuery([]query.Query{
query.NewDisjunctionQuery([]query.Query{
wildcardQuery("Name", `moby\ di*`),
query.NewQueryStringQuery(`Tags:bestseller`),
query.NewQueryStringQuery(`Name_lowercase:moby\ di*`),
query.NewQueryStringQuery(`Tags_lowercase:bestseller`),
}),
query.NewQueryStringQuery(`Tags:book`),
query.NewQueryStringQuery(`Tags_lowercase:book`),
}),
wantErr: false,
},
@@ -228,11 +238,11 @@ func Test_compile(t *testing.T) {
},
want: query.NewConjunctionQuery([]query.Query{
query.NewDisjunctionQuery([]query.Query{
wildcardQuery("Name", `moby\ di*`),
query.NewQueryStringQuery(`Tags:bestseller`),
query.NewQueryStringQuery(`Name_lowercase:moby\ di*`),
query.NewQueryStringQuery(`Tags_lowercase:bestseller`),
}),
query.NewQueryStringQuery(`Tags:book`),
query.NewBooleanQuery(nil, nil, []query.Query{query.NewQueryStringQuery(`Tags:read`)}),
query.NewQueryStringQuery(`Tags_lowercase:book`),
query.NewBooleanQuery(nil, nil, []query.Query{query.NewQueryStringQuery(`Tags_lowercase:read`)}),
}),
wantErr: false,
},
@@ -251,7 +261,7 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
phraseQuery("author", "John Smith"),
query.NewQueryStringQuery(`author:John\ Smith`),
query.NewQueryStringQuery(`author:Jane`),
}),
wantErr: false,
@@ -273,9 +283,9 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
phraseQuery("author", "John Smith"),
query.NewQueryStringQuery(`author:John\ Smith`),
query.NewQueryStringQuery(`author:Jane`),
query.NewQueryStringQuery(`Tags:bestseller`),
query.NewQueryStringQuery(`Tags_lowercase:bestseller`),
}),
wantErr: false,
},
@@ -317,47 +327,12 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
phraseQuery("Name", "John Smith"),
query.NewQueryStringQuery(`Name_words:"john smith"`),
boolFieldQuery("Hidden", true),
boolFieldQuery("Hidden", true),
}),
wantErr: false,
},
{
name: `hidden:banana`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "hidden", Value: "banana"},
},
},
want: query.NewConjunctionQuery([]query.Query{query.NewMatchNoneQuery()}),
wantErr: false,
},
{
name: `name="Report.txt"`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "name", Value: "Report.txt", Exact: true},
},
},
want: query.NewConjunctionQuery([]query.Query{query.NewQueryStringQuery(`Name.wildcard:report.txt`)}),
wantErr: false,
},
{
name: `type:File`,
args: &ast.Ast{
Nodes: []ast.Node{
&ast.StringNode{Key: "type", Value: "File"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "type", Value: "FOLDER"},
},
},
want: query.NewDisjunctionQuery([]query.Query{
query.NewQueryStringQuery(`Type:1`),
query.NewQueryStringQuery(`Type:2`),
}),
wantErr: false,
},
{
name: `NOT tag:physik`,
args: &ast.Ast{
@@ -367,7 +342,7 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
query.NewBooleanQuery(nil, nil, []query.Query{query.NewQueryStringQuery(`Tags:physik`)}),
query.NewBooleanQuery(nil, nil, []query.Query{query.NewQueryStringQuery(`Tags_lowercase:physik`)}),
}),
wantErr: false,
},
@@ -483,7 +458,7 @@ func Test_compile(t *testing.T) {
query.NewQueryStringQuery(`MimeType:application/rtf`),
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
}),
wildcardQuery("Name", `*tdd*`),
query.NewQueryStringQuery(`Name_lowercase:*tdd*`),
}),
wantErr: false,
},
@@ -509,7 +484,7 @@ func Test_compile(t *testing.T) {
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
query.NewConjunctionQuery([]query.Query{
query.NewQueryStringQuery(`MimeType:application/pdf`),
wildcardQuery("Name", `*tdd*`),
query.NewQueryStringQuery(`Name_lowercase:*tdd*`),
}),
}),
wantErr: false,
@@ -539,7 +514,7 @@ func Test_compile(t *testing.T) {
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
query.NewQueryStringQuery(`MimeType:application/pdf`),
}),
wildcardQuery("Name", `*tdd*`),
query.NewQueryStringQuery(`Name_lowercase:*tdd*`),
}),
wantErr: false,
},
@@ -568,7 +543,7 @@ func Test_compile(t *testing.T) {
query.NewQueryStringQuery(`MimeType:application/rtf`),
query.NewQueryStringQuery(`MimeType:application/vnd.apple.pages`),
}),
wildcardQuery("Name", `*tdd*`),
query.NewQueryStringQuery(`Name_lowercase:*tdd*`),
}),
wantErr: false,
},
@@ -580,7 +555,7 @@ func Test_compile(t *testing.T) {
},
},
want: query.NewConjunctionQuery([]query.Query{
phraseQuery("Name", "John Smith +-=&|><!(){}[]^\"~: "),
query.NewQueryStringQuery(`Name_words:"john smith +-=&|><!(){}[]^\"~: "`),
}),
wantErr: false,
},
@@ -590,7 +565,7 @@ func Test_compile(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := compile(tt.args)
got, err := compile(searchquery.Normalize(tt.args, searchquery.ResolveField))
if (err != nil) != tt.wantErr {
t.Errorf("compile() error = %v, wantErr %v", err, tt.wantErr)
services/search/pkg/query/mimetype/mimetype.go
+102
View File
@@ -0,0 +1,102 @@
// Package mimetype maps the "mediatype" KQL restriction (field name and value)
// to a concrete MimeType query.
package mimetype
import (
"strings"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/pkg/kql"
)
// field is the real index field a mediatype restriction targets.
const field = "MimeType"
// Expand turns mediatype:<value> into the MimeType query it stands for: category
// values (file/document/image/...) expand to their MIME set, anything else is a
// literal MimeType:<value>. Returns nil for non-mediatype keys. Categories and
// MIME types are case-insensitive, so the value is lowercased.
func Expand(key, value string) []ast.Node {
if strings.ToLower(key) != "mediatype" {
return nil
}
value = strings.ToLower(value)
switch value {
case "file":
// grouped so the negation stays atomic when it composes with other
// terms (mediatype:file OR ...)
return []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.OperatorNode{Value: kql.BoolNOT},
&ast.StringNode{Key: field, Value: "httpd/unix-directory"},
}},
}
case "folder":
return term("httpd/unix-directory")
case "document":
return group(
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.openxmlformats-officedocument.wordprocessingml.form",
"application/vnd.oasis.opendocument.text",
"text/plain",
"text/markdown",
"application/rtf",
"application/vnd.apple.pages",
)
case "spreadsheet":
return group(
"application/vnd.ms-excel",
"application/vnd.oasis.opendocument.spreadsheet",
"text/csv",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.apple.numbers",
)
case "presentation":
return group(
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"application/vnd.oasis.opendocument.presentation",
"application/vnd.ms-powerpoint",
"application/vnd.apple.keynote",
)
case "pdf":
return term("application/pdf")
case "image":
return term("image/*")
case "video":
return term("video/*")
case "audio":
return term("audio/*")
case "archive":
return group(
"application/zip",
"application/gzip",
"application/x-gzip",
"application/x-7z-compressed",
"application/x-rar-compressed",
"application/x-tar",
"application/x-bzip2",
"application/x-bzip",
"application/x-tgz",
)
}
// not a category: treat the value as a literal MIME type.
return term(value)
}
// term is a single MimeType:value restriction.
func term(value string) []ast.Node {
return []ast.Node{&ast.StringNode{Key: field, Value: value}}
}
// group is a single OR group of MimeType:value restrictions.
func group(values ...string) []ast.Node {
nodes := make([]ast.Node, 0, len(values)*2-1)
for i, v := range values {
if i > 0 {
nodes = append(nodes, &ast.OperatorNode{Value: kql.BoolOR})
}
nodes = append(nodes, &ast.StringNode{Key: field, Value: v})
}
return []ast.Node{&ast.GroupNode{Nodes: nodes}}
}
services/search/pkg/query/mimetype/mimetype_suite_test.go
+13
View File
@@ -0,0 +1,13 @@
package mimetype_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestMimetype(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Mimetype Suite")
}
services/search/pkg/query/mimetype/mimetype_test.go
+110
View File
@@ -0,0 +1,110 @@
package mimetype_test
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/pkg/query/mimetype"
)
// This is the single place the mediatype -> MimeType mapping is tested. The
// query pipeline consumes Expand via query.Normalize and must NOT re-test it.
// mimeValues extracts the StringNode values from an OR group, dropping operators.
func mimeValues(group *ast.GroupNode) []string {
var out []string
for _, n := range group.Nodes {
if s, ok := n.(*ast.StringNode); ok {
out = append(out, s.Value)
}
}
return out
}
var _ = Describe("Expand", func() {
It("only triggers on the mediatype key", func() {
Expect(mimetype.Expand("Name", "document")).To(BeNil())
Expect(mimetype.Expand("MimeType", "file")).To(BeNil()) // the real field name is not the trigger
Expect(mimetype.Expand("Tags", "file")).To(BeNil())
})
It("matches the key case-insensitively", func() {
Expect(mimetype.Expand("MediaType", "file")).ToNot(BeNil())
})
It("matches the value case-insensitively", func() {
// a category matches regardless of case
Expect(mimetype.Expand("mediatype", "Folder")).To(Equal([]ast.Node{
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
}))
// a literal MIME type is lowercased too (MIME types are case-insensitive)
Expect(mimetype.Expand("mediatype", "Image/SVG+XML")).To(Equal([]ast.Node{
&ast.StringNode{Key: "MimeType", Value: "image/svg+xml"},
}))
})
// A non-category value is a literal MIME type and targets the MimeType field.
It("passes literal values through to MimeType", func() {
Expect(mimetype.Expand("mediatype", "application/pdf")).To(Equal([]ast.Node{
&ast.StringNode{Key: "MimeType", Value: "application/pdf"},
}))
Expect(mimetype.Expand("mediatype", "image/jpeg")).To(Equal([]ast.Node{
&ast.StringNode{Key: "MimeType", Value: "image/jpeg"},
}))
})
It("expands file to not-a-folder", func() {
Expect(mimetype.Expand("mediatype", "file")).To(Equal([]ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
}},
}))
})
It("expands folder to a single term", func() {
Expect(mimetype.Expand("mediatype", "folder")).To(Equal([]ast.Node{
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
}))
})
It("expands wildcard categories", func() {
for value, mime := range map[string]string{
"image": "image/*", "video": "video/*", "audio": "audio/*", "pdf": "application/pdf",
} {
Expect(mimetype.Expand("mediatype", value)).To(Equal([]ast.Node{
&ast.StringNode{Key: "MimeType", Value: mime},
}), value)
}
})
It("expands the document group", func() {
got := mimetype.Expand("mediatype", "document")
Expect(got).To(HaveLen(1))
group, ok := got[0].(*ast.GroupNode)
Expect(ok).To(BeTrue())
Expect(mimeValues(group)).To(Equal([]string{
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.openxmlformats-officedocument.wordprocessingml.form",
"application/vnd.oasis.opendocument.text",
"text/plain",
"text/markdown",
"application/rtf",
"application/vnd.apple.pages",
}))
})
// spreadsheet asserts the exact MIME set, in order, with no duplicate entry.
It("expands the spreadsheet group", func() {
group := mimetype.Expand("mediatype", "spreadsheet")[0].(*ast.GroupNode)
Expect(mimeValues(group)).To(Equal([]string{
"application/vnd.ms-excel",
"application/vnd.oasis.opendocument.spreadsheet",
"text/csv",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.apple.numbers",
}))
})
})
services/search/pkg/query/normalize.go
+100
View File
@@ -0,0 +1,100 @@
package query
import (
"strconv"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"reflect"
"strings"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/pkg/query/mimetype"
)
// Normalize is the shared KQL lowering pass between parse and compile: it
// resolves keys to real field names (via resolve) and expands media-type
// restrictions, so the backends compile a plain field:value AST.
func Normalize(a *ast.Ast, resolve func(string) string) *ast.Ast {
a.Nodes = normalizeNodes(a.Nodes, resolve, "")
return a
}
// normalizeNodes rewrites nodes in place. defaultKey is what a bare restriction
// inherits: its enclosing group's key, or "" at the top level.
func normalizeNodes(nodes []ast.Node, resolve func(string) string, defaultKey string) []ast.Node {
resolveKey := func(key string) string {
if key == "" && defaultKey != "" {
return defaultKey // bare child inherits the group key
}
return resolve(key)
}
out := make([]ast.Node, 0, len(nodes))
for _, n := range nodes {
n = toPointer(n) // ensure a pointer so in-place key rewrites persist
switch node := n.(type) {
case *ast.StringNode:
node.Key = resolveKey(node.Key)
if FieldValueIsNormalized(node.Key) {
node.Value = strings.ToLower(node.Value)
}
if node.Key == "Type" {
node.Value = resourceType(node.Value)
}
if exp := mimetype.Expand(node.Key, node.Value); exp != nil {
out = append(out, normalizeNodes(exp, resolve, defaultKey)...)
continue
}
node.CaseInsensitive = FieldIsCaseInsensitive(node.Key)
out = append(out, node)
case *ast.DateTimeNode:
node.Key = resolveKey(node.Key)
out = append(out, node)
case *ast.BooleanNode:
node.Key = resolveKey(node.Key)
out = append(out, node)
case *ast.NumberNode:
node.Key = resolveKey(node.Key)
out = append(out, node)
case *ast.GroupNode:
groupKey := defaultKey
if node.Key != "" {
node.Key = resolve(node.Key)
groupKey = node.Key
}
node.Nodes = normalizeNodes(node.Nodes, resolve, groupKey)
out = append(out, node)
default:
out = append(out, n)
}
}
return out
}
// toPointer returns n as a pointer; the parser emits some nodes by value and the
// in-place key rewrites would be lost on those.
func toPointer(n ast.Node) ast.Node {
rv := reflect.ValueOf(n)
if rv.Kind() == reflect.Ptr {
return n
}
ptr := reflect.New(rv.Type())
ptr.Elem().Set(rv)
if pn, ok := ptr.Interface().(ast.Node); ok {
return pn
}
return n
}
// resourceType maps the type categories to the stored resource type value;
// unknown values pass through and become dead term queries.
func resourceType(value string) string {
switch strings.ToLower(value) {
case "file":
return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_FILE), 10)
case "folder":
return strconv.FormatUint(uint64(provider.ResourceType_RESOURCE_TYPE_CONTAINER), 10)
default:
return value
}
}
services/search/pkg/query/normalize_test.go
+125
View File
@@ -0,0 +1,125 @@
package query_test
import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/services/search/pkg/query"
)
// This is the single place the shared KQL lowering pass is tested (field
// resolution, media-type expansion, group-key defaulting, pointer conversion).
// The backend query compilers consume its canonical output and must not re-test
// it.
func norm(nodes ...ast.Node) []ast.Node {
return query.Normalize(&ast.Ast{Nodes: nodes}, query.ResolveField).Nodes
}
var _ = Describe("ResolveField", func() {
It("resolves keys to canonical field names", func() {
Expect(query.ResolveField("")).To(Equal("Name")) // empty -> free-text default
Expect(query.ResolveField("NAME")).To(Equal("Name")) // canonical, case-insensitive key match
Expect(query.ResolveField("tag")).To(Equal("Tags")) // singular alias
Expect(query.ResolveField("mimetype")).To(Equal("MimeType")) // real field
Expect(query.ResolveField("photo.CAMERAMAKE")).To(Equal("photo.cameraMake")) // facet, case-insensitive key match
Expect(query.ResolveField("unknown.field")).To(Equal("unknown.field")) // unknown key: unchanged, becomes a dead query
})
})
var _ = Describe("FieldIsCaseInsensitive", func() {
It("reports the CaseInsensitive override fields", func() {
// keyword fields are case-insensitive by default, facets included
for _, f := range []string{"Name", "Title", "Tags", "audio.artist", "photo.cameraMake"} {
Expect(query.FieldIsCaseInsensitive(f)).To(BeTrue(), f)
}
// opted out (ids, favorites, path, mime type) or not a keyword at all
for _, f := range []string{"MimeType", "ID", "RootID", "ParentID", "Favorites", "Content", "Path", "Size", "unknown"} {
Expect(query.FieldIsCaseInsensitive(f)).To(BeFalse(), f)
}
})
})
var _ = Describe("FieldIsWordBroken", func() {
It("reports the word-broken fields, keywords unless opted out", func() {
for _, f := range []string{"Name", "Title", "audio.artist", "photo.cameraMake"} {
Expect(query.FieldIsWordBroken(f)).To(BeTrue(), f)
}
// opted out (labels, ids, mime type), paths and full text are not
for _, f := range []string{"Tags", "Favorites", "MimeType", "ID", "Content", "Path", "unknown"} {
Expect(query.FieldIsWordBroken(f)).To(BeFalse(), f)
}
})
})
var _ = Describe("Normalize", func() {
It("resolves fields and expands mediatype", func() {
got := norm(
&ast.StringNode{Key: "", Value: "free"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "TAG", Value: "x"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "photo.cameramake", Value: "canon"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "mediatype", Value: "file"},
&ast.OperatorNode{Value: "AND"},
ast.NumberNode{Key: "size", Value: 100},
)
Expect(got).To(Equal([]ast.Node{
&ast.StringNode{Key: "Name", Value: "free", CaseInsensitive: true},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "Tags", Value: "x", CaseInsensitive: true},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "photo.cameraMake", Value: "canon", CaseInsensitive: true},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "MimeType", Value: "httpd/unix-directory"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.NumberNode{Key: "Size", Value: 100},
}))
})
// A bare restriction inside a named group inherits the group key; a keyed
// child keeps its own key; a bare restriction in an unnamed group falls
// back to Name.
It("defaults group keys", func() {
got := norm(
&ast.GroupNode{Key: "author", Nodes: []ast.Node{
&ast.StringNode{Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "name", Value: "d"},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Value: "e"},
}},
)
Expect(got).To(Equal([]ast.Node{
&ast.GroupNode{Key: "author", Nodes: []ast.Node{
&ast.StringNode{Key: "author", Value: "b"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "Name", Value: "d", CaseInsensitive: true},
}},
&ast.OperatorNode{Value: "AND"},
&ast.GroupNode{Nodes: []ast.Node{
&ast.StringNode{Key: "Name", Value: "e", CaseInsensitive: true},
}},
}))
})
It("converts value nodes to pointers", func() {
got := norm(
ast.StringNode{Key: "name", Value: "x"},
ast.OperatorNode{Value: "AND"},
ast.DateTimeNode{Key: "mtime"},
)
Expect(got).To(Equal([]ast.Node{
&ast.StringNode{Key: "Name", Value: "x", CaseInsensitive: true},
&ast.OperatorNode{Value: "AND"},
&ast.DateTimeNode{Key: "Mtime"},
}))
})
})
services/search/pkg/query/query_suite_test.go
+13
View File
@@ -0,0 +1,13 @@
package query_test
import (
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
func TestQuery(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Query Suite")
}
services/search/pkg/query/resolver.go
+103
View File
@@ -0,0 +1,103 @@
package query
import (
"reflect"
"strings"
"sync"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
"github.com/opencloud-eu/opencloud/services/search/pkg/search"
)
// aliases are KQL spellings the derived index can't produce (fields are plural).
var aliases = map[string]string{
"tag": "Tags",
"favorite": "Favorites",
}
// fieldIndex maps a lowercased KQL key to its canonical field name ("" is the
// bare-search default).
var fieldIndex = sync.OnceValue(func() map[string]string {
idx := mapping.FieldNameIndex(reflect.TypeFor[search.Resource](), search.Resource{}.SearchFieldOverrides())
for k, v := range aliases {
idx[k] = v
}
idx[""] = idx["name"]
return idx
})
// siblingFields lists which search siblings every field carries, from the
// resource struct and its overrides.
var siblingFields = sync.OnceValue(func() map[string]mapping.Siblings {
return mapping.SearchSiblings(reflect.TypeFor[search.Resource](), search.Resource{}.SearchFieldOverrides())
})
// pathFields are hierarchical path fields (TypePath), derived from the overrides.
var pathFields = sync.OnceValue(func() map[string]struct{} {
out := map[string]struct{}{}
for field, opts := range (search.Resource{}).SearchFieldOverrides() {
if opts.Type == mapping.TypePath {
out[field] = struct{}{}
}
}
return out
})
// fulltextFields are analyzed full-text fields (TypeFulltext), derived from the
// overrides.
var fulltextFields = sync.OnceValue(func() map[string]struct{} {
out := map[string]struct{}{}
for field, opts := range (search.Resource{}).SearchFieldOverrides() {
if opts.Type == mapping.TypeFulltext {
out[field] = struct{}{}
}
}
return out
})
// ResolveField maps a KQL key to its canonical field name; unknown keys pass through.
func ResolveField(name string) string {
if v, ok := fieldIndex()[strings.ToLower(name)]; ok {
return v
}
return name
}
// normalizedValueFields have their stored values normalized to lowercase at
// index time, so query values fold to match even though the fields themselves
// are case-preserved keywords.
var normalizedValueFields = map[string]struct{}{
"MimeType": {},
"Type": {},
"Hidden": {},
}
// FieldValueIsNormalized reports whether a field's stored values are
// normalized lowercase.
func FieldValueIsNormalized(field string) bool {
_, ok := normalizedValueFields[field]
return ok
}
// FieldIsCaseInsensitive reports whether a field's default search is case-insensitive.
func FieldIsCaseInsensitive(field string) bool {
return siblingFields()[field].Lowercase
}
// FieldIsPath reports whether a field is a hierarchical path field.
func FieldIsPath(field string) bool {
_, ok := pathFields()[field]
return ok
}
// FieldIsFulltext reports whether a field is an analyzed full-text field.
func FieldIsFulltext(field string) bool {
_, ok := fulltextFields()[field]
return ok
}
// FieldIsWordBroken reports whether a field is split into words, so a value
// without a wildcard matches it as a phrase of those words instead of as a whole.
func FieldIsWordBroken(field string) bool {
return siblingFields()[field].Words
}
services/search/pkg/search/search.go
+42 -7
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"regexp"
"strings"
"sync"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
@@ -19,8 +20,15 @@ import (
searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/content"
"github.com/opencloud-eu/opencloud/services/search/pkg/mapping"
)
// SchemaVersion is the shared schema version for both search backends. Bump it
// on a breaking mapping change: each version gets its own index (OpenSearch name
// suffix, bleve path suffix), so the service builds a fresh index instead of
// colliding with the old one. No migration; reindex to populate.
const SchemaVersion = 4
var scopeRegex = regexp.MustCompile(`scope:\s*([^" "\n\r]*)`)
// Engine is the interface to the search engine
@@ -52,13 +60,40 @@ type BatchOperator interface {
type Resource struct {
content.Document
ID string
RootID string
Path string
ParentID string
Type uint64
Deleted bool
Hidden bool
ID string `json:"ID"`
RootID string `json:"RootID"`
Path string `json:"Path"`
ParentID string `json:"ParentID"`
Type uint64 `json:"Type"`
Deleted bool `json:"Deleted"`
Hidden bool `json:"Hidden"`
}
// resourceFieldOverrides is built once (it never changes) and reused on hot
// paths instead of reallocating per call.
var resourceFieldOverrides = sync.OnceValue(func() map[string]mapping.FieldOpts {
True, False := true, false
return map[string]mapping.FieldOpts{
// every keyword field searches case-insensitively and by word (name,
// title, the facets) unless opted out: ids are opaque, paths are POSIX,
// the mime type is normalized already, a tag is one label
"ID": {CaseInsensitive: &False, NoWordBreaker: &True},
"RootID": {CaseInsensitive: &False, NoWordBreaker: &True},
"ParentID": {CaseInsensitive: &False, NoWordBreaker: &True},
"Path": {Type: mapping.TypePath, CaseInsensitive: &False},
"MimeType": {CaseInsensitive: &False, NoWordBreaker: &True},
"Content": {Type: mapping.TypeFulltext},
"Tags": {NoWordBreaker: &True, IncludeInAll: &False},
"Favorites": {NoWordBreaker: &True, IncludeInAll: &False, CaseInsensitive: &False}, // opaque user ids
"location": {Type: mapping.TypeGeopoint},
}
})
// SearchFieldOverrides returns the field options the mapping package needs to
// build per-backend index mappings for a Resource (keys are json-tag names).
// The map is shared and read-only; clone it before mutating.
func (Resource) SearchFieldOverrides() map[string]mapping.FieldOpts {
return resourceFieldOverrides()
}
// ResolveReference makes sure the path is relative to the space root
services/search/pkg/search/service.go
+19 -36
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
@@ -666,10 +667,10 @@ func (s *Service) doUpsertItem(ref *provider.Reference, batch BatchOperator) {
// determine if metadata needs to be stored in storage as well
metadata := map[string]string{}
addAudioMetadata(metadata, doc.Audio)
addImageMetadata(metadata, doc.Image)
addLocationMetadata(metadata, doc.Location)
addPhotoMetadata(metadata, doc.Photo)
facetToMetadata(metadata, doc.Audio, "libre.graph.audio.")
facetToMetadata(metadata, doc.Image, "libre.graph.image.")
facetToMetadata(metadata, doc.Location, "libre.graph.location.")
facetToMetadata(metadata, doc.Photo, "libre.graph.photo.")
if len(metadata) == 0 {
return
}
@@ -705,43 +706,25 @@ func IsHidden(path string) bool {
return false
}
func addAudioMetadata(metadata map[string]string, audio *libregraph.Audio) {
if audio == nil {
return
// facetToMetadata flattens a libregraph facet (Audio / Image / Location / Photo
// pointer) into the metadata map under the given prefix via the model's ToMap.
// No-op when the facet is nil.
func facetToMetadata[T libregraph.MappedNullable](metadata map[string]string, facet T, prefix string) {
// Only nilable kinds can be nil; IsNil panics on a value type (some
// libregraph models satisfy MappedNullable with a value receiver).
switch v := reflect.ValueOf(facet); v.Kind() {
case reflect.Ptr, reflect.Map, reflect.Slice, reflect.Interface, reflect.Chan, reflect.Func:
if v.IsNil() {
return
}
}
marshalToStringMap(audio, metadata, "libre.graph.audio.")
}
func addImageMetadata(metadata map[string]string, image *libregraph.Image) {
if image == nil {
return
}
marshalToStringMap(image, metadata, "libre.graph.image.")
}
func addLocationMetadata(metadata map[string]string, location *libregraph.GeoCoordinates) {
if location == nil {
return
}
marshalToStringMap(location, metadata, "libre.graph.location.")
}
func addPhotoMetadata(metadata map[string]string, photo *libregraph.Photo) {
if photo == nil {
return
}
marshalToStringMap(photo, metadata, "libre.graph.photo.")
}
func marshalToStringMap[T libregraph.MappedNullable](source T, target map[string]string, prefix string) {
// ToMap never returns a non-nil error ...
m, _ := source.ToMap()
// ToMap never returns a non-nil error.
m, _ := facet.ToMap()
for k, v := range m {
if v == nil {
continue
}
target[prefix+k] = valueToString(v)
metadata[prefix+k] = valueToString(v)
}
}
vendor/modules.txt
Vendored
+1 -1
View File
@@ -125,8 +125,8 @@ github.com/blevesearch/bleve/v2/analysis
github.com/blevesearch/bleve/v2/analysis/analyzer/custom
github.com/blevesearch/bleve/v2/analysis/analyzer/keyword
github.com/blevesearch/bleve/v2/analysis/analyzer/standard
github.com/blevesearch/bleve/v2/analysis/datetime/flexible
github.com/blevesearch/bleve/v2/analysis/char/regexp
github.com/blevesearch/bleve/v2/analysis/datetime/flexible
github.com/blevesearch/bleve/v2/analysis/datetime/optional
github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds
github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds