diff --git a/pkg/conversions/conversions.go b/pkg/conversions/conversions.go
new file mode 100644
index 0000000000..6931f6e723
--- /dev/null
+++ b/pkg/conversions/conversions.go
@@ -0,0 +1,24 @@
+package conversions
+
+import (
+ "encoding/json"
+)
+
+func To[T any](v any) (T, error) {
+ var t T
+
+ if v == nil {
+ return t, nil
+ }
+
+ j, err := json.Marshal(v)
+ if err != nil {
+ return t, err
+ }
+
+ if err := json.Unmarshal(j, &t); err != nil {
+ return t, err
+ }
+
+ return t, nil
+}
diff --git a/services/search/pkg/opensearch/engine.go b/services/search/pkg/opensearch/backend.go
similarity index 60%
rename from services/search/pkg/opensearch/engine.go
rename to services/search/pkg/opensearch/backend.go
index 98b5df0b00..072ea5d2ee 100644
--- a/services/search/pkg/opensearch/engine.go
+++ b/services/search/pkg/opensearch/backend.go
@@ -7,6 +7,7 @@ import (
"fmt"
"path"
"strings"
+ "time"
storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
@@ -14,18 +15,23 @@ import (
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/opencloud-eu/opencloud/pkg/conversions"
- "github.com/opencloud-eu/opencloud/pkg/kql"
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/engine"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
)
-type Engine struct {
+var (
+ ErrUnhealthyCluster = fmt.Errorf("cluster is not healthy")
+)
+
+type Backend struct {
index string
client *opensearchgoAPI.Client
}
-func NewEngine(index string, client *opensearchgoAPI.Client) (*Engine, error) {
+func NewBackend(index string, client *opensearchgoAPI.Client) (*Backend, error) {
pingResp, err := client.Ping(context.TODO(), &opensearchgoAPI.PingReq{})
switch {
case err != nil:
@@ -35,45 +41,46 @@ func NewEngine(index string, client *opensearchgoAPI.Client) (*Engine, error) {
}
// apply the index template
- if err := IndexManagerLatest.Apply(context.TODO(), index, client); err != nil {
+ if err := osu.IndexManagerLatest.Apply(context.TODO(), index, client); err != nil {
return nil, fmt.Errorf("failed to apply index template: %w", err)
}
// first check if the cluster is healthy
- _, healthy, err := clusterHealth(context.TODO(), client, []string{index})
+
+ resp, err := client.Cluster.Health(context.TODO(), &opensearchgoAPI.ClusterHealthReq{
+ Indices: []string{index},
+ Params: opensearchgoAPI.ClusterHealthParams{
+ Local: opensearchgoAPI.ToPointer(true),
+ Timeout: 5 * time.Second,
+ },
+ })
switch {
case err != nil:
- return nil, fmt.Errorf("failed to get cluster health: %w", err)
- case !healthy:
- return nil, fmt.Errorf("cluster health is not healthy")
+ return nil, fmt.Errorf("%w, failed to get cluster health: %w", ErrUnhealthyCluster, err)
+ case resp.TimedOut:
+ return nil, fmt.Errorf("%w, cluster health request timed out", ErrUnhealthyCluster)
+ case resp.Status != "green" && resp.Status != "yellow":
+ return nil, fmt.Errorf("%w, cluster health is not green or yellow: %s", ErrUnhealthyCluster, resp.Status)
}
- return &Engine{index: index, client: client}, nil
+ return &Backend{index: index, client: client}, nil
}
-func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) {
- ast, err := kql.Builder{}.Build(sir.Query)
+func (be *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) {
+ boolQuery, err := convert.KQLToOpenSearchBoolQuery(sir.Query)
if err != nil {
- return nil, fmt.Errorf("failed to build query: %w", err)
+ return nil, fmt.Errorf("failed to convert KQL query to OpenSearch bool query: %w", err)
}
- transpiler, err := NewKQLToOsDSL()
- if err != nil {
- return nil, fmt.Errorf("failed to create KQL compiler: %w", err)
- }
-
- builder, err := transpiler.Compile(ast)
- if err != nil {
- return nil, fmt.Errorf("failed to compile query: %w", err)
- }
-
- boolQuery := builderToBoolQuery(builder).Filter(
- NewTermQuery[bool]("Deleted").Value(false),
+ // filter out deleted resources
+ boolQuery.Filter(
+ osu.NewTermQuery[bool]("Deleted").Value(false),
)
if sir.Ref != nil {
+ // if a reference is provided, filter by the root ID
boolQuery.Filter(
- NewTermQuery[string]("RootID").Value(
+ osu.NewTermQuery[string]("RootID").Value(
storagespace.FormatResourceID(
&storageProvider.ResourceId{
StorageId: sir.Ref.GetResourceId().GetStorageId(),
@@ -96,16 +103,16 @@ func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexReque
searchParams.Size = conversions.ToPointer(int(sir.PageSize))
}
- req, err := BuildSearchReq(&opensearchgoAPI.SearchReq{
- Indices: []string{e.index},
+ req, err := osu.BuildSearchReq(&opensearchgoAPI.SearchReq{
+ Indices: []string{be.index},
Params: searchParams,
},
boolQuery,
- SearchReqOptions{
- Highlight: &HighlightOption{
+ osu.SearchReqOptions{
+ Highlight: &osu.HighlightOption{
PreTags: []string{""},
PostTags: []string{""},
- Fields: map[string]HighlightOption{
+ Fields: map[string]osu.HighlightOption{
"Content": {},
},
},
@@ -115,7 +122,7 @@ func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexReque
return nil, fmt.Errorf("failed to build search request: %w", err)
}
- resp, err := e.client.Search(ctx, req)
+ resp, err := be.client.Search(ctx, req)
if err != nil {
return nil, fmt.Errorf("failed to search: %w", err)
}
@@ -123,7 +130,7 @@ func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexReque
matches := make([]*searchMessage.Match, 0, len(resp.Hits.Hits))
totalMatches := resp.Hits.Total.Value
for _, hit := range resp.Hits.Hits {
- match, err := searchHitToSearchMessageMatch(hit)
+ match, err := convert.OpenSearchHitToMatch(hit)
if err != nil {
return nil, fmt.Errorf("failed to convert hit to match: %w", err)
}
@@ -148,14 +155,14 @@ func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexReque
}, nil
}
-func (e *Engine) Upsert(id string, r engine.Resource) error {
+func (be *Backend) Upsert(id string, r engine.Resource) error {
body, err := json.Marshal(r)
if err != nil {
return fmt.Errorf("failed to marshal resource: %w", err)
}
- _, err = e.client.Index(context.TODO(), opensearchgoAPI.IndexReq{
- Index: e.index,
+ _, err = be.client.Index(context.TODO(), opensearchgoAPI.IndexReq{
+ Index: be.index,
DocumentID: id,
Body: bytes.NewReader(body),
})
@@ -166,9 +173,9 @@ func (e *Engine) Upsert(id string, r engine.Resource) error {
return nil
}
-func (e *Engine) Move(id string, parentID string, target string) error {
- return e.updateSelfAndDescendants(id, func(rootResource engine.Resource) *ScriptOption {
- return &ScriptOption{
+func (be *Backend) Move(id string, parentID string, target string) error {
+ return be.updateSelfAndDescendants(id, func(rootResource engine.Resource) *osu.ScriptOption {
+ return &osu.ScriptOption{
Source: `
if (ctx._source.ID == params.id ) { ctx._source.Name = params.newName; ctx._source.ParentID = params.parentID; }
ctx._source.Path = ctx._source.Path.replace(params.oldPath, params.newPath)
@@ -185,9 +192,9 @@ func (e *Engine) Move(id string, parentID string, target string) error {
})
}
-func (e *Engine) Delete(id string) error {
- return e.updateSelfAndDescendants(id, func(_ engine.Resource) *ScriptOption {
- return &ScriptOption{
+func (be *Backend) Delete(id string) error {
+ return be.updateSelfAndDescendants(id, func(_ engine.Resource) *osu.ScriptOption {
+ return &osu.ScriptOption{
Source: "ctx._source.Deleted = params.deleted",
Lang: "painless",
Params: map[string]any{
@@ -197,9 +204,9 @@ func (e *Engine) Delete(id string) error {
})
}
-func (e *Engine) Restore(id string) error {
- return e.updateSelfAndDescendants(id, func(_ engine.Resource) *ScriptOption {
- return &ScriptOption{
+func (be *Backend) Restore(id string) error {
+ return be.updateSelfAndDescendants(id, func(_ engine.Resource) *osu.ScriptOption {
+ return &osu.ScriptOption{
Source: "ctx._source.Deleted = params.deleted",
Lang: "painless",
Params: map[string]any{
@@ -209,26 +216,26 @@ func (e *Engine) Restore(id string) error {
})
}
-func (e *Engine) Purge(id string) error {
- resource, err := e.getResource(id)
+func (be *Backend) Purge(id string) error {
+ resource, err := be.getResource(id)
if err != nil {
return fmt.Errorf("failed to get resource: %w", err)
}
- req, err := BuildDocumentDeleteByQueryReq(
+ req, err := osu.BuildDocumentDeleteByQueryReq(
opensearchgoAPI.DocumentDeleteByQueryReq{
- Indices: []string{e.index},
+ Indices: []string{be.index},
Params: opensearchgoAPI.DocumentDeleteByQueryParams{
WaitForCompletion: conversions.ToPointer(true),
},
},
- NewTermQuery[string]("Path").Value(resource.Path),
+ osu.NewTermQuery[string]("Path").Value(resource.Path),
)
if err != nil {
return fmt.Errorf("failed to build delete by query request: %w", err)
}
- resp, err := e.client.Document.DeleteByQuery(context.TODO(), req)
+ resp, err := be.client.Document.DeleteByQuery(context.TODO(), req)
switch {
case err != nil:
return fmt.Errorf("failed to delete by query: %w", err)
@@ -239,18 +246,18 @@ func (e *Engine) Purge(id string) error {
return nil
}
-func (e *Engine) DocCount() (uint64, error) {
- req, err := BuildIndicesCountReq(
+func (be *Backend) DocCount() (uint64, error) {
+ req, err := osu.BuildIndicesCountReq(
&opensearchgoAPI.IndicesCountReq{
- Indices: []string{e.index},
+ Indices: []string{be.index},
},
- NewTermQuery[bool]("Deleted").Value(false),
+ osu.NewTermQuery[bool]("Deleted").Value(false),
)
if err != nil {
return 0, fmt.Errorf("failed to build count request: %w", err)
}
- resp, err := e.client.Indices.Count(context.TODO(), req)
+ resp, err := be.client.Indices.Count(context.TODO(), req)
if err != nil {
return 0, fmt.Errorf("failed to count documents: %w", err)
}
@@ -258,25 +265,25 @@ func (e *Engine) DocCount() (uint64, error) {
return uint64(resp.Count), nil
}
-func (e *Engine) updateSelfAndDescendants(id string, scriptProvider func(engine.Resource) *ScriptOption) error {
+func (be *Backend) updateSelfAndDescendants(id string, scriptProvider func(engine.Resource) *osu.ScriptOption) error {
if scriptProvider == nil {
return fmt.Errorf("script cannot be nil")
}
- resource, err := e.getResource(id)
+ resource, err := be.getResource(id)
if err != nil {
return fmt.Errorf("failed to get resource: %w", err)
}
- req, err := BuildUpdateByQueryReq(
+ req, err := osu.BuildUpdateByQueryReq(
opensearchgoAPI.UpdateByQueryReq{
- Indices: []string{e.index},
+ Indices: []string{be.index},
Params: opensearchgoAPI.UpdateByQueryParams{
WaitForCompletion: conversions.ToPointer(true),
},
},
- NewTermQuery[string]("Path").Value(resource.Path),
- UpdateByQueryReqOptions{
+ osu.NewTermQuery[string]("Path").Value(resource.Path),
+ osu.UpdateByQueryReqOptions{
Script: scriptProvider(resource),
},
)
@@ -284,7 +291,7 @@ func (e *Engine) updateSelfAndDescendants(id string, scriptProvider func(engine.
return fmt.Errorf("failed to build update by query request: %w", err)
}
- resp, err := e.client.UpdateByQuery(context.TODO(), req)
+ resp, err := be.client.UpdateByQuery(context.TODO(), req)
switch {
case err != nil:
return fmt.Errorf("failed to update by query: %w", err)
@@ -295,18 +302,18 @@ func (e *Engine) updateSelfAndDescendants(id string, scriptProvider func(engine.
return nil
}
-func (e *Engine) getResource(id string) (engine.Resource, error) {
- req, err := BuildSearchReq(
+func (be *Backend) getResource(id string) (engine.Resource, error) {
+ req, err := osu.BuildSearchReq(
&opensearchgoAPI.SearchReq{
- Indices: []string{e.index},
+ Indices: []string{be.index},
},
- NewIDsQuery([]string{id}),
+ osu.NewIDsQuery(id),
)
if err != nil {
return engine.Resource{}, fmt.Errorf("failed to build search request: %w", err)
}
- resp, err := e.client.Search(context.TODO(), req)
+ resp, err := be.client.Search(context.TODO(), req)
switch {
case err != nil:
return engine.Resource{}, fmt.Errorf("failed to search for resource: %w", err)
@@ -314,7 +321,7 @@ func (e *Engine) getResource(id string) (engine.Resource, error) {
return engine.Resource{}, fmt.Errorf("document with id %s not found", id)
}
- resource, err := convert[engine.Resource](resp.Hits.Hits[0].Source)
+ resource, err := conversions.To[engine.Resource](resp.Hits.Hits[0].Source)
if err != nil {
return engine.Resource{}, fmt.Errorf("failed to convert hit source: %w", err)
}
@@ -322,10 +329,10 @@ func (e *Engine) getResource(id string) (engine.Resource, error) {
return resource, nil
}
-func (e *Engine) StartBatch(_ int) error {
+func (be *Backend) StartBatch(_ int) error {
return nil // todo: implement batch processing
}
-func (e *Engine) EndBatch() error {
+func (be *Backend) EndBatch() error {
return nil // todo: implement batch processing
}
diff --git a/services/search/pkg/opensearch/engine_test.go b/services/search/pkg/opensearch/backend_test.go
similarity index 90%
rename from services/search/pkg/opensearch/engine_test.go
rename to services/search/pkg/opensearch/backend_test.go
index 6d0578ff90..69371fc758 100644
--- a/services/search/pkg/opensearch/engine_test.go
+++ b/services/search/pkg/opensearch/backend_test.go
@@ -15,7 +15,7 @@ import (
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
-func TestNewEngine(t *testing.T) {
+func TestNewBackend(t *testing.T) {
t.Run("fails to create if the cluster is not healthy", func(t *testing.T) {
client, err := opensearchgoAPI.NewClient(opensearchgoAPI.Config{
Client: opensearchgo.Config{
@@ -24,21 +24,21 @@ func TestNewEngine(t *testing.T) {
})
require.NoError(t, err, "failed to create OpenSearch client")
- backend, err := opensearch.NewEngine("test-engine-new-engine", client)
+ backend, err := opensearch.NewBackend("test-engine-new-engine", client)
require.Nil(t, backend)
require.ErrorIs(t, err, opensearch.ErrUnhealthyCluster)
})
}
func TestEngine_Search(t *testing.T) {
- indexName := "opencloud-test-resource"
+ indexName := "opencloud-test-engine-search"
tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
defer tc.Require.IndicesDelete([]string{indexName})
- backend, err := opensearch.NewEngine(indexName, tc.Client())
+ backend, err := opensearch.NewBackend(indexName, tc.Client())
require.NoError(t, err)
document := opensearchtest.Testdata.Resources.File
@@ -74,14 +74,14 @@ func TestEngine_Search(t *testing.T) {
}
func TestEngine_Upsert(t *testing.T) {
- indexName := "opencloud-test-resource"
+ indexName := "opencloud-test-engine-upsert"
tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
defer tc.Require.IndicesDelete([]string{indexName})
- backend, err := opensearch.NewEngine(indexName, tc.Client())
+ backend, err := opensearch.NewBackend(indexName, tc.Client())
require.NoError(t, err)
t.Run("upsert with full document", func(t *testing.T) {
@@ -93,14 +93,14 @@ func TestEngine_Upsert(t *testing.T) {
}
func TestEngine_Move(t *testing.T) {
- indexName := "opencloud-test-resource"
+ indexName := "opencloud-test-engine-move"
tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
defer tc.Require.IndicesDelete([]string{indexName})
- backend, err := opensearch.NewEngine(indexName, tc.Client())
+ backend, err := opensearch.NewBackend(indexName, tc.Client())
require.NoError(t, err)
t.Run("moves the document to a new path", func(t *testing.T) {
@@ -130,14 +130,14 @@ func TestEngine_Move(t *testing.T) {
}
func TestEngine_Delete(t *testing.T) {
- indexName := "opencloud-test-resource"
+ indexName := "opencloud-test-engine-delete"
tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
defer tc.Require.IndicesDelete([]string{indexName})
- backend, err := opensearch.NewEngine(indexName, tc.Client())
+ backend, err := opensearch.NewBackend(indexName, tc.Client())
require.NoError(t, err)
t.Run("mark document as deleted", func(t *testing.T) {
@@ -163,14 +163,14 @@ func TestEngine_Delete(t *testing.T) {
}
func TestEngine_Restore(t *testing.T) {
- indexName := "opencloud-test-resource"
+ indexName := "opencloud-test-engine-restore"
tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
defer tc.Require.IndicesDelete([]string{indexName})
- backend, err := opensearch.NewEngine(indexName, tc.Client())
+ backend, err := opensearch.NewBackend(indexName, tc.Client())
require.NoError(t, err)
t.Run("mark document as not deleted", func(t *testing.T) {
@@ -197,14 +197,14 @@ func TestEngine_Restore(t *testing.T) {
}
func TestEngine_Purge(t *testing.T) {
- indexName := "opencloud-test-resource"
+ indexName := "opencloud-test-engine-purge"
tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
defer tc.Require.IndicesDelete([]string{indexName})
- backend, err := opensearch.NewEngine(indexName, tc.Client())
+ backend, err := opensearch.NewBackend(indexName, tc.Client())
require.NoError(t, err)
t.Run("purge with full document", func(t *testing.T) {
@@ -219,14 +219,14 @@ func TestEngine_Purge(t *testing.T) {
}
func TestEngine_DocCount(t *testing.T) {
- indexName := "opencloud-test-resource"
+ indexName := "opencloud-test-engine-doc-count"
tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{indexName}, nil, 0)
defer tc.Require.IndicesDelete([]string{indexName})
- backend, err := opensearch.NewEngine(indexName, tc.Client())
+ backend, err := opensearch.NewBackend(indexName, tc.Client())
require.NoError(t, err)
t.Run("ignore deleted documents", func(t *testing.T) {
diff --git a/services/search/pkg/opensearch/export_test.go b/services/search/pkg/opensearch/export_test.go
deleted file mode 100644
index 6e02286943..0000000000
--- a/services/search/pkg/opensearch/export_test.go
+++ /dev/null
@@ -1,11 +0,0 @@
-package opensearch
-
-var (
- SearchHitToSearchMessageMatch = searchHitToSearchMessageMatch
- BuilderToBoolQuery = builderToBoolQuery
- ExpandKQLASTNodes = expandKQLASTNodes
-)
-
-func Convert[T any](v any) (T, error) {
- return convert[T](v)
-}
diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand.go b/services/search/pkg/opensearch/internal/convert/kql_expand.go
new file mode 100644
index 0000000000..5688f6dfe6
--- /dev/null
+++ b/services/search/pkg/opensearch/internal/convert/kql_expand.go
@@ -0,0 +1,197 @@
+package convert
+
+import (
+ "fmt"
+ "reflect"
+ "slices"
+ "strings"
+
+ "github.com/opencloud-eu/opencloud/pkg/ast"
+)
+
+func ExpandKQL(nodes []ast.Node) ([]ast.Node, error) {
+ return kqlExpander{}.expand(nodes, "")
+}
+
+type kqlExpander struct{}
+
+func (e kqlExpander) expand(nodes []ast.Node, defaultKey string) ([]ast.Node, error) {
+ for i, node := range nodes {
+ rnode := reflect.ValueOf(node)
+
+ // we need to ensure that the node is a pointer to an ast.Node in every case
+ if rnode.Kind() != reflect.Ptr {
+ ptr := reflect.New(rnode.Type())
+ ptr.Elem().Set(rnode)
+ rnode = ptr
+ cnode, ok := rnode.Interface().(ast.Node)
+ if !ok {
+ return nil, fmt.Errorf("expected node to be of type ast.Node, got %T", rnode.Interface())
+ }
+
+ node = cnode // Update the original node to the pointer
+ nodes[i] = node // Update the original slice with the pointer
+ }
+
+ var unfoldedNodes []ast.Node
+ switch cnode := node.(type) {
+ case *ast.GroupNode:
+ if cnode.Key != "" { // group nodes should not get a default key
+ cnode.Key = e.remapKey(cnode.Key, defaultKey)
+ }
+
+ groupNodes, err := e.expand(cnode.Nodes, cnode.Key)
+ if err != nil {
+ return nil, err
+ }
+ cnode.Nodes = groupNodes
+ case *ast.StringNode:
+ cnode.Key = e.remapKey(cnode.Key, defaultKey)
+ cnode.Value = e.lowerValue(cnode.Key, cnode.Value)
+ unfoldedNodes = e.unfoldValue(cnode.Key, cnode.Value)
+ case *ast.DateTimeNode:
+ cnode.Key = e.remapKey(cnode.Key, defaultKey)
+ case *ast.BooleanNode:
+ cnode.Key = e.remapKey(cnode.Key, defaultKey)
+ }
+
+ if unfoldedNodes != nil {
+ // Insert unfolded nodes at the current index
+ nodes = append(nodes[:i], append(unfoldedNodes, nodes[i+1:]...)...)
+ // Adjust index to account for new nodes
+ i += len(unfoldedNodes) - 1
+ }
+ }
+
+ return nodes, nil
+}
+
+func (_ kqlExpander) remapKey(current string, defaultKey string) string {
+ if defaultKey == "" {
+ defaultKey = "Name" // Set a default key if none is provided
+ }
+
+ key, ok := map[string]string{
+ "": defaultKey, // Default case if current is empty
+ "rootid": "RootID",
+ "path": "Path",
+ "id": "ID",
+ "name": "Name",
+ "size": "Size",
+ "mtime": "Mtime",
+ "mediatype": "MimeType",
+ "type": "Type",
+ "tag": "Tags",
+ "tags": "Tags",
+ "content": "Content",
+ "hidden": "Hidden",
+ }[current]
+ if !ok {
+ return current // Return the original key if not found
+ }
+
+ return key
+}
+
+func (_ kqlExpander) lowerValue(key, value string) string {
+ if slices.Contains([]string{"Hidden"}, key) {
+ return value // ignore certain keys and return the original value
+ }
+
+ return strings.ToLower(value)
+}
+
+func (_ kqlExpander) unfoldValue(key, value string) []ast.Node {
+ result, ok := map[string][]ast.Node{
+ "MimeType:file": {
+ &ast.OperatorNode{Value: "NOT"},
+ &ast.StringNode{Key: key, Value: "httpd/unix-directory"},
+ },
+ "MimeType:folder": {
+ &ast.StringNode{Key: key, Value: "httpd/unix-directory"},
+ },
+ "MimeType:document": {
+ &ast.GroupNode{Nodes: []ast.Node{
+ &ast.StringNode{Key: key, Value: "application/msword"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.text"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "text/plain"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "text/markdown"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/rtf"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/vnd.apple.pages"},
+ }},
+ },
+ "MimeType:spreadsheet": {
+ &ast.GroupNode{Nodes: []ast.Node{
+ &ast.StringNode{Key: key, Value: "application/vnd.ms-excel"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadsheet"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "text/csv"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"},
+ }},
+ },
+ "MimeType:presentation": {
+ &ast.GroupNode{Nodes: []ast.Node{
+ &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.presentation"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/vnd.ms-powerpoint"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/vnd.apple.keynote"},
+ }},
+ },
+ "MimeType:pdf": {
+ &ast.StringNode{Key: key, Value: "application/pdf"},
+ },
+ "MimeType:image": {
+ &ast.StringNode{Key: key, Value: "image/*"},
+ },
+ "MimeType:video": {
+ &ast.StringNode{Key: key, Value: "video/*"},
+ },
+ "MimeType:audio": {
+ &ast.StringNode{Key: key, Value: "audio/*"},
+ },
+ "MimeType:archive": {
+ &ast.GroupNode{Nodes: []ast.Node{
+ &ast.StringNode{Key: key, Value: "application/zip"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/gzip"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/x-gzip"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/x-7z-compressed"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/x-rar-compressed"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/x-tar"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/x-bzip2"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/x-bzip"},
+ &ast.OperatorNode{Value: "OR"},
+ &ast.StringNode{Key: key, Value: "application/x-tgz"},
+ }},
+ },
+ }[fmt.Sprintf("%s:%s", key, value)]
+ if !ok {
+ return nil
+ }
+
+ return result
+}
diff --git a/services/search/pkg/opensearch/kql_ast_test.go b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go
similarity index 97%
rename from services/search/pkg/opensearch/kql_ast_test.go
rename to services/search/pkg/opensearch/internal/convert/kql_expand_test.go
index f742b44e93..49218d5aa5 100644
--- a/services/search/pkg/opensearch/kql_ast_test.go
+++ b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go
@@ -1,4 +1,4 @@
-package opensearch_test
+package convert_test
import (
"fmt"
@@ -6,12 +6,13 @@ import (
"github.com/stretchr/testify/require"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
+
"github.com/opencloud-eu/opencloud/pkg/ast"
- "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
- opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
-func TestExpandKQLASTNodes(t *testing.T) {
+func TestExpandKQLAST(t *testing.T) {
t.Run("always converts a value node to a pointer node", func(t *testing.T) {
tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{
{
@@ -117,7 +118,7 @@ func TestExpandKQLASTNodes(t *testing.T) {
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
- result, err := opensearch.ExpandKQLASTNodes(test.Got)
+ result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.Equal(t, test.Want, result)
})
@@ -233,7 +234,7 @@ func TestExpandKQLASTNodes(t *testing.T) {
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
- result, err := opensearch.ExpandKQLASTNodes(test.Got)
+ result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.Equal(t, test.Want, result)
})
@@ -276,7 +277,7 @@ func TestExpandKQLASTNodes(t *testing.T) {
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
- result, err := opensearch.ExpandKQLASTNodes(test.Got)
+ result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.Equal(t, test.Want, result)
})
@@ -526,7 +527,7 @@ func TestExpandKQLASTNodes(t *testing.T) {
if test.Skip {
t.Skip("Skipping test due to known issue")
}
- result, err := opensearch.ExpandKQLASTNodes(test.Got)
+ result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.EqualValues(t, test.Want, result)
})
@@ -597,7 +598,7 @@ func TestExpandKQLASTNodes(t *testing.T) {
if test.Skip {
t.Skip("Skipping test due to known issue")
}
- result, err := opensearch.ExpandKQLASTNodes(test.Got)
+ result, err := convert.ExpandKQL(test.Got)
require.NoError(t, err)
require.EqualValues(t, test.Want, result)
})
diff --git a/services/search/pkg/opensearch/internal/convert/kql_query.go b/services/search/pkg/opensearch/internal/convert/kql_query.go
new file mode 100644
index 0000000000..f04e577077
--- /dev/null
+++ b/services/search/pkg/opensearch/internal/convert/kql_query.go
@@ -0,0 +1,35 @@
+package convert
+
+import (
+ "fmt"
+
+ "github.com/opencloud-eu/opencloud/pkg/kql"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
+)
+
+var (
+ ErrUnsupportedNodeType = fmt.Errorf("unsupported node type")
+)
+
+func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) {
+ kqlAst, err := kql.Builder{}.Build(kqlQuery)
+ if err != nil {
+ return nil, fmt.Errorf("failed to build query: %w", err)
+ }
+
+ kqlNodes, err := ExpandKQL(kqlAst.Nodes)
+ if err != nil {
+ return nil, fmt.Errorf("failed to expand KQL AST nodes: %w", err)
+ }
+
+ builder, err := TranspileKQLToOpenSearch(kqlNodes)
+ if err != nil {
+ return nil, fmt.Errorf("failed to compile query: %w", err)
+ }
+
+ if q, ok := builder.(*osu.BoolQuery); !ok {
+ return osu.NewBoolQuery().Must(builder), nil
+ } else {
+ return q, nil
+ }
+}
diff --git a/services/search/pkg/opensearch/internal/convert/kql_query_test.go b/services/search/pkg/opensearch/internal/convert/kql_query_test.go
new file mode 100644
index 0000000000..3c03b5e0cd
--- /dev/null
+++ b/services/search/pkg/opensearch/internal/convert/kql_query_test.go
@@ -0,0 +1 @@
+package convert_test
diff --git a/services/search/pkg/opensearch/kql_to_os_dsl.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go
similarity index 56%
rename from services/search/pkg/opensearch/kql_to_os_dsl.go
rename to services/search/pkg/opensearch/internal/convert/kql_transpile.go
index b299f21cd4..4b86d80ca6 100644
--- a/services/search/pkg/opensearch/kql_to_os_dsl.go
+++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go
@@ -1,4 +1,4 @@
-package opensearch
+package convert
import (
"errors"
@@ -8,20 +8,17 @@ import (
"github.com/opencloud-eu/opencloud/pkg/ast"
"github.com/opencloud-eu/opencloud/pkg/kql"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
)
-var (
- ErrUnsupportedNodeType = fmt.Errorf("unsupported node type")
-)
-
-type KQLToOsDSL struct{}
-
-func NewKQLToOsDSL() (*KQLToOsDSL, error) {
- return &KQLToOsDSL{}, nil
+func TranspileKQLToOpenSearch(nodes []ast.Node) (osu.Builder, error) {
+ return kqlOpensearchTranspiler{}.Transpile(nodes)
}
-func (k *KQLToOsDSL) Compile(tree *ast.Ast) (Builder, error) {
- q, err := k.transpile(tree.Nodes)
+type kqlOpensearchTranspiler struct{}
+
+func (t kqlOpensearchTranspiler) Transpile(nodes []ast.Node) (osu.Builder, error) {
+ q, err := t.transpile(nodes)
if err != nil {
return nil, err
}
@@ -29,41 +26,36 @@ func (k *KQLToOsDSL) Compile(tree *ast.Ast) (Builder, error) {
return q, nil
}
-func (k *KQLToOsDSL) transpile(nodes []ast.Node) (Builder, error) {
+func (t kqlOpensearchTranspiler) transpile(nodes []ast.Node) (osu.Builder, error) {
if len(nodes) == 0 {
return nil, fmt.Errorf("no nodes to compile")
}
- expandedNodes, err := expandKQLASTNodes(nodes)
- if err != nil {
- return nil, fmt.Errorf("failed to expand KQL AST nodes: %w", err)
- }
-
- if len(expandedNodes) == 1 {
- builder, err := k.toBuilder(expandedNodes[0])
+ if len(nodes) == 1 {
+ builder, err := t.toBuilder(nodes[0])
if err != nil {
return nil, fmt.Errorf("failed to get builder for single node: %w", err)
}
return builder, nil
}
- boolQuery := NewBoolQuery()
- add := boolQuery.Must
-
- for i, node := range expandedNodes {
- nextOp := k.getOperatorValueAt(expandedNodes, i+1)
- prevOp := k.getOperatorValueAt(expandedNodes, i-1)
+ boolQueryOptions := &osu.BoolQueryOptions{}
+ boolQuery := osu.NewBoolQuery().Options(boolQueryOptions)
+ boolQueryAdd := boolQuery.Must
+ for i, node := range nodes {
+ nextOp := t.getOperatorValueAt(nodes, i+1)
+ prevOp := t.getOperatorValueAt(nodes, i-1)
switch {
case nextOp == kql.BoolOR:
- add = boolQuery.Should
+ boolQueryAdd = boolQuery.Should
case nextOp == kql.BoolAND:
- add = boolQuery.Must
+ boolQueryAdd = boolQuery.Must
case prevOp == kql.BoolNOT:
- add = boolQuery.MustNot
+ boolQueryAdd = boolQuery.MustNot
}
- builder, err := k.toBuilder(node)
+ builder, err := t.toBuilder(node)
switch {
// if the node is not known, we skip it, such as an operator node
case errors.Is(err, ErrUnsupportedNodeType):
@@ -77,17 +69,18 @@ func (k *KQLToOsDSL) transpile(nodes []ast.Node) (Builder, error) {
continue
}
- add(builder)
- }
+ if nextOp == kql.BoolOR {
+ // if there are should clauses, we set the minimum should match to 1
+ boolQueryOptions.MinimumShouldMatch = 1
+ }
- if len(boolQuery.should) != 0 {
- boolQuery.options.MinimumShouldMatch = 1
+ boolQueryAdd(builder)
}
return boolQuery, nil
}
-func (k *KQLToOsDSL) getOperatorValueAt(nodes []ast.Node, i int) string {
+func (t kqlOpensearchTranspiler) getOperatorValueAt(nodes []ast.Node, i int) string {
if i < 0 || i >= len(nodes) {
return ""
}
@@ -99,16 +92,16 @@ func (k *KQLToOsDSL) getOperatorValueAt(nodes []ast.Node, i int) string {
return ""
}
-func (k *KQLToOsDSL) toBuilder(node ast.Node) (Builder, error) {
- var builder Builder
+func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) {
+ var builder osu.Builder
switch node := node.(type) {
case *ast.BooleanNode:
- return NewTermQuery[bool](node.Key).Value(node.Value), nil
+ return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil
case *ast.StringNode:
isWildcard := strings.Contains(node.Value, "*")
if isWildcard {
- return NewWildcardQuery(node.Key).Value(node.Value), nil
+ return osu.NewWildcardQuery(node.Key).Value(node.Value), nil
}
totalTerms := strings.Split(node.Value, " ")
@@ -116,9 +109,9 @@ func (k *KQLToOsDSL) toBuilder(node ast.Node) (Builder, error) {
isMultiTerm := len(totalTerms) >= 1
switch {
case isSingleTerm:
- return NewTermQuery[string](node.Key).Value(node.Value), nil
+ return osu.NewTermQuery[string](node.Key).Value(node.Value), nil
case isMultiTerm:
- return NewMatchPhraseQuery(node.Key).Query(node.Value), nil
+ return osu.NewMatchPhraseQuery(node.Key).Query(node.Value), nil
}
return nil, fmt.Errorf("unsupported string node value: %s", node.Value)
@@ -127,7 +120,7 @@ func (k *KQLToOsDSL) toBuilder(node ast.Node) (Builder, error) {
return builder, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType)
}
- query := NewRangeQuery[time.Time](node.Key)
+ query := osu.NewRangeQuery[time.Time](node.Key)
switch node.Operator.Value {
case ">":
@@ -142,7 +135,7 @@ func (k *KQLToOsDSL) toBuilder(node ast.Node) (Builder, error) {
return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType)
case *ast.GroupNode:
- group, err := k.transpile(node.Nodes)
+ group, err := t.transpile(node.Nodes)
if err != nil {
return nil, fmt.Errorf("failed to build group: %w", err)
}
diff --git a/services/search/pkg/opensearch/kql_to_os_dsl_test.go b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go
similarity index 57%
rename from services/search/pkg/opensearch/kql_to_os_dsl_test.go
rename to services/search/pkg/opensearch/internal/convert/kql_transpile_test.go
index 500d18300a..86c02d1cdc 100644
--- a/services/search/pkg/opensearch/kql_to_os_dsl_test.go
+++ b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go
@@ -1,4 +1,4 @@
-package opensearch_test
+package convert_test
import (
"testing"
@@ -7,22 +7,13 @@ import (
"github.com/stretchr/testify/assert"
"github.com/opencloud-eu/opencloud/pkg/ast"
- "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
-func TestKQL_Compile(t *testing.T) {
- tests := []opensearchtest.TableTest[*ast.Ast, opensearch.Builder]{
- // field name tests
- {
- Name: "Name is the default field",
- Got: &ast.Ast{
- Nodes: []ast.Node{
- &ast.StringNode{Value: "openCloud"},
- },
- },
- Want: opensearch.NewTermQuery[string]("Name").Value("opencloud"),
- },
+func TestTranspileKQLToOpenSearch(t *testing.T) {
+ tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{
// kql to os dsl - type tests
{
Name: "term query - string node",
@@ -31,7 +22,7 @@ func TestKQL_Compile(t *testing.T) {
&ast.StringNode{Key: "Name", Value: "openCloud"},
},
},
- Want: opensearch.NewTermQuery[string]("Name").Value("opencloud"),
+ Want: osu.NewTermQuery[string]("Name").Value("openCloud"),
},
{
Name: "term query - boolean node - true",
@@ -40,7 +31,7 @@ func TestKQL_Compile(t *testing.T) {
&ast.BooleanNode{Key: "Deleted", Value: true},
},
},
- Want: opensearch.NewTermQuery[bool]("Deleted").Value(true),
+ Want: osu.NewTermQuery[bool]("Deleted").Value(true),
},
{
Name: "term query - boolean node - false",
@@ -49,7 +40,7 @@ func TestKQL_Compile(t *testing.T) {
&ast.BooleanNode{Key: "Deleted", Value: false},
},
},
- Want: opensearch.NewTermQuery[bool]("Deleted").Value(false),
+ Want: osu.NewTermQuery[bool]("Deleted").Value(false),
},
{
Name: "match-phrase query - string node",
@@ -58,7 +49,7 @@ func TestKQL_Compile(t *testing.T) {
&ast.StringNode{Key: "Name", Value: "open cloud"},
},
},
- Want: opensearch.NewMatchPhraseQuery("Name").Query(`open cloud`),
+ Want: osu.NewMatchPhraseQuery("Name").Query(`open cloud`),
},
{
Name: "wildcard query - string node",
@@ -67,21 +58,21 @@ func TestKQL_Compile(t *testing.T) {
&ast.StringNode{Key: "Name", Value: "open*"},
},
},
- Want: opensearch.NewWildcardQuery("Name").Value("open*"),
+ Want: osu.NewWildcardQuery("Name").Value("open*"),
},
{
Name: "bool query",
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
- &ast.StringNode{Value: "a"},
- &ast.StringNode{Value: "b"},
+ &ast.StringNode{Key: "Name", Value: "a"},
+ &ast.StringNode{Key: "Name", Value: "b"},
}},
},
},
- Want: opensearch.NewBoolQuery().Must(
- opensearch.NewTermQuery[string]("Name").Value("a"),
- opensearch.NewTermQuery[string]("Name").Value("b"),
+ Want: osu.NewBoolQuery().Must(
+ osu.NewTermQuery[string]("Name").Value("a"),
+ osu.NewTermQuery[string]("Name").Value("b"),
),
},
{
@@ -89,11 +80,11 @@ func TestKQL_Compile(t *testing.T) {
Got: &ast.Ast{
Nodes: []ast.Node{
&ast.GroupNode{Nodes: []ast.Node{
- &ast.StringNode{Value: "any"},
+ &ast.StringNode{Key: "Name", Value: "any"},
}},
},
},
- Want: opensearch.NewTermQuery[string]("Name").Value("any"),
+ Want: osu.NewTermQuery[string]("Name").Value("any"),
},
{
Name: "range query >",
@@ -106,7 +97,7 @@ func TestKQL_Compile(t *testing.T) {
},
},
},
- Want: opensearch.NewRangeQuery[time.Time]("Mtime").Gt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
+ Want: osu.NewRangeQuery[time.Time]("Mtime").Gt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
{
Name: "range query >=",
@@ -119,7 +110,7 @@ func TestKQL_Compile(t *testing.T) {
},
},
},
- Want: opensearch.NewRangeQuery[time.Time]("Mtime").Gte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
+ Want: osu.NewRangeQuery[time.Time]("Mtime").Gte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
{
Name: "range query <",
@@ -132,7 +123,7 @@ func TestKQL_Compile(t *testing.T) {
},
},
},
- Want: opensearch.NewRangeQuery[time.Time]("Mtime").Lt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
+ Want: osu.NewRangeQuery[time.Time]("Mtime").Lt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
{
Name: "range query <=",
@@ -145,60 +136,61 @@ func TestKQL_Compile(t *testing.T) {
},
},
},
- Want: opensearch.NewRangeQuery[time.Time]("Mtime").Lte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
+ Want: osu.NewRangeQuery[time.Time]("Mtime").Lte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")),
},
// kql to os dsl - structure tests
{
Name: "[*]",
Got: &ast.Ast{
Nodes: []ast.Node{
- &ast.StringNode{Key: "name", Value: "openCloud"},
+ &ast.StringNode{Key: "Name", Value: "openCloud"},
},
},
- Want: opensearch.NewTermQuery[string]("Name").Value("opencloud"),
+ Want: osu.NewTermQuery[string]("Name").Value("openCloud"),
},
{
Name: "[* *]",
Got: &ast.Ast{
Nodes: []ast.Node{
- &ast.StringNode{Key: "name", Value: "openCloud"},
+ &ast.StringNode{Key: "Name", Value: "openCloud"},
&ast.StringNode{Key: "age", Value: "32"},
},
},
- Want: opensearch.NewBoolQuery().
+ Want: osu.NewBoolQuery().
Must(
- opensearch.NewTermQuery[string]("Name").Value("opencloud"),
- opensearch.NewTermQuery[string]("age").Value("32"),
+ osu.NewTermQuery[string]("Name").Value("openCloud"),
+ osu.NewTermQuery[string]("age").Value("32"),
),
},
{
Name: "[* AND *]",
Got: &ast.Ast{
Nodes: []ast.Node{
- &ast.StringNode{Key: "name", Value: "openCloud"},
+ &ast.StringNode{Key: "Name", Value: "openCloud"},
&ast.OperatorNode{Value: "AND"},
&ast.StringNode{Key: "age", Value: "32"},
},
},
- Want: opensearch.NewBoolQuery().
+ Want: osu.NewBoolQuery().
Must(
- opensearch.NewTermQuery[string]("Name").Value("opencloud"),
- opensearch.NewTermQuery[string]("age").Value("32"),
+ osu.NewTermQuery[string]("Name").Value("openCloud"),
+ osu.NewTermQuery[string]("age").Value("32"),
),
},
{
Name: "[* OR *]",
Got: &ast.Ast{
Nodes: []ast.Node{
- &ast.StringNode{Key: "name", Value: "openCloud"},
+ &ast.StringNode{Key: "Name", Value: "openCloud"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "age", Value: "32"},
},
},
- Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}).
+ Want: osu.NewBoolQuery().
+ Options(&osu.BoolQueryOptions{MinimumShouldMatch: 1}).
Should(
- opensearch.NewTermQuery[string]("Name").Value("opencloud"),
- opensearch.NewTermQuery[string]("age").Value("32"),
+ osu.NewTermQuery[string]("Name").Value("openCloud"),
+ osu.NewTermQuery[string]("age").Value("32"),
),
},
{
@@ -209,44 +201,45 @@ func TestKQL_Compile(t *testing.T) {
&ast.StringNode{Key: "age", Value: "32"},
},
},
- Want: opensearch.NewBoolQuery().
+ Want: osu.NewBoolQuery().
MustNot(
- opensearch.NewTermQuery[string]("age").Value("32"),
+ osu.NewTermQuery[string]("age").Value("32"),
),
},
{
Name: "[* NOT *]",
Got: &ast.Ast{
Nodes: []ast.Node{
- &ast.StringNode{Key: "name", Value: "openCloud"},
+ &ast.StringNode{Key: "Name", Value: "openCloud"},
&ast.OperatorNode{Value: "NOT"},
&ast.StringNode{Key: "age", Value: "32"},
},
},
- Want: opensearch.NewBoolQuery().
+ Want: osu.NewBoolQuery().
Must(
- opensearch.NewTermQuery[string]("Name").Value("opencloud"),
+ osu.NewTermQuery[string]("Name").Value("openCloud"),
).
MustNot(
- opensearch.NewTermQuery[string]("age").Value("32"),
+ osu.NewTermQuery[string]("age").Value("32"),
),
},
{
Name: "[* OR * OR *]",
Got: &ast.Ast{
Nodes: []ast.Node{
- &ast.StringNode{Key: "name", Value: "openCloud"},
+ &ast.StringNode{Key: "Name", Value: "openCloud"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "age", Value: "32"},
&ast.OperatorNode{Value: "OR"},
&ast.StringNode{Key: "age", Value: "44"},
},
},
- Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}).
+ Want: osu.NewBoolQuery().
+ Options(&osu.BoolQueryOptions{MinimumShouldMatch: 1}).
Should(
- opensearch.NewTermQuery[string]("Name").Value("opencloud"),
- opensearch.NewTermQuery[string]("age").Value("32"),
- opensearch.NewTermQuery[string]("age").Value("44"),
+ osu.NewTermQuery[string]("Name").Value("openCloud"),
+ osu.NewTermQuery[string]("age").Value("32"),
+ osu.NewTermQuery[string]("age").Value("44"),
),
},
{
@@ -260,13 +253,14 @@ func TestKQL_Compile(t *testing.T) {
&ast.StringNode{Key: "c", Value: "c"},
},
},
- Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}).
+ Want: osu.NewBoolQuery().
+ Options(&osu.BoolQueryOptions{MinimumShouldMatch: 1}).
Must(
- opensearch.NewTermQuery[string]("a").Value("a"),
+ osu.NewTermQuery[string]("a").Value("a"),
).
Should(
- opensearch.NewTermQuery[string]("b").Value("b"),
- opensearch.NewTermQuery[string]("c").Value("c"),
+ osu.NewTermQuery[string]("b").Value("b"),
+ osu.NewTermQuery[string]("c").Value("c"),
),
},
{
@@ -280,13 +274,14 @@ func TestKQL_Compile(t *testing.T) {
&ast.StringNode{Key: "c", Value: "c"},
},
},
- Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}).
+ Want: osu.NewBoolQuery().
+ Options(&osu.BoolQueryOptions{MinimumShouldMatch: 1}).
Must(
- opensearch.NewTermQuery[string]("b").Value("b"),
- opensearch.NewTermQuery[string]("c").Value("c"),
+ osu.NewTermQuery[string]("b").Value("b"),
+ osu.NewTermQuery[string]("c").Value("c"),
).
Should(
- opensearch.NewTermQuery[string]("a").Value("a"),
+ osu.NewTermQuery[string]("a").Value("a"),
),
},
{
@@ -300,13 +295,14 @@ func TestKQL_Compile(t *testing.T) {
&ast.StringNode{Key: "c", Value: "c"},
},
},
- Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}).
+ Want: osu.NewBoolQuery().
+ Options(&osu.BoolQueryOptions{MinimumShouldMatch: 1}).
Should(
- opensearch.NewTermQuery[string]("a").Value("a"),
+ osu.NewTermQuery[string]("a").Value("a"),
).
Must(
- opensearch.NewTermQuery[string]("b").Value("b"),
- opensearch.NewTermQuery[string]("c").Value("c"),
+ osu.NewTermQuery[string]("b").Value("b"),
+ osu.NewTermQuery[string]("c").Value("c"),
),
},
{
@@ -324,15 +320,16 @@ func TestKQL_Compile(t *testing.T) {
&ast.StringNode{Key: "d", Value: "d"},
},
},
- Want: opensearch.NewBoolQuery().
+ Want: osu.NewBoolQuery().
Must(
- opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}).
+ osu.NewBoolQuery().
+ Options(&osu.BoolQueryOptions{MinimumShouldMatch: 1}).
Should(
- opensearch.NewTermQuery[string]("a").Value("a"),
- opensearch.NewTermQuery[string]("b").Value("b"),
- opensearch.NewTermQuery[string]("c").Value("c"),
+ osu.NewTermQuery[string]("a").Value("a"),
+ osu.NewTermQuery[string]("b").Value("b"),
+ osu.NewTermQuery[string]("c").Value("c"),
),
- opensearch.NewTermQuery[string]("d").Value("d"),
+ osu.NewTermQuery[string]("d").Value("d"),
),
},
{
@@ -351,16 +348,17 @@ func TestKQL_Compile(t *testing.T) {
&ast.StringNode{Key: "d", Value: "d"},
},
},
- Want: opensearch.NewBoolQuery().
+ Want: osu.NewBoolQuery().
Must(
- opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}).
+ osu.NewBoolQuery().
+ Options(&osu.BoolQueryOptions{MinimumShouldMatch: 1}).
Should(
- opensearch.NewTermQuery[string]("a").Value("a"),
- opensearch.NewTermQuery[string]("b").Value("b"),
- opensearch.NewTermQuery[string]("c").Value("c"),
+ osu.NewTermQuery[string]("a").Value("a"),
+ osu.NewTermQuery[string]("b").Value("b"),
+ osu.NewTermQuery[string]("c").Value("c"),
),
).MustNot(
- opensearch.NewTermQuery[string]("d").Value("d"),
+ osu.NewTermQuery[string]("d").Value("d"),
),
},
}
@@ -371,10 +369,7 @@ func TestKQL_Compile(t *testing.T) {
t.Skip("skipping test: " + test.Name)
}
- transpiler, err := opensearch.NewKQLToOsDSL()
- assert.NoError(t, err)
-
- dsl, err := transpiler.Compile(test.Got)
+ dsl, err := convert.TranspileKQLToOpenSearch(test.Got.Nodes)
assert.NoError(t, err)
assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, dsl))
diff --git a/services/search/pkg/opensearch/engine_convert.go b/services/search/pkg/opensearch/internal/convert/opensearch.go
similarity index 81%
rename from services/search/pkg/opensearch/engine_convert.go
rename to services/search/pkg/opensearch/internal/convert/opensearch.go
index 0c09b7e8e4..8febaaf7ec 100644
--- a/services/search/pkg/opensearch/engine_convert.go
+++ b/services/search/pkg/opensearch/internal/convert/opensearch.go
@@ -1,4 +1,4 @@
-package opensearch
+package convert
import (
"fmt"
@@ -9,12 +9,13 @@ import (
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"google.golang.org/protobuf/types/known/timestamppb"
+ "github.com/opencloud-eu/opencloud/pkg/conversions"
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
"github.com/opencloud-eu/opencloud/services/search/pkg/engine"
)
-func searchHitToSearchMessageMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match, error) {
- resource, err := convert[engine.Resource](hit.Source)
+func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match, error) {
+ resource, err := conversions.To[engine.Resource](hit.Source)
if err != nil {
return nil, fmt.Errorf("failed to convert hit source: %w", err)
}
@@ -71,19 +72,19 @@ func searchHitToSearchMessageMatch(hit opensearchgoAPI.SearchHit) (*searchMessag
return nil
}
- audio, _ := convert[*searchMessage.Audio](resource.Audio)
+ audio, _ := conversions.To[*searchMessage.Audio](resource.Audio)
return audio
}(),
Image: func() *searchMessage.Image {
- image, _ := convert[*searchMessage.Image](resource.Image)
+ image, _ := conversions.To[*searchMessage.Image](resource.Image)
return image
}(),
Location: func() *searchMessage.GeoCoordinates {
- geoCoordinates, _ := convert[*searchMessage.GeoCoordinates](resource.Location)
+ geoCoordinates, _ := conversions.To[*searchMessage.GeoCoordinates](resource.Location)
return geoCoordinates
}(),
Photo: func() *searchMessage.Photo {
- photo, _ := convert[*searchMessage.Photo](resource.Photo)
+ photo, _ := conversions.To[*searchMessage.Photo](resource.Photo)
return photo
}(),
},
diff --git a/services/search/pkg/opensearch/engine_convert_test.go b/services/search/pkg/opensearch/internal/convert/opensearch_test.go
similarity index 81%
rename from services/search/pkg/opensearch/engine_convert_test.go
rename to services/search/pkg/opensearch/internal/convert/opensearch_test.go
index 9bca812115..afff90411e 100644
--- a/services/search/pkg/opensearch/engine_convert_test.go
+++ b/services/search/pkg/opensearch/internal/convert/opensearch_test.go
@@ -1,4 +1,4 @@
-package opensearch_test
+package convert_test
import (
"encoding/json"
@@ -7,12 +7,13 @@ import (
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/stretchr/testify/assert"
+ "github.com/opencloud-eu/opencloud/pkg/conversions"
searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0"
- "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
-func TestSearchHitToSearchMessageMatch(t *testing.T) {
+func TestOpenSearchHitToMatch(t *testing.T) {
resource := opensearchtest.Testdata.Resources.File
resource.MimeType = "audio/anything"
@@ -20,15 +21,15 @@ func TestSearchHitToSearchMessageMatch(t *testing.T) {
Score: 1.1,
Source: json.RawMessage(opensearchtest.JSONMustMarshal(t, resource)),
}
- match, err := opensearch.SearchHitToSearchMessageMatch(hit)
+ match, err := convert.OpenSearchHitToMatch(hit)
assert.NoError(t, err)
assert.Equal(t, hit.Score, match.Score)
assert.Equal(t, resource.Name, match.Entity.Name)
-
+ t.Parallel()
t.Run("converts the audio field to the expected type", func(t *testing.T) {
// searchMessage.Audio contains int64, int32 ... values that are converted to strings by the JSON marshaler,
// so we need to convert the resource.Audio to align the expectations for the JSON comparison.
- audio, err := opensearch.Convert[*searchMessage.Audio](resource.Audio)
+ audio, err := conversions.To[*searchMessage.Audio](resource.Audio)
assert.NoError(t, err)
assert.Equal(t, resource.Audio.Bitrate, match.Entity.Audio.Bitrate)
diff --git a/services/search/pkg/opensearch/os_index.go b/services/search/pkg/opensearch/internal/osu/index.go
similarity index 96%
rename from services/search/pkg/opensearch/os_index.go
rename to services/search/pkg/opensearch/internal/osu/index.go
index ef69cf9041..2b1d3e5985 100644
--- a/services/search/pkg/opensearch/os_index.go
+++ b/services/search/pkg/opensearch/internal/osu/index.go
@@ -1,4 +1,4 @@
-package opensearch
+package osu
import (
"bytes"
@@ -20,7 +20,7 @@ var (
IndexIndexManagerResourceV1 IndexManager = "resource_v1.json"
)
-//go:embed internal/indexes/*.json
+//go:embed indexes/*.json
var indexes embed.FS
type IndexManager string
@@ -36,7 +36,7 @@ func (m IndexManager) String() string {
func (m IndexManager) MarshalJSON() ([]byte, error) {
filePath := string(m)
- body, err := indexes.ReadFile(path.Join("./internal/indexes", filePath))
+ body, err := indexes.ReadFile(path.Join("./indexes", filePath))
switch {
case err != nil:
return nil, fmt.Errorf("failed to read index file %s: %w", filePath, err)
diff --git a/services/search/pkg/opensearch/os_index_test.go b/services/search/pkg/opensearch/internal/osu/index_test.go
similarity index 85%
rename from services/search/pkg/opensearch/os_index_test.go
rename to services/search/pkg/opensearch/internal/osu/index_test.go
index 789e3d92d3..112708a91c 100644
--- a/services/search/pkg/opensearch/os_index_test.go
+++ b/services/search/pkg/opensearch/internal/osu/index_test.go
@@ -1,4 +1,4 @@
-package opensearch_test
+package osu_test
import (
"strings"
@@ -7,16 +7,16 @@ import (
"github.com/stretchr/testify/require"
"github.com/tidwall/sjson"
- "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestIndexManager(t *testing.T) {
t.Run("index plausibility", func(t *testing.T) {
- tests := []opensearchtest.TableTest[opensearch.IndexManager, struct{}]{
+ tests := []opensearchtest.TableTest[osu.IndexManager, struct{}]{
{
Name: "empty",
- Got: opensearch.IndexManagerLatest,
+ Got: osu.IndexManagerLatest,
},
}
tc := opensearchtest.NewDefaultTestClient(t)
@@ -37,7 +37,7 @@ func TestIndexManager(t *testing.T) {
})
t.Run("does not create index if it already exists and is up to date", func(t *testing.T) {
- indexManager := opensearch.IndexManagerLatest
+ indexManager := osu.IndexManagerLatest
indexName := "opencloud-test-resource"
tc := opensearchtest.NewDefaultTestClient(t)
@@ -48,7 +48,7 @@ func TestIndexManager(t *testing.T) {
})
t.Run("fails to create index if it already exists but is not up to date", func(t *testing.T) {
- indexManager := opensearch.IndexManagerLatest
+ indexManager := osu.IndexManagerLatest
indexName := "opencloud-test-resource"
tc := opensearchtest.NewDefaultTestClient(t)
@@ -58,6 +58,6 @@ func TestIndexManager(t *testing.T) {
require.NoError(t, err)
tc.Require.IndicesCreate(indexName, strings.NewReader(body))
- require.ErrorIs(t, indexManager.Apply(t.Context(), indexName, tc.Client()), opensearch.ErrManualActionRequired)
+ require.ErrorIs(t, indexManager.Apply(t.Context(), indexName, tc.Client()), osu.ErrManualActionRequired)
})
}
diff --git a/services/search/pkg/opensearch/internal/indexes/resource_v1.json b/services/search/pkg/opensearch/internal/osu/indexes/resource_v1.json
similarity index 100%
rename from services/search/pkg/opensearch/internal/indexes/resource_v1.json
rename to services/search/pkg/opensearch/internal/osu/indexes/resource_v1.json
diff --git a/services/search/pkg/opensearch/os_dsl.go b/services/search/pkg/opensearch/internal/osu/osu.go
similarity index 53%
rename from services/search/pkg/opensearch/os_dsl.go
rename to services/search/pkg/opensearch/internal/osu/osu.go
index dad92b3292..820065f409 100644
--- a/services/search/pkg/opensearch/os_dsl.go
+++ b/services/search/pkg/opensearch/internal/osu/osu.go
@@ -1,46 +1,39 @@
-package opensearch
+package osu
import (
"encoding/json"
"fmt"
+ "reflect"
+
+ "dario.cat/mergo"
+
+ "github.com/opencloud-eu/opencloud/pkg/conversions"
)
-type Rewrite string
-
-const (
- ConstantScore Rewrite = "constant_score"
- ScoringBoolean Rewrite = "scoring_boolean"
- ConstantScoreBoolean Rewrite = "constant_score_boolean"
- TopTermsN Rewrite = "top_terms_N"
- TopTermsBoostN Rewrite = "top_terms_boost_N"
- TopTermsBlendedFreqsN Rewrite = "top_terms_blended_freqs_N"
-)
-
-type Analyzer string
-
type Builder interface {
json.Marshaler
fmt.Stringer
Map() (map[string]any, error)
}
-type BuilderFunc func() (map[string]any, error)
+func newBase(v ...any) (map[string]any, error) {
+ base := make(map[string]any)
+ for _, value := range v {
+ data, err := conversions.To[map[string]any](value)
+ if err != nil {
+ return nil, fmt.Errorf("failed to convert value to map: %w", err)
+ }
-func (f BuilderFunc) Map() (map[string]any, error) {
- return f()
-}
+ if isEmpty(data) {
+ continue
+ }
-func (f BuilderFunc) MarshalJSON() ([]byte, error) {
- data, err := f.Map()
- if err != nil {
- return nil, err
+ if err := mergo.Merge(&base, data); err != nil {
+ return nil, fmt.Errorf("failed to merge value into base: %w", err)
+ }
}
- return json.Marshal(data)
-}
-func (f BuilderFunc) String() string {
- b, _ := f.MarshalJSON()
- return string(b)
+ return base, nil
}
func applyValue[T any](target map[string]any, key string, v T) {
@@ -104,14 +97,34 @@ func applyBuilders(target map[string]any, key string, bs ...Builder) error {
return nil
}
-func builderToBoolQuery(b Builder) *BoolQuery {
- var bq *BoolQuery
+func isEmpty(x any) bool {
+ switch {
+ case x == nil:
+ return true
+ case reflect.ValueOf(x).Kind() == reflect.Bool:
+ return false
+ case reflect.DeepEqual(x, reflect.Zero(reflect.TypeOf(x)).Interface()):
+ return true
+ case reflect.ValueOf(x).Kind() == reflect.Map && reflect.ValueOf(x).Len() == 0:
+ return true
+ default:
+ return false
+ }
+}
- if q, ok := b.(*BoolQuery); !ok {
- bq = NewBoolQuery().Must(b)
- } else {
- bq = q
+func merge[T any](options ...T) T {
+ mapOptions := make(map[string]any)
+
+ for _, option := range options {
+ data, err := conversions.To[map[string]any](option)
+ if err != nil {
+ continue
+ }
+
+ _ = mergo.Merge(&mapOptions, data)
}
- return bq
+ data, _ := conversions.To[T](mapOptions)
+
+ return data
}
diff --git a/services/search/pkg/opensearch/os_dsl_query_bool.go b/services/search/pkg/opensearch/internal/osu/query_bool.go
similarity index 70%
rename from services/search/pkg/opensearch/os_dsl_query_bool.go
rename to services/search/pkg/opensearch/internal/osu/query_bool.go
index 4471c422b5..30a06d176f 100644
--- a/services/search/pkg/opensearch/os_dsl_query_bool.go
+++ b/services/search/pkg/opensearch/internal/osu/query_bool.go
@@ -1,4 +1,4 @@
-package opensearch
+package osu
import (
"encoding/json"
@@ -9,7 +9,7 @@ type BoolQuery struct {
mustNot []Builder
should []Builder
filter []Builder
- options BoolQueryOptions
+ options *BoolQueryOptions
}
type BoolQueryOptions struct {
@@ -18,8 +18,13 @@ type BoolQueryOptions struct {
Name string `json:"_name,omitempty"`
}
-func NewBoolQuery(o ...BoolQueryOptions) *BoolQuery {
- return &BoolQuery{options: merge(o...)}
+func NewBoolQuery() *BoolQuery {
+ return &BoolQuery{}
+}
+
+func (q *BoolQuery) Options(v *BoolQueryOptions) *BoolQuery {
+ q.options = v
+ return q
}
func (q *BoolQuery) Must(v ...Builder) *BoolQuery {
@@ -43,33 +48,33 @@ func (q *BoolQuery) Filter(v ...Builder) *BoolQuery {
}
func (q *BoolQuery) Map() (map[string]any, error) {
- data, err := convert[map[string]any](q.options)
+ base, err := newBase(q.options)
if err != nil {
return nil, err
}
- if err := applyBuilders(data, "must", q.must...); err != nil {
+ if err := applyBuilders(base, "must", q.must...); err != nil {
return nil, err
}
- if err := applyBuilders(data, "must_not", q.mustNot...); err != nil {
+ if err := applyBuilders(base, "must_not", q.mustNot...); err != nil {
return nil, err
}
- if err := applyBuilders(data, "should", q.should...); err != nil {
+ if err := applyBuilders(base, "should", q.should...); err != nil {
return nil, err
}
- if err := applyBuilders(data, "filter", q.filter...); err != nil {
+ if err := applyBuilders(base, "filter", q.filter...); err != nil {
return nil, err
}
- if isEmpty(data) {
+ if isEmpty(base) {
return nil, nil
}
return map[string]any{
- "bool": data,
+ "bool": base,
}, nil
}
diff --git a/services/search/pkg/opensearch/os_dsl_query_bool_test.go b/services/search/pkg/opensearch/internal/osu/query_bool_test.go
similarity index 74%
rename from services/search/pkg/opensearch/os_dsl_query_bool_test.go
rename to services/search/pkg/opensearch/internal/osu/query_bool_test.go
index 1125d33da1..34cba77dc2 100644
--- a/services/search/pkg/opensearch/os_dsl_query_bool_test.go
+++ b/services/search/pkg/opensearch/internal/osu/query_bool_test.go
@@ -1,24 +1,24 @@
-package opensearch_test
+package osu_test
import (
"testing"
"github.com/stretchr/testify/assert"
- "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestBoolQuery(t *testing.T) {
- tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{
+ tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
{
Name: "empty",
- Got: opensearch.NewBoolQuery(),
+ Got: osu.NewBoolQuery(),
Want: nil,
},
{
Name: "with-options",
- Got: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{
+ Got: osu.NewBoolQuery().Options(&osu.BoolQueryOptions{
MinimumShouldMatch: 10,
Boost: 10,
Name: "some-name",
@@ -33,7 +33,7 @@ func TestBoolQuery(t *testing.T) {
},
{
Name: "must",
- Got: opensearch.NewBoolQuery().Must(opensearch.NewTermQuery[string]("name").Value("tom")),
+ Got: osu.NewBoolQuery().Must(osu.NewTermQuery[string]("name").Value("tom")),
Want: map[string]any{
"bool": map[string]any{
"must": []map[string]any{
@@ -50,7 +50,7 @@ func TestBoolQuery(t *testing.T) {
},
{
Name: "must_not",
- Got: opensearch.NewBoolQuery().MustNot(opensearch.NewTermQuery[string]("name").Value("tom")),
+ Got: osu.NewBoolQuery().MustNot(osu.NewTermQuery[string]("name").Value("tom")),
Want: map[string]any{
"bool": map[string]any{
"must_not": []map[string]any{
@@ -67,7 +67,7 @@ func TestBoolQuery(t *testing.T) {
},
{
Name: "should",
- Got: opensearch.NewBoolQuery().Should(opensearch.NewTermQuery[string]("name").Value("tom")),
+ Got: osu.NewBoolQuery().Should(osu.NewTermQuery[string]("name").Value("tom")),
Want: map[string]any{
"bool": map[string]any{
"should": []map[string]any{
@@ -84,7 +84,7 @@ func TestBoolQuery(t *testing.T) {
},
{
Name: "filter",
- Got: opensearch.NewBoolQuery().Filter(opensearch.NewTermQuery[string]("name").Value("tom")),
+ Got: osu.NewBoolQuery().Filter(osu.NewTermQuery[string]("name").Value("tom")),
Want: map[string]any{
"bool": map[string]any{
"filter": []map[string]any{
@@ -101,11 +101,11 @@ func TestBoolQuery(t *testing.T) {
},
{
Name: "full",
- Got: opensearch.NewBoolQuery().
- Must(opensearch.NewTermQuery[string]("name").Value("tom")).
- MustNot(opensearch.NewTermQuery[bool]("deleted").Value(true)).
- Should(opensearch.NewTermQuery[string]("gender").Value("male")).
- Filter(opensearch.NewTermQuery[int]("age").Value(42)),
+ Got: osu.NewBoolQuery().
+ Must(osu.NewTermQuery[string]("name").Value("tom")).
+ MustNot(osu.NewTermQuery[bool]("deleted").Value(true)).
+ Should(osu.NewTermQuery[string]("gender").Value("male")).
+ Filter(osu.NewTermQuery[int]("age").Value(42)),
Want: map[string]any{
"bool": map[string]any{
"must": []map[string]any{
diff --git a/services/search/pkg/opensearch/os_dsl_query_full_text_match_phrase.go b/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase.go
similarity index 56%
rename from services/search/pkg/opensearch/os_dsl_query_full_text_match_phrase.go
rename to services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase.go
index a8ca72902d..2def251a8c 100644
--- a/services/search/pkg/opensearch/os_dsl_query_full_text_match_phrase.go
+++ b/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase.go
@@ -1,4 +1,4 @@
-package opensearch
+package osu
import (
"encoding/json"
@@ -7,17 +7,22 @@ import (
type MatchPhraseQuery struct {
field string
query string
- options MatchPhraseQueryOptions
+ options *MatchPhraseQueryOptions
}
type MatchPhraseQueryOptions struct {
- Analyzer Analyzer `json:"analyzer,omitempty"`
- Slop int `json:"slop,omitempty"`
- ZeroTermsQuery string `json:"zero_terms_query,omitempty"`
+ Analyzer string `json:"analyzer,omitempty"`
+ Slop int `json:"slop,omitempty"`
+ ZeroTermsQuery string `json:"zero_terms_query,omitempty"`
}
-func NewMatchPhraseQuery(field string, o ...MatchPhraseQueryOptions) *MatchPhraseQuery {
- return &MatchPhraseQuery{field: field, options: merge(o...)}
+func NewMatchPhraseQuery(field string) *MatchPhraseQuery {
+ return &MatchPhraseQuery{field: field}
+}
+
+func (q *MatchPhraseQuery) Options(v *MatchPhraseQueryOptions) *MatchPhraseQuery {
+ q.options = v
+ return q
}
func (q *MatchPhraseQuery) Query(v string) *MatchPhraseQuery {
@@ -26,20 +31,20 @@ func (q *MatchPhraseQuery) Query(v string) *MatchPhraseQuery {
}
func (q *MatchPhraseQuery) Map() (map[string]any, error) {
- data, err := convert[map[string]any](q.options)
+ base, err := newBase(q.options)
if err != nil {
return nil, err
}
- applyValue(data, "query", q.query)
+ applyValue(base, "query", q.query)
- if isEmpty(data) {
+ if isEmpty(base) {
return nil, nil
}
return map[string]any{
"match_phrase": map[string]any{
- q.field: data,
+ q.field: base,
},
}, nil
}
diff --git a/services/search/pkg/opensearch/os_dsl_query_full_text_match_phrase_test.go b/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase_test.go
similarity index 77%
rename from services/search/pkg/opensearch/os_dsl_query_full_text_match_phrase_test.go
rename to services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase_test.go
index 0c58bc8cdf..32d28630d4 100644
--- a/services/search/pkg/opensearch/os_dsl_query_full_text_match_phrase_test.go
+++ b/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase_test.go
@@ -1,24 +1,24 @@
-package opensearch_test
+package osu_test
import (
"testing"
"github.com/stretchr/testify/assert"
- "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestNewMatchPhraseQuery(t *testing.T) {
- tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{
+ tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
{
Name: "empty",
- Got: opensearch.NewMatchPhraseQuery("empty"),
+ Got: osu.NewMatchPhraseQuery("empty"),
Want: nil,
},
{
Name: "options",
- Got: opensearch.NewMatchPhraseQuery("name", opensearch.MatchPhraseQueryOptions{
+ Got: osu.NewMatchPhraseQuery("name").Options(&osu.MatchPhraseQueryOptions{
Analyzer: "analyzer",
Slop: 2,
ZeroTermsQuery: "all",
@@ -35,7 +35,7 @@ func TestNewMatchPhraseQuery(t *testing.T) {
},
{
Name: "query",
- Got: opensearch.NewMatchPhraseQuery("name").Query("some match query"),
+ Got: osu.NewMatchPhraseQuery("name").Query("some match query"),
Want: map[string]any{
"match_phrase": map[string]any{
"name": map[string]any{
@@ -46,7 +46,7 @@ func TestNewMatchPhraseQuery(t *testing.T) {
},
{
Name: "full",
- Got: opensearch.NewMatchPhraseQuery("name", opensearch.MatchPhraseQueryOptions{
+ Got: osu.NewMatchPhraseQuery("name").Options(&osu.MatchPhraseQueryOptions{
Analyzer: "analyzer",
Slop: 2,
ZeroTermsQuery: "all",
diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_ids.go b/services/search/pkg/opensearch/internal/osu/query_term_level_ids.go
similarity index 62%
rename from services/search/pkg/opensearch/os_dsl_query_term_level_ids.go
rename to services/search/pkg/opensearch/internal/osu/query_term_level_ids.go
index 0c0b7a0f96..409f3f94cf 100644
--- a/services/search/pkg/opensearch/os_dsl_query_term_level_ids.go
+++ b/services/search/pkg/opensearch/internal/osu/query_term_level_ids.go
@@ -1,4 +1,4 @@
-package opensearch
+package osu
import (
"encoding/json"
@@ -7,31 +7,36 @@ import (
type IDsQuery struct {
values []string
- options IDsQueryOptions
+ options *IDsQueryOptions
}
type IDsQueryOptions struct {
Boost float32 `json:"boost,omitempty"`
}
-func NewIDsQuery(v []string, o ...IDsQueryOptions) *IDsQuery {
- return &IDsQuery{values: slices.Compact(v), options: merge(o...)}
+func NewIDsQuery(v ...string) *IDsQuery {
+ return &IDsQuery{values: slices.Compact(v)}
+}
+
+func (q *IDsQuery) Options(v *IDsQueryOptions) *IDsQuery {
+ q.options = v
+ return q
}
func (q *IDsQuery) Map() (map[string]any, error) {
- data, err := convert[map[string]any](q.options)
+ base, err := newBase(q.options)
if err != nil {
return nil, err
}
- applyValue(data, "values", q.values)
+ applyValue(base, "values", q.values)
- if isEmpty(data) {
+ if isEmpty(base) {
return nil, nil
}
return map[string]any{
- "ids": data,
+ "ids": base,
}, nil
}
diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_ids_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_ids_test.go
similarity index 72%
rename from services/search/pkg/opensearch/os_dsl_query_term_level_ids_test.go
rename to services/search/pkg/opensearch/internal/osu/query_term_level_ids_test.go
index d8342bf93b..bc693ebb49 100644
--- a/services/search/pkg/opensearch/os_dsl_query_term_level_ids_test.go
+++ b/services/search/pkg/opensearch/internal/osu/query_term_level_ids_test.go
@@ -1,24 +1,24 @@
-package opensearch_test
+package osu_test
import (
"testing"
"github.com/stretchr/testify/assert"
- "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestIDsQuery(t *testing.T) {
- tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{
+ tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
{
Name: "empty",
- Got: opensearch.NewIDsQuery(nil),
+ Got: osu.NewIDsQuery(),
Want: nil,
},
{
Name: "ids",
- Got: opensearch.NewIDsQuery([]string{"1", "2", "3", "3"}, opensearch.IDsQueryOptions{Boost: 1.0}),
+ Got: osu.NewIDsQuery("1", "2", "3", "3").Options(&osu.IDsQueryOptions{Boost: 1.0}),
Want: map[string]any{
"ids": map[string]any{
"values": []string{"1", "2", "3"},
diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_range.go b/services/search/pkg/opensearch/internal/osu/query_term_level_range.go
similarity index 79%
rename from services/search/pkg/opensearch/os_dsl_query_term_level_range.go
rename to services/search/pkg/opensearch/internal/osu/query_term_level_range.go
index a1853162fb..083e3e0cfc 100644
--- a/services/search/pkg/opensearch/os_dsl_query_term_level_range.go
+++ b/services/search/pkg/opensearch/internal/osu/query_term_level_range.go
@@ -1,4 +1,4 @@
-package opensearch
+package osu
import (
"encoding/json"
@@ -12,7 +12,7 @@ type RangeQuery[T time.Time | string] struct {
gte T
lt T
lte T
- options RangeQueryOptions
+ options *RangeQueryOptions
}
type RangeQueryOptions struct {
@@ -22,8 +22,13 @@ type RangeQueryOptions struct {
TimeZone string `json:"time_zone,omitempty"`
}
-func NewRangeQuery[T time.Time | string](field string, o ...RangeQueryOptions) *RangeQuery[T] {
- return &RangeQuery[T]{field: field, options: merge(o...)}
+func NewRangeQuery[T time.Time | string](field string) *RangeQuery[T] {
+ return &RangeQuery[T]{field: field}
+}
+
+func (q *RangeQuery[T]) Options(v *RangeQueryOptions) *RangeQuery[T] {
+ q.options = v
+ return q
}
func (q *RangeQuery[T]) Gt(v T) *RangeQuery[T] {
@@ -47,11 +52,6 @@ func (q *RangeQuery[T]) Lte(v T) *RangeQuery[T] {
}
func (q *RangeQuery[T]) Map() (map[string]any, error) {
- data, err := convert[map[string]any](q.options)
- if err != nil {
- return nil, err
- }
-
if !isEmpty(q.gt) && !isEmpty(q.gte) {
return nil, errors.New("cannot set both gt and gte in RangeQuery")
}
@@ -60,20 +60,25 @@ func (q *RangeQuery[T]) Map() (map[string]any, error) {
return nil, errors.New("cannot set both lt and lte in RangeQuery")
}
- applyValues(data, map[string]T{
+ base, err := newBase(q.options)
+ if err != nil {
+ return nil, err
+ }
+
+ applyValues(base, map[string]T{
"gt": q.gt,
"gte": q.gte,
"lt": q.lt,
"lte": q.lte,
})
- if isEmpty(data) {
+ if isEmpty(base) {
return nil, nil
}
return map[string]any{
"range": map[string]any{
- q.field: data,
+ q.field: base,
},
}, nil
}
diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_range_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_range_test.go
similarity index 74%
rename from services/search/pkg/opensearch/os_dsl_query_term_level_range_test.go
rename to services/search/pkg/opensearch/internal/osu/query_term_level_range_test.go
index 4498c39be5..dce6727fbd 100644
--- a/services/search/pkg/opensearch/os_dsl_query_term_level_range_test.go
+++ b/services/search/pkg/opensearch/internal/osu/query_term_level_range_test.go
@@ -1,4 +1,4 @@
-package opensearch_test
+package osu_test
import (
"errors"
@@ -8,21 +8,21 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestRangeQuery(t *testing.T) {
now := time.Now()
- tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{
+ tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
{
Name: "empty",
- Got: opensearch.NewRangeQuery[string]("empty"),
+ Got: osu.NewRangeQuery[string]("empty"),
Want: nil,
},
{
Name: "gt string",
- Got: opensearch.NewRangeQuery[string]("created").Gt("2023-01-01T00:00:00Z"),
+ Got: osu.NewRangeQuery[string]("created").Gt("2023-01-01T00:00:00Z"),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
@@ -33,7 +33,7 @@ func TestRangeQuery(t *testing.T) {
},
{
Name: "gt time",
- Got: opensearch.NewRangeQuery[time.Time]("created").Gt(now),
+ Got: osu.NewRangeQuery[time.Time]("created").Gt(now),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
@@ -44,7 +44,7 @@ func TestRangeQuery(t *testing.T) {
},
{
Name: "gte string",
- Got: opensearch.NewRangeQuery[string]("created").Gte("2023-01-01T00:00:00Z"),
+ Got: osu.NewRangeQuery[string]("created").Gte("2023-01-01T00:00:00Z"),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
@@ -55,7 +55,7 @@ func TestRangeQuery(t *testing.T) {
},
{
Name: "gte time",
- Got: opensearch.NewRangeQuery[time.Time]("created").Gte(now),
+ Got: osu.NewRangeQuery[time.Time]("created").Gte(now),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
@@ -66,13 +66,13 @@ func TestRangeQuery(t *testing.T) {
},
{
Name: "gt & gte",
- Got: opensearch.NewRangeQuery[time.Time]("created").Gt(now).Gte(now),
+ Got: osu.NewRangeQuery[time.Time]("created").Gt(now).Gte(now),
Want: nil,
Err: errors.New(""),
},
{
Name: "gt string",
- Got: opensearch.NewRangeQuery[string]("created").Lt("2023-01-01T00:00:00Z"),
+ Got: osu.NewRangeQuery[string]("created").Lt("2023-01-01T00:00:00Z"),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
@@ -83,7 +83,7 @@ func TestRangeQuery(t *testing.T) {
},
{
Name: "lt time",
- Got: opensearch.NewRangeQuery[time.Time]("created").Lt(now),
+ Got: osu.NewRangeQuery[time.Time]("created").Lt(now),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
@@ -94,7 +94,7 @@ func TestRangeQuery(t *testing.T) {
},
{
Name: "lte string",
- Got: opensearch.NewRangeQuery[string]("created").Lte("2023-01-01T00:00:00Z"),
+ Got: osu.NewRangeQuery[string]("created").Lte("2023-01-01T00:00:00Z"),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
@@ -105,7 +105,7 @@ func TestRangeQuery(t *testing.T) {
},
{
Name: "lte time",
- Got: opensearch.NewRangeQuery[time.Time]("created").Lte(now),
+ Got: osu.NewRangeQuery[time.Time]("created").Lte(now),
Want: map[string]any{
"range": map[string]any{
"created": map[string]any{
@@ -116,13 +116,13 @@ func TestRangeQuery(t *testing.T) {
},
{
Name: "lt & lte",
- Got: opensearch.NewRangeQuery[time.Time]("created").Lt(now).Lte(now),
+ Got: osu.NewRangeQuery[time.Time]("created").Lt(now).Lte(now),
Want: nil,
Err: errors.New(""),
},
{
Name: "options",
- Got: opensearch.NewRangeQuery[time.Time]("created", opensearch.RangeQueryOptions{
+ Got: osu.NewRangeQuery[time.Time]("created").Options(&osu.RangeQueryOptions{
Format: "strict_date_optional_time",
Relation: "within",
Boost: 1.0,
diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_term.go b/services/search/pkg/opensearch/internal/osu/query_term_level_term.go
similarity index 69%
rename from services/search/pkg/opensearch/os_dsl_query_term_level_term.go
rename to services/search/pkg/opensearch/internal/osu/query_term_level_term.go
index a9620e8fcb..39eaf3be6c 100644
--- a/services/search/pkg/opensearch/os_dsl_query_term_level_term.go
+++ b/services/search/pkg/opensearch/internal/osu/query_term_level_term.go
@@ -1,4 +1,4 @@
-package opensearch
+package osu
import (
"encoding/json"
@@ -7,7 +7,7 @@ import (
type TermQuery[T comparable] struct {
field string
value T
- options TermQueryOptions
+ options *TermQueryOptions
}
type TermQueryOptions struct {
@@ -16,8 +16,13 @@ type TermQueryOptions struct {
Name string `json:"_name,omitempty"`
}
-func NewTermQuery[T comparable](field string, o ...TermQueryOptions) *TermQuery[T] {
- return &TermQuery[T]{field: field, options: merge(o...)}
+func NewTermQuery[T comparable](field string) *TermQuery[T] {
+ return &TermQuery[T]{field: field}
+}
+
+func (q *TermQuery[T]) Options(v *TermQueryOptions) *TermQuery[T] {
+ q.options = v
+ return q
}
func (q *TermQuery[T]) Value(v T) *TermQuery[T] {
@@ -26,20 +31,20 @@ func (q *TermQuery[T]) Value(v T) *TermQuery[T] {
}
func (q *TermQuery[T]) Map() (map[string]any, error) {
- data, err := convert[map[string]any](q.options)
+ base, err := newBase(q.options)
if err != nil {
return nil, err
}
- applyValue(data, "value", q.value)
+ applyValue(base, "value", q.value)
- if isEmpty(data) {
+ if isEmpty(base) {
return nil, nil
}
return map[string]any{
"term": map[string]any{
- q.field: data,
+ q.field: base,
},
}, nil
}
diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_term_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_term_test.go
similarity index 75%
rename from services/search/pkg/opensearch/os_dsl_query_term_level_term_test.go
rename to services/search/pkg/opensearch/internal/osu/query_term_level_term_test.go
index 53f2cfb390..0a5a7d35d6 100644
--- a/services/search/pkg/opensearch/os_dsl_query_term_level_term_test.go
+++ b/services/search/pkg/opensearch/internal/osu/query_term_level_term_test.go
@@ -1,24 +1,24 @@
-package opensearch_test
+package osu_test
import (
"testing"
"github.com/stretchr/testify/assert"
- "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestTermQuery(t *testing.T) {
- tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{
+ tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
{
Name: "empty",
- Got: opensearch.NewTermQuery[string]("empty"),
+ Got: osu.NewTermQuery[string]("empty"),
Want: nil,
},
{
- Name: "op-options",
- Got: opensearch.NewTermQuery[bool]("deleted").Value(false),
+ Name: "no-options",
+ Got: osu.NewTermQuery[bool]("deleted").Value(false),
Want: map[string]any{
"term": map[string]any{
"deleted": map[string]any{
@@ -29,7 +29,7 @@ func TestTermQuery(t *testing.T) {
},
{
Name: "with-options",
- Got: opensearch.NewTermQuery[bool]("deleted", opensearch.TermQueryOptions{
+ Got: osu.NewTermQuery[bool]("deleted").Options(&osu.TermQueryOptions{
Boost: 1.0,
CaseInsensitive: true,
Name: "is-deleted",
diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_wildcard.go b/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard.go
similarity index 65%
rename from services/search/pkg/opensearch/os_dsl_query_term_level_wildcard.go
rename to services/search/pkg/opensearch/internal/osu/query_term_level_wildcard.go
index b6010e1f85..f21255c64f 100644
--- a/services/search/pkg/opensearch/os_dsl_query_term_level_wildcard.go
+++ b/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard.go
@@ -1,4 +1,4 @@
-package opensearch
+package osu
import (
"encoding/json"
@@ -7,17 +7,22 @@ import (
type WildcardQuery struct {
field string
value string
- options WildcardQueryOptions
+ options *WildcardQueryOptions
}
type WildcardQueryOptions struct {
Boost float32 `json:"boost,omitempty"`
CaseInsensitive bool `json:"case_insensitive,omitempty"`
- Rewrite Rewrite `json:"rewrite,omitempty"`
+ Rewrite string `json:"rewrite,omitempty"`
}
-func NewWildcardQuery(field string, o ...WildcardQueryOptions) *WildcardQuery {
- return &WildcardQuery{field: field, options: merge(o...)}
+func NewWildcardQuery(field string) *WildcardQuery {
+ return &WildcardQuery{field: field}
+}
+
+func (q *WildcardQuery) Options(v *WildcardQueryOptions) *WildcardQuery {
+ q.options = v
+ return q
}
func (q *WildcardQuery) Value(v string) *WildcardQuery {
@@ -26,20 +31,20 @@ func (q *WildcardQuery) Value(v string) *WildcardQuery {
}
func (q *WildcardQuery) Map() (map[string]any, error) {
- data, err := convert[map[string]any](q.options)
+ base, err := newBase(q.options)
if err != nil {
return nil, err
}
- applyValue(data, "value", q.value)
+ applyValue(base, "value", q.value)
- if isEmpty(data) {
+ if isEmpty(base) {
return nil, nil
}
return map[string]any{
"wildcard": map[string]any{
- q.field: data,
+ q.field: base,
},
}, nil
}
diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_wildcard_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard_test.go
similarity index 75%
rename from services/search/pkg/opensearch/os_dsl_query_term_level_wildcard_test.go
rename to services/search/pkg/opensearch/internal/osu/query_term_level_wildcard_test.go
index 5bcd51626a..b23f3e47c4 100644
--- a/services/search/pkg/opensearch/os_dsl_query_term_level_wildcard_test.go
+++ b/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard_test.go
@@ -1,27 +1,27 @@
-package opensearch_test
+package osu_test
import (
"testing"
"github.com/stretchr/testify/assert"
- "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestWildcardQuery(t *testing.T) {
- tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{
+ tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
{
Name: "empty",
- Got: opensearch.NewWildcardQuery("empty"),
+ Got: osu.NewWildcardQuery("empty"),
Want: nil,
},
{
Name: "wildcard",
- Got: opensearch.NewWildcardQuery("name", opensearch.WildcardQueryOptions{
+ Got: osu.NewWildcardQuery("name").Options(&osu.WildcardQueryOptions{
Boost: 1.0,
CaseInsensitive: true,
- Rewrite: opensearch.TopTermsBlendedFreqsN,
+ Rewrite: "top_terms_blended_freqs_N",
}).Value("opencl*"),
Want: map[string]any{
"wildcard": map[string]any{
diff --git a/services/search/pkg/opensearch/os_request.go b/services/search/pkg/opensearch/internal/osu/request.go
similarity index 96%
rename from services/search/pkg/opensearch/os_request.go
rename to services/search/pkg/opensearch/internal/osu/request.go
index ef798d0a55..fc33c65eaf 100644
--- a/services/search/pkg/opensearch/os_request.go
+++ b/services/search/pkg/opensearch/internal/osu/request.go
@@ -1,4 +1,4 @@
-package opensearch
+package osu
import (
"bytes"
@@ -7,6 +7,8 @@ import (
"strings"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
+
+ "github.com/opencloud-eu/opencloud/pkg/conversions"
)
type RequestBody[O any] struct {
@@ -19,7 +21,7 @@ func NewRequestBody[O any](q Builder, o ...O) *RequestBody[O] {
}
func (q RequestBody[O]) Map() (map[string]any, error) {
- data, err := convert[map[string]any](q.options)
+ data, err := conversions.To[map[string]any](q.options)
if err != nil {
return nil, err
}
diff --git a/services/search/pkg/opensearch/os_request_test.go b/services/search/pkg/opensearch/internal/osu/request_test.go
similarity index 79%
rename from services/search/pkg/opensearch/os_request_test.go
rename to services/search/pkg/opensearch/internal/osu/request_test.go
index d547472abf..3b607c6591 100644
--- a/services/search/pkg/opensearch/os_request_test.go
+++ b/services/search/pkg/opensearch/internal/osu/request_test.go
@@ -1,4 +1,4 @@
-package opensearch_test
+package osu_test
import (
"io"
@@ -8,15 +8,15 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
+ "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestRequestBody(t *testing.T) {
- tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{
+ tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{
{
Name: "simple",
- Got: opensearch.NewRequestBody[any](opensearch.NewTermQuery[string]("name").Value("tom")),
+ Got: osu.NewRequestBody[any](osu.NewTermQuery[string]("name").Value("tom")),
Want: map[string]any{
"query": map[string]any{
"term": map[string]any{
@@ -41,14 +41,14 @@ func TestBuildSearchReq(t *testing.T) {
{
Name: "highlight",
Got: func() io.Reader {
- req, _ := opensearch.BuildSearchReq(
+ req, _ := osu.BuildSearchReq(
&opensearchgoAPI.SearchReq{},
- opensearch.NewTermQuery[string]("content").Value("content"),
- opensearch.SearchReqOptions{
- Highlight: &opensearch.HighlightOption{
+ osu.NewTermQuery[string]("content").Value("content"),
+ osu.SearchReqOptions{
+ Highlight: &osu.HighlightOption{
PreTags: []string{""},
PostTags: []string{""},
- Fields: map[string]opensearch.HighlightOption{
+ Fields: map[string]osu.HighlightOption{
"content": {},
},
},
diff --git a/services/search/pkg/opensearch/internal/test/helper.go b/services/search/pkg/opensearch/internal/test/helper.go
index a7e7e26d1d..1b1b0e2497 100644
--- a/services/search/pkg/opensearch/internal/test/helper.go
+++ b/services/search/pkg/opensearch/internal/test/helper.go
@@ -8,6 +8,8 @@ import (
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/samber/lo"
"github.com/stretchr/testify/require"
+
+ "github.com/opencloud-eu/opencloud/pkg/conversions"
)
var TimeMustParse = func(t *testing.T, ts string) time.Time {
@@ -25,27 +27,8 @@ func JSONMustMarshal(t *testing.T, data any) string {
func SearchHitsMustBeConverted[T any](t *testing.T, hits []opensearchgoAPI.SearchHit) []T {
return lo.ReduceRight(hits, func(agg []T, item opensearchgoAPI.SearchHit, _ int) []T {
- resource, err := convert[T](item.Source)
+ resource, err := conversions.To[T](item.Source)
require.NoError(t, err)
return append(agg, resource)
}, []T{})
}
-
-func convert[T any](v any) (T, error) {
- var t T
-
- if v == nil {
- return t, nil
- }
-
- j, err := json.Marshal(v)
- if err != nil {
- return t, err
- }
-
- if err := json.Unmarshal(j, &t); err != nil {
- return t, err
- }
-
- return t, nil
-}
diff --git a/services/search/pkg/opensearch/internal/test/testdata.go b/services/search/pkg/opensearch/internal/test/testdata.go
index faf3709d36..68e14af9f5 100644
--- a/services/search/pkg/opensearch/internal/test/testdata.go
+++ b/services/search/pkg/opensearch/internal/test/testdata.go
@@ -1,14 +1,17 @@
package opensearchtest
import (
+ "embed"
"encoding/json"
"fmt"
- "os"
"path"
"github.com/opencloud-eu/opencloud/services/search/pkg/engine"
)
+//go:embed testdata/*.json
+var testdata embed.FS
+
var Testdata = struct {
Resources resourceTestdata
}{
@@ -22,8 +25,8 @@ type resourceTestdata struct {
}
func loadTestdata[D any](name string) D {
- name = path.Join("internal/test/testdata", name)
- data, err := os.ReadFile(name)
+ name = path.Join("./testdata", name)
+ data, err := testdata.ReadFile(name)
if err != nil {
panic(fmt.Sprintf("failed to read testdata file %s: %v", name, err))
}
diff --git a/services/search/pkg/opensearch/kql_ast.go b/services/search/pkg/opensearch/kql_ast.go
deleted file mode 100644
index 672379b4e9..0000000000
--- a/services/search/pkg/opensearch/kql_ast.go
+++ /dev/null
@@ -1,196 +0,0 @@
-package opensearch
-
-import (
- "fmt"
- "reflect"
- "slices"
- "strings"
-
- "github.com/opencloud-eu/opencloud/pkg/ast"
-)
-
-func expandKQLASTNodes(nodes []ast.Node) ([]ast.Node, error) {
- remapKey := func(current string, defaultKey string) string {
- if defaultKey == "" {
- defaultKey = "Name" // Set a default key if none is provided
- }
-
- key, ok := map[string]string{
- "": defaultKey, // Default case if current is empty
- "rootid": "RootID",
- "path": "Path",
- "id": "ID",
- "name": "Name",
- "size": "Size",
- "mtime": "Mtime",
- "mediatype": "MimeType",
- "type": "Type",
- "tag": "Tags",
- "tags": "Tags",
- "content": "Content",
- "hidden": "Hidden",
- }[current]
- if !ok {
- return current // Return the original key if not found
- }
-
- return key
- }
-
- lowerValue := func(key, value string) string {
- if slices.Contains([]string{"Hidden"}, key) {
- return value // ignore certain keys and return the original value
- }
-
- return strings.ToLower(value)
- }
-
- unfoldValue := func(key, value string) []ast.Node {
- result, ok := map[string][]ast.Node{
- "MimeType:file": {
- &ast.OperatorNode{Value: "NOT"},
- &ast.StringNode{Key: key, Value: "httpd/unix-directory"},
- },
- "MimeType:folder": {
- &ast.StringNode{Key: key, Value: "httpd/unix-directory"},
- },
- "MimeType:document": {
- &ast.GroupNode{Nodes: []ast.Node{
- &ast.StringNode{Key: key, Value: "application/msword"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.text"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "text/plain"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "text/markdown"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/rtf"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/vnd.apple.pages"},
- }},
- },
- "MimeType:spreadsheet": {
- &ast.GroupNode{Nodes: []ast.Node{
- &ast.StringNode{Key: key, Value: "application/vnd.ms-excel"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadsheet"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "text/csv"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"},
- }},
- },
- "MimeType:presentation": {
- &ast.GroupNode{Nodes: []ast.Node{
- &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.presentation"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/vnd.ms-powerpoint"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/vnd.apple.keynote"},
- }},
- },
- "MimeType:pdf": {
- &ast.StringNode{Key: key, Value: "application/pdf"},
- },
- "MimeType:image": {
- &ast.StringNode{Key: key, Value: "image/*"},
- },
- "MimeType:video": {
- &ast.StringNode{Key: key, Value: "video/*"},
- },
- "MimeType:audio": {
- &ast.StringNode{Key: key, Value: "audio/*"},
- },
- "MimeType:archive": {
- &ast.GroupNode{Nodes: []ast.Node{
- &ast.StringNode{Key: key, Value: "application/zip"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/gzip"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/x-gzip"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/x-7z-compressed"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/x-rar-compressed"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/x-tar"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/x-bzip2"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/x-bzip"},
- &ast.OperatorNode{Value: "OR"},
- &ast.StringNode{Key: key, Value: "application/x-tgz"},
- }},
- },
- }[fmt.Sprintf("%s:%s", key, value)]
- if !ok {
- return nil
- }
-
- return result
- }
-
- var expand func([]ast.Node, string) ([]ast.Node, error)
- expand = func(nodes []ast.Node, defaultKey string) ([]ast.Node, error) {
- for i, node := range nodes {
- rnode := reflect.ValueOf(node)
-
- // we need to ensure that the node is a pointer to an ast.Node in every case
- if rnode.Kind() != reflect.Ptr {
- ptr := reflect.New(rnode.Type())
- ptr.Elem().Set(rnode)
- rnode = ptr
- cnode, ok := rnode.Interface().(ast.Node)
- if !ok {
- return nil, fmt.Errorf("expected node to be of type ast.Node, got %T", rnode.Interface())
- }
-
- node = cnode // Update the original node to the pointer
- nodes[i] = node // Update the original slice with the pointer
- }
-
- var unfoldedNodes []ast.Node
- switch cnode := node.(type) {
- case *ast.GroupNode:
- if cnode.Key != "" { // group nodes should not get a default key
- cnode.Key = remapKey(cnode.Key, defaultKey)
- }
-
- groupNodes, err := expand(cnode.Nodes, cnode.Key)
- if err != nil {
- return nil, err
- }
- cnode.Nodes = groupNodes
- case *ast.StringNode:
- cnode.Key = remapKey(cnode.Key, defaultKey)
- cnode.Value = lowerValue(cnode.Key, cnode.Value)
- unfoldedNodes = unfoldValue(cnode.Key, cnode.Value)
- case *ast.DateTimeNode:
- cnode.Key = remapKey(cnode.Key, defaultKey)
- case *ast.BooleanNode:
- cnode.Key = remapKey(cnode.Key, defaultKey)
- }
-
- if unfoldedNodes != nil {
- // Insert unfolded nodes at the current index
- nodes = append(nodes[:i], append(unfoldedNodes, nodes[i+1:]...)...)
- // Adjust index to account for new nodes
- i += len(unfoldedNodes) - 1
- }
- }
-
- return nodes, nil
- }
-
- return expand(nodes, "")
-}
diff --git a/services/search/pkg/opensearch/opensearch.go b/services/search/pkg/opensearch/opensearch.go
deleted file mode 100644
index f4089736a3..0000000000
--- a/services/search/pkg/opensearch/opensearch.go
+++ /dev/null
@@ -1,64 +0,0 @@
-package opensearch
-
-import (
- "encoding/json"
- "fmt"
- "reflect"
-
- "dario.cat/mergo"
-)
-
-var (
- ErrUnhealthyCluster = fmt.Errorf("cluster is not healthy")
-)
-
-func isEmpty(x any) bool {
- switch {
- case x == nil:
- return true
- case reflect.ValueOf(x).Kind() == reflect.Bool:
- return false
- case reflect.DeepEqual(x, reflect.Zero(reflect.TypeOf(x)).Interface()):
- return true
- case reflect.ValueOf(x).Kind() == reflect.Map && reflect.ValueOf(x).Len() == 0:
- return true
- default:
- return false
- }
-}
-
-func merge[T any](options ...T) T {
- mapOptions := make(map[string]any)
-
- for _, option := range options {
- data, err := convert[map[string]any](option)
- if err != nil {
- continue
- }
-
- _ = mergo.Merge(&mapOptions, data)
- }
-
- data, _ := convert[T](mapOptions)
-
- return data
-}
-
-func convert[T any](v any) (T, error) {
- var t T
-
- if v == nil {
- return t, nil
- }
-
- j, err := json.Marshal(v)
- if err != nil {
- return t, err
- }
-
- if err := json.Unmarshal(j, &t); err != nil {
- return t, err
- }
-
- return t, nil
-}
diff --git a/services/search/pkg/opensearch/opensearch_test.go b/services/search/pkg/opensearch/opensearch_test.go
deleted file mode 100644
index 5cfac2a0d0..0000000000
--- a/services/search/pkg/opensearch/opensearch_test.go
+++ /dev/null
@@ -1 +0,0 @@
-package opensearch_test
diff --git a/services/search/pkg/opensearch/os.go b/services/search/pkg/opensearch/os.go
deleted file mode 100644
index 01ee40c65e..0000000000
--- a/services/search/pkg/opensearch/os.go
+++ /dev/null
@@ -1,30 +0,0 @@
-package opensearch
-
-import (
- "context"
- "fmt"
- "time"
-
- opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
-)
-
-func clusterHealth(ctx context.Context, client *opensearchgoAPI.Client, indices []string) (*opensearchgoAPI.ClusterHealthResp, bool, error) {
- resp, err := client.Cluster.Health(ctx, &opensearchgoAPI.ClusterHealthReq{
- Indices: indices,
- Params: opensearchgoAPI.ClusterHealthParams{
- Local: opensearchgoAPI.ToPointer(true),
- Timeout: 5 * time.Second,
- },
- })
- switch {
- case err != nil:
- return nil, false, fmt.Errorf("%w, failed to get cluster health: %w", ErrUnhealthyCluster, err)
- case resp.TimedOut:
- return resp, false, fmt.Errorf("%w, cluster health request timed out", ErrUnhealthyCluster)
-
- case resp.Status != "green" && resp.Status != "yellow":
- return resp, false, fmt.Errorf("%w, cluster health is not green or yellow: %s", ErrUnhealthyCluster, resp.Status)
- default:
- return resp, true, nil
- }
-}
diff --git a/services/search/pkg/opensearch/os_dsl_test.go b/services/search/pkg/opensearch/os_dsl_test.go
deleted file mode 100644
index 9bb99173b4..0000000000
--- a/services/search/pkg/opensearch/os_dsl_test.go
+++ /dev/null
@@ -1,31 +0,0 @@
-package opensearch_test
-
-import (
- "testing"
-
- "github.com/stretchr/testify/assert"
-
- "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
- "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
-)
-
-func TestBuilderToBoolQuery(t *testing.T) {
- tests := []opensearchtest.TableTest[opensearch.Builder, *opensearch.BoolQuery]{
- {
- Name: "term-query",
- Got: opensearch.NewTermQuery[string]("Name").Value("openCloud"),
- Want: opensearch.NewBoolQuery().Must(opensearch.NewTermQuery[string]("Name").Value("openCloud")),
- },
- {
- Name: "bool-query",
- Got: opensearch.NewBoolQuery().Must(opensearch.NewTermQuery[string]("Name").Value("openCloud")),
- Want: opensearch.NewBoolQuery().Must(opensearch.NewTermQuery[string]("Name").Value("openCloud")),
- },
- }
-
- for _, test := range tests {
- t.Run(test.Name, func(t *testing.T) {
- assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, opensearch.BuilderToBoolQuery(test.Got)))
- })
- }
-}
diff --git a/services/search/pkg/service/grpc/v0/service.go b/services/search/pkg/service/grpc/v0/service.go
index 52d5acd87e..3b53ba9fc4 100644
--- a/services/search/pkg/service/grpc/v0/service.go
+++ b/services/search/pkg/service/grpc/v0/service.go
@@ -80,12 +80,12 @@ func NewHandler(opts ...Option) (searchsvc.SearchProviderHandler, func(), error)
return nil, teardown, fmt.Errorf("failed to create OpenSearch client: %w", err)
}
- backend, err := opensearch.NewEngine(cfg.Engine.OpenSearch.ResourceIndex.Name, client)
+ openSearchBackend, err := opensearch.NewBackend(cfg.Engine.OpenSearch.ResourceIndex.Name, client)
if err != nil {
- return nil, teardown, fmt.Errorf("failed to create OpenSearch engine: %w", err)
+ return nil, teardown, fmt.Errorf("failed to create OpenSearch backend: %w", err)
}
- eng = backend
+ eng = openSearchBackend
default:
return nil, teardown, fmt.Errorf("unknown search engine: %s", cfg.Engine.Type)
}