From 42b794e01aeedf8850d1dccb56481790f964fdf1 Mon Sep 17 00:00:00 2001 From: fschade Date: Fri, 8 Aug 2025 13:44:43 +0200 Subject: [PATCH] refactor(search): cleanup for review --- pkg/conversions/conversions.go | 24 +++ .../pkg/opensearch/{engine.go => backend.go} | 147 ++++++------- .../{engine_test.go => backend_test.go} | 32 +-- services/search/pkg/opensearch/export_test.go | 11 - .../opensearch/internal/convert/kql_expand.go | 197 ++++++++++++++++++ .../convert/kql_expand_test.go} | 19 +- .../opensearch/internal/convert/kql_query.go | 35 ++++ .../internal/convert/kql_query_test.go | 1 + .../convert/kql_transpile.go} | 77 ++++--- .../convert/kql_transpile_test.go} | 161 +++++++------- .../convert/opensearch.go} | 15 +- .../convert/opensearch_test.go} | 13 +- .../{os_index.go => internal/osu/index.go} | 6 +- .../osu/index_test.go} | 14 +- .../{ => osu}/indexes/resource_v1.json | 0 .../{os_dsl.go => internal/osu/osu.go} | 81 ++++--- .../osu/query_bool.go} | 27 ++- .../osu/query_bool_test.go} | 28 +-- .../osu/query_full_text_match_phrase.go} | 27 ++- .../osu/query_full_text_match_phrase_test.go} | 14 +- .../osu/query_term_level_ids.go} | 21 +- .../osu/query_term_level_ids_test.go} | 10 +- .../osu/query_term_level_range.go} | 29 +-- .../osu/query_term_level_range_test.go} | 30 +-- .../osu/query_term_level_term.go} | 21 +- .../osu/query_term_level_term_test.go} | 14 +- .../osu/query_term_level_wildcard.go} | 23 +- .../osu/query_term_level_wildcard_test.go} | 12 +- .../osu/request.go} | 6 +- .../osu/request_test.go} | 18 +- .../pkg/opensearch/internal/test/helper.go | 23 +- .../pkg/opensearch/internal/test/testdata.go | 9 +- services/search/pkg/opensearch/kql_ast.go | 196 ----------------- services/search/pkg/opensearch/opensearch.go | 64 ------ .../search/pkg/opensearch/opensearch_test.go | 1 - services/search/pkg/opensearch/os.go | 30 --- services/search/pkg/opensearch/os_dsl_test.go | 31 --- .../search/pkg/service/grpc/v0/service.go | 6 +- 38 files changed, 713 insertions(+), 760 deletions(-) create mode 100644 pkg/conversions/conversions.go rename services/search/pkg/opensearch/{engine.go => backend.go} (60%) rename services/search/pkg/opensearch/{engine_test.go => backend_test.go} (90%) delete mode 100644 services/search/pkg/opensearch/export_test.go create mode 100644 services/search/pkg/opensearch/internal/convert/kql_expand.go rename services/search/pkg/opensearch/{kql_ast_test.go => internal/convert/kql_expand_test.go} (97%) create mode 100644 services/search/pkg/opensearch/internal/convert/kql_query.go create mode 100644 services/search/pkg/opensearch/internal/convert/kql_query_test.go rename services/search/pkg/opensearch/{kql_to_os_dsl.go => internal/convert/kql_transpile.go} (56%) rename services/search/pkg/opensearch/{kql_to_os_dsl_test.go => internal/convert/kql_transpile_test.go} (57%) rename services/search/pkg/opensearch/{engine_convert.go => internal/convert/opensearch.go} (81%) rename services/search/pkg/opensearch/{engine_convert_test.go => internal/convert/opensearch_test.go} (81%) rename services/search/pkg/opensearch/{os_index.go => internal/osu/index.go} (96%) rename services/search/pkg/opensearch/{os_index_test.go => internal/osu/index_test.go} (85%) rename services/search/pkg/opensearch/internal/{ => osu}/indexes/resource_v1.json (100%) rename services/search/pkg/opensearch/{os_dsl.go => internal/osu/osu.go} (53%) rename services/search/pkg/opensearch/{os_dsl_query_bool.go => internal/osu/query_bool.go} (70%) rename services/search/pkg/opensearch/{os_dsl_query_bool_test.go => internal/osu/query_bool_test.go} (74%) rename services/search/pkg/opensearch/{os_dsl_query_full_text_match_phrase.go => internal/osu/query_full_text_match_phrase.go} (56%) rename services/search/pkg/opensearch/{os_dsl_query_full_text_match_phrase_test.go => internal/osu/query_full_text_match_phrase_test.go} (77%) rename services/search/pkg/opensearch/{os_dsl_query_term_level_ids.go => internal/osu/query_term_level_ids.go} (62%) rename services/search/pkg/opensearch/{os_dsl_query_term_level_ids_test.go => internal/osu/query_term_level_ids_test.go} (72%) rename services/search/pkg/opensearch/{os_dsl_query_term_level_range.go => internal/osu/query_term_level_range.go} (79%) rename services/search/pkg/opensearch/{os_dsl_query_term_level_range_test.go => internal/osu/query_term_level_range_test.go} (74%) rename services/search/pkg/opensearch/{os_dsl_query_term_level_term.go => internal/osu/query_term_level_term.go} (69%) rename services/search/pkg/opensearch/{os_dsl_query_term_level_term_test.go => internal/osu/query_term_level_term_test.go} (75%) rename services/search/pkg/opensearch/{os_dsl_query_term_level_wildcard.go => internal/osu/query_term_level_wildcard.go} (65%) rename services/search/pkg/opensearch/{os_dsl_query_term_level_wildcard_test.go => internal/osu/query_term_level_wildcard_test.go} (75%) rename services/search/pkg/opensearch/{os_request.go => internal/osu/request.go} (96%) rename services/search/pkg/opensearch/{os_request_test.go => internal/osu/request_test.go} (79%) delete mode 100644 services/search/pkg/opensearch/kql_ast.go delete mode 100644 services/search/pkg/opensearch/opensearch.go delete mode 100644 services/search/pkg/opensearch/opensearch_test.go delete mode 100644 services/search/pkg/opensearch/os.go delete mode 100644 services/search/pkg/opensearch/os_dsl_test.go diff --git a/pkg/conversions/conversions.go b/pkg/conversions/conversions.go new file mode 100644 index 0000000000..6931f6e723 --- /dev/null +++ b/pkg/conversions/conversions.go @@ -0,0 +1,24 @@ +package conversions + +import ( + "encoding/json" +) + +func To[T any](v any) (T, error) { + var t T + + if v == nil { + return t, nil + } + + j, err := json.Marshal(v) + if err != nil { + return t, err + } + + if err := json.Unmarshal(j, &t); err != nil { + return t, err + } + + return t, nil +} diff --git a/services/search/pkg/opensearch/engine.go b/services/search/pkg/opensearch/backend.go similarity index 60% rename from services/search/pkg/opensearch/engine.go rename to services/search/pkg/opensearch/backend.go index 98b5df0b00..072ea5d2ee 100644 --- a/services/search/pkg/opensearch/engine.go +++ b/services/search/pkg/opensearch/backend.go @@ -7,6 +7,7 @@ import ( "fmt" "path" "strings" + "time" storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/opencloud-eu/reva/v2/pkg/storagespace" @@ -14,18 +15,23 @@ import ( opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "github.com/opencloud-eu/opencloud/pkg/conversions" - "github.com/opencloud-eu/opencloud/pkg/kql" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" "github.com/opencloud-eu/opencloud/services/search/pkg/engine" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" ) -type Engine struct { +var ( + ErrUnhealthyCluster = fmt.Errorf("cluster is not healthy") +) + +type Backend struct { index string client *opensearchgoAPI.Client } -func NewEngine(index string, client *opensearchgoAPI.Client) (*Engine, error) { +func NewBackend(index string, client *opensearchgoAPI.Client) (*Backend, error) { pingResp, err := client.Ping(context.TODO(), &opensearchgoAPI.PingReq{}) switch { case err != nil: @@ -35,45 +41,46 @@ func NewEngine(index string, client *opensearchgoAPI.Client) (*Engine, error) { } // apply the index template - if err := IndexManagerLatest.Apply(context.TODO(), index, client); err != nil { + if err := osu.IndexManagerLatest.Apply(context.TODO(), index, client); err != nil { return nil, fmt.Errorf("failed to apply index template: %w", err) } // first check if the cluster is healthy - _, healthy, err := clusterHealth(context.TODO(), client, []string{index}) + + resp, err := client.Cluster.Health(context.TODO(), &opensearchgoAPI.ClusterHealthReq{ + Indices: []string{index}, + Params: opensearchgoAPI.ClusterHealthParams{ + Local: opensearchgoAPI.ToPointer(true), + Timeout: 5 * time.Second, + }, + }) switch { case err != nil: - return nil, fmt.Errorf("failed to get cluster health: %w", err) - case !healthy: - return nil, fmt.Errorf("cluster health is not healthy") + return nil, fmt.Errorf("%w, failed to get cluster health: %w", ErrUnhealthyCluster, err) + case resp.TimedOut: + return nil, fmt.Errorf("%w, cluster health request timed out", ErrUnhealthyCluster) + case resp.Status != "green" && resp.Status != "yellow": + return nil, fmt.Errorf("%w, cluster health is not green or yellow: %s", ErrUnhealthyCluster, resp.Status) } - return &Engine{index: index, client: client}, nil + return &Backend{index: index, client: client}, nil } -func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) { - ast, err := kql.Builder{}.Build(sir.Query) +func (be *Backend) Search(ctx context.Context, sir *searchService.SearchIndexRequest) (*searchService.SearchIndexResponse, error) { + boolQuery, err := convert.KQLToOpenSearchBoolQuery(sir.Query) if err != nil { - return nil, fmt.Errorf("failed to build query: %w", err) + return nil, fmt.Errorf("failed to convert KQL query to OpenSearch bool query: %w", err) } - transpiler, err := NewKQLToOsDSL() - if err != nil { - return nil, fmt.Errorf("failed to create KQL compiler: %w", err) - } - - builder, err := transpiler.Compile(ast) - if err != nil { - return nil, fmt.Errorf("failed to compile query: %w", err) - } - - boolQuery := builderToBoolQuery(builder).Filter( - NewTermQuery[bool]("Deleted").Value(false), + // filter out deleted resources + boolQuery.Filter( + osu.NewTermQuery[bool]("Deleted").Value(false), ) if sir.Ref != nil { + // if a reference is provided, filter by the root ID boolQuery.Filter( - NewTermQuery[string]("RootID").Value( + osu.NewTermQuery[string]("RootID").Value( storagespace.FormatResourceID( &storageProvider.ResourceId{ StorageId: sir.Ref.GetResourceId().GetStorageId(), @@ -96,16 +103,16 @@ func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexReque searchParams.Size = conversions.ToPointer(int(sir.PageSize)) } - req, err := BuildSearchReq(&opensearchgoAPI.SearchReq{ - Indices: []string{e.index}, + req, err := osu.BuildSearchReq(&opensearchgoAPI.SearchReq{ + Indices: []string{be.index}, Params: searchParams, }, boolQuery, - SearchReqOptions{ - Highlight: &HighlightOption{ + osu.SearchReqOptions{ + Highlight: &osu.HighlightOption{ PreTags: []string{""}, PostTags: []string{""}, - Fields: map[string]HighlightOption{ + Fields: map[string]osu.HighlightOption{ "Content": {}, }, }, @@ -115,7 +122,7 @@ func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexReque return nil, fmt.Errorf("failed to build search request: %w", err) } - resp, err := e.client.Search(ctx, req) + resp, err := be.client.Search(ctx, req) if err != nil { return nil, fmt.Errorf("failed to search: %w", err) } @@ -123,7 +130,7 @@ func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexReque matches := make([]*searchMessage.Match, 0, len(resp.Hits.Hits)) totalMatches := resp.Hits.Total.Value for _, hit := range resp.Hits.Hits { - match, err := searchHitToSearchMessageMatch(hit) + match, err := convert.OpenSearchHitToMatch(hit) if err != nil { return nil, fmt.Errorf("failed to convert hit to match: %w", err) } @@ -148,14 +155,14 @@ func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexReque }, nil } -func (e *Engine) Upsert(id string, r engine.Resource) error { +func (be *Backend) Upsert(id string, r engine.Resource) error { body, err := json.Marshal(r) if err != nil { return fmt.Errorf("failed to marshal resource: %w", err) } - _, err = e.client.Index(context.TODO(), opensearchgoAPI.IndexReq{ - Index: e.index, + _, err = be.client.Index(context.TODO(), opensearchgoAPI.IndexReq{ + Index: be.index, DocumentID: id, Body: bytes.NewReader(body), }) @@ -166,9 +173,9 @@ func (e *Engine) Upsert(id string, r engine.Resource) error { return nil } -func (e *Engine) Move(id string, parentID string, target string) error { - return e.updateSelfAndDescendants(id, func(rootResource engine.Resource) *ScriptOption { - return &ScriptOption{ +func (be *Backend) Move(id string, parentID string, target string) error { + return be.updateSelfAndDescendants(id, func(rootResource engine.Resource) *osu.ScriptOption { + return &osu.ScriptOption{ Source: ` if (ctx._source.ID == params.id ) { ctx._source.Name = params.newName; ctx._source.ParentID = params.parentID; } ctx._source.Path = ctx._source.Path.replace(params.oldPath, params.newPath) @@ -185,9 +192,9 @@ func (e *Engine) Move(id string, parentID string, target string) error { }) } -func (e *Engine) Delete(id string) error { - return e.updateSelfAndDescendants(id, func(_ engine.Resource) *ScriptOption { - return &ScriptOption{ +func (be *Backend) Delete(id string) error { + return be.updateSelfAndDescendants(id, func(_ engine.Resource) *osu.ScriptOption { + return &osu.ScriptOption{ Source: "ctx._source.Deleted = params.deleted", Lang: "painless", Params: map[string]any{ @@ -197,9 +204,9 @@ func (e *Engine) Delete(id string) error { }) } -func (e *Engine) Restore(id string) error { - return e.updateSelfAndDescendants(id, func(_ engine.Resource) *ScriptOption { - return &ScriptOption{ +func (be *Backend) Restore(id string) error { + return be.updateSelfAndDescendants(id, func(_ engine.Resource) *osu.ScriptOption { + return &osu.ScriptOption{ Source: "ctx._source.Deleted = params.deleted", Lang: "painless", Params: map[string]any{ @@ -209,26 +216,26 @@ func (e *Engine) Restore(id string) error { }) } -func (e *Engine) Purge(id string) error { - resource, err := e.getResource(id) +func (be *Backend) Purge(id string) error { + resource, err := be.getResource(id) if err != nil { return fmt.Errorf("failed to get resource: %w", err) } - req, err := BuildDocumentDeleteByQueryReq( + req, err := osu.BuildDocumentDeleteByQueryReq( opensearchgoAPI.DocumentDeleteByQueryReq{ - Indices: []string{e.index}, + Indices: []string{be.index}, Params: opensearchgoAPI.DocumentDeleteByQueryParams{ WaitForCompletion: conversions.ToPointer(true), }, }, - NewTermQuery[string]("Path").Value(resource.Path), + osu.NewTermQuery[string]("Path").Value(resource.Path), ) if err != nil { return fmt.Errorf("failed to build delete by query request: %w", err) } - resp, err := e.client.Document.DeleteByQuery(context.TODO(), req) + resp, err := be.client.Document.DeleteByQuery(context.TODO(), req) switch { case err != nil: return fmt.Errorf("failed to delete by query: %w", err) @@ -239,18 +246,18 @@ func (e *Engine) Purge(id string) error { return nil } -func (e *Engine) DocCount() (uint64, error) { - req, err := BuildIndicesCountReq( +func (be *Backend) DocCount() (uint64, error) { + req, err := osu.BuildIndicesCountReq( &opensearchgoAPI.IndicesCountReq{ - Indices: []string{e.index}, + Indices: []string{be.index}, }, - NewTermQuery[bool]("Deleted").Value(false), + osu.NewTermQuery[bool]("Deleted").Value(false), ) if err != nil { return 0, fmt.Errorf("failed to build count request: %w", err) } - resp, err := e.client.Indices.Count(context.TODO(), req) + resp, err := be.client.Indices.Count(context.TODO(), req) if err != nil { return 0, fmt.Errorf("failed to count documents: %w", err) } @@ -258,25 +265,25 @@ func (e *Engine) DocCount() (uint64, error) { return uint64(resp.Count), nil } -func (e *Engine) updateSelfAndDescendants(id string, scriptProvider func(engine.Resource) *ScriptOption) error { +func (be *Backend) updateSelfAndDescendants(id string, scriptProvider func(engine.Resource) *osu.ScriptOption) error { if scriptProvider == nil { return fmt.Errorf("script cannot be nil") } - resource, err := e.getResource(id) + resource, err := be.getResource(id) if err != nil { return fmt.Errorf("failed to get resource: %w", err) } - req, err := BuildUpdateByQueryReq( + req, err := osu.BuildUpdateByQueryReq( opensearchgoAPI.UpdateByQueryReq{ - Indices: []string{e.index}, + Indices: []string{be.index}, Params: opensearchgoAPI.UpdateByQueryParams{ WaitForCompletion: conversions.ToPointer(true), }, }, - NewTermQuery[string]("Path").Value(resource.Path), - UpdateByQueryReqOptions{ + osu.NewTermQuery[string]("Path").Value(resource.Path), + osu.UpdateByQueryReqOptions{ Script: scriptProvider(resource), }, ) @@ -284,7 +291,7 @@ func (e *Engine) updateSelfAndDescendants(id string, scriptProvider func(engine. return fmt.Errorf("failed to build update by query request: %w", err) } - resp, err := e.client.UpdateByQuery(context.TODO(), req) + resp, err := be.client.UpdateByQuery(context.TODO(), req) switch { case err != nil: return fmt.Errorf("failed to update by query: %w", err) @@ -295,18 +302,18 @@ func (e *Engine) updateSelfAndDescendants(id string, scriptProvider func(engine. return nil } -func (e *Engine) getResource(id string) (engine.Resource, error) { - req, err := BuildSearchReq( +func (be *Backend) getResource(id string) (engine.Resource, error) { + req, err := osu.BuildSearchReq( &opensearchgoAPI.SearchReq{ - Indices: []string{e.index}, + Indices: []string{be.index}, }, - NewIDsQuery([]string{id}), + osu.NewIDsQuery(id), ) if err != nil { return engine.Resource{}, fmt.Errorf("failed to build search request: %w", err) } - resp, err := e.client.Search(context.TODO(), req) + resp, err := be.client.Search(context.TODO(), req) switch { case err != nil: return engine.Resource{}, fmt.Errorf("failed to search for resource: %w", err) @@ -314,7 +321,7 @@ func (e *Engine) getResource(id string) (engine.Resource, error) { return engine.Resource{}, fmt.Errorf("document with id %s not found", id) } - resource, err := convert[engine.Resource](resp.Hits.Hits[0].Source) + resource, err := conversions.To[engine.Resource](resp.Hits.Hits[0].Source) if err != nil { return engine.Resource{}, fmt.Errorf("failed to convert hit source: %w", err) } @@ -322,10 +329,10 @@ func (e *Engine) getResource(id string) (engine.Resource, error) { return resource, nil } -func (e *Engine) StartBatch(_ int) error { +func (be *Backend) StartBatch(_ int) error { return nil // todo: implement batch processing } -func (e *Engine) EndBatch() error { +func (be *Backend) EndBatch() error { return nil // todo: implement batch processing } diff --git a/services/search/pkg/opensearch/engine_test.go b/services/search/pkg/opensearch/backend_test.go similarity index 90% rename from services/search/pkg/opensearch/engine_test.go rename to services/search/pkg/opensearch/backend_test.go index 6d0578ff90..69371fc758 100644 --- a/services/search/pkg/opensearch/engine_test.go +++ b/services/search/pkg/opensearch/backend_test.go @@ -15,7 +15,7 @@ import ( "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) -func TestNewEngine(t *testing.T) { +func TestNewBackend(t *testing.T) { t.Run("fails to create if the cluster is not healthy", func(t *testing.T) { client, err := opensearchgoAPI.NewClient(opensearchgoAPI.Config{ Client: opensearchgo.Config{ @@ -24,21 +24,21 @@ func TestNewEngine(t *testing.T) { }) require.NoError(t, err, "failed to create OpenSearch client") - backend, err := opensearch.NewEngine("test-engine-new-engine", client) + backend, err := opensearch.NewBackend("test-engine-new-engine", client) require.Nil(t, backend) require.ErrorIs(t, err, opensearch.ErrUnhealthyCluster) }) } func TestEngine_Search(t *testing.T) { - indexName := "opencloud-test-resource" + indexName := "opencloud-test-engine-search" tc := opensearchtest.NewDefaultTestClient(t) tc.Require.IndicesReset([]string{indexName}) tc.Require.IndicesCount([]string{indexName}, nil, 0) defer tc.Require.IndicesDelete([]string{indexName}) - backend, err := opensearch.NewEngine(indexName, tc.Client()) + backend, err := opensearch.NewBackend(indexName, tc.Client()) require.NoError(t, err) document := opensearchtest.Testdata.Resources.File @@ -74,14 +74,14 @@ func TestEngine_Search(t *testing.T) { } func TestEngine_Upsert(t *testing.T) { - indexName := "opencloud-test-resource" + indexName := "opencloud-test-engine-upsert" tc := opensearchtest.NewDefaultTestClient(t) tc.Require.IndicesReset([]string{indexName}) tc.Require.IndicesCount([]string{indexName}, nil, 0) defer tc.Require.IndicesDelete([]string{indexName}) - backend, err := opensearch.NewEngine(indexName, tc.Client()) + backend, err := opensearch.NewBackend(indexName, tc.Client()) require.NoError(t, err) t.Run("upsert with full document", func(t *testing.T) { @@ -93,14 +93,14 @@ func TestEngine_Upsert(t *testing.T) { } func TestEngine_Move(t *testing.T) { - indexName := "opencloud-test-resource" + indexName := "opencloud-test-engine-move" tc := opensearchtest.NewDefaultTestClient(t) tc.Require.IndicesReset([]string{indexName}) tc.Require.IndicesCount([]string{indexName}, nil, 0) defer tc.Require.IndicesDelete([]string{indexName}) - backend, err := opensearch.NewEngine(indexName, tc.Client()) + backend, err := opensearch.NewBackend(indexName, tc.Client()) require.NoError(t, err) t.Run("moves the document to a new path", func(t *testing.T) { @@ -130,14 +130,14 @@ func TestEngine_Move(t *testing.T) { } func TestEngine_Delete(t *testing.T) { - indexName := "opencloud-test-resource" + indexName := "opencloud-test-engine-delete" tc := opensearchtest.NewDefaultTestClient(t) tc.Require.IndicesReset([]string{indexName}) tc.Require.IndicesCount([]string{indexName}, nil, 0) defer tc.Require.IndicesDelete([]string{indexName}) - backend, err := opensearch.NewEngine(indexName, tc.Client()) + backend, err := opensearch.NewBackend(indexName, tc.Client()) require.NoError(t, err) t.Run("mark document as deleted", func(t *testing.T) { @@ -163,14 +163,14 @@ func TestEngine_Delete(t *testing.T) { } func TestEngine_Restore(t *testing.T) { - indexName := "opencloud-test-resource" + indexName := "opencloud-test-engine-restore" tc := opensearchtest.NewDefaultTestClient(t) tc.Require.IndicesReset([]string{indexName}) tc.Require.IndicesCount([]string{indexName}, nil, 0) defer tc.Require.IndicesDelete([]string{indexName}) - backend, err := opensearch.NewEngine(indexName, tc.Client()) + backend, err := opensearch.NewBackend(indexName, tc.Client()) require.NoError(t, err) t.Run("mark document as not deleted", func(t *testing.T) { @@ -197,14 +197,14 @@ func TestEngine_Restore(t *testing.T) { } func TestEngine_Purge(t *testing.T) { - indexName := "opencloud-test-resource" + indexName := "opencloud-test-engine-purge" tc := opensearchtest.NewDefaultTestClient(t) tc.Require.IndicesReset([]string{indexName}) tc.Require.IndicesCount([]string{indexName}, nil, 0) defer tc.Require.IndicesDelete([]string{indexName}) - backend, err := opensearch.NewEngine(indexName, tc.Client()) + backend, err := opensearch.NewBackend(indexName, tc.Client()) require.NoError(t, err) t.Run("purge with full document", func(t *testing.T) { @@ -219,14 +219,14 @@ func TestEngine_Purge(t *testing.T) { } func TestEngine_DocCount(t *testing.T) { - indexName := "opencloud-test-resource" + indexName := "opencloud-test-engine-doc-count" tc := opensearchtest.NewDefaultTestClient(t) tc.Require.IndicesReset([]string{indexName}) tc.Require.IndicesCount([]string{indexName}, nil, 0) defer tc.Require.IndicesDelete([]string{indexName}) - backend, err := opensearch.NewEngine(indexName, tc.Client()) + backend, err := opensearch.NewBackend(indexName, tc.Client()) require.NoError(t, err) t.Run("ignore deleted documents", func(t *testing.T) { diff --git a/services/search/pkg/opensearch/export_test.go b/services/search/pkg/opensearch/export_test.go deleted file mode 100644 index 6e02286943..0000000000 --- a/services/search/pkg/opensearch/export_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package opensearch - -var ( - SearchHitToSearchMessageMatch = searchHitToSearchMessageMatch - BuilderToBoolQuery = builderToBoolQuery - ExpandKQLASTNodes = expandKQLASTNodes -) - -func Convert[T any](v any) (T, error) { - return convert[T](v) -} diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand.go b/services/search/pkg/opensearch/internal/convert/kql_expand.go new file mode 100644 index 0000000000..5688f6dfe6 --- /dev/null +++ b/services/search/pkg/opensearch/internal/convert/kql_expand.go @@ -0,0 +1,197 @@ +package convert + +import ( + "fmt" + "reflect" + "slices" + "strings" + + "github.com/opencloud-eu/opencloud/pkg/ast" +) + +func ExpandKQL(nodes []ast.Node) ([]ast.Node, error) { + return kqlExpander{}.expand(nodes, "") +} + +type kqlExpander struct{} + +func (e kqlExpander) expand(nodes []ast.Node, defaultKey string) ([]ast.Node, error) { + for i, node := range nodes { + rnode := reflect.ValueOf(node) + + // we need to ensure that the node is a pointer to an ast.Node in every case + if rnode.Kind() != reflect.Ptr { + ptr := reflect.New(rnode.Type()) + ptr.Elem().Set(rnode) + rnode = ptr + cnode, ok := rnode.Interface().(ast.Node) + if !ok { + return nil, fmt.Errorf("expected node to be of type ast.Node, got %T", rnode.Interface()) + } + + node = cnode // Update the original node to the pointer + nodes[i] = node // Update the original slice with the pointer + } + + var unfoldedNodes []ast.Node + switch cnode := node.(type) { + case *ast.GroupNode: + if cnode.Key != "" { // group nodes should not get a default key + cnode.Key = e.remapKey(cnode.Key, defaultKey) + } + + groupNodes, err := e.expand(cnode.Nodes, cnode.Key) + if err != nil { + return nil, err + } + cnode.Nodes = groupNodes + case *ast.StringNode: + cnode.Key = e.remapKey(cnode.Key, defaultKey) + cnode.Value = e.lowerValue(cnode.Key, cnode.Value) + unfoldedNodes = e.unfoldValue(cnode.Key, cnode.Value) + case *ast.DateTimeNode: + cnode.Key = e.remapKey(cnode.Key, defaultKey) + case *ast.BooleanNode: + cnode.Key = e.remapKey(cnode.Key, defaultKey) + } + + if unfoldedNodes != nil { + // Insert unfolded nodes at the current index + nodes = append(nodes[:i], append(unfoldedNodes, nodes[i+1:]...)...) + // Adjust index to account for new nodes + i += len(unfoldedNodes) - 1 + } + } + + return nodes, nil +} + +func (_ kqlExpander) remapKey(current string, defaultKey string) string { + if defaultKey == "" { + defaultKey = "Name" // Set a default key if none is provided + } + + key, ok := map[string]string{ + "": defaultKey, // Default case if current is empty + "rootid": "RootID", + "path": "Path", + "id": "ID", + "name": "Name", + "size": "Size", + "mtime": "Mtime", + "mediatype": "MimeType", + "type": "Type", + "tag": "Tags", + "tags": "Tags", + "content": "Content", + "hidden": "Hidden", + }[current] + if !ok { + return current // Return the original key if not found + } + + return key +} + +func (_ kqlExpander) lowerValue(key, value string) string { + if slices.Contains([]string{"Hidden"}, key) { + return value // ignore certain keys and return the original value + } + + return strings.ToLower(value) +} + +func (_ kqlExpander) unfoldValue(key, value string) []ast.Node { + result, ok := map[string][]ast.Node{ + "MimeType:file": { + &ast.OperatorNode{Value: "NOT"}, + &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, + }, + "MimeType:folder": { + &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, + }, + "MimeType:document": { + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: key, Value: "application/msword"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.text"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "text/plain"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "text/markdown"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/rtf"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.apple.pages"}, + }}, + }, + "MimeType:spreadsheet": { + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: key, Value: "application/vnd.ms-excel"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadsheet"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "text/csv"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"}, + }}, + }, + "MimeType:presentation": { + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.presentation"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.ms-powerpoint"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/vnd.apple.keynote"}, + }}, + }, + "MimeType:pdf": { + &ast.StringNode{Key: key, Value: "application/pdf"}, + }, + "MimeType:image": { + &ast.StringNode{Key: key, Value: "image/*"}, + }, + "MimeType:video": { + &ast.StringNode{Key: key, Value: "video/*"}, + }, + "MimeType:audio": { + &ast.StringNode{Key: key, Value: "audio/*"}, + }, + "MimeType:archive": { + &ast.GroupNode{Nodes: []ast.Node{ + &ast.StringNode{Key: key, Value: "application/zip"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/gzip"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/x-gzip"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/x-7z-compressed"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/x-rar-compressed"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/x-tar"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/x-bzip2"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/x-bzip"}, + &ast.OperatorNode{Value: "OR"}, + &ast.StringNode{Key: key, Value: "application/x-tgz"}, + }}, + }, + }[fmt.Sprintf("%s:%s", key, value)] + if !ok { + return nil + } + + return result +} diff --git a/services/search/pkg/opensearch/kql_ast_test.go b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go similarity index 97% rename from services/search/pkg/opensearch/kql_ast_test.go rename to services/search/pkg/opensearch/internal/convert/kql_expand_test.go index f742b44e93..49218d5aa5 100644 --- a/services/search/pkg/opensearch/kql_ast_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go @@ -1,4 +1,4 @@ -package opensearch_test +package convert_test import ( "fmt" @@ -6,12 +6,13 @@ import ( "github.com/stretchr/testify/require" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" + "github.com/opencloud-eu/opencloud/pkg/ast" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) -func TestExpandKQLASTNodes(t *testing.T) { +func TestExpandKQLAST(t *testing.T) { t.Run("always converts a value node to a pointer node", func(t *testing.T) { tests := []opensearchtest.TableTest[[]ast.Node, []ast.Node]{ { @@ -117,7 +118,7 @@ func TestExpandKQLASTNodes(t *testing.T) { for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - result, err := opensearch.ExpandKQLASTNodes(test.Got) + result, err := convert.ExpandKQL(test.Got) require.NoError(t, err) require.Equal(t, test.Want, result) }) @@ -233,7 +234,7 @@ func TestExpandKQLASTNodes(t *testing.T) { for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - result, err := opensearch.ExpandKQLASTNodes(test.Got) + result, err := convert.ExpandKQL(test.Got) require.NoError(t, err) require.Equal(t, test.Want, result) }) @@ -276,7 +277,7 @@ func TestExpandKQLASTNodes(t *testing.T) { for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - result, err := opensearch.ExpandKQLASTNodes(test.Got) + result, err := convert.ExpandKQL(test.Got) require.NoError(t, err) require.Equal(t, test.Want, result) }) @@ -526,7 +527,7 @@ func TestExpandKQLASTNodes(t *testing.T) { if test.Skip { t.Skip("Skipping test due to known issue") } - result, err := opensearch.ExpandKQLASTNodes(test.Got) + result, err := convert.ExpandKQL(test.Got) require.NoError(t, err) require.EqualValues(t, test.Want, result) }) @@ -597,7 +598,7 @@ func TestExpandKQLASTNodes(t *testing.T) { if test.Skip { t.Skip("Skipping test due to known issue") } - result, err := opensearch.ExpandKQLASTNodes(test.Got) + result, err := convert.ExpandKQL(test.Got) require.NoError(t, err) require.EqualValues(t, test.Want, result) }) diff --git a/services/search/pkg/opensearch/internal/convert/kql_query.go b/services/search/pkg/opensearch/internal/convert/kql_query.go new file mode 100644 index 0000000000..f04e577077 --- /dev/null +++ b/services/search/pkg/opensearch/internal/convert/kql_query.go @@ -0,0 +1,35 @@ +package convert + +import ( + "fmt" + + "github.com/opencloud-eu/opencloud/pkg/kql" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" +) + +var ( + ErrUnsupportedNodeType = fmt.Errorf("unsupported node type") +) + +func KQLToOpenSearchBoolQuery(kqlQuery string) (*osu.BoolQuery, error) { + kqlAst, err := kql.Builder{}.Build(kqlQuery) + if err != nil { + return nil, fmt.Errorf("failed to build query: %w", err) + } + + kqlNodes, err := ExpandKQL(kqlAst.Nodes) + if err != nil { + return nil, fmt.Errorf("failed to expand KQL AST nodes: %w", err) + } + + builder, err := TranspileKQLToOpenSearch(kqlNodes) + if err != nil { + return nil, fmt.Errorf("failed to compile query: %w", err) + } + + if q, ok := builder.(*osu.BoolQuery); !ok { + return osu.NewBoolQuery().Must(builder), nil + } else { + return q, nil + } +} diff --git a/services/search/pkg/opensearch/internal/convert/kql_query_test.go b/services/search/pkg/opensearch/internal/convert/kql_query_test.go new file mode 100644 index 0000000000..3c03b5e0cd --- /dev/null +++ b/services/search/pkg/opensearch/internal/convert/kql_query_test.go @@ -0,0 +1 @@ +package convert_test diff --git a/services/search/pkg/opensearch/kql_to_os_dsl.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go similarity index 56% rename from services/search/pkg/opensearch/kql_to_os_dsl.go rename to services/search/pkg/opensearch/internal/convert/kql_transpile.go index b299f21cd4..4b86d80ca6 100644 --- a/services/search/pkg/opensearch/kql_to_os_dsl.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -1,4 +1,4 @@ -package opensearch +package convert import ( "errors" @@ -8,20 +8,17 @@ import ( "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/pkg/kql" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" ) -var ( - ErrUnsupportedNodeType = fmt.Errorf("unsupported node type") -) - -type KQLToOsDSL struct{} - -func NewKQLToOsDSL() (*KQLToOsDSL, error) { - return &KQLToOsDSL{}, nil +func TranspileKQLToOpenSearch(nodes []ast.Node) (osu.Builder, error) { + return kqlOpensearchTranspiler{}.Transpile(nodes) } -func (k *KQLToOsDSL) Compile(tree *ast.Ast) (Builder, error) { - q, err := k.transpile(tree.Nodes) +type kqlOpensearchTranspiler struct{} + +func (t kqlOpensearchTranspiler) Transpile(nodes []ast.Node) (osu.Builder, error) { + q, err := t.transpile(nodes) if err != nil { return nil, err } @@ -29,41 +26,36 @@ func (k *KQLToOsDSL) Compile(tree *ast.Ast) (Builder, error) { return q, nil } -func (k *KQLToOsDSL) transpile(nodes []ast.Node) (Builder, error) { +func (t kqlOpensearchTranspiler) transpile(nodes []ast.Node) (osu.Builder, error) { if len(nodes) == 0 { return nil, fmt.Errorf("no nodes to compile") } - expandedNodes, err := expandKQLASTNodes(nodes) - if err != nil { - return nil, fmt.Errorf("failed to expand KQL AST nodes: %w", err) - } - - if len(expandedNodes) == 1 { - builder, err := k.toBuilder(expandedNodes[0]) + if len(nodes) == 1 { + builder, err := t.toBuilder(nodes[0]) if err != nil { return nil, fmt.Errorf("failed to get builder for single node: %w", err) } return builder, nil } - boolQuery := NewBoolQuery() - add := boolQuery.Must - - for i, node := range expandedNodes { - nextOp := k.getOperatorValueAt(expandedNodes, i+1) - prevOp := k.getOperatorValueAt(expandedNodes, i-1) + boolQueryOptions := &osu.BoolQueryOptions{} + boolQuery := osu.NewBoolQuery().Options(boolQueryOptions) + boolQueryAdd := boolQuery.Must + for i, node := range nodes { + nextOp := t.getOperatorValueAt(nodes, i+1) + prevOp := t.getOperatorValueAt(nodes, i-1) switch { case nextOp == kql.BoolOR: - add = boolQuery.Should + boolQueryAdd = boolQuery.Should case nextOp == kql.BoolAND: - add = boolQuery.Must + boolQueryAdd = boolQuery.Must case prevOp == kql.BoolNOT: - add = boolQuery.MustNot + boolQueryAdd = boolQuery.MustNot } - builder, err := k.toBuilder(node) + builder, err := t.toBuilder(node) switch { // if the node is not known, we skip it, such as an operator node case errors.Is(err, ErrUnsupportedNodeType): @@ -77,17 +69,18 @@ func (k *KQLToOsDSL) transpile(nodes []ast.Node) (Builder, error) { continue } - add(builder) - } + if nextOp == kql.BoolOR { + // if there are should clauses, we set the minimum should match to 1 + boolQueryOptions.MinimumShouldMatch = 1 + } - if len(boolQuery.should) != 0 { - boolQuery.options.MinimumShouldMatch = 1 + boolQueryAdd(builder) } return boolQuery, nil } -func (k *KQLToOsDSL) getOperatorValueAt(nodes []ast.Node, i int) string { +func (t kqlOpensearchTranspiler) getOperatorValueAt(nodes []ast.Node, i int) string { if i < 0 || i >= len(nodes) { return "" } @@ -99,16 +92,16 @@ func (k *KQLToOsDSL) getOperatorValueAt(nodes []ast.Node, i int) string { return "" } -func (k *KQLToOsDSL) toBuilder(node ast.Node) (Builder, error) { - var builder Builder +func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { + var builder osu.Builder switch node := node.(type) { case *ast.BooleanNode: - return NewTermQuery[bool](node.Key).Value(node.Value), nil + return osu.NewTermQuery[bool](node.Key).Value(node.Value), nil case *ast.StringNode: isWildcard := strings.Contains(node.Value, "*") if isWildcard { - return NewWildcardQuery(node.Key).Value(node.Value), nil + return osu.NewWildcardQuery(node.Key).Value(node.Value), nil } totalTerms := strings.Split(node.Value, " ") @@ -116,9 +109,9 @@ func (k *KQLToOsDSL) toBuilder(node ast.Node) (Builder, error) { isMultiTerm := len(totalTerms) >= 1 switch { case isSingleTerm: - return NewTermQuery[string](node.Key).Value(node.Value), nil + return osu.NewTermQuery[string](node.Key).Value(node.Value), nil case isMultiTerm: - return NewMatchPhraseQuery(node.Key).Query(node.Value), nil + return osu.NewMatchPhraseQuery(node.Key).Query(node.Value), nil } return nil, fmt.Errorf("unsupported string node value: %s", node.Value) @@ -127,7 +120,7 @@ func (k *KQLToOsDSL) toBuilder(node ast.Node) (Builder, error) { return builder, fmt.Errorf("date time node without operator: %w", ErrUnsupportedNodeType) } - query := NewRangeQuery[time.Time](node.Key) + query := osu.NewRangeQuery[time.Time](node.Key) switch node.Operator.Value { case ">": @@ -142,7 +135,7 @@ func (k *KQLToOsDSL) toBuilder(node ast.Node) (Builder, error) { return nil, fmt.Errorf("unsupported operator %s for date time node: %w", node.Operator.Value, ErrUnsupportedNodeType) case *ast.GroupNode: - group, err := k.transpile(node.Nodes) + group, err := t.transpile(node.Nodes) if err != nil { return nil, fmt.Errorf("failed to build group: %w", err) } diff --git a/services/search/pkg/opensearch/kql_to_os_dsl_test.go b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go similarity index 57% rename from services/search/pkg/opensearch/kql_to_os_dsl_test.go rename to services/search/pkg/opensearch/internal/convert/kql_transpile_test.go index 500d18300a..86c02d1cdc 100644 --- a/services/search/pkg/opensearch/kql_to_os_dsl_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go @@ -1,4 +1,4 @@ -package opensearch_test +package convert_test import ( "testing" @@ -7,22 +7,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/opencloud-eu/opencloud/pkg/ast" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) -func TestKQL_Compile(t *testing.T) { - tests := []opensearchtest.TableTest[*ast.Ast, opensearch.Builder]{ - // field name tests - { - Name: "Name is the default field", - Got: &ast.Ast{ - Nodes: []ast.Node{ - &ast.StringNode{Value: "openCloud"}, - }, - }, - Want: opensearch.NewTermQuery[string]("Name").Value("opencloud"), - }, +func TestTranspileKQLToOpenSearch(t *testing.T) { + tests := []opensearchtest.TableTest[*ast.Ast, osu.Builder]{ // kql to os dsl - type tests { Name: "term query - string node", @@ -31,7 +22,7 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "Name", Value: "openCloud"}, }, }, - Want: opensearch.NewTermQuery[string]("Name").Value("opencloud"), + Want: osu.NewTermQuery[string]("Name").Value("openCloud"), }, { Name: "term query - boolean node - true", @@ -40,7 +31,7 @@ func TestKQL_Compile(t *testing.T) { &ast.BooleanNode{Key: "Deleted", Value: true}, }, }, - Want: opensearch.NewTermQuery[bool]("Deleted").Value(true), + Want: osu.NewTermQuery[bool]("Deleted").Value(true), }, { Name: "term query - boolean node - false", @@ -49,7 +40,7 @@ func TestKQL_Compile(t *testing.T) { &ast.BooleanNode{Key: "Deleted", Value: false}, }, }, - Want: opensearch.NewTermQuery[bool]("Deleted").Value(false), + Want: osu.NewTermQuery[bool]("Deleted").Value(false), }, { Name: "match-phrase query - string node", @@ -58,7 +49,7 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "Name", Value: "open cloud"}, }, }, - Want: opensearch.NewMatchPhraseQuery("Name").Query(`open cloud`), + Want: osu.NewMatchPhraseQuery("Name").Query(`open cloud`), }, { Name: "wildcard query - string node", @@ -67,21 +58,21 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "Name", Value: "open*"}, }, }, - Want: opensearch.NewWildcardQuery("Name").Value("open*"), + Want: osu.NewWildcardQuery("Name").Value("open*"), }, { Name: "bool query", Got: &ast.Ast{ Nodes: []ast.Node{ &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Value: "a"}, - &ast.StringNode{Value: "b"}, + &ast.StringNode{Key: "Name", Value: "a"}, + &ast.StringNode{Key: "Name", Value: "b"}, }}, }, }, - Want: opensearch.NewBoolQuery().Must( - opensearch.NewTermQuery[string]("Name").Value("a"), - opensearch.NewTermQuery[string]("Name").Value("b"), + Want: osu.NewBoolQuery().Must( + osu.NewTermQuery[string]("Name").Value("a"), + osu.NewTermQuery[string]("Name").Value("b"), ), }, { @@ -89,11 +80,11 @@ func TestKQL_Compile(t *testing.T) { Got: &ast.Ast{ Nodes: []ast.Node{ &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Value: "any"}, + &ast.StringNode{Key: "Name", Value: "any"}, }}, }, }, - Want: opensearch.NewTermQuery[string]("Name").Value("any"), + Want: osu.NewTermQuery[string]("Name").Value("any"), }, { Name: "range query >", @@ -106,7 +97,7 @@ func TestKQL_Compile(t *testing.T) { }, }, }, - Want: opensearch.NewRangeQuery[time.Time]("Mtime").Gt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")), + Want: osu.NewRangeQuery[time.Time]("Mtime").Gt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")), }, { Name: "range query >=", @@ -119,7 +110,7 @@ func TestKQL_Compile(t *testing.T) { }, }, }, - Want: opensearch.NewRangeQuery[time.Time]("Mtime").Gte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")), + Want: osu.NewRangeQuery[time.Time]("Mtime").Gte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")), }, { Name: "range query <", @@ -132,7 +123,7 @@ func TestKQL_Compile(t *testing.T) { }, }, }, - Want: opensearch.NewRangeQuery[time.Time]("Mtime").Lt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")), + Want: osu.NewRangeQuery[time.Time]("Mtime").Lt(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")), }, { Name: "range query <=", @@ -145,60 +136,61 @@ func TestKQL_Compile(t *testing.T) { }, }, }, - Want: opensearch.NewRangeQuery[time.Time]("Mtime").Lte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")), + Want: osu.NewRangeQuery[time.Time]("Mtime").Lte(opensearchtest.TimeMustParse(t, "2023-09-05T08:42:11.23554+02:00")), }, // kql to os dsl - structure tests { Name: "[*]", Got: &ast.Ast{ Nodes: []ast.Node{ - &ast.StringNode{Key: "name", Value: "openCloud"}, + &ast.StringNode{Key: "Name", Value: "openCloud"}, }, }, - Want: opensearch.NewTermQuery[string]("Name").Value("opencloud"), + Want: osu.NewTermQuery[string]("Name").Value("openCloud"), }, { Name: "[* *]", Got: &ast.Ast{ Nodes: []ast.Node{ - &ast.StringNode{Key: "name", Value: "openCloud"}, + &ast.StringNode{Key: "Name", Value: "openCloud"}, &ast.StringNode{Key: "age", Value: "32"}, }, }, - Want: opensearch.NewBoolQuery(). + Want: osu.NewBoolQuery(). Must( - opensearch.NewTermQuery[string]("Name").Value("opencloud"), - opensearch.NewTermQuery[string]("age").Value("32"), + osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewTermQuery[string]("age").Value("32"), ), }, { Name: "[* AND *]", Got: &ast.Ast{ Nodes: []ast.Node{ - &ast.StringNode{Key: "name", Value: "openCloud"}, + &ast.StringNode{Key: "Name", Value: "openCloud"}, &ast.OperatorNode{Value: "AND"}, &ast.StringNode{Key: "age", Value: "32"}, }, }, - Want: opensearch.NewBoolQuery(). + Want: osu.NewBoolQuery(). Must( - opensearch.NewTermQuery[string]("Name").Value("opencloud"), - opensearch.NewTermQuery[string]("age").Value("32"), + osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewTermQuery[string]("age").Value("32"), ), }, { Name: "[* OR *]", Got: &ast.Ast{ Nodes: []ast.Node{ - &ast.StringNode{Key: "name", Value: "openCloud"}, + &ast.StringNode{Key: "Name", Value: "openCloud"}, &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: "age", Value: "32"}, }, }, - Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). + Want: osu.NewBoolQuery(). + Options(&osu.BoolQueryOptions{MinimumShouldMatch: 1}). Should( - opensearch.NewTermQuery[string]("Name").Value("opencloud"), - opensearch.NewTermQuery[string]("age").Value("32"), + osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewTermQuery[string]("age").Value("32"), ), }, { @@ -209,44 +201,45 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "age", Value: "32"}, }, }, - Want: opensearch.NewBoolQuery(). + Want: osu.NewBoolQuery(). MustNot( - opensearch.NewTermQuery[string]("age").Value("32"), + osu.NewTermQuery[string]("age").Value("32"), ), }, { Name: "[* NOT *]", Got: &ast.Ast{ Nodes: []ast.Node{ - &ast.StringNode{Key: "name", Value: "openCloud"}, + &ast.StringNode{Key: "Name", Value: "openCloud"}, &ast.OperatorNode{Value: "NOT"}, &ast.StringNode{Key: "age", Value: "32"}, }, }, - Want: opensearch.NewBoolQuery(). + Want: osu.NewBoolQuery(). Must( - opensearch.NewTermQuery[string]("Name").Value("opencloud"), + osu.NewTermQuery[string]("Name").Value("openCloud"), ). MustNot( - opensearch.NewTermQuery[string]("age").Value("32"), + osu.NewTermQuery[string]("age").Value("32"), ), }, { Name: "[* OR * OR *]", Got: &ast.Ast{ Nodes: []ast.Node{ - &ast.StringNode{Key: "name", Value: "openCloud"}, + &ast.StringNode{Key: "Name", Value: "openCloud"}, &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: "age", Value: "32"}, &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: "age", Value: "44"}, }, }, - Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). + Want: osu.NewBoolQuery(). + Options(&osu.BoolQueryOptions{MinimumShouldMatch: 1}). Should( - opensearch.NewTermQuery[string]("Name").Value("opencloud"), - opensearch.NewTermQuery[string]("age").Value("32"), - opensearch.NewTermQuery[string]("age").Value("44"), + osu.NewTermQuery[string]("Name").Value("openCloud"), + osu.NewTermQuery[string]("age").Value("32"), + osu.NewTermQuery[string]("age").Value("44"), ), }, { @@ -260,13 +253,14 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "c", Value: "c"}, }, }, - Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). + Want: osu.NewBoolQuery(). + Options(&osu.BoolQueryOptions{MinimumShouldMatch: 1}). Must( - opensearch.NewTermQuery[string]("a").Value("a"), + osu.NewTermQuery[string]("a").Value("a"), ). Should( - opensearch.NewTermQuery[string]("b").Value("b"), - opensearch.NewTermQuery[string]("c").Value("c"), + osu.NewTermQuery[string]("b").Value("b"), + osu.NewTermQuery[string]("c").Value("c"), ), }, { @@ -280,13 +274,14 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "c", Value: "c"}, }, }, - Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). + Want: osu.NewBoolQuery(). + Options(&osu.BoolQueryOptions{MinimumShouldMatch: 1}). Must( - opensearch.NewTermQuery[string]("b").Value("b"), - opensearch.NewTermQuery[string]("c").Value("c"), + osu.NewTermQuery[string]("b").Value("b"), + osu.NewTermQuery[string]("c").Value("c"), ). Should( - opensearch.NewTermQuery[string]("a").Value("a"), + osu.NewTermQuery[string]("a").Value("a"), ), }, { @@ -300,13 +295,14 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "c", Value: "c"}, }, }, - Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). + Want: osu.NewBoolQuery(). + Options(&osu.BoolQueryOptions{MinimumShouldMatch: 1}). Should( - opensearch.NewTermQuery[string]("a").Value("a"), + osu.NewTermQuery[string]("a").Value("a"), ). Must( - opensearch.NewTermQuery[string]("b").Value("b"), - opensearch.NewTermQuery[string]("c").Value("c"), + osu.NewTermQuery[string]("b").Value("b"), + osu.NewTermQuery[string]("c").Value("c"), ), }, { @@ -324,15 +320,16 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "d", Value: "d"}, }, }, - Want: opensearch.NewBoolQuery(). + Want: osu.NewBoolQuery(). Must( - opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). + osu.NewBoolQuery(). + Options(&osu.BoolQueryOptions{MinimumShouldMatch: 1}). Should( - opensearch.NewTermQuery[string]("a").Value("a"), - opensearch.NewTermQuery[string]("b").Value("b"), - opensearch.NewTermQuery[string]("c").Value("c"), + osu.NewTermQuery[string]("a").Value("a"), + osu.NewTermQuery[string]("b").Value("b"), + osu.NewTermQuery[string]("c").Value("c"), ), - opensearch.NewTermQuery[string]("d").Value("d"), + osu.NewTermQuery[string]("d").Value("d"), ), }, { @@ -351,16 +348,17 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "d", Value: "d"}, }, }, - Want: opensearch.NewBoolQuery(). + Want: osu.NewBoolQuery(). Must( - opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). + osu.NewBoolQuery(). + Options(&osu.BoolQueryOptions{MinimumShouldMatch: 1}). Should( - opensearch.NewTermQuery[string]("a").Value("a"), - opensearch.NewTermQuery[string]("b").Value("b"), - opensearch.NewTermQuery[string]("c").Value("c"), + osu.NewTermQuery[string]("a").Value("a"), + osu.NewTermQuery[string]("b").Value("b"), + osu.NewTermQuery[string]("c").Value("c"), ), ).MustNot( - opensearch.NewTermQuery[string]("d").Value("d"), + osu.NewTermQuery[string]("d").Value("d"), ), }, } @@ -371,10 +369,7 @@ func TestKQL_Compile(t *testing.T) { t.Skip("skipping test: " + test.Name) } - transpiler, err := opensearch.NewKQLToOsDSL() - assert.NoError(t, err) - - dsl, err := transpiler.Compile(test.Got) + dsl, err := convert.TranspileKQLToOpenSearch(test.Got.Nodes) assert.NoError(t, err) assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, dsl)) diff --git a/services/search/pkg/opensearch/engine_convert.go b/services/search/pkg/opensearch/internal/convert/opensearch.go similarity index 81% rename from services/search/pkg/opensearch/engine_convert.go rename to services/search/pkg/opensearch/internal/convert/opensearch.go index 0c09b7e8e4..8febaaf7ec 100644 --- a/services/search/pkg/opensearch/engine_convert.go +++ b/services/search/pkg/opensearch/internal/convert/opensearch.go @@ -1,4 +1,4 @@ -package opensearch +package convert import ( "fmt" @@ -9,12 +9,13 @@ import ( opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "google.golang.org/protobuf/types/known/timestamppb" + "github.com/opencloud-eu/opencloud/pkg/conversions" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" "github.com/opencloud-eu/opencloud/services/search/pkg/engine" ) -func searchHitToSearchMessageMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match, error) { - resource, err := convert[engine.Resource](hit.Source) +func OpenSearchHitToMatch(hit opensearchgoAPI.SearchHit) (*searchMessage.Match, error) { + resource, err := conversions.To[engine.Resource](hit.Source) if err != nil { return nil, fmt.Errorf("failed to convert hit source: %w", err) } @@ -71,19 +72,19 @@ func searchHitToSearchMessageMatch(hit opensearchgoAPI.SearchHit) (*searchMessag return nil } - audio, _ := convert[*searchMessage.Audio](resource.Audio) + audio, _ := conversions.To[*searchMessage.Audio](resource.Audio) return audio }(), Image: func() *searchMessage.Image { - image, _ := convert[*searchMessage.Image](resource.Image) + image, _ := conversions.To[*searchMessage.Image](resource.Image) return image }(), Location: func() *searchMessage.GeoCoordinates { - geoCoordinates, _ := convert[*searchMessage.GeoCoordinates](resource.Location) + geoCoordinates, _ := conversions.To[*searchMessage.GeoCoordinates](resource.Location) return geoCoordinates }(), Photo: func() *searchMessage.Photo { - photo, _ := convert[*searchMessage.Photo](resource.Photo) + photo, _ := conversions.To[*searchMessage.Photo](resource.Photo) return photo }(), }, diff --git a/services/search/pkg/opensearch/engine_convert_test.go b/services/search/pkg/opensearch/internal/convert/opensearch_test.go similarity index 81% rename from services/search/pkg/opensearch/engine_convert_test.go rename to services/search/pkg/opensearch/internal/convert/opensearch_test.go index 9bca812115..afff90411e 100644 --- a/services/search/pkg/opensearch/engine_convert_test.go +++ b/services/search/pkg/opensearch/internal/convert/opensearch_test.go @@ -1,4 +1,4 @@ -package opensearch_test +package convert_test import ( "encoding/json" @@ -7,12 +7,13 @@ import ( opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "github.com/stretchr/testify/assert" + "github.com/opencloud-eu/opencloud/pkg/conversions" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) -func TestSearchHitToSearchMessageMatch(t *testing.T) { +func TestOpenSearchHitToMatch(t *testing.T) { resource := opensearchtest.Testdata.Resources.File resource.MimeType = "audio/anything" @@ -20,15 +21,15 @@ func TestSearchHitToSearchMessageMatch(t *testing.T) { Score: 1.1, Source: json.RawMessage(opensearchtest.JSONMustMarshal(t, resource)), } - match, err := opensearch.SearchHitToSearchMessageMatch(hit) + match, err := convert.OpenSearchHitToMatch(hit) assert.NoError(t, err) assert.Equal(t, hit.Score, match.Score) assert.Equal(t, resource.Name, match.Entity.Name) - + t.Parallel() t.Run("converts the audio field to the expected type", func(t *testing.T) { // searchMessage.Audio contains int64, int32 ... values that are converted to strings by the JSON marshaler, // so we need to convert the resource.Audio to align the expectations for the JSON comparison. - audio, err := opensearch.Convert[*searchMessage.Audio](resource.Audio) + audio, err := conversions.To[*searchMessage.Audio](resource.Audio) assert.NoError(t, err) assert.Equal(t, resource.Audio.Bitrate, match.Entity.Audio.Bitrate) diff --git a/services/search/pkg/opensearch/os_index.go b/services/search/pkg/opensearch/internal/osu/index.go similarity index 96% rename from services/search/pkg/opensearch/os_index.go rename to services/search/pkg/opensearch/internal/osu/index.go index ef69cf9041..2b1d3e5985 100644 --- a/services/search/pkg/opensearch/os_index.go +++ b/services/search/pkg/opensearch/internal/osu/index.go @@ -1,4 +1,4 @@ -package opensearch +package osu import ( "bytes" @@ -20,7 +20,7 @@ var ( IndexIndexManagerResourceV1 IndexManager = "resource_v1.json" ) -//go:embed internal/indexes/*.json +//go:embed indexes/*.json var indexes embed.FS type IndexManager string @@ -36,7 +36,7 @@ func (m IndexManager) String() string { func (m IndexManager) MarshalJSON() ([]byte, error) { filePath := string(m) - body, err := indexes.ReadFile(path.Join("./internal/indexes", filePath)) + body, err := indexes.ReadFile(path.Join("./indexes", filePath)) switch { case err != nil: return nil, fmt.Errorf("failed to read index file %s: %w", filePath, err) diff --git a/services/search/pkg/opensearch/os_index_test.go b/services/search/pkg/opensearch/internal/osu/index_test.go similarity index 85% rename from services/search/pkg/opensearch/os_index_test.go rename to services/search/pkg/opensearch/internal/osu/index_test.go index 789e3d92d3..112708a91c 100644 --- a/services/search/pkg/opensearch/os_index_test.go +++ b/services/search/pkg/opensearch/internal/osu/index_test.go @@ -1,4 +1,4 @@ -package opensearch_test +package osu_test import ( "strings" @@ -7,16 +7,16 @@ import ( "github.com/stretchr/testify/require" "github.com/tidwall/sjson" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestIndexManager(t *testing.T) { t.Run("index plausibility", func(t *testing.T) { - tests := []opensearchtest.TableTest[opensearch.IndexManager, struct{}]{ + tests := []opensearchtest.TableTest[osu.IndexManager, struct{}]{ { Name: "empty", - Got: opensearch.IndexManagerLatest, + Got: osu.IndexManagerLatest, }, } tc := opensearchtest.NewDefaultTestClient(t) @@ -37,7 +37,7 @@ func TestIndexManager(t *testing.T) { }) t.Run("does not create index if it already exists and is up to date", func(t *testing.T) { - indexManager := opensearch.IndexManagerLatest + indexManager := osu.IndexManagerLatest indexName := "opencloud-test-resource" tc := opensearchtest.NewDefaultTestClient(t) @@ -48,7 +48,7 @@ func TestIndexManager(t *testing.T) { }) t.Run("fails to create index if it already exists but is not up to date", func(t *testing.T) { - indexManager := opensearch.IndexManagerLatest + indexManager := osu.IndexManagerLatest indexName := "opencloud-test-resource" tc := opensearchtest.NewDefaultTestClient(t) @@ -58,6 +58,6 @@ func TestIndexManager(t *testing.T) { require.NoError(t, err) tc.Require.IndicesCreate(indexName, strings.NewReader(body)) - require.ErrorIs(t, indexManager.Apply(t.Context(), indexName, tc.Client()), opensearch.ErrManualActionRequired) + require.ErrorIs(t, indexManager.Apply(t.Context(), indexName, tc.Client()), osu.ErrManualActionRequired) }) } diff --git a/services/search/pkg/opensearch/internal/indexes/resource_v1.json b/services/search/pkg/opensearch/internal/osu/indexes/resource_v1.json similarity index 100% rename from services/search/pkg/opensearch/internal/indexes/resource_v1.json rename to services/search/pkg/opensearch/internal/osu/indexes/resource_v1.json diff --git a/services/search/pkg/opensearch/os_dsl.go b/services/search/pkg/opensearch/internal/osu/osu.go similarity index 53% rename from services/search/pkg/opensearch/os_dsl.go rename to services/search/pkg/opensearch/internal/osu/osu.go index dad92b3292..820065f409 100644 --- a/services/search/pkg/opensearch/os_dsl.go +++ b/services/search/pkg/opensearch/internal/osu/osu.go @@ -1,46 +1,39 @@ -package opensearch +package osu import ( "encoding/json" "fmt" + "reflect" + + "dario.cat/mergo" + + "github.com/opencloud-eu/opencloud/pkg/conversions" ) -type Rewrite string - -const ( - ConstantScore Rewrite = "constant_score" - ScoringBoolean Rewrite = "scoring_boolean" - ConstantScoreBoolean Rewrite = "constant_score_boolean" - TopTermsN Rewrite = "top_terms_N" - TopTermsBoostN Rewrite = "top_terms_boost_N" - TopTermsBlendedFreqsN Rewrite = "top_terms_blended_freqs_N" -) - -type Analyzer string - type Builder interface { json.Marshaler fmt.Stringer Map() (map[string]any, error) } -type BuilderFunc func() (map[string]any, error) +func newBase(v ...any) (map[string]any, error) { + base := make(map[string]any) + for _, value := range v { + data, err := conversions.To[map[string]any](value) + if err != nil { + return nil, fmt.Errorf("failed to convert value to map: %w", err) + } -func (f BuilderFunc) Map() (map[string]any, error) { - return f() -} + if isEmpty(data) { + continue + } -func (f BuilderFunc) MarshalJSON() ([]byte, error) { - data, err := f.Map() - if err != nil { - return nil, err + if err := mergo.Merge(&base, data); err != nil { + return nil, fmt.Errorf("failed to merge value into base: %w", err) + } } - return json.Marshal(data) -} -func (f BuilderFunc) String() string { - b, _ := f.MarshalJSON() - return string(b) + return base, nil } func applyValue[T any](target map[string]any, key string, v T) { @@ -104,14 +97,34 @@ func applyBuilders(target map[string]any, key string, bs ...Builder) error { return nil } -func builderToBoolQuery(b Builder) *BoolQuery { - var bq *BoolQuery +func isEmpty(x any) bool { + switch { + case x == nil: + return true + case reflect.ValueOf(x).Kind() == reflect.Bool: + return false + case reflect.DeepEqual(x, reflect.Zero(reflect.TypeOf(x)).Interface()): + return true + case reflect.ValueOf(x).Kind() == reflect.Map && reflect.ValueOf(x).Len() == 0: + return true + default: + return false + } +} - if q, ok := b.(*BoolQuery); !ok { - bq = NewBoolQuery().Must(b) - } else { - bq = q +func merge[T any](options ...T) T { + mapOptions := make(map[string]any) + + for _, option := range options { + data, err := conversions.To[map[string]any](option) + if err != nil { + continue + } + + _ = mergo.Merge(&mapOptions, data) } - return bq + data, _ := conversions.To[T](mapOptions) + + return data } diff --git a/services/search/pkg/opensearch/os_dsl_query_bool.go b/services/search/pkg/opensearch/internal/osu/query_bool.go similarity index 70% rename from services/search/pkg/opensearch/os_dsl_query_bool.go rename to services/search/pkg/opensearch/internal/osu/query_bool.go index 4471c422b5..30a06d176f 100644 --- a/services/search/pkg/opensearch/os_dsl_query_bool.go +++ b/services/search/pkg/opensearch/internal/osu/query_bool.go @@ -1,4 +1,4 @@ -package opensearch +package osu import ( "encoding/json" @@ -9,7 +9,7 @@ type BoolQuery struct { mustNot []Builder should []Builder filter []Builder - options BoolQueryOptions + options *BoolQueryOptions } type BoolQueryOptions struct { @@ -18,8 +18,13 @@ type BoolQueryOptions struct { Name string `json:"_name,omitempty"` } -func NewBoolQuery(o ...BoolQueryOptions) *BoolQuery { - return &BoolQuery{options: merge(o...)} +func NewBoolQuery() *BoolQuery { + return &BoolQuery{} +} + +func (q *BoolQuery) Options(v *BoolQueryOptions) *BoolQuery { + q.options = v + return q } func (q *BoolQuery) Must(v ...Builder) *BoolQuery { @@ -43,33 +48,33 @@ func (q *BoolQuery) Filter(v ...Builder) *BoolQuery { } func (q *BoolQuery) Map() (map[string]any, error) { - data, err := convert[map[string]any](q.options) + base, err := newBase(q.options) if err != nil { return nil, err } - if err := applyBuilders(data, "must", q.must...); err != nil { + if err := applyBuilders(base, "must", q.must...); err != nil { return nil, err } - if err := applyBuilders(data, "must_not", q.mustNot...); err != nil { + if err := applyBuilders(base, "must_not", q.mustNot...); err != nil { return nil, err } - if err := applyBuilders(data, "should", q.should...); err != nil { + if err := applyBuilders(base, "should", q.should...); err != nil { return nil, err } - if err := applyBuilders(data, "filter", q.filter...); err != nil { + if err := applyBuilders(base, "filter", q.filter...); err != nil { return nil, err } - if isEmpty(data) { + if isEmpty(base) { return nil, nil } return map[string]any{ - "bool": data, + "bool": base, }, nil } diff --git a/services/search/pkg/opensearch/os_dsl_query_bool_test.go b/services/search/pkg/opensearch/internal/osu/query_bool_test.go similarity index 74% rename from services/search/pkg/opensearch/os_dsl_query_bool_test.go rename to services/search/pkg/opensearch/internal/osu/query_bool_test.go index 1125d33da1..34cba77dc2 100644 --- a/services/search/pkg/opensearch/os_dsl_query_bool_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_bool_test.go @@ -1,24 +1,24 @@ -package opensearch_test +package osu_test import ( "testing" "github.com/stretchr/testify/assert" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestBoolQuery(t *testing.T) { - tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{ + tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{ { Name: "empty", - Got: opensearch.NewBoolQuery(), + Got: osu.NewBoolQuery(), Want: nil, }, { Name: "with-options", - Got: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{ + Got: osu.NewBoolQuery().Options(&osu.BoolQueryOptions{ MinimumShouldMatch: 10, Boost: 10, Name: "some-name", @@ -33,7 +33,7 @@ func TestBoolQuery(t *testing.T) { }, { Name: "must", - Got: opensearch.NewBoolQuery().Must(opensearch.NewTermQuery[string]("name").Value("tom")), + Got: osu.NewBoolQuery().Must(osu.NewTermQuery[string]("name").Value("tom")), Want: map[string]any{ "bool": map[string]any{ "must": []map[string]any{ @@ -50,7 +50,7 @@ func TestBoolQuery(t *testing.T) { }, { Name: "must_not", - Got: opensearch.NewBoolQuery().MustNot(opensearch.NewTermQuery[string]("name").Value("tom")), + Got: osu.NewBoolQuery().MustNot(osu.NewTermQuery[string]("name").Value("tom")), Want: map[string]any{ "bool": map[string]any{ "must_not": []map[string]any{ @@ -67,7 +67,7 @@ func TestBoolQuery(t *testing.T) { }, { Name: "should", - Got: opensearch.NewBoolQuery().Should(opensearch.NewTermQuery[string]("name").Value("tom")), + Got: osu.NewBoolQuery().Should(osu.NewTermQuery[string]("name").Value("tom")), Want: map[string]any{ "bool": map[string]any{ "should": []map[string]any{ @@ -84,7 +84,7 @@ func TestBoolQuery(t *testing.T) { }, { Name: "filter", - Got: opensearch.NewBoolQuery().Filter(opensearch.NewTermQuery[string]("name").Value("tom")), + Got: osu.NewBoolQuery().Filter(osu.NewTermQuery[string]("name").Value("tom")), Want: map[string]any{ "bool": map[string]any{ "filter": []map[string]any{ @@ -101,11 +101,11 @@ func TestBoolQuery(t *testing.T) { }, { Name: "full", - Got: opensearch.NewBoolQuery(). - Must(opensearch.NewTermQuery[string]("name").Value("tom")). - MustNot(opensearch.NewTermQuery[bool]("deleted").Value(true)). - Should(opensearch.NewTermQuery[string]("gender").Value("male")). - Filter(opensearch.NewTermQuery[int]("age").Value(42)), + Got: osu.NewBoolQuery(). + Must(osu.NewTermQuery[string]("name").Value("tom")). + MustNot(osu.NewTermQuery[bool]("deleted").Value(true)). + Should(osu.NewTermQuery[string]("gender").Value("male")). + Filter(osu.NewTermQuery[int]("age").Value(42)), Want: map[string]any{ "bool": map[string]any{ "must": []map[string]any{ diff --git a/services/search/pkg/opensearch/os_dsl_query_full_text_match_phrase.go b/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase.go similarity index 56% rename from services/search/pkg/opensearch/os_dsl_query_full_text_match_phrase.go rename to services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase.go index a8ca72902d..2def251a8c 100644 --- a/services/search/pkg/opensearch/os_dsl_query_full_text_match_phrase.go +++ b/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase.go @@ -1,4 +1,4 @@ -package opensearch +package osu import ( "encoding/json" @@ -7,17 +7,22 @@ import ( type MatchPhraseQuery struct { field string query string - options MatchPhraseQueryOptions + options *MatchPhraseQueryOptions } type MatchPhraseQueryOptions struct { - Analyzer Analyzer `json:"analyzer,omitempty"` - Slop int `json:"slop,omitempty"` - ZeroTermsQuery string `json:"zero_terms_query,omitempty"` + Analyzer string `json:"analyzer,omitempty"` + Slop int `json:"slop,omitempty"` + ZeroTermsQuery string `json:"zero_terms_query,omitempty"` } -func NewMatchPhraseQuery(field string, o ...MatchPhraseQueryOptions) *MatchPhraseQuery { - return &MatchPhraseQuery{field: field, options: merge(o...)} +func NewMatchPhraseQuery(field string) *MatchPhraseQuery { + return &MatchPhraseQuery{field: field} +} + +func (q *MatchPhraseQuery) Options(v *MatchPhraseQueryOptions) *MatchPhraseQuery { + q.options = v + return q } func (q *MatchPhraseQuery) Query(v string) *MatchPhraseQuery { @@ -26,20 +31,20 @@ func (q *MatchPhraseQuery) Query(v string) *MatchPhraseQuery { } func (q *MatchPhraseQuery) Map() (map[string]any, error) { - data, err := convert[map[string]any](q.options) + base, err := newBase(q.options) if err != nil { return nil, err } - applyValue(data, "query", q.query) + applyValue(base, "query", q.query) - if isEmpty(data) { + if isEmpty(base) { return nil, nil } return map[string]any{ "match_phrase": map[string]any{ - q.field: data, + q.field: base, }, }, nil } diff --git a/services/search/pkg/opensearch/os_dsl_query_full_text_match_phrase_test.go b/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase_test.go similarity index 77% rename from services/search/pkg/opensearch/os_dsl_query_full_text_match_phrase_test.go rename to services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase_test.go index 0c58bc8cdf..32d28630d4 100644 --- a/services/search/pkg/opensearch/os_dsl_query_full_text_match_phrase_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase_test.go @@ -1,24 +1,24 @@ -package opensearch_test +package osu_test import ( "testing" "github.com/stretchr/testify/assert" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestNewMatchPhraseQuery(t *testing.T) { - tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{ + tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{ { Name: "empty", - Got: opensearch.NewMatchPhraseQuery("empty"), + Got: osu.NewMatchPhraseQuery("empty"), Want: nil, }, { Name: "options", - Got: opensearch.NewMatchPhraseQuery("name", opensearch.MatchPhraseQueryOptions{ + Got: osu.NewMatchPhraseQuery("name").Options(&osu.MatchPhraseQueryOptions{ Analyzer: "analyzer", Slop: 2, ZeroTermsQuery: "all", @@ -35,7 +35,7 @@ func TestNewMatchPhraseQuery(t *testing.T) { }, { Name: "query", - Got: opensearch.NewMatchPhraseQuery("name").Query("some match query"), + Got: osu.NewMatchPhraseQuery("name").Query("some match query"), Want: map[string]any{ "match_phrase": map[string]any{ "name": map[string]any{ @@ -46,7 +46,7 @@ func TestNewMatchPhraseQuery(t *testing.T) { }, { Name: "full", - Got: opensearch.NewMatchPhraseQuery("name", opensearch.MatchPhraseQueryOptions{ + Got: osu.NewMatchPhraseQuery("name").Options(&osu.MatchPhraseQueryOptions{ Analyzer: "analyzer", Slop: 2, ZeroTermsQuery: "all", diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_ids.go b/services/search/pkg/opensearch/internal/osu/query_term_level_ids.go similarity index 62% rename from services/search/pkg/opensearch/os_dsl_query_term_level_ids.go rename to services/search/pkg/opensearch/internal/osu/query_term_level_ids.go index 0c0b7a0f96..409f3f94cf 100644 --- a/services/search/pkg/opensearch/os_dsl_query_term_level_ids.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_ids.go @@ -1,4 +1,4 @@ -package opensearch +package osu import ( "encoding/json" @@ -7,31 +7,36 @@ import ( type IDsQuery struct { values []string - options IDsQueryOptions + options *IDsQueryOptions } type IDsQueryOptions struct { Boost float32 `json:"boost,omitempty"` } -func NewIDsQuery(v []string, o ...IDsQueryOptions) *IDsQuery { - return &IDsQuery{values: slices.Compact(v), options: merge(o...)} +func NewIDsQuery(v ...string) *IDsQuery { + return &IDsQuery{values: slices.Compact(v)} +} + +func (q *IDsQuery) Options(v *IDsQueryOptions) *IDsQuery { + q.options = v + return q } func (q *IDsQuery) Map() (map[string]any, error) { - data, err := convert[map[string]any](q.options) + base, err := newBase(q.options) if err != nil { return nil, err } - applyValue(data, "values", q.values) + applyValue(base, "values", q.values) - if isEmpty(data) { + if isEmpty(base) { return nil, nil } return map[string]any{ - "ids": data, + "ids": base, }, nil } diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_ids_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_ids_test.go similarity index 72% rename from services/search/pkg/opensearch/os_dsl_query_term_level_ids_test.go rename to services/search/pkg/opensearch/internal/osu/query_term_level_ids_test.go index d8342bf93b..bc693ebb49 100644 --- a/services/search/pkg/opensearch/os_dsl_query_term_level_ids_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_ids_test.go @@ -1,24 +1,24 @@ -package opensearch_test +package osu_test import ( "testing" "github.com/stretchr/testify/assert" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestIDsQuery(t *testing.T) { - tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{ + tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{ { Name: "empty", - Got: opensearch.NewIDsQuery(nil), + Got: osu.NewIDsQuery(), Want: nil, }, { Name: "ids", - Got: opensearch.NewIDsQuery([]string{"1", "2", "3", "3"}, opensearch.IDsQueryOptions{Boost: 1.0}), + Got: osu.NewIDsQuery("1", "2", "3", "3").Options(&osu.IDsQueryOptions{Boost: 1.0}), Want: map[string]any{ "ids": map[string]any{ "values": []string{"1", "2", "3"}, diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_range.go b/services/search/pkg/opensearch/internal/osu/query_term_level_range.go similarity index 79% rename from services/search/pkg/opensearch/os_dsl_query_term_level_range.go rename to services/search/pkg/opensearch/internal/osu/query_term_level_range.go index a1853162fb..083e3e0cfc 100644 --- a/services/search/pkg/opensearch/os_dsl_query_term_level_range.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_range.go @@ -1,4 +1,4 @@ -package opensearch +package osu import ( "encoding/json" @@ -12,7 +12,7 @@ type RangeQuery[T time.Time | string] struct { gte T lt T lte T - options RangeQueryOptions + options *RangeQueryOptions } type RangeQueryOptions struct { @@ -22,8 +22,13 @@ type RangeQueryOptions struct { TimeZone string `json:"time_zone,omitempty"` } -func NewRangeQuery[T time.Time | string](field string, o ...RangeQueryOptions) *RangeQuery[T] { - return &RangeQuery[T]{field: field, options: merge(o...)} +func NewRangeQuery[T time.Time | string](field string) *RangeQuery[T] { + return &RangeQuery[T]{field: field} +} + +func (q *RangeQuery[T]) Options(v *RangeQueryOptions) *RangeQuery[T] { + q.options = v + return q } func (q *RangeQuery[T]) Gt(v T) *RangeQuery[T] { @@ -47,11 +52,6 @@ func (q *RangeQuery[T]) Lte(v T) *RangeQuery[T] { } func (q *RangeQuery[T]) Map() (map[string]any, error) { - data, err := convert[map[string]any](q.options) - if err != nil { - return nil, err - } - if !isEmpty(q.gt) && !isEmpty(q.gte) { return nil, errors.New("cannot set both gt and gte in RangeQuery") } @@ -60,20 +60,25 @@ func (q *RangeQuery[T]) Map() (map[string]any, error) { return nil, errors.New("cannot set both lt and lte in RangeQuery") } - applyValues(data, map[string]T{ + base, err := newBase(q.options) + if err != nil { + return nil, err + } + + applyValues(base, map[string]T{ "gt": q.gt, "gte": q.gte, "lt": q.lt, "lte": q.lte, }) - if isEmpty(data) { + if isEmpty(base) { return nil, nil } return map[string]any{ "range": map[string]any{ - q.field: data, + q.field: base, }, }, nil } diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_range_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_range_test.go similarity index 74% rename from services/search/pkg/opensearch/os_dsl_query_term_level_range_test.go rename to services/search/pkg/opensearch/internal/osu/query_term_level_range_test.go index 4498c39be5..dce6727fbd 100644 --- a/services/search/pkg/opensearch/os_dsl_query_term_level_range_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_range_test.go @@ -1,4 +1,4 @@ -package opensearch_test +package osu_test import ( "errors" @@ -8,21 +8,21 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestRangeQuery(t *testing.T) { now := time.Now() - tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{ + tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{ { Name: "empty", - Got: opensearch.NewRangeQuery[string]("empty"), + Got: osu.NewRangeQuery[string]("empty"), Want: nil, }, { Name: "gt string", - Got: opensearch.NewRangeQuery[string]("created").Gt("2023-01-01T00:00:00Z"), + Got: osu.NewRangeQuery[string]("created").Gt("2023-01-01T00:00:00Z"), Want: map[string]any{ "range": map[string]any{ "created": map[string]any{ @@ -33,7 +33,7 @@ func TestRangeQuery(t *testing.T) { }, { Name: "gt time", - Got: opensearch.NewRangeQuery[time.Time]("created").Gt(now), + Got: osu.NewRangeQuery[time.Time]("created").Gt(now), Want: map[string]any{ "range": map[string]any{ "created": map[string]any{ @@ -44,7 +44,7 @@ func TestRangeQuery(t *testing.T) { }, { Name: "gte string", - Got: opensearch.NewRangeQuery[string]("created").Gte("2023-01-01T00:00:00Z"), + Got: osu.NewRangeQuery[string]("created").Gte("2023-01-01T00:00:00Z"), Want: map[string]any{ "range": map[string]any{ "created": map[string]any{ @@ -55,7 +55,7 @@ func TestRangeQuery(t *testing.T) { }, { Name: "gte time", - Got: opensearch.NewRangeQuery[time.Time]("created").Gte(now), + Got: osu.NewRangeQuery[time.Time]("created").Gte(now), Want: map[string]any{ "range": map[string]any{ "created": map[string]any{ @@ -66,13 +66,13 @@ func TestRangeQuery(t *testing.T) { }, { Name: "gt & gte", - Got: opensearch.NewRangeQuery[time.Time]("created").Gt(now).Gte(now), + Got: osu.NewRangeQuery[time.Time]("created").Gt(now).Gte(now), Want: nil, Err: errors.New(""), }, { Name: "gt string", - Got: opensearch.NewRangeQuery[string]("created").Lt("2023-01-01T00:00:00Z"), + Got: osu.NewRangeQuery[string]("created").Lt("2023-01-01T00:00:00Z"), Want: map[string]any{ "range": map[string]any{ "created": map[string]any{ @@ -83,7 +83,7 @@ func TestRangeQuery(t *testing.T) { }, { Name: "lt time", - Got: opensearch.NewRangeQuery[time.Time]("created").Lt(now), + Got: osu.NewRangeQuery[time.Time]("created").Lt(now), Want: map[string]any{ "range": map[string]any{ "created": map[string]any{ @@ -94,7 +94,7 @@ func TestRangeQuery(t *testing.T) { }, { Name: "lte string", - Got: opensearch.NewRangeQuery[string]("created").Lte("2023-01-01T00:00:00Z"), + Got: osu.NewRangeQuery[string]("created").Lte("2023-01-01T00:00:00Z"), Want: map[string]any{ "range": map[string]any{ "created": map[string]any{ @@ -105,7 +105,7 @@ func TestRangeQuery(t *testing.T) { }, { Name: "lte time", - Got: opensearch.NewRangeQuery[time.Time]("created").Lte(now), + Got: osu.NewRangeQuery[time.Time]("created").Lte(now), Want: map[string]any{ "range": map[string]any{ "created": map[string]any{ @@ -116,13 +116,13 @@ func TestRangeQuery(t *testing.T) { }, { Name: "lt & lte", - Got: opensearch.NewRangeQuery[time.Time]("created").Lt(now).Lte(now), + Got: osu.NewRangeQuery[time.Time]("created").Lt(now).Lte(now), Want: nil, Err: errors.New(""), }, { Name: "options", - Got: opensearch.NewRangeQuery[time.Time]("created", opensearch.RangeQueryOptions{ + Got: osu.NewRangeQuery[time.Time]("created").Options(&osu.RangeQueryOptions{ Format: "strict_date_optional_time", Relation: "within", Boost: 1.0, diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_term.go b/services/search/pkg/opensearch/internal/osu/query_term_level_term.go similarity index 69% rename from services/search/pkg/opensearch/os_dsl_query_term_level_term.go rename to services/search/pkg/opensearch/internal/osu/query_term_level_term.go index a9620e8fcb..39eaf3be6c 100644 --- a/services/search/pkg/opensearch/os_dsl_query_term_level_term.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_term.go @@ -1,4 +1,4 @@ -package opensearch +package osu import ( "encoding/json" @@ -7,7 +7,7 @@ import ( type TermQuery[T comparable] struct { field string value T - options TermQueryOptions + options *TermQueryOptions } type TermQueryOptions struct { @@ -16,8 +16,13 @@ type TermQueryOptions struct { Name string `json:"_name,omitempty"` } -func NewTermQuery[T comparable](field string, o ...TermQueryOptions) *TermQuery[T] { - return &TermQuery[T]{field: field, options: merge(o...)} +func NewTermQuery[T comparable](field string) *TermQuery[T] { + return &TermQuery[T]{field: field} +} + +func (q *TermQuery[T]) Options(v *TermQueryOptions) *TermQuery[T] { + q.options = v + return q } func (q *TermQuery[T]) Value(v T) *TermQuery[T] { @@ -26,20 +31,20 @@ func (q *TermQuery[T]) Value(v T) *TermQuery[T] { } func (q *TermQuery[T]) Map() (map[string]any, error) { - data, err := convert[map[string]any](q.options) + base, err := newBase(q.options) if err != nil { return nil, err } - applyValue(data, "value", q.value) + applyValue(base, "value", q.value) - if isEmpty(data) { + if isEmpty(base) { return nil, nil } return map[string]any{ "term": map[string]any{ - q.field: data, + q.field: base, }, }, nil } diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_term_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_term_test.go similarity index 75% rename from services/search/pkg/opensearch/os_dsl_query_term_level_term_test.go rename to services/search/pkg/opensearch/internal/osu/query_term_level_term_test.go index 53f2cfb390..0a5a7d35d6 100644 --- a/services/search/pkg/opensearch/os_dsl_query_term_level_term_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_term_test.go @@ -1,24 +1,24 @@ -package opensearch_test +package osu_test import ( "testing" "github.com/stretchr/testify/assert" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestTermQuery(t *testing.T) { - tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{ + tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{ { Name: "empty", - Got: opensearch.NewTermQuery[string]("empty"), + Got: osu.NewTermQuery[string]("empty"), Want: nil, }, { - Name: "op-options", - Got: opensearch.NewTermQuery[bool]("deleted").Value(false), + Name: "no-options", + Got: osu.NewTermQuery[bool]("deleted").Value(false), Want: map[string]any{ "term": map[string]any{ "deleted": map[string]any{ @@ -29,7 +29,7 @@ func TestTermQuery(t *testing.T) { }, { Name: "with-options", - Got: opensearch.NewTermQuery[bool]("deleted", opensearch.TermQueryOptions{ + Got: osu.NewTermQuery[bool]("deleted").Options(&osu.TermQueryOptions{ Boost: 1.0, CaseInsensitive: true, Name: "is-deleted", diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_wildcard.go b/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard.go similarity index 65% rename from services/search/pkg/opensearch/os_dsl_query_term_level_wildcard.go rename to services/search/pkg/opensearch/internal/osu/query_term_level_wildcard.go index b6010e1f85..f21255c64f 100644 --- a/services/search/pkg/opensearch/os_dsl_query_term_level_wildcard.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard.go @@ -1,4 +1,4 @@ -package opensearch +package osu import ( "encoding/json" @@ -7,17 +7,22 @@ import ( type WildcardQuery struct { field string value string - options WildcardQueryOptions + options *WildcardQueryOptions } type WildcardQueryOptions struct { Boost float32 `json:"boost,omitempty"` CaseInsensitive bool `json:"case_insensitive,omitempty"` - Rewrite Rewrite `json:"rewrite,omitempty"` + Rewrite string `json:"rewrite,omitempty"` } -func NewWildcardQuery(field string, o ...WildcardQueryOptions) *WildcardQuery { - return &WildcardQuery{field: field, options: merge(o...)} +func NewWildcardQuery(field string) *WildcardQuery { + return &WildcardQuery{field: field} +} + +func (q *WildcardQuery) Options(v *WildcardQueryOptions) *WildcardQuery { + q.options = v + return q } func (q *WildcardQuery) Value(v string) *WildcardQuery { @@ -26,20 +31,20 @@ func (q *WildcardQuery) Value(v string) *WildcardQuery { } func (q *WildcardQuery) Map() (map[string]any, error) { - data, err := convert[map[string]any](q.options) + base, err := newBase(q.options) if err != nil { return nil, err } - applyValue(data, "value", q.value) + applyValue(base, "value", q.value) - if isEmpty(data) { + if isEmpty(base) { return nil, nil } return map[string]any{ "wildcard": map[string]any{ - q.field: data, + q.field: base, }, }, nil } diff --git a/services/search/pkg/opensearch/os_dsl_query_term_level_wildcard_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard_test.go similarity index 75% rename from services/search/pkg/opensearch/os_dsl_query_term_level_wildcard_test.go rename to services/search/pkg/opensearch/internal/osu/query_term_level_wildcard_test.go index 5bcd51626a..b23f3e47c4 100644 --- a/services/search/pkg/opensearch/os_dsl_query_term_level_wildcard_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard_test.go @@ -1,27 +1,27 @@ -package opensearch_test +package osu_test import ( "testing" "github.com/stretchr/testify/assert" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestWildcardQuery(t *testing.T) { - tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{ + tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{ { Name: "empty", - Got: opensearch.NewWildcardQuery("empty"), + Got: osu.NewWildcardQuery("empty"), Want: nil, }, { Name: "wildcard", - Got: opensearch.NewWildcardQuery("name", opensearch.WildcardQueryOptions{ + Got: osu.NewWildcardQuery("name").Options(&osu.WildcardQueryOptions{ Boost: 1.0, CaseInsensitive: true, - Rewrite: opensearch.TopTermsBlendedFreqsN, + Rewrite: "top_terms_blended_freqs_N", }).Value("opencl*"), Want: map[string]any{ "wildcard": map[string]any{ diff --git a/services/search/pkg/opensearch/os_request.go b/services/search/pkg/opensearch/internal/osu/request.go similarity index 96% rename from services/search/pkg/opensearch/os_request.go rename to services/search/pkg/opensearch/internal/osu/request.go index ef798d0a55..fc33c65eaf 100644 --- a/services/search/pkg/opensearch/os_request.go +++ b/services/search/pkg/opensearch/internal/osu/request.go @@ -1,4 +1,4 @@ -package opensearch +package osu import ( "bytes" @@ -7,6 +7,8 @@ import ( "strings" opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" + + "github.com/opencloud-eu/opencloud/pkg/conversions" ) type RequestBody[O any] struct { @@ -19,7 +21,7 @@ func NewRequestBody[O any](q Builder, o ...O) *RequestBody[O] { } func (q RequestBody[O]) Map() (map[string]any, error) { - data, err := convert[map[string]any](q.options) + data, err := conversions.To[map[string]any](q.options) if err != nil { return nil, err } diff --git a/services/search/pkg/opensearch/os_request_test.go b/services/search/pkg/opensearch/internal/osu/request_test.go similarity index 79% rename from services/search/pkg/opensearch/os_request_test.go rename to services/search/pkg/opensearch/internal/osu/request_test.go index d547472abf..3b607c6591 100644 --- a/services/search/pkg/opensearch/os_request_test.go +++ b/services/search/pkg/opensearch/internal/osu/request_test.go @@ -1,4 +1,4 @@ -package opensearch_test +package osu_test import ( "io" @@ -8,15 +8,15 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestRequestBody(t *testing.T) { - tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{ + tests := []opensearchtest.TableTest[osu.Builder, map[string]any]{ { Name: "simple", - Got: opensearch.NewRequestBody[any](opensearch.NewTermQuery[string]("name").Value("tom")), + Got: osu.NewRequestBody[any](osu.NewTermQuery[string]("name").Value("tom")), Want: map[string]any{ "query": map[string]any{ "term": map[string]any{ @@ -41,14 +41,14 @@ func TestBuildSearchReq(t *testing.T) { { Name: "highlight", Got: func() io.Reader { - req, _ := opensearch.BuildSearchReq( + req, _ := osu.BuildSearchReq( &opensearchgoAPI.SearchReq{}, - opensearch.NewTermQuery[string]("content").Value("content"), - opensearch.SearchReqOptions{ - Highlight: &opensearch.HighlightOption{ + osu.NewTermQuery[string]("content").Value("content"), + osu.SearchReqOptions{ + Highlight: &osu.HighlightOption{ PreTags: []string{""}, PostTags: []string{""}, - Fields: map[string]opensearch.HighlightOption{ + Fields: map[string]osu.HighlightOption{ "content": {}, }, }, diff --git a/services/search/pkg/opensearch/internal/test/helper.go b/services/search/pkg/opensearch/internal/test/helper.go index a7e7e26d1d..1b1b0e2497 100644 --- a/services/search/pkg/opensearch/internal/test/helper.go +++ b/services/search/pkg/opensearch/internal/test/helper.go @@ -8,6 +8,8 @@ import ( opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "github.com/samber/lo" "github.com/stretchr/testify/require" + + "github.com/opencloud-eu/opencloud/pkg/conversions" ) var TimeMustParse = func(t *testing.T, ts string) time.Time { @@ -25,27 +27,8 @@ func JSONMustMarshal(t *testing.T, data any) string { func SearchHitsMustBeConverted[T any](t *testing.T, hits []opensearchgoAPI.SearchHit) []T { return lo.ReduceRight(hits, func(agg []T, item opensearchgoAPI.SearchHit, _ int) []T { - resource, err := convert[T](item.Source) + resource, err := conversions.To[T](item.Source) require.NoError(t, err) return append(agg, resource) }, []T{}) } - -func convert[T any](v any) (T, error) { - var t T - - if v == nil { - return t, nil - } - - j, err := json.Marshal(v) - if err != nil { - return t, err - } - - if err := json.Unmarshal(j, &t); err != nil { - return t, err - } - - return t, nil -} diff --git a/services/search/pkg/opensearch/internal/test/testdata.go b/services/search/pkg/opensearch/internal/test/testdata.go index faf3709d36..68e14af9f5 100644 --- a/services/search/pkg/opensearch/internal/test/testdata.go +++ b/services/search/pkg/opensearch/internal/test/testdata.go @@ -1,14 +1,17 @@ package opensearchtest import ( + "embed" "encoding/json" "fmt" - "os" "path" "github.com/opencloud-eu/opencloud/services/search/pkg/engine" ) +//go:embed testdata/*.json +var testdata embed.FS + var Testdata = struct { Resources resourceTestdata }{ @@ -22,8 +25,8 @@ type resourceTestdata struct { } func loadTestdata[D any](name string) D { - name = path.Join("internal/test/testdata", name) - data, err := os.ReadFile(name) + name = path.Join("./testdata", name) + data, err := testdata.ReadFile(name) if err != nil { panic(fmt.Sprintf("failed to read testdata file %s: %v", name, err)) } diff --git a/services/search/pkg/opensearch/kql_ast.go b/services/search/pkg/opensearch/kql_ast.go deleted file mode 100644 index 672379b4e9..0000000000 --- a/services/search/pkg/opensearch/kql_ast.go +++ /dev/null @@ -1,196 +0,0 @@ -package opensearch - -import ( - "fmt" - "reflect" - "slices" - "strings" - - "github.com/opencloud-eu/opencloud/pkg/ast" -) - -func expandKQLASTNodes(nodes []ast.Node) ([]ast.Node, error) { - remapKey := func(current string, defaultKey string) string { - if defaultKey == "" { - defaultKey = "Name" // Set a default key if none is provided - } - - key, ok := map[string]string{ - "": defaultKey, // Default case if current is empty - "rootid": "RootID", - "path": "Path", - "id": "ID", - "name": "Name", - "size": "Size", - "mtime": "Mtime", - "mediatype": "MimeType", - "type": "Type", - "tag": "Tags", - "tags": "Tags", - "content": "Content", - "hidden": "Hidden", - }[current] - if !ok { - return current // Return the original key if not found - } - - return key - } - - lowerValue := func(key, value string) string { - if slices.Contains([]string{"Hidden"}, key) { - return value // ignore certain keys and return the original value - } - - return strings.ToLower(value) - } - - unfoldValue := func(key, value string) []ast.Node { - result, ok := map[string][]ast.Node{ - "MimeType:file": { - &ast.OperatorNode{Value: "NOT"}, - &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, - }, - "MimeType:folder": { - &ast.StringNode{Key: key, Value: "httpd/unix-directory"}, - }, - "MimeType:document": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/msword"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.document"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.wordprocessingml.form"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.text"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "text/plain"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "text/markdown"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/rtf"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.apple.pages"}, - }}, - }, - "MimeType:spreadsheet": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/vnd.ms-excel"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadsheet"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "text/csv"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.spreadshee"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.apple.numbers"}, - }}, - }, - "MimeType:presentation": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/vnd.openxmlformats-officedocument.presentationml.presentation"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.oasis.opendocument.presentation"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.ms-powerpoint"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/vnd.apple.keynote"}, - }}, - }, - "MimeType:pdf": { - &ast.StringNode{Key: key, Value: "application/pdf"}, - }, - "MimeType:image": { - &ast.StringNode{Key: key, Value: "image/*"}, - }, - "MimeType:video": { - &ast.StringNode{Key: key, Value: "video/*"}, - }, - "MimeType:audio": { - &ast.StringNode{Key: key, Value: "audio/*"}, - }, - "MimeType:archive": { - &ast.GroupNode{Nodes: []ast.Node{ - &ast.StringNode{Key: key, Value: "application/zip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/gzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-gzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-7z-compressed"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-rar-compressed"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-tar"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-bzip2"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-bzip"}, - &ast.OperatorNode{Value: "OR"}, - &ast.StringNode{Key: key, Value: "application/x-tgz"}, - }}, - }, - }[fmt.Sprintf("%s:%s", key, value)] - if !ok { - return nil - } - - return result - } - - var expand func([]ast.Node, string) ([]ast.Node, error) - expand = func(nodes []ast.Node, defaultKey string) ([]ast.Node, error) { - for i, node := range nodes { - rnode := reflect.ValueOf(node) - - // we need to ensure that the node is a pointer to an ast.Node in every case - if rnode.Kind() != reflect.Ptr { - ptr := reflect.New(rnode.Type()) - ptr.Elem().Set(rnode) - rnode = ptr - cnode, ok := rnode.Interface().(ast.Node) - if !ok { - return nil, fmt.Errorf("expected node to be of type ast.Node, got %T", rnode.Interface()) - } - - node = cnode // Update the original node to the pointer - nodes[i] = node // Update the original slice with the pointer - } - - var unfoldedNodes []ast.Node - switch cnode := node.(type) { - case *ast.GroupNode: - if cnode.Key != "" { // group nodes should not get a default key - cnode.Key = remapKey(cnode.Key, defaultKey) - } - - groupNodes, err := expand(cnode.Nodes, cnode.Key) - if err != nil { - return nil, err - } - cnode.Nodes = groupNodes - case *ast.StringNode: - cnode.Key = remapKey(cnode.Key, defaultKey) - cnode.Value = lowerValue(cnode.Key, cnode.Value) - unfoldedNodes = unfoldValue(cnode.Key, cnode.Value) - case *ast.DateTimeNode: - cnode.Key = remapKey(cnode.Key, defaultKey) - case *ast.BooleanNode: - cnode.Key = remapKey(cnode.Key, defaultKey) - } - - if unfoldedNodes != nil { - // Insert unfolded nodes at the current index - nodes = append(nodes[:i], append(unfoldedNodes, nodes[i+1:]...)...) - // Adjust index to account for new nodes - i += len(unfoldedNodes) - 1 - } - } - - return nodes, nil - } - - return expand(nodes, "") -} diff --git a/services/search/pkg/opensearch/opensearch.go b/services/search/pkg/opensearch/opensearch.go deleted file mode 100644 index f4089736a3..0000000000 --- a/services/search/pkg/opensearch/opensearch.go +++ /dev/null @@ -1,64 +0,0 @@ -package opensearch - -import ( - "encoding/json" - "fmt" - "reflect" - - "dario.cat/mergo" -) - -var ( - ErrUnhealthyCluster = fmt.Errorf("cluster is not healthy") -) - -func isEmpty(x any) bool { - switch { - case x == nil: - return true - case reflect.ValueOf(x).Kind() == reflect.Bool: - return false - case reflect.DeepEqual(x, reflect.Zero(reflect.TypeOf(x)).Interface()): - return true - case reflect.ValueOf(x).Kind() == reflect.Map && reflect.ValueOf(x).Len() == 0: - return true - default: - return false - } -} - -func merge[T any](options ...T) T { - mapOptions := make(map[string]any) - - for _, option := range options { - data, err := convert[map[string]any](option) - if err != nil { - continue - } - - _ = mergo.Merge(&mapOptions, data) - } - - data, _ := convert[T](mapOptions) - - return data -} - -func convert[T any](v any) (T, error) { - var t T - - if v == nil { - return t, nil - } - - j, err := json.Marshal(v) - if err != nil { - return t, err - } - - if err := json.Unmarshal(j, &t); err != nil { - return t, err - } - - return t, nil -} diff --git a/services/search/pkg/opensearch/opensearch_test.go b/services/search/pkg/opensearch/opensearch_test.go deleted file mode 100644 index 5cfac2a0d0..0000000000 --- a/services/search/pkg/opensearch/opensearch_test.go +++ /dev/null @@ -1 +0,0 @@ -package opensearch_test diff --git a/services/search/pkg/opensearch/os.go b/services/search/pkg/opensearch/os.go deleted file mode 100644 index 01ee40c65e..0000000000 --- a/services/search/pkg/opensearch/os.go +++ /dev/null @@ -1,30 +0,0 @@ -package opensearch - -import ( - "context" - "fmt" - "time" - - opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" -) - -func clusterHealth(ctx context.Context, client *opensearchgoAPI.Client, indices []string) (*opensearchgoAPI.ClusterHealthResp, bool, error) { - resp, err := client.Cluster.Health(ctx, &opensearchgoAPI.ClusterHealthReq{ - Indices: indices, - Params: opensearchgoAPI.ClusterHealthParams{ - Local: opensearchgoAPI.ToPointer(true), - Timeout: 5 * time.Second, - }, - }) - switch { - case err != nil: - return nil, false, fmt.Errorf("%w, failed to get cluster health: %w", ErrUnhealthyCluster, err) - case resp.TimedOut: - return resp, false, fmt.Errorf("%w, cluster health request timed out", ErrUnhealthyCluster) - - case resp.Status != "green" && resp.Status != "yellow": - return resp, false, fmt.Errorf("%w, cluster health is not green or yellow: %s", ErrUnhealthyCluster, resp.Status) - default: - return resp, true, nil - } -} diff --git a/services/search/pkg/opensearch/os_dsl_test.go b/services/search/pkg/opensearch/os_dsl_test.go deleted file mode 100644 index 9bb99173b4..0000000000 --- a/services/search/pkg/opensearch/os_dsl_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package opensearch_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" -) - -func TestBuilderToBoolQuery(t *testing.T) { - tests := []opensearchtest.TableTest[opensearch.Builder, *opensearch.BoolQuery]{ - { - Name: "term-query", - Got: opensearch.NewTermQuery[string]("Name").Value("openCloud"), - Want: opensearch.NewBoolQuery().Must(opensearch.NewTermQuery[string]("Name").Value("openCloud")), - }, - { - Name: "bool-query", - Got: opensearch.NewBoolQuery().Must(opensearch.NewTermQuery[string]("Name").Value("openCloud")), - Want: opensearch.NewBoolQuery().Must(opensearch.NewTermQuery[string]("Name").Value("openCloud")), - }, - } - - for _, test := range tests { - t.Run(test.Name, func(t *testing.T) { - assert.JSONEq(t, opensearchtest.JSONMustMarshal(t, test.Want), opensearchtest.JSONMustMarshal(t, opensearch.BuilderToBoolQuery(test.Got))) - }) - } -} diff --git a/services/search/pkg/service/grpc/v0/service.go b/services/search/pkg/service/grpc/v0/service.go index 52d5acd87e..3b53ba9fc4 100644 --- a/services/search/pkg/service/grpc/v0/service.go +++ b/services/search/pkg/service/grpc/v0/service.go @@ -80,12 +80,12 @@ func NewHandler(opts ...Option) (searchsvc.SearchProviderHandler, func(), error) return nil, teardown, fmt.Errorf("failed to create OpenSearch client: %w", err) } - backend, err := opensearch.NewEngine(cfg.Engine.OpenSearch.ResourceIndex.Name, client) + openSearchBackend, err := opensearch.NewBackend(cfg.Engine.OpenSearch.ResourceIndex.Name, client) if err != nil { - return nil, teardown, fmt.Errorf("failed to create OpenSearch engine: %w", err) + return nil, teardown, fmt.Errorf("failed to create OpenSearch backend: %w", err) } - eng = backend + eng = openSearchBackend default: return nil, teardown, fmt.Errorf("unknown search engine: %s", cfg.Engine.Type) }