From 2d325d70b89d3f76ffd7b17d2fe5917a3c84e2c8 Mon Sep 17 00:00:00 2001 From: fschade Date: Thu, 31 Jul 2025 13:14:16 +0200 Subject: [PATCH] enhancement(search): implement search engine match to pg-hit conversion --- .../pkg/opensearch/dsl_query_bool_test.go | 51 ++++---- .../dsl_query_full_text_match_phrase_test.go | 34 +++--- .../dsl_query_term_level_ids_test.go | 22 ++-- .../dsl_query_term_level_term_test.go | 28 ++--- .../dsl_query_term_level_wildcard_test.go | 22 ++-- .../search/pkg/opensearch/dsl_query_test.go | 16 ++- services/search/pkg/opensearch/engine.go | 25 ++-- services/search/pkg/opensearch/engine_test.go | 36 +++--- .../search/pkg/opensearch/internal/test/os.go | 9 +- .../pkg/opensearch/internal/test/test.go | 21 +++- .../pkg/opensearch/internal/test/testdata.go | 2 +- services/search/pkg/opensearch/kql_test.go | 111 +++++++++--------- .../search/pkg/opensearch/opensearch_test.go | 19 --- 13 files changed, 198 insertions(+), 198 deletions(-) diff --git a/services/search/pkg/opensearch/dsl_query_bool_test.go b/services/search/pkg/opensearch/dsl_query_bool_test.go index 650ea1f033..aaa069e867 100644 --- a/services/search/pkg/opensearch/dsl_query_bool_test.go +++ b/services/search/pkg/opensearch/dsl_query_bool_test.go @@ -6,23 +6,24 @@ import ( "github.com/stretchr/testify/assert" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestBoolQuery(t *testing.T) { - tests := []tableTest[opensearch.Builder, map[string]any]{ + tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{ { - name: "empty", - got: opensearch.NewBoolQuery(), - want: nil, + Name: "empty", + Got: opensearch.NewBoolQuery(), + Want: nil, }, { - name: "naked", - got: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{ + Name: "naked", + Got: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{ MinimumShouldMatch: 10, Boost: 10, Name: "some-name", }), - want: map[string]any{ + Want: map[string]any{ "bool": map[string]any{ "minimum_should_match": 10, "boost": 10, @@ -31,9 +32,9 @@ func TestBoolQuery(t *testing.T) { }, }, { - name: "must", - got: opensearch.NewBoolQuery().Must(opensearch.NewTermQuery[string]("name").Value("tom")), - want: map[string]any{ + Name: "must", + Got: opensearch.NewBoolQuery().Must(opensearch.NewTermQuery[string]("name").Value("tom")), + Want: map[string]any{ "bool": map[string]any{ "must": []map[string]any{ { @@ -48,9 +49,9 @@ func TestBoolQuery(t *testing.T) { }, }, { - name: "must_not", - got: opensearch.NewBoolQuery().MustNot(opensearch.NewTermQuery[string]("name").Value("tom")), - want: map[string]any{ + Name: "must_not", + Got: opensearch.NewBoolQuery().MustNot(opensearch.NewTermQuery[string]("name").Value("tom")), + Want: map[string]any{ "bool": map[string]any{ "must_not": []map[string]any{ { @@ -65,9 +66,9 @@ func TestBoolQuery(t *testing.T) { }, }, { - name: "should", - got: opensearch.NewBoolQuery().Should(opensearch.NewTermQuery[string]("name").Value("tom")), - want: map[string]any{ + Name: "should", + Got: opensearch.NewBoolQuery().Should(opensearch.NewTermQuery[string]("name").Value("tom")), + Want: map[string]any{ "bool": map[string]any{ "should": []map[string]any{ { @@ -82,9 +83,9 @@ func TestBoolQuery(t *testing.T) { }, }, { - name: "filter", - got: opensearch.NewBoolQuery().Filter(opensearch.NewTermQuery[string]("name").Value("tom")), - want: map[string]any{ + Name: "filter", + Got: opensearch.NewBoolQuery().Filter(opensearch.NewTermQuery[string]("name").Value("tom")), + Want: map[string]any{ "bool": map[string]any{ "filter": []map[string]any{ { @@ -99,13 +100,13 @@ func TestBoolQuery(t *testing.T) { }, }, { - name: "full", - got: opensearch.NewBoolQuery(). + Name: "full", + Got: opensearch.NewBoolQuery(). Must(opensearch.NewTermQuery[string]("name").Value("tom")). MustNot(opensearch.NewTermQuery[bool]("deleted").Value(true)). Should(opensearch.NewTermQuery[string]("gender").Value("male")). Filter(opensearch.NewTermQuery[int]("age").Value(42)), - want: map[string]any{ + Want: map[string]any{ "bool": map[string]any{ "must": []map[string]any{ { @@ -149,11 +150,9 @@ func TestBoolQuery(t *testing.T) { } for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - gotJSON, err := test.got.MarshalJSON() - assert.NoError(t, err) + t.Run(test.Name, func(t *testing.T) { - assert.JSONEq(t, toJSON(t, test.want), string(gotJSON)) + assert.JSONEq(t, opensearchtest.ToJSON(t, test.Want), opensearchtest.ToJSON(t, test.Got)) }) } } diff --git a/services/search/pkg/opensearch/dsl_query_full_text_match_phrase_test.go b/services/search/pkg/opensearch/dsl_query_full_text_match_phrase_test.go index c3e7fbf7fc..1ef0d1be56 100644 --- a/services/search/pkg/opensearch/dsl_query_full_text_match_phrase_test.go +++ b/services/search/pkg/opensearch/dsl_query_full_text_match_phrase_test.go @@ -6,23 +6,24 @@ import ( "github.com/stretchr/testify/assert" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestNewMatchPhraseQuery(t *testing.T) { - tests := []tableTest[opensearch.Builder, map[string]any]{ + tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{ { - name: "empty", - got: opensearch.NewMatchPhraseQuery("empty"), - want: nil, + Name: "empty", + Got: opensearch.NewMatchPhraseQuery("empty"), + Want: nil, }, { - name: "options", - got: opensearch.NewMatchPhraseQuery("name", opensearch.MatchPhraseQueryOptions{ + Name: "options", + Got: opensearch.NewMatchPhraseQuery("name", opensearch.MatchPhraseQueryOptions{ Analyzer: "analyzer", Slop: 2, ZeroTermsQuery: "all", }), - want: map[string]any{ + Want: map[string]any{ "match_phrase": map[string]any{ "name": map[string]any{ "analyzer": "analyzer", @@ -33,9 +34,9 @@ func TestNewMatchPhraseQuery(t *testing.T) { }, }, { - name: "query", - got: opensearch.NewMatchPhraseQuery("name").Query("some match query"), - want: map[string]any{ + Name: "query", + Got: opensearch.NewMatchPhraseQuery("name").Query("some match query"), + Want: map[string]any{ "match_phrase": map[string]any{ "name": map[string]any{ "query": "some match query", @@ -44,13 +45,13 @@ func TestNewMatchPhraseQuery(t *testing.T) { }, }, { - name: "full", - got: opensearch.NewMatchPhraseQuery("name", opensearch.MatchPhraseQueryOptions{ + Name: "full", + Got: opensearch.NewMatchPhraseQuery("name", opensearch.MatchPhraseQueryOptions{ Analyzer: "analyzer", Slop: 2, ZeroTermsQuery: "all", }).Query("some match query"), - want: map[string]any{ + Want: map[string]any{ "match_phrase": map[string]any{ "name": map[string]any{ "query": "some match query", @@ -64,11 +65,8 @@ func TestNewMatchPhraseQuery(t *testing.T) { } for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - gotJSON, err := test.got.MarshalJSON() - assert.NoError(t, err) - - assert.JSONEq(t, toJSON(t, test.want), string(gotJSON)) + t.Run(test.Name, func(t *testing.T) { + assert.JSONEq(t, opensearchtest.ToJSON(t, test.Want), opensearchtest.ToJSON(t, test.Got)) }) } } diff --git a/services/search/pkg/opensearch/dsl_query_term_level_ids_test.go b/services/search/pkg/opensearch/dsl_query_term_level_ids_test.go index ecc824ad98..4cf5d8a1d4 100644 --- a/services/search/pkg/opensearch/dsl_query_term_level_ids_test.go +++ b/services/search/pkg/opensearch/dsl_query_term_level_ids_test.go @@ -6,19 +6,20 @@ import ( "github.com/stretchr/testify/assert" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestIDsQuery(t *testing.T) { - tests := []tableTest[opensearch.Builder, map[string]any]{ + tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{ { - name: "empty", - got: opensearch.NewIDsQuery(nil), - want: nil, + Name: "empty", + Got: opensearch.NewIDsQuery(nil), + Want: nil, }, { - name: "ids", - got: opensearch.NewIDsQuery([]string{"1", "2", "3", "3"}, opensearch.IDsQueryOptions{Boost: 1.0}), - want: map[string]any{ + Name: "ids", + Got: opensearch.NewIDsQuery([]string{"1", "2", "3", "3"}, opensearch.IDsQueryOptions{Boost: 1.0}), + Want: map[string]any{ "ids": map[string]any{ "values": []string{"1", "2", "3"}, "boost": 1.0, @@ -28,11 +29,8 @@ func TestIDsQuery(t *testing.T) { } for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - gotJSON, err := test.got.MarshalJSON() - assert.NoError(t, err) - - assert.JSONEq(t, toJSON(t, test.want), string(gotJSON)) + t.Run(test.Name, func(t *testing.T) { + assert.JSONEq(t, opensearchtest.ToJSON(t, test.Want), opensearchtest.ToJSON(t, test.Got)) }) } } diff --git a/services/search/pkg/opensearch/dsl_query_term_level_term_test.go b/services/search/pkg/opensearch/dsl_query_term_level_term_test.go index cc899ea644..4d36cba091 100644 --- a/services/search/pkg/opensearch/dsl_query_term_level_term_test.go +++ b/services/search/pkg/opensearch/dsl_query_term_level_term_test.go @@ -6,19 +6,20 @@ import ( "github.com/stretchr/testify/assert" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestTermQuery(t *testing.T) { - tests := []tableTest[opensearch.Builder, map[string]any]{ + tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{ { - name: "empty", - got: opensearch.NewTermQuery[string]("empty"), - want: nil, + Name: "empty", + Got: opensearch.NewTermQuery[string]("empty"), + Want: nil, }, { - name: "naked", - got: opensearch.NewTermQuery[bool]("deleted").Value(false), - want: map[string]any{ + Name: "naked", + Got: opensearch.NewTermQuery[bool]("deleted").Value(false), + Want: map[string]any{ "term": map[string]any{ "deleted": map[string]any{ "value": false, @@ -27,13 +28,13 @@ func TestTermQuery(t *testing.T) { }, }, { - name: "term", - got: opensearch.NewTermQuery[bool]("deleted", opensearch.TermQueryOptions{ + Name: "term", + Got: opensearch.NewTermQuery[bool]("deleted", opensearch.TermQueryOptions{ Boost: 1.0, CaseInsensitive: true, Name: "is-deleted", }).Value(true), - want: map[string]any{ + Want: map[string]any{ "term": map[string]any{ "deleted": map[string]any{ "value": true, @@ -47,11 +48,8 @@ func TestTermQuery(t *testing.T) { } for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - gotJSON, err := test.got.MarshalJSON() - assert.NoError(t, err) - - assert.JSONEq(t, toJSON(t, test.want), string(gotJSON)) + t.Run(test.Name, func(t *testing.T) { + assert.JSONEq(t, opensearchtest.ToJSON(t, test.Want), opensearchtest.ToJSON(t, test.Got)) }) } } diff --git a/services/search/pkg/opensearch/dsl_query_term_level_wildcard_test.go b/services/search/pkg/opensearch/dsl_query_term_level_wildcard_test.go index 4253ee2026..e66201fea6 100644 --- a/services/search/pkg/opensearch/dsl_query_term_level_wildcard_test.go +++ b/services/search/pkg/opensearch/dsl_query_term_level_wildcard_test.go @@ -6,23 +6,24 @@ import ( "github.com/stretchr/testify/assert" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestWildcardQuery(t *testing.T) { - tests := []tableTest[opensearch.Builder, map[string]any]{ + tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{ { - name: "empty", - got: opensearch.NewWildcardQuery("empty"), - want: nil, + Name: "empty", + Got: opensearch.NewWildcardQuery("empty"), + Want: nil, }, { - name: "wildcard", - got: opensearch.NewWildcardQuery("name", opensearch.WildcardQueryOptions{ + Name: "wildcard", + Got: opensearch.NewWildcardQuery("name", opensearch.WildcardQueryOptions{ Boost: 1.0, CaseInsensitive: true, Rewrite: opensearch.TopTermsBlendedFreqsN, }).Value("opencl*"), - want: map[string]any{ + Want: map[string]any{ "wildcard": map[string]any{ "name": map[string]any{ "value": "opencl*", @@ -36,11 +37,8 @@ func TestWildcardQuery(t *testing.T) { } for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - gotJSON, err := test.got.MarshalJSON() - assert.NoError(t, err) - - assert.JSONEq(t, toJSON(t, test.want), string(gotJSON)) + t.Run(test.Name, func(t *testing.T) { + assert.JSONEq(t, opensearchtest.ToJSON(t, test.Want), opensearchtest.ToJSON(t, test.Got)) }) } } diff --git a/services/search/pkg/opensearch/dsl_query_test.go b/services/search/pkg/opensearch/dsl_query_test.go index d77a66bb22..6e17bea201 100644 --- a/services/search/pkg/opensearch/dsl_query_test.go +++ b/services/search/pkg/opensearch/dsl_query_test.go @@ -6,14 +6,15 @@ import ( "github.com/stretchr/testify/assert" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestQuery(t *testing.T) { - tests := []tableTest[opensearch.Builder, map[string]any]{ + tests := []opensearchtest.TableTest[opensearch.Builder, map[string]any]{ { - name: "simple", - got: opensearch.NewRootQuery(opensearch.NewTermQuery[string]("name").Value("tom")), - want: map[string]any{ + Name: "simple", + Got: opensearch.NewRootQuery(opensearch.NewTermQuery[string]("name").Value("tom")), + Want: map[string]any{ "query": map[string]any{ "term": map[string]any{ "name": map[string]any{ @@ -26,11 +27,8 @@ func TestQuery(t *testing.T) { } for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - gotJSON, err := test.got.MarshalJSON() - assert.NoError(t, err) - - assert.JSONEq(t, toJSON(t, test.want), string(gotJSON)) + t.Run(test.Name, func(t *testing.T) { + assert.JSONEq(t, opensearchtest.ToJSON(t, test.Want), opensearchtest.ToJSON(t, test.Got)) }) } } diff --git a/services/search/pkg/opensearch/engine.go b/services/search/pkg/opensearch/engine.go index 584d8ddff3..f78342c167 100644 --- a/services/search/pkg/opensearch/engine.go +++ b/services/search/pkg/opensearch/engine.go @@ -5,7 +5,9 @@ import ( "context" "encoding/json" "fmt" + "strings" + "github.com/opencloud-eu/reva/v2/pkg/utils" opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "github.com/opencloud-eu/opencloud/pkg/kql" @@ -44,6 +46,8 @@ func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexReque return nil, fmt.Errorf("failed to marshal query: %w", err) } + // todo: ignore deleted resources + resp, err := e.client.Search(context.Background(), &opensearchgoAPI.SearchReq{ Indices: []string{e.index}, Body: bytes.NewReader(body), @@ -53,23 +57,30 @@ func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexReque } matches := make([]*searchMessage.Match, len(resp.Hits.Hits)) + totalMatches := resp.Hits.Total.Value for i, hit := range resp.Hits.Hits { - resource, err := convert[engine.Resource](hit.Source) + match, err := searchHitToSearchMessageMatch(hit) if err != nil { return nil, fmt.Errorf("failed to convert hit %d: %w", i, err) } - matches[i] = &searchMessage.Match{ - Score: hit.Score, - Entity: &searchMessage.Entity{ - Name: resource.Name, - }, + if sir.Ref != nil { + hitPath := strings.TrimSuffix(match.GetEntity().GetRef().GetPath(), "/") + requestedPath := utils.MakeRelativePath(sir.Ref.Path) + isRoot := hitPath == requestedPath + + if !isRoot && requestedPath != "." && !strings.HasPrefix(hitPath, requestedPath+"/") { + totalMatches-- + continue + } } + + matches[i] = match } return &searchService.SearchIndexResponse{ Matches: matches, - TotalMatches: int32(resp.Hits.Total.Value), + TotalMatches: int32(totalMatches), }, nil } diff --git a/services/search/pkg/opensearch/engine_test.go b/services/search/pkg/opensearch/engine_test.go index fcf146f36b..b4ddfbdc10 100644 --- a/services/search/pkg/opensearch/engine_test.go +++ b/services/search/pkg/opensearch/engine_test.go @@ -14,14 +14,14 @@ import ( func TestEngine_Search(t *testing.T) { index := "test-engine-search" - tc := ostest.NewDefaultTestClient(t) + tc := opensearchtest.NewDefaultTestClient(t) tc.Require.IndicesReset([]string{index}) tc.Require.IndicesCount([]string{index}, "", 0) defer tc.Require.IndicesDelete([]string{index}) - document := ostest.Testdata.Resources.Full - tc.Require.DocumentCreate(index, document.ID, toJSON(t, document)) + document := opensearchtest.Testdata.Resources.Full + tc.Require.DocumentCreate(index, document.ID, opensearchtest.ToJSON(t, document)) tc.Require.IndicesCount([]string{index}, "", 1) engine, err := opensearch.NewEngine(index, tc.Client()) @@ -40,7 +40,7 @@ func TestEngine_Search(t *testing.T) { func TestEngine_Upsert(t *testing.T) { index := "test-engine-upsert" - tc := ostest.NewDefaultTestClient(t) + tc := opensearchtest.NewDefaultTestClient(t) tc.Require.IndicesReset([]string{index}) tc.Require.IndicesCount([]string{index}, "", 0) @@ -50,7 +50,7 @@ func TestEngine_Upsert(t *testing.T) { assert.NoError(t, err) t.Run("upsert with full document", func(t *testing.T) { - document := ostest.Testdata.Resources.Full + document := opensearchtest.Testdata.Resources.Full assert.NoError(t, engine.Upsert(document.ID, document)) tc.Require.IndicesCount([]string{index}, "", 1) @@ -61,7 +61,7 @@ func TestEngine_Move(t *testing.T) {} func TestEngine_Delete(t *testing.T) { index := "test-engine-delete" - tc := ostest.NewDefaultTestClient(t) + tc := opensearchtest.NewDefaultTestClient(t) tc.Require.IndicesReset([]string{index}) tc.Require.IndicesCount([]string{index}, "", 0) @@ -71,8 +71,8 @@ func TestEngine_Delete(t *testing.T) { assert.NoError(t, err) t.Run("mark document as deleted", func(t *testing.T) { - document := ostest.Testdata.Resources.Full - tc.Require.DocumentCreate(index, document.ID, toJSON(t, document)) + document := opensearchtest.Testdata.Resources.Full + tc.Require.DocumentCreate(index, document.ID, opensearchtest.ToJSON(t, document)) tc.Require.IndicesCount([]string{index}, "", 1) tc.Require.IndicesCount([]string{index}, opensearch.NewRootQuery( @@ -88,7 +88,7 @@ func TestEngine_Delete(t *testing.T) { func TestEngine_Restore(t *testing.T) { index := "test-engine-restore" - tc := ostest.NewDefaultTestClient(t) + tc := opensearchtest.NewDefaultTestClient(t) tc.Require.IndicesReset([]string{index}) tc.Require.IndicesCount([]string{index}, "", 0) @@ -98,9 +98,9 @@ func TestEngine_Restore(t *testing.T) { assert.NoError(t, err) t.Run("mark document as not deleted", func(t *testing.T) { - document := ostest.Testdata.Resources.Full + document := opensearchtest.Testdata.Resources.Full document.Deleted = true - tc.Require.DocumentCreate(index, document.ID, toJSON(t, document)) + tc.Require.DocumentCreate(index, document.ID, opensearchtest.ToJSON(t, document)) tc.Require.IndicesCount([]string{index}, "", 1) tc.Require.IndicesCount([]string{index}, opensearch.NewRootQuery( @@ -116,7 +116,7 @@ func TestEngine_Restore(t *testing.T) { func TestEngine_Purge(t *testing.T) { index := "test-engine-purge" - tc := ostest.NewDefaultTestClient(t) + tc := opensearchtest.NewDefaultTestClient(t) tc.Require.IndicesReset([]string{index}) tc.Require.IndicesCount([]string{index}, "", 0) @@ -126,8 +126,8 @@ func TestEngine_Purge(t *testing.T) { assert.NoError(t, err) t.Run("purge with full document", func(t *testing.T) { - document := ostest.Testdata.Resources.Full - tc.Require.DocumentCreate(index, document.ID, toJSON(t, document)) + document := opensearchtest.Testdata.Resources.Full + tc.Require.DocumentCreate(index, document.ID, opensearchtest.ToJSON(t, document)) tc.Require.IndicesCount([]string{index}, "", 1) assert.NoError(t, engine.Purge(document.ID)) @@ -138,7 +138,7 @@ func TestEngine_Purge(t *testing.T) { func TestEngine_DocCount(t *testing.T) { index := "test-engine-doc-count" - tc := ostest.NewDefaultTestClient(t) + tc := opensearchtest.NewDefaultTestClient(t) tc.Require.IndicesReset([]string{index}) tc.Require.IndicesCount([]string{index}, "", 0) @@ -148,15 +148,15 @@ func TestEngine_DocCount(t *testing.T) { assert.NoError(t, err) t.Run("ignore deleted documents", func(t *testing.T) { - document := ostest.Testdata.Resources.Full - tc.Require.DocumentCreate(index, document.ID, toJSON(t, document)) + document := opensearchtest.Testdata.Resources.Full + tc.Require.DocumentCreate(index, document.ID, opensearchtest.ToJSON(t, document)) tc.Require.IndicesCount([]string{index}, "", 1) count, err := engine.DocCount() assert.NoError(t, err) assert.Equal(t, uint64(1), count) - tc.Require.Update(index, document.ID, toJSON(t, map[string]any{ + tc.Require.Update(index, document.ID, opensearchtest.ToJSON(t, map[string]any{ "doc": map[string]any{ "Deleted": true, }, diff --git a/services/search/pkg/opensearch/internal/test/os.go b/services/search/pkg/opensearch/internal/test/os.go index 1f10741e8f..24492f50c1 100644 --- a/services/search/pkg/opensearch/internal/test/os.go +++ b/services/search/pkg/opensearch/internal/test/os.go @@ -1,4 +1,4 @@ -package ostest +package opensearchtest import ( "context" @@ -8,6 +8,7 @@ import ( "strings" "testing" + opensearchgo "github.com/opensearch-project/opensearch-go/v4" opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "github.com/stretchr/testify/require" ) @@ -18,7 +19,11 @@ type TestClient struct { } func NewDefaultTestClient(t *testing.T) *TestClient { - client, err := opensearchgoAPI.NewDefaultClient() + client, err := opensearchgoAPI.NewClient(opensearchgoAPI.Config{ + Client: opensearchgo.Config{ + Addresses: []string{"http://localhost:9200"}, + }, + }) require.NoError(t, err, "failed to create OpenSearch client") return NewTestClient(t, client) diff --git a/services/search/pkg/opensearch/internal/test/test.go b/services/search/pkg/opensearch/internal/test/test.go index 163278acb7..d19ea47cd7 100644 --- a/services/search/pkg/opensearch/internal/test/test.go +++ b/services/search/pkg/opensearch/internal/test/test.go @@ -1 +1,20 @@ -package ostest +package opensearchtest + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +type TableTest[G any, W any] struct { + Name string + Got G + Want W +} + +func ToJSON(t *testing.T, data any) string { + jsonData, err := json.Marshal(data) + require.NoError(t, err, "failed to marshal data to JSON") + return string(jsonData) +} diff --git a/services/search/pkg/opensearch/internal/test/testdata.go b/services/search/pkg/opensearch/internal/test/testdata.go index cf002be41b..c5e61da07c 100644 --- a/services/search/pkg/opensearch/internal/test/testdata.go +++ b/services/search/pkg/opensearch/internal/test/testdata.go @@ -1,4 +1,4 @@ -package ostest +package opensearchtest import ( "encoding/json" diff --git a/services/search/pkg/opensearch/kql_test.go b/services/search/pkg/opensearch/kql_test.go index 8dfb6a2c61..102abafebe 100644 --- a/services/search/pkg/opensearch/kql_test.go +++ b/services/search/pkg/opensearch/kql_test.go @@ -7,60 +7,61 @@ import ( "github.com/opencloud-eu/opencloud/pkg/ast" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestKQL_Compile(t *testing.T) { - tests := []tableTest[*ast.Ast, opensearch.Builder]{ + tests := []opensearchtest.TableTest[*ast.Ast, opensearch.Builder]{ // field name tests { - name: "Name is the default field", - got: &ast.Ast{ + Name: "Name is the default field", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Value: "openCloud"}, }, }, - want: opensearch.NewTermQuery[string]("Name").Value("openCloud"), + Want: opensearch.NewTermQuery[string]("Name").Value("openCloud"), }, { - name: "remaps known field names", - got: &ast.Ast{ + Name: "remaps known field names", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "mediatype", Value: "application/gzip"}, }, }, - want: opensearch.NewTermQuery[string]("MimeType").Value("application/gzip"), + Want: opensearch.NewTermQuery[string]("MimeType").Value("application/gzip"), }, // kql to os dsl - type tests { - name: "term query", - got: &ast.Ast{ + Name: "term query", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "Name", Value: "openCloud"}, }, }, - want: opensearch.NewTermQuery[string]("Name").Value("openCloud"), + Want: opensearch.NewTermQuery[string]("Name").Value("openCloud"), }, { - name: "match-phrase query", - got: &ast.Ast{ + Name: "match-phrase query", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "Name", Value: "open cloud"}, }, }, - want: opensearch.NewMatchPhraseQuery("Name").Query("open cloud"), + Want: opensearch.NewMatchPhraseQuery("Name").Query("open cloud"), }, { - name: "wildcard query", - got: &ast.Ast{ + Name: "wildcard query", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "Name", Value: "open*"}, }, }, - want: opensearch.NewWildcardQuery("Name").Value("open*"), + Want: opensearch.NewWildcardQuery("Name").Value("open*"), }, { - name: "bool query", - got: &ast.Ast{ + Name: "bool query", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.GroupNode{Nodes: []ast.Node{ &ast.StringNode{Value: "a"}, @@ -68,79 +69,79 @@ func TestKQL_Compile(t *testing.T) { }}, }, }, - want: opensearch.NewBoolQuery().Must( + Want: opensearch.NewBoolQuery().Must( opensearch.NewTermQuery[string]("Name").Value("a"), opensearch.NewTermQuery[string]("Name").Value("b"), ), }, { - name: "no bool query for single term", - got: &ast.Ast{ + Name: "no bool query for single term", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.GroupNode{Nodes: []ast.Node{ &ast.StringNode{Value: "any"}, }}, }, }, - want: opensearch.NewWildcardQuery("Name").Value("open*"), + Want: opensearch.NewTermQuery[string]("Name").Value("any"), }, // kql to os dsl - structure tests { - name: "[*]", - got: &ast.Ast{ + Name: "[*]", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "name", Value: "openCloud"}, }, }, - want: opensearch.NewTermQuery[string]("Name").Value("openCloud"), + Want: opensearch.NewTermQuery[string]("Name").Value("openCloud"), }, { - name: "[* *]", - got: &ast.Ast{ + Name: "[* *]", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "name", Value: "openCloud"}, &ast.StringNode{Key: "age", Value: "32"}, }, }, - want: opensearch.NewBoolQuery(). + Want: opensearch.NewBoolQuery(). Must( opensearch.NewTermQuery[string]("Name").Value("openCloud"), opensearch.NewTermQuery[string]("age").Value("32"), ), }, { - name: "[* AND *]", - got: &ast.Ast{ + Name: "[* AND *]", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "name", Value: "openCloud"}, &ast.OperatorNode{Value: "AND"}, &ast.StringNode{Key: "age", Value: "32"}, }, }, - want: opensearch.NewBoolQuery(). + Want: opensearch.NewBoolQuery(). Must( opensearch.NewTermQuery[string]("Name").Value("openCloud"), opensearch.NewTermQuery[string]("age").Value("32"), ), }, { - name: "[* OR *]", - got: &ast.Ast{ + Name: "[* OR *]", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "name", Value: "openCloud"}, &ast.OperatorNode{Value: "OR"}, &ast.StringNode{Key: "age", Value: "32"}, }, }, - want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). + Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). Should( opensearch.NewTermQuery[string]("Name").Value("openCloud"), opensearch.NewTermQuery[string]("age").Value("32"), ), }, { - name: "[* OR * OR *]", - got: &ast.Ast{ + Name: "[* OR * OR *]", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "name", Value: "openCloud"}, &ast.OperatorNode{Value: "OR"}, @@ -149,7 +150,7 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "age", Value: "44"}, }, }, - want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). + Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). Should( opensearch.NewTermQuery[string]("Name").Value("openCloud"), opensearch.NewTermQuery[string]("age").Value("32"), @@ -157,8 +158,8 @@ func TestKQL_Compile(t *testing.T) { ), }, { - name: "[* AND * OR *]", - got: &ast.Ast{ + Name: "[* AND * OR *]", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "a", Value: "a"}, &ast.OperatorNode{Value: "AND"}, @@ -167,7 +168,7 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "c", Value: "c"}, }, }, - want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). + Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). Must( opensearch.NewTermQuery[string]("a").Value("a"), ). @@ -177,8 +178,8 @@ func TestKQL_Compile(t *testing.T) { ), }, { - name: "[* OR * AND *]", - got: &ast.Ast{ + Name: "[* OR * AND *]", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "a", Value: "a"}, &ast.OperatorNode{Value: "OR"}, @@ -187,7 +188,7 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "c", Value: "c"}, }, }, - want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). + Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). Must( opensearch.NewTermQuery[string]("b").Value("b"), opensearch.NewTermQuery[string]("c").Value("c"), @@ -197,8 +198,8 @@ func TestKQL_Compile(t *testing.T) { ), }, { - name: "NEW[* OR * AND *]", - got: &ast.Ast{ + Name: "NEW[* OR * AND *]", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "a", Value: "a"}, &ast.OperatorNode{Value: "OR"}, @@ -207,7 +208,7 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "c", Value: "c"}, }, }, - want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). + Want: opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). Should( opensearch.NewTermQuery[string]("a").Value("a"), ). @@ -217,8 +218,8 @@ func TestKQL_Compile(t *testing.T) { ), }, { - name: "[[* OR * OR *] AND *]", - got: &ast.Ast{ + Name: "[[* OR * OR *] AND *]", + Got: &ast.Ast{ Nodes: []ast.Node{ &ast.GroupNode{Nodes: []ast.Node{ &ast.StringNode{Key: "a", Value: "a"}, @@ -231,7 +232,7 @@ func TestKQL_Compile(t *testing.T) { &ast.StringNode{Key: "d", Value: "d"}, }, }, - want: opensearch.NewBoolQuery(). + Want: opensearch.NewBoolQuery(). Must( opensearch.NewBoolQuery(opensearch.BoolQueryOptions{MinimumShouldMatch: 1}). Should( @@ -245,20 +246,14 @@ func TestKQL_Compile(t *testing.T) { } for _, test := range tests { - t.Run(test.name, func(t *testing.T) { + t.Run(test.Name, func(t *testing.T) { compiler, err := opensearch.NewKQL() assert.NoError(t, err) - got, err := compiler.Compile(test.got) + dsl, err := compiler.Compile(test.Got) assert.NoError(t, err) - gotJSON, err := got.MarshalJSON() - assert.NoError(t, err) - - wantJSON, err := test.want.MarshalJSON() - assert.NoError(t, err) - - assert.JSONEq(t, string(wantJSON), string(gotJSON)) + assert.JSONEq(t, opensearchtest.ToJSON(t, test.Want), opensearchtest.ToJSON(t, dsl)) }) } } diff --git a/services/search/pkg/opensearch/opensearch_test.go b/services/search/pkg/opensearch/opensearch_test.go index db18b48098..5cfac2a0d0 100644 --- a/services/search/pkg/opensearch/opensearch_test.go +++ b/services/search/pkg/opensearch/opensearch_test.go @@ -1,20 +1 @@ package opensearch_test - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/require" -) - -type tableTest[G any, W any] struct { - name string - got G - want W -} - -func toJSON(t *testing.T, data any) string { - jsonData, err := json.Marshal(data) - require.NoError(t, err, "failed to marshal data to JSON") - return string(jsonData) -}