diff --git a/services/graph/pkg/l10n/locale/sv/LC_MESSAGES/graph.po b/services/graph/pkg/l10n/locale/sv/LC_MESSAGES/graph.po index 57323d07e4..ddb83fd0a4 100644 --- a/services/graph/pkg/l10n/locale/sv/LC_MESSAGES/graph.po +++ b/services/graph/pkg/l10n/locale/sv/LC_MESSAGES/graph.po @@ -11,7 +11,7 @@ msgid "" msgstr "" "Project-Id-Version: \n" "Report-Msgid-Bugs-To: EMAIL\n" -"POT-Creation-Date: 2026-10-10 23:16+0000\n" +"POT-Creation-Date: 2026-10-31 23:16+0000\n" "PO-Revision-Date: 2025-01-27 10:17+0000\n" "Last-Translator: Daniel Nylander , 2026\n" "Language-Team: Swedish (https://app.transifex.com/opencloud-eu/teams/204053/sv/)\n" diff --git a/services/search/pkg/opensearch/batch.go b/services/search/pkg/opensearch/batch.go index 9d53bce1bb..698aba8562 100644 --- a/services/search/pkg/opensearch/batch.go +++ b/services/search/pkg/opensearch/batch.go @@ -197,7 +197,12 @@ func (b *Batch) Purge(id string, onlyDeleted bool) error { case err != nil: return fmt.Errorf("failed to delete by query: %w", err) case len(resp.Failures) != 0: - return fmt.Errorf("failed to delete by query, failures: %v", resp.Failures) + failures, err := json.Marshal(resp.Failures) + if err != nil { + return fmt.Errorf("failed to marshal delete by query failures: %w", err) + } + + return fmt.Errorf("failed to delete by query, failures: %s", failures) } return nil @@ -223,6 +228,7 @@ func (b *Batch) Push() error { if len(bulkOperations) == 0 { return nil } + defer func() { bulkOperations = nil }() var body strings.Builder for _, operation := range bulkOperations { @@ -234,14 +240,31 @@ func (b *Batch) Push() error { body.WriteString("\n") } - if _, err := b.client.Bulk(context.Background(), opensearchgoAPI.BulkReq{ + resp, err := b.client.Bulk(context.Background(), opensearchgoAPI.BulkReq{ Body: strings.NewReader(body.String()), Params: opensearchgoAPI.BulkParams{Refresh: "wait_for"}, - }); err != nil { + }) + switch { + case err != nil: return fmt.Errorf("failed to execute bulk operations: %w", err) + case resp.Errors: + var failed []opensearchgoAPI.BulkRespItem + for _, item := range resp.Items { + for _, result := range item { + if result.Error != nil { + failed = append(failed, result) + } + } + } + + failures, err := json.Marshal(failed) + if err != nil { + return fmt.Errorf("failed to marshal bulk failures: %w", err) + } + + return fmt.Errorf("failed to execute bulk operations, failures: %s", failures) } - bulkOperations = nil return nil } @@ -249,21 +272,25 @@ func (b *Batch) Push() error { // unfortunately, operations like DeleteByQuery cannot be part of the bulk API, // so we need to push the previous bulk operations before executing such operations // this might lead to smaller bulks than the configured size, but ensures correct order + var errs []error for _, operation := range b.operations { switch op := operation.(type) { case func() []map[string]any: bulkOperations = append(bulkOperations, op()...) case func() error: if err := pushBulkOperations(); err != nil { - return fmt.Errorf("failed to push operations: %w", err) + errs = append(errs, fmt.Errorf("failed to push operations: %w", err)) } if err := op(); err != nil { - return fmt.Errorf("failed to execute operation: %w", err) + errs = append(errs, fmt.Errorf("failed to execute operation: %w", err)) } } } + if err := pushBulkOperations(); err != nil { + errs = append(errs, fmt.Errorf("failed to push operations: %w", err)) + } - return pushBulkOperations() + return errors.Join(errs...) } func (b *Batch) withSizeLimit(f func() error) error { diff --git a/services/search/pkg/opensearch/batch_test.go b/services/search/pkg/opensearch/batch_test.go new file mode 100644 index 0000000000..22cdcbfb06 --- /dev/null +++ b/services/search/pkg/opensearch/batch_test.go @@ -0,0 +1,89 @@ +package opensearch_test + +import ( + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" +) + +var _ = Describe("Batch", func() { + Describe("Push", func() { + var tc *opensearchtest.TestClient + + BeforeEach(func() { + tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) + }) + + It("reports the documents the bulk API rejected", func() { + const indexName = "opencloud-test-batch-push-rejected" + tc.Require.IndicesReset([]string{indexName}) + deleteIndexOnCleanup(tc, indexName) + + // Name is a string, mapping it as a long makes every document fail to parse. + tc.Require.IndicesCreate(indexName, strings.NewReader(`{"mappings":{"properties":{"Name":{"type":"long"}}}}`)) + + batch, err := opensearch.NewBatch(tc.Client(), indexName, 10) + Expect(err).ToNot(HaveOccurred()) + + document := opensearchtest.Testdata.Resources.File + Expect(batch.Upsert(document.ID, document)).To(Succeed()) + + err = batch.Push() + Expect(err).To(MatchError(ContainSubstring(document.ID))) + Expect(err).To(MatchError(ContainSubstring("mapper_parsing_exception"))) + tc.Require.IndicesCount([]string{indexName}, nil, 0) + }) + + It("keeps executing the operations queued after a rejected bulk", func() { + const indexName = "opencloud-test-batch-push-continue" + tc.Require.IndicesReset([]string{indexName}) + deleteIndexOnCleanup(tc, indexName) + + tc.Require.IndicesCreate(indexName, strings.NewReader(opensearch.IndexManagerLatest.String())) + + seed, err := opensearch.NewBatch(tc.Client(), indexName, 10) + Expect(err).ToNot(HaveOccurred()) + document := opensearchtest.Testdata.Resources.File + Expect(seed.Upsert(document.ID, document)).To(Succeed()) + Expect(seed.Push()).To(Succeed()) + + batch, err := opensearch.NewBatch(tc.Client(), indexName, 10) + Expect(err).ToNot(HaveOccurred()) + + // OpenSearch rejects ids longer than 512 bytes per item, the bulk itself goes through. + rejectedID := strings.Repeat("x", 513) + Expect(batch.Upsert(rejectedID, document)).To(Succeed()) + Expect(batch.Delete(document.ID)).To(Succeed()) + sibling := document + sibling.ID = "1$1!4" + Expect(batch.Upsert(sibling.ID, sibling)).To(Succeed()) + + err = batch.Push() + Expect(err).To(MatchError(ContainSubstring(rejectedID))) + Expect(err).ToNot(MatchError(ContainSubstring(sibling.ID))) + tc.Require.IndicesCount([]string{indexName}, nil, 2) + tc.Require.IndicesCount([]string{indexName}, strings.NewReader(`{"query":{"term":{"Deleted":true}}}`), 1) + }) + + It("pushes the documents the bulk API accepted", func() { + const indexName = "opencloud-test-batch-push-accepted" + tc.Require.IndicesReset([]string{indexName}) + deleteIndexOnCleanup(tc, indexName) + + tc.Require.IndicesCreate(indexName, strings.NewReader(opensearch.IndexManagerLatest.String())) + + batch, err := opensearch.NewBatch(tc.Client(), indexName, 10) + Expect(err).ToNot(HaveOccurred()) + + document := opensearchtest.Testdata.Resources.File + Expect(batch.Upsert(document.ID, document)).To(Succeed()) + Expect(batch.Push()).To(Succeed()) + + tc.Require.IndicesCount([]string{indexName}, nil, 1) + }) + }) +}) diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile.go b/services/search/pkg/opensearch/internal/convert/kql_transpile.go index 9747c7e2b3..101677651e 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile.go @@ -119,9 +119,6 @@ func (t kqlOpensearchTranspiler) toBuilder(node ast.Node) (osu.Builder, error) { } field, value := node.Key, node.Value - if query.FieldIsPath(node.Key) { - value = strings.TrimSuffix(value, "/") - } if node.CaseInsensitive { field += mapping.LowercaseSuffix value = strings.ToLower(value) diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index 05b205dea6..6845457ede 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -250,6 +250,15 @@ Fixtures: | PATH-06 | `path:"./DOCUMENTS"` | docs-upper | docs-upper | docs-upper | ✅ | | PATH-07 | `path:"./Documents"` | docs-mixed | docs-mixed | docs-mixed | ✅ | | PATH-08 | `path:"./parent/"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ | +| PATH-09 | `path:"/"` | child.jpg, docs-lower, docs-mixed, docs-upper, parent | child.jpg, docs-lower, docs-mixed, docs-upper, parent | child.jpg, docs-lower, docs-mixed, docs-upper, parent | ✅ | +| PATH-10 | `path:""` | child.jpg, docs-lower, docs-mixed, docs-upper, parent | child.jpg, docs-lower, docs-mixed, docs-upper, parent | child.jpg, docs-lower, docs-mixed, docs-upper, parent | ✅ | +| PATH-11 | `path:"/parent"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ | +| PATH-12 | `path:"parent"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ | +| PATH-13 | `path:"parent/*"` | child.jpg | child.jpg | child.jpg | ✅ | +| PATH-14 | `path:"*/parent"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ | +| PATH-15 | `path:"?/parent"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ | +| PATH-16 | `path:"*/Documents"` | docs-mixed | docs-mixed | docs-mixed | ✅ | +| PATH-17 | `path:"*/parent/"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ | ### fields diff --git a/services/search/pkg/parity/query_path_test.go b/services/search/pkg/parity/query_path_test.go index e97e8e1596..0200a98f68 100644 --- a/services/search/pkg/parity/query_path_test.go +++ b/services/search/pkg/parity/query_path_test.go @@ -23,6 +23,15 @@ func pathGroup() queryGroup { {id: 6, query: `path:"./DOCUMENTS"`, want: []string{"docs-upper"}}, {id: 7, query: `path:"./Documents"`, want: []string{"docs-mixed"}}, {id: 8, query: `path:"./parent/"`, want: []string{"parent", "child.jpg"}}, + {id: 9, query: `path:"/"`, want: []string{"parent", "child.jpg", "docs-lower", "docs-upper", "docs-mixed"}}, + {id: 10, query: `path:""`, want: []string{"parent", "child.jpg", "docs-lower", "docs-upper", "docs-mixed"}}, + {id: 11, query: `path:"/parent"`, want: []string{"parent", "child.jpg"}}, + {id: 12, query: `path:"parent"`, want: []string{"parent", "child.jpg"}}, + {id: 13, query: `path:"parent/*"`, want: []string{"child.jpg"}}, + {id: 14, query: `path:"*/parent"`, want: []string{"parent", "child.jpg"}}, + {id: 15, query: `path:"?/parent"`, want: []string{"parent", "child.jpg"}}, + {id: 16, query: `path:"*/Documents"`, want: []string{"docs-mixed"}}, + {id: 17, query: `path:"*/parent/"`, want: []string{"parent", "child.jpg"}}, }, } } diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index a1dba9a65a..74f7d60dc6 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -95,9 +95,6 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { // bleve treats `/` and `+` as literals mid-term, so a literal MIME like // image/svg+xml still matches exactly. val := n.Value - if searchQuery.FieldIsPath(n.Key) { - val = strings.TrimSuffix(val, "/") - } k := n.Key v := val if k != "ID" && k != "Size" && k != "MimeType" { diff --git a/services/search/pkg/query/bleve/compiler_test.go b/services/search/pkg/query/bleve/compiler_test.go index 632a29b387..606c953050 100644 --- a/services/search/pkg/query/bleve/compiler_test.go +++ b/services/search/pkg/query/bleve/compiler_test.go @@ -59,7 +59,7 @@ func Test_compile(t *testing.T) { }, }, want: func() query.Query { - tq := query.NewTermQuery("/Foo") + tq := query.NewTermQuery("./Foo") tq.SetField("Path") return query.NewConjunctionQuery([]query.Query{tq}) }(), diff --git a/services/search/pkg/query/normalize.go b/services/search/pkg/query/normalize.go index bce1fd1be6..364a23f683 100644 --- a/services/search/pkg/query/normalize.go +++ b/services/search/pkg/query/normalize.go @@ -5,6 +5,7 @@ import ( "strings" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/opencloud-eu/reva/v2/pkg/utils" "reflect" "github.com/opencloud-eu/opencloud/pkg/ast" @@ -36,6 +37,14 @@ func normalizeNodes(nodes []ast.Node, resolve func(string) string, defaultKey st switch node := n.(type) { case *ast.StringNode: node.Key = resolveKey(node.Key) + if FieldIsPath(node.Key) { + if strings.IndexAny(node.Value, "*?") == 0 { + // a leading wildcard stands in for the "./" prefix, anchoring would break it + node.Value = strings.TrimSuffix(node.Value, "/") + } else { + node.Value = utils.MakeRelativePath(node.Value) + } + } if FieldValueIsNormalized(node.Key) { node.Value = strings.ToLower(node.Value) } diff --git a/services/search/pkg/query/normalize_test.go b/services/search/pkg/query/normalize_test.go index 94eecfb300..afc9182c3e 100644 --- a/services/search/pkg/query/normalize_test.go +++ b/services/search/pkg/query/normalize_test.go @@ -127,6 +127,27 @@ var _ = Describe("Normalize", func() { })) }) + DescribeTable("normalizes path values to the stored relative form", + func(in, want string) { + got := norm(&ast.StringNode{Key: "path", Value: in}) + Expect(got).To(Equal([]ast.Node{&ast.StringNode{Key: "Path", Value: want}})) + }, + Entry("relative", "./parent", "./parent"), + Entry("trailing slash", "./parent/", "./parent"), + Entry("absolute", "/parent", "./parent"), + Entry("bare", "parent", "./parent"), + Entry("nested bare", "parent/child.jpg", "./parent/child.jpg"), + Entry("root", "/", "."), + Entry("empty", "", "."), + Entry("dot", ".", "."), + Entry("wildcard", "./par*/", "./par*"), + Entry("bare wildcard", "parent/*", "./parent/*"), + Entry("leading wildcard", "*child*", "*child*"), + Entry("leading wildcard covers the prefix", "*/parent", "*/parent"), + Entry("leading single-char wildcard covers the prefix", "?/parent", "?/parent"), + Entry("leading wildcard trailing slash", "*/parent/", "*/parent"), + ) + It("converts value nodes to pointers", func() { got := norm( ast.StringNode{Key: "name", Value: "x"},