From 5601be127644f95a0581a58129b361d5ba8c1000 Mon Sep 17 00:00:00 2001 From: Florian Schade Date: Fri, 28 Aug 2026 09:54:35 +0200 Subject: [PATCH 1/9] test(search): introduce engine parity suite --- services/search/pkg/parity/README.md | 619 ++++++++++++++++++ services/search/pkg/parity/engines_test.go | 127 ++++ services/search/pkg/parity/fixtures_test.go | 108 +++ .../search/pkg/parity/lifecycle_batch_test.go | 83 +++ .../pkg/parity/lifecycle_casepath_test.go | 48 ++ .../pkg/parity/lifecycle_delete_test.go | 44 ++ .../pkg/parity/lifecycle_hidden_test.go | 48 ++ .../pkg/parity/lifecycle_idempotency_test.go | 76 +++ .../search/pkg/parity/lifecycle_move_test.go | 37 ++ .../search/pkg/parity/lifecycle_purge_test.go | 40 ++ .../pkg/parity/lifecycle_purgespace_test.go | 36 + .../pkg/parity/lifecycle_restore_test.go | 40 ++ .../pkg/parity/lifecycle_rootscope_test.go | 50 ++ .../pkg/parity/lifecycle_upsert_test.go | 27 + services/search/pkg/parity/matrix_test.go | 395 +++++++++++ services/search/pkg/parity/parity_test.go | 318 +++++++++ .../search/pkg/parity/query_boolean_test.go | 24 + .../search/pkg/parity/query_content_test.go | 27 + .../search/pkg/parity/query_deleted_test.go | 26 + .../pkg/parity/query_everything_test.go | 23 + .../search/pkg/parity/query_extension_test.go | 22 + .../search/pkg/parity/query_favorites_test.go | 22 + .../search/pkg/parity/query_fields_test.go | 35 + .../search/pkg/parity/query_invalid_test.go | 18 + .../search/pkg/parity/query_mediatype_test.go | 25 + services/search/pkg/parity/query_name_test.go | 66 ++ services/search/pkg/parity/query_path_test.go | 28 + .../search/pkg/parity/query_range_test.go | 26 + .../search/pkg/parity/query_samename_test.go | 34 + .../search/pkg/parity/query_scope_test.go | 32 + .../search/pkg/parity/query_stress_test.go | 38 ++ services/search/pkg/parity/query_tags_test.go | 34 + .../search/pkg/parity/query_title_test.go | 23 + .../pkg/parity/query_visibility_test.go | 26 + .../search/pkg/parity/response_entity_test.go | 102 +++ .../pkg/parity/response_metadata_test.go | 143 ++++ 36 files changed, 2870 insertions(+) create mode 100644 services/search/pkg/parity/README.md create mode 100644 services/search/pkg/parity/engines_test.go create mode 100644 services/search/pkg/parity/fixtures_test.go create mode 100644 services/search/pkg/parity/lifecycle_batch_test.go create mode 100644 services/search/pkg/parity/lifecycle_casepath_test.go create mode 100644 services/search/pkg/parity/lifecycle_delete_test.go create mode 100644 services/search/pkg/parity/lifecycle_hidden_test.go create mode 100644 services/search/pkg/parity/lifecycle_idempotency_test.go create mode 100644 services/search/pkg/parity/lifecycle_move_test.go create mode 100644 services/search/pkg/parity/lifecycle_purge_test.go create mode 100644 services/search/pkg/parity/lifecycle_purgespace_test.go create mode 100644 services/search/pkg/parity/lifecycle_restore_test.go create mode 100644 services/search/pkg/parity/lifecycle_rootscope_test.go create mode 100644 services/search/pkg/parity/lifecycle_upsert_test.go create mode 100644 services/search/pkg/parity/matrix_test.go create mode 100644 services/search/pkg/parity/parity_test.go create mode 100644 services/search/pkg/parity/query_boolean_test.go create mode 100644 services/search/pkg/parity/query_content_test.go create mode 100644 services/search/pkg/parity/query_deleted_test.go create mode 100644 services/search/pkg/parity/query_everything_test.go create mode 100644 services/search/pkg/parity/query_extension_test.go create mode 100644 services/search/pkg/parity/query_favorites_test.go create mode 100644 services/search/pkg/parity/query_fields_test.go create mode 100644 services/search/pkg/parity/query_invalid_test.go create mode 100644 services/search/pkg/parity/query_mediatype_test.go create mode 100644 services/search/pkg/parity/query_name_test.go create mode 100644 services/search/pkg/parity/query_path_test.go create mode 100644 services/search/pkg/parity/query_range_test.go create mode 100644 services/search/pkg/parity/query_samename_test.go create mode 100644 services/search/pkg/parity/query_scope_test.go create mode 100644 services/search/pkg/parity/query_stress_test.go create mode 100644 services/search/pkg/parity/query_tags_test.go create mode 100644 services/search/pkg/parity/query_title_test.go create mode 100644 services/search/pkg/parity/query_visibility_test.go create mode 100644 services/search/pkg/parity/response_entity_test.go create mode 100644 services/search/pkg/parity/response_metadata_test.go diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md new file mode 100644 index 0000000000..39fb5589dc --- /dev/null +++ b/services/search/pkg/parity/README.md @@ -0,0 +1,619 @@ +## Queries + +### name + +Fixtures: + +- `new-folder`, folder +- `quarterly notes.txt` +- `Report.txt` +- `Übung.txt` +- `a+b.txt` +- `c(d).txt` +- `e&f.txt` +- `v1.2.3.txt` +- `foo bar.txt` +- `aaaaaaaaaa...edle.txt` + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| NAME-01 | `new` | new-folder | new-folder | new-folder | ✅ | +| NAME-02 | `quarterly` | quarterly notes.txt | quarterly notes.txt | quarterly notes.txt | ✅ | +| NAME-03 | `report` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-04 | `name:"*new-folder*"` | new-folder | new-folder | new-folder | ✅ | +| NAME-05 | `name:"*w-fol*"` | new-folder | new-folder | new-folder | ✅ | +| NAME-06 | `name:"*oo ba*"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | +| NAME-07 | `name:"*REPORT*"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-08 | `name:"*übung*"` | Übung.txt | Übung.txt | Übung.txt | ✅ | +| NAME-09 | `name:"*ÜBUNG*"` | Übung.txt | Übung.txt | Übung.txt | ✅ | +| NAME-10 | `name:"*a+b*"` | a+b.txt | a+b.txt | a+b.txt | ✅ | +| NAME-11 | `name:"*c(d)*"` | c(d).txt | c(d).txt | c(d).txt | ✅ | +| NAME-12 | `name:"*e&f*"` | e&f.txt | e&f.txt | e&f.txt | ✅ | +| NAME-13 | `name:"*v1.2*"` | v1.2.3.txt | v1.2.3.txt | v1.2.3.txt | ✅ | +| NAME-14 | `new-folder` | new-folder | new-folder | new-folder | ✅ | +| NAME-15 | `*folder*` | new-folder | new-folder | new-folder | ✅ | +| NAME-16 | `name:"*foo bar*"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | +| NAME-17 | `name:"foo bar.txt"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | +| NAME-18 | `name:"*needle*"` | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | ✅ | +| NAME-19 | `name:"report*"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-20 | `name:"*report"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-21 | `name:"Rep*rt.txt"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-22 | `Name:"*report*"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-23 | `NAME:"*report*"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-24 | `name:Rep?rt.txt` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-25 | `name:"*eport"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-26 | `name:"repor*"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-27 | `REPORT` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-28 | `name:REPORT` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-29 | `name:"REPORT.TXT"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-30 | `name:"FOO BAR.TXT"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | +| NAME-31 | `name:"ÜBUNG.TXT"` | Übung.txt | Übung.txt | Übung.txt | ✅ | +| NAME-32 | `name:"folder*"` | no match | no match | no match | ✅ | +| NAME-33 | `name:"*new"` | no match | no match | no match | ✅ | +| NAME-34 | `name:new` | new-folder | new-folder | new-folder | ✅ | +| NAME-35 | `name:"new"` | no match | no match | no match | ✅ | +| NAME-36 | `name:"new-folder"` | new-folder | new-folder | new-folder | ✅ | +| NAME-37 | `name:"new-*"` | new-folder | new-folder | new-folder | ✅ | +| NAME-38 | `name:"new*"` | new-folder | new-folder | new-folder | ✅ | +| NAME-39 | `name:new-*` | new-folder | new-folder | new-folder | ✅ | +| NAME-40 | `name:"*-folder"` | new-folder | new-folder | new-folder | ✅ | +| NAME-41 | `name:"Rep?rt.txt"` | Report.txt | Report.txt | Report.txt | ✅ | + +### extension + +Fixtures: + +- `report.txt` +- `notes.md`, MimeType = text/markdown +- `archive`, folder + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| EXTENSION-01 | `txt` | report.txt | report.txt | report.txt | ✅ | +| EXTENSION-02 | `md` | notes.md | notes.md | notes.md | ✅ | +| EXTENSION-03 | `name:"*.txt"` | report.txt | report.txt | report.txt | ✅ | +| EXTENSION-04 | `report` | report.txt | report.txt | report.txt | ✅ | + +### tags + +Fixtures: + +- `invoice.txt`, Tags = foo-bar +- `memo.txt`, Tags = foo +- `spaced.txt`, Tags = spaced tag +- `project`, folder, Tags = work +- `draft.txt`, Path = ./project/draft.txt +- `longtag.txt`, Tags = zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzneedle + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| TAGS-01 | `name:"*foo-bar*"` | no match | no match | no match | ✅ | +| TAGS-02 | `tag:("foo-bar")` | invoice.txt | invoice.txt | invoice.txt | ✅ | +| TAGS-03 | `tag:("foo")` | memo.txt | memo.txt | memo.txt | ✅ | +| TAGS-04 | `tag:("FOO-BAR")` | invoice.txt | invoice.txt | invoice.txt | ✅ | +| TAGS-05 | `tag:("*foo*")` | invoice.txt, memo.txt | invoice.txt, memo.txt | invoice.txt, memo.txt | ✅ | +| TAGS-06 | `tag:("spaced tag")` | spaced.txt | spaced.txt | spaced.txt | ✅ | +| TAGS-07 | `tag:("*paced ta*")` | spaced.txt | spaced.txt | spaced.txt | ✅ | +| TAGS-08 | `tag:("work")` | project | project | project | ✅ | +| TAGS-09 | `tag:("zzzzzzzzzzzzzzzzzzzzzzzzzz...zzzzzzzzzzzzzzzzneedle")` | longtag.txt | longtag.txt | longtag.txt | ✅ | + +### title + +Fixtures: + +- `q1.html`, MimeType = text/html, Title = "quarterly report" + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| TITLE-01 | `Title:"quarterly report"` | q1.html | q1.html | q1.html | ✅ | +| TITLE-02 | `Title:quarterly` | q1.html | q1.html | q1.html | ✅ | +| TITLE-03 | `Title:QUARTERLY` | q1.html | q1.html | q1.html | ✅ | +| TITLE-04 | `Title:quarterl*` | q1.html | q1.html | q1.html | ✅ | +| TITLE-05 | `Title:"*ly rep*"` | q1.html | q1.html | q1.html | ✅ | +| TITLE-06 | `title:quarterly` | q1.html | q1.html | q1.html | ✅ | +| TITLE-07 | `Title:"QUARTERLY REPORT"` | q1.html | q1.html | q1.html | ✅ | + +### content + +Fixtures: + +- `monthly.txt`, Content = "the monthly reports are due" +- `links.txt`, Content = "see https://opencloud.example.com/help or write to alan@example.org" + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| CONTENT-01 | `Content:report` | no match | no match | no match | ✅ | +| CONTENT-02 | `Content:REPORTS` | monthly.txt | monthly.txt | monthly.txt | ✅ | +| CONTENT-03 | `Content:"monthly reports"` | monthly.txt | monthly.txt | monthly.txt | ✅ | +| CONTENT-04 | `Content:"reports monthly"` | no match | no match | no match | ✅ | +| CONTENT-05 | `Content:report*` | monthly.txt | monthly.txt | monthly.txt | ✅ | +| CONTENT-06 | `Content:*eport*` | monthly.txt | monthly.txt | monthly.txt | ✅ | +| CONTENT-07 | `Content:month*` | monthly.txt | monthly.txt | monthly.txt | ✅ | +| CONTENT-08 | `Content:"https://opencloud.example.com/help"` | links.txt | links.txt | links.txt | ✅ | +| CONTENT-09 | `Content:"alan@example.org"` | links.txt | links.txt | links.txt | ✅ | +| CONTENT-10 | `Content:opencloud` | links.txt | links.txt | links.txt | ✅ | + +### favorites + +Fixtures: + +- `starred.txt`, Favorites = A1B2-Upper +- `plain.txt` +- `keepsakes`, folder, Favorites = A1B2-Upper +- `photo.jpg`, MimeType = image/jpeg, Path = ./keepsakes/photo.jpg + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| FAVORITES-01 | `Favorites:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | keepsakes, starred.txt | ✅ | +| FAVORITES-02 | `favorite:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | keepsakes, starred.txt | ✅ | +| FAVORITES-03 | `Favorites:"somebody-else"` | no match | no match | no match | ✅ | + +### mediatype + +Fixtures: + +- `notes.md`, MimeType = text/markdown +- `photo.jpg`, MimeType = image/jpeg +- `albums`, folder +- `drafts`, folder + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| MEDIATYPE-01 | `mediatype:text/markdown` | notes.md | notes.md | notes.md | ✅ | +| MEDIATYPE-02 | `mediatype:TEXT/MARKDOWN` | notes.md | notes.md | notes.md | ✅ | +| MEDIATYPE-03 | `mediatype:image/jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ | +| MEDIATYPE-04 | `mediatype:*jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ | +| MEDIATYPE-05 | `mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ | +| MEDIATYPE-06 | `mediatype:folder` | albums, drafts | albums, drafts | albums, drafts | ✅ | + +### path + +Fixtures: + +- `parent`, folder +- `child.jpg`, MimeType = image/jpeg, Path = ./parent/child.jpg +- `docs-lower`, folder, Path = ./documents +- `docs-upper`, folder, Path = ./DOCUMENTS +- `docs-mixed`, folder, Path = ./Documents + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| PATH-01 | `path:"./parent"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ | +| PATH-02 | `path:"./parent/child.jpg"` | child.jpg | child.jpg | child.jpg | ✅ | +| PATH-03 | `path:"./Parent"` | no match | no match | no match | ✅ | +| PATH-04 | `path:"*child*"` | child.jpg | child.jpg | child.jpg | ✅ | +| PATH-05 | `path:"./documents"` | docs-lower | docs-lower | docs-lower | ✅ | +| PATH-06 | `path:"./DOCUMENTS"` | docs-upper | docs-upper | docs-upper | ✅ | +| PATH-07 | `path:"./Documents"` | docs-mixed | docs-mixed | docs-mixed | ✅ | +| PATH-08 | `path:"./parent/"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ | + +### fields + +Fixtures: + +- `small.txt`, ID = 1$1!small.txt, Size = 42 +- `old.txt`, ID = 1$1!old.txt, Mtime = 2020-01-01T00:00:00Z +- `known.txt`, ID = 1$1!23 +- `cased.txt`, ID = 1$1!AB-23 +- `hidden.txt`, ID = 1$1!hidden.txt, hidden +- `plain.txt`, ID = 1$1!plain.txt +- `box`, ID = 1$1!box, folder +- `boxed.txt`, ID = 1$1!boxed.txt, Path = ./box/boxed.txt + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| FIELDS-01 | `size:42` | small.txt | small.txt | small.txt | ✅ | +| FIELDS-02 | `mtime<"2021-01-01T00:00:00Z"` | old.txt | old.txt | old.txt | ✅ | +| FIELDS-03 | `id:"1$1!23"` | known.txt | known.txt | known.txt | ✅ | +| FIELDS-04 | `hidden:true` | hidden.txt | hidden.txt | hidden.txt | ✅ | +| FIELDS-05 | `type:file` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | ✅ | +| FIELDS-06 | `type:folder` | box | box | box | ✅ | +| FIELDS-07 | `unknown:field` | no match | no match | no match | ✅ | +| FIELDS-08 | `type:File` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | ✅ | +| FIELDS-09 | `type:FOLDER` | box | box | box | ✅ | +| FIELDS-10 | `hidden:TRUE` | hidden.txt | hidden.txt | hidden.txt | ✅ | +| FIELDS-11 | `id:"1$1!AB-23"` | cased.txt | cased.txt | cased.txt | ✅ | +| FIELDS-12 | `id:"1$1!ab-23"` | no match | no match | no match | ✅ | + +### deleted + +Fixtures: + +- `trashed.txt`, deleted +- `kept.txt` +- `bin`, folder, deleted +- `receipt.txt`, Path = ./bin/receipt.txt, deleted +- `shelf`, folder +- `book.txt`, Path = ./shelf/book.txt + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| DELETED-01 | `name:"*trashed*"` | no match | no match | no match | ✅ | +| DELETED-02 | `name:"*.txt"` | book.txt, kept.txt | book.txt, kept.txt | book.txt, kept.txt | ✅ | +| DELETED-03 | `name:"*receipt*"` | no match | no match | no match | ✅ | +| DELETED-04 | `path:"./bin"` | no match | no match | no match | ✅ | +| DELETED-05 | `path:"./shelf"` | book.txt, shelf | book.txt, shelf | book.txt, shelf | ✅ | + +### visibility + +Fixtures: + +- `visible.txt` +- `dotfile.txt`, hidden +- `.private`, folder, hidden +- `secret.txt`, Path = ./.private/secret.txt, hidden + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| VISIBILITY-01 | `hidden:true` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ | +| VISIBILITY-02 | `hidden:TRUE` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ | +| VISIBILITY-03 | `hidden:false` | visible.txt | visible.txt | visible.txt | ✅ | +| VISIBILITY-04 | `name:"*secret*"` | secret.txt | secret.txt | secret.txt | ✅ | +| VISIBILITY-05 | `path:"./.private"` | .private, secret.txt | .private, secret.txt | .private, secret.txt | ✅ | +| VISIBILITY-06 | `hidden:banana` | no match | no match | no match | ✅ | +| VISIBILITY-07 | `hidden:"true"` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ | + +### boolean + +Fixtures: + +- `alpha.txt`, Tags = red +- `beta.txt`, Tags = blue +- `gamma.md`, MimeType = text/markdown + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| BOOLEAN-01 | `name:"*alpha*" AND name:"*txt*"` | alpha.txt | alpha.txt | alpha.txt | ✅ | +| BOOLEAN-02 | `name:"*alpha*" OR name:"*beta*"` | alpha.txt, beta.txt | alpha.txt, beta.txt | alpha.txt, beta.txt | ✅ | +| BOOLEAN-03 | `name:"*a*" AND NOT name:"*alpha*"` | beta.txt, gamma.md | beta.txt, gamma.md | beta.txt, gamma.md | ✅ | +| BOOLEAN-04 | `name:"*a*" AND tag:("red")` | alpha.txt | alpha.txt | alpha.txt | ✅ | +| BOOLEAN-05 | `(name:"*alpha*" OR name:"*beta*") AND mediatype:text/plain` | alpha.txt, beta.txt | alpha.txt, beta.txt | alpha.txt, beta.txt | ✅ | +| BOOLEAN-06 | `name:"*a*" AND mediatype:text/markdown` | gamma.md | gamma.md | gamma.md | ✅ | + +### samename + +Fixtures: + +- `doc`, folder +- `doc.pdf` +- `file.pdf` +- `doc.pdf`, Path = ./doc/doc.pdf +- `file.pdf`, Path = ./doc/file.pdf + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| SAMENAME-01 | `name:"*doc*"` | 3 items | doc, doc.pdf, doc.pdf | doc, doc.pdf, doc.pdf | ✅ | +| SAMENAME-02 | `name:"*doc*"` in 1$1!1 under ./doc | 2 items | doc, doc.pdf | doc, doc.pdf | ✅ | +| SAMENAME-03 | `name:"*file*"` | 2 items | file.pdf, file.pdf | file.pdf, file.pdf | ✅ | +| SAMENAME-04 | `name:"*file*"` in 1$1!1 under ./doc | 1 items | file.pdf | file.pdf | ✅ | + +### stress + +Fixtures: + +- `quarterly report.docx`, MimeType = application/vnd.openxmlformats-officedocument.wordprocessingml.document, Tags = final, Size = 2000 +- `draft report.txt`, Tags = draft, Size = 50, Mtime = 2020-01-01T00:00:00Z +- `photo.jpg`, MimeType = image/jpeg, Tags = final +- `notes.md`, MimeType = text/markdown, hidden +- `archive`, folder + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| STRESS-01 | `name:"*report*" AND mediatype:document` | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | ✅ | +| STRESS-02 | `name:"*report*" AND NOT tag:("draft")` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | +| STRESS-03 | `(tag:("final") OR tag:("draft")) AND mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ | +| STRESS-04 | `mediatype:document AND mtime>"2021-01-01T00:00:00Z"` | notes.md, quarterly report.docx | notes.md, quarterly report.docx | notes.md, quarterly report.docx | ✅ | +| STRESS-05 | `name:"*report*" AND size>100` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | +| STRESS-06 | `tag:("final") AND NOT mediatype:folder` | photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | ✅ | +| STRESS-07 | `hidden:true AND name:"*notes*"` | notes.md | notes.md | notes.md | ✅ | +| STRESS-08 | `name:quarterly report` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | +| STRESS-09 | `name:"quarterly report"` | no match | no match | no match | ✅ | +| STRESS-10 | `name:"quarterly report.docx"` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | +| STRESS-11 | `NOT tag:("draft")` | archive, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | ✅ | +| STRESS-12 | `tag:("final") OR hidden:true` | notes.md, photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | ✅ | +| STRESS-13 | `(name:"*report*" OR name:"*notes..."draft") OR hidden:true)` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | +| STRESS-14 | `mediatype:image OR (mediatype:document AND tag:("draft"))` | draft report.txt, photo.jpg | draft report.txt, photo.jpg | draft report.txt, photo.jpg | ✅ | +| STRESS-15 | `NOT (mediatype:folder OR hidden:true)` | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | ✅ | +| STRESS-16 | `name:"*report*" AND (size>100 OR tag:("draft"))` | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | ✅ | + +### everything + +Fixtures: + +- `alpha.txt` +- `beta.txt` +- `box`, folder + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| EVERYTHING-01 | `*` | alpha.txt, beta.txt, box | alpha.txt, beta.txt, box | alpha.txt, beta.txt, box | ✅ | +| EVERYTHING-02 | `name:"*"` | alpha.txt, beta.txt, box | alpha.txt, beta.txt, box | alpha.txt, beta.txt, box | ✅ | +| EVERYTHING-03 | `*` with a limit of 2 | 2 items | alpha.txt, beta.txt | alpha.txt, beta.txt | ✅ | +| EVERYTHING-04 | `*` with a limit of -1 | 3 items | alpha.txt, beta.txt, box | alpha.txt, beta.txt, box | ✅ | + +### range + +Fixtures: + +- `small.txt`, Size = 50 +- `big.txt`, Size = 500 +- `ancient.txt`, Size = 10, Mtime = 2020-01-01T00:00:00Z + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| RANGE-01 | `size>100` | big.txt | big.txt | big.txt | ✅ | +| RANGE-02 | `size<100` | ancient.txt, small.txt | ancient.txt, small.txt | ancient.txt, small.txt | ✅ | +| RANGE-03 | `mtime>"2021-01-01T00:00:00Z"` | big.txt, small.txt | big.txt, small.txt | big.txt, small.txt | ✅ | +| RANGE-04 | `mtime<"2021-01-01T00:00:00Z"` | ancient.txt | ancient.txt | ancient.txt | ✅ | +| RANGE-05 | `Mtime:"today"` | big.txt, small.txt | big.txt, small.txt | big.txt, small.txt | ✅ | +| RANGE-06 | `Mtime:"yesterday"` | no match | no match | no match | ✅ | +| RANGE-07 | `mtime>2021` | no match | no match | no match | ✅ | +| RANGE-08 | `name>100` | no match | no match | no match | ✅ | + +### scope + +Fixtures: + +- `parent`, folder +- `child.pdf`, Path = ./parent/child.pdf +- `outside.txt` +- `elsewhere.txt` + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| SCOPE-01 | `*` | child.pdf, elsewhere.txt, outside.txt, parent | child.pdf, elsewhere.txt, outside.txt, parent | child.pdf, elsewhere.txt, outside.txt, parent | ✅ | +| SCOPE-02 | `*` in 1$1!1 | child.pdf, outside.txt, parent | child.pdf, outside.txt, parent | child.pdf, outside.txt, parent | ✅ | +| SCOPE-03 | `*` in 1$1!1 under ./parent | child.pdf, parent | child.pdf, parent | child.pdf, parent | ✅ | +| SCOPE-04 | `*` in 1$1!1 under ./parent/child.pdf | child.pdf | child.pdf | child.pdf | ✅ | +| SCOPE-05 | `name:"*elsewhere*"` in 1$1!1 | no match | no match | no match | ✅ | + +### invalid + +Fixtures: + +- `alpha.txt` + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| INVALID-01 | `AND mediatype:document` | bad request | bad request | bad request | ✅ | +| INVALID-02 | `mediatype:document AND` | alpha.txt | alpha.txt | alpha.txt | ✅ | + +## Operations + +### delete + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| DELETE-01 | takes the resource out of the results, then `name:"*parent*"` | parent | parent | parent | ✅ | +| DELETE-01 | takes the resource out of the results, then `name:"*child*"` | no match | no match | no match | ✅ | +| DELETE-02 | takes the descendants along, then `name:"*parent*"` | no match | no match | no match | ✅ | +| DELETE-02 | takes the descendants along, then `name:"*child*"` | no match | no match | no match | ✅ | +| DELETE-03 | leaves the resource in the index, then `DocCount()` | 2 | 2 | 2 | ✅ | +| DELETE-04 | takes a resource out that was just written, then `name:"*fresh*"` | no match | no match | no match | ✅ | + +### restore + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf +- `file.txt`, ID = 1$1!5, Path = ./.secret/file.txt, hidden + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| RESTORE-01 | brings the descendants back, then `name:"*parent*"` | parent | parent | parent | ✅ | +| RESTORE-01 | brings the descendants back, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ | +| RESTORE-02 | leaves the hidden flag alone, then `hidden:true` | file.txt | file.txt | file.txt | ✅ | + +### purge + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| PURGE-01 | removes one resource, then `name:"*parent*"` | parent | parent | parent | ✅ | +| PURGE-01 | removes one resource, then `name:"*child*"` | no match | no match | no match | ✅ | +| PURGE-01 | removes one resource, then `DocCount()` | 1 | 1 | 1 | ✅ | +| PURGE-02 | removes the tree, then `name:"*parent*"` | no match | no match | no match | ✅ | +| PURGE-02 | removes the tree, then `name:"*child*"` | no match | no match | no match | ✅ | +| PURGE-02 | removes the tree, then `DocCount()` | 0 | 0 | 0 | ✅ | +| PURGE-03 | takes only the deleted ones when it is told to, then `name:"*parent*"` | parent | parent | parent | ✅ | +| PURGE-03 | takes only the deleted ones when it is told to, then `name:"*child*"` | no match | no match | no match | ✅ | + +### purgespace + +Fixtures: + +- `inSpace.txt`, ID = 1$1!4 +- `elsewhere.txt`, ID = 2$2!4 +- `bulk-0.txt`, ID = 1$1!bulk-0.txt +- `bulk-1.txt`, ID = 1$1!bulk-1.txt +- `bulk-2.txt`, ID = 1$1!bulk-2.txt +- `bulk-3.txt`, ID = 1$1!bulk-3.txt +- `bulk-4.txt`, ID = 1$1!bulk-4.txt +- `bulk-5.txt`, ID = 1$1!bulk-5.txt +- `bulk-6.txt`, ID = 1$1!bulk-6.txt +- `bulk-7.txt`, ID = 1$1!bulk-7.txt +- ... and 52 more of the same + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| PURGESPACE-01 | leaves the other space alone, then `name:"*inSpace*"` | no match | no match | no match | ✅ | +| PURGESPACE-01 | leaves the other space alone, then `name:"*elsewhere*"` | elsewhere.txt | elsewhere.txt | elsewhere.txt | ✅ | +| PURGESPACE-02 | takes a space out that holds more than one round, then `name:"*bulk*"` | no match | no match | no match | ✅ | +| PURGESPACE-02 | takes a space out that holds more than one round, then `name:"*elsewhere*"` | elsewhere.txt | elsewhere.txt | elsewhere.txt | ✅ | +| PURGESPACE-02 | takes a space out that holds more than one round, then `DocCount()` | 1 | 1 | 1 | ✅ | + +### move + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| MOVE-01 | carries the descendants to the new path, then `path:"./my/newname/child.pdf"` | child.pdf | child.pdf | child.pdf | ✅ | +| MOVE-01 | carries the descendants to the new path, then `path:"./parent/child.pdf"` | no match | no match | no match | ✅ | +| MOVE-02 | through the trash and back leaves the flag behind, then `hidden:true` | no match | no match | no match | ✅ | + +### rootscope + +Fixtures: + +- `target.txt`, ID = 1$1!3, Path = ./same/path.txt +- `twin.txt`, ID = 2$2!3, Path = ./same/path.txt +- `target.txt`, ID = 1$1!3, Path = ./same/path.txt, deleted +- `twin.txt`, ID = 2$2!3, Path = ./same/path.txt, deleted + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| ROOTSCOPE-01 | deletes only the one in the target root, then `name:"*target*"` | no match | no match | no match | ✅ | +| ROOTSCOPE-01 | deletes only the one in the target root, then `name:"*twin*"` | twin.txt | twin.txt | twin.txt | ✅ | +| ROOTSCOPE-02 | restores only the one in the target root, then `name:"*target*"` | target.txt | target.txt | target.txt | ✅ | +| ROOTSCOPE-02 | restores only the one in the target root, then `name:"*twin*"` | no match | no match | no match | ✅ | +| ROOTSCOPE-03 | moves only the one in the target root, then `path:"./moved.txt"` | moved.txt | moved.txt | moved.txt | ✅ | +| ROOTSCOPE-03 | moves only the one in the target root, then `path:"./same/path.txt"` | twin.txt | twin.txt | twin.txt | ✅ | + +### casepath + +Fixtures: + +- `Documents`, ID = 1$1!2, folder +- `Picture.jpg`, ID = 1$1!3, MimeType = image/jpeg, Path = ./Documents/Picture.jpg + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| CASEPATH-01 | takes the descendants along when deleting, then `path:"./Documents"` | no match | no match | no match | ✅ | +| CASEPATH-02 | takes the descendants along when moving, then `path:"./Other Documents"` | Other Documents, Picture.jpg | Other Documents, Picture.jpg | Other Documents, Picture.jpg | ✅ | +| CASEPATH-02 | takes the descendants along when moving, then `path:"./Documents"` | no match | no match | no match | ✅ | +| CASEPATH-03 | reaches the descendants when purging, then `path:"./Documents"` | no match | no match | no match | ✅ | +| CASEPATH-03 | reaches the descendants when purging, then `DocCount()` | 0 | 0 | 0 | ✅ | + +### hidden + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf +- `parent`, ID = 1$1!2, folder, Path = ./.trash/parent, hidden +- `child.pdf`, ID = 1$1!3, Path = ./.trash/parent/child.pdf, hidden +- `parent`, ID = 1$1!2, folder, Path = ./.parent, hidden +- `child.pdf`, ID = 1$1!3, Path = ./.parent/child.pdf, hidden + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| HIDDEN-01 | follows a move into a dot folder, then `hidden:true` | child.pdf, parent | child.pdf, parent | child.pdf, parent | ✅ | +| HIDDEN-02 | follows a move into a plain folder, then `hidden:true` | no match | no match | no match | ✅ | +| HIDDEN-03 | follows a move renamed with a leading dot, then `hidden:true` | .parent, child.pdf | .parent, child.pdf | .parent, child.pdf | ✅ | +| HIDDEN-04 | follows a move out of a dot folder, then `hidden:true` | no match | no match | no match | ✅ | +| HIDDEN-05 | follows a move renamed without the leading dot, then `hidden:true` | no match | no match | no match | ✅ | +| HIDDEN-06 | follows a move within the same dot folder, then `hidden:true` | child.pdf, moved | child.pdf, moved | child.pdf, moved | ✅ | + +### upsert + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| UPSERT-01 | replaces the resource it already knows, then `name:"*parent*"` | parent | parent | parent | ✅ | +| UPSERT-01 | replaces the resource it already knows, then `name:"*child*"` | no match | no match | no match | ✅ | +| UPSERT-01 | replaces the resource it already knows, then `name:"*renamed*"` | renamed.pdf | renamed.pdf | renamed.pdf | ✅ | +| UPSERT-01 | replaces the resource it already knows, then `DocCount()` | 2 | 2 | 2 | ✅ | + +### idempotency + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| IDEMPOTENCY-01 | deleting the same resource twice is not an error, then `name:"*parent*"` | parent | parent | parent | ✅ | +| IDEMPOTENCY-01 | deleting the same resource twice is not an error, then `name:"*child*"` | no match | no match | no match | ✅ | +| IDEMPOTENCY-02 | deleting a resource the index does not have reports it, then `name:"*parent*"` | parent | parent | parent | ✅ | +| IDEMPOTENCY-02 | deleting a resource the index does not have reports it, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ | +| IDEMPOTENCY-03 | restoring a resource that was never deleted leaves it alone, then `name:"*parent*"` | parent | parent | parent | ✅ | +| IDEMPOTENCY-03 | restoring a resource that was never deleted leaves it alone, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ | +| IDEMPOTENCY-04 | purging the same resource twice reports the second one, then `name:"*parent*"` | parent | parent | parent | ✅ | +| IDEMPOTENCY-04 | purging the same resource twice reports the second one, then `name:"*child*"` | no match | no match | no match | ✅ | +| IDEMPOTENCY-04 | purging the same resource twice reports the second one, then `DocCount()` | 1 | 1 | 1 | ✅ | +| IDEMPOTENCY-05 | purging a resource the index does not have reports it, then `name:"*parent*"` | parent | parent | parent | ✅ | +| IDEMPOTENCY-05 | purging a resource the index does not have reports it, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ | +| IDEMPOTENCY-05 | purging a resource the index does not have reports it, then `DocCount()` | 2 | 2 | 2 | ✅ | +| IDEMPOTENCY-06 | moving a resource onto its own path leaves it where it is, then `path:"./parent/child.pdf"` | child.pdf | child.pdf | child.pdf | ✅ | +| IDEMPOTENCY-06 | moving a resource onto its own path leaves it where it is, then `name:"*parent*"` | parent | parent | parent | ✅ | +| IDEMPOTENCY-07 | purging a whole space that holds nothing is not an error, then `name:"*parent*"` | parent | parent | parent | ✅ | +| IDEMPOTENCY-07 | purging a whole space that holds nothing is not an error, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ | +| IDEMPOTENCY-07 | purging a whole space that holds nothing is not an error, then `DocCount()` | 2 | 2 | 2 | ✅ | + +### batch + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `child.pdf`, ID = 1$1!3, Path = ./parent/child.pdf + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| BATCH-01 | holds what it was given until it is pushed, then `name:"*added*"` | no match | no match | no match | ✅ | +| BATCH-02 | writes what it was given once it is pushed, then `name:"*added*"` | added.pdf | added.pdf | added.pdf | ✅ | +| BATCH-02 | writes what it was given once it is pushed, then `DocCount()` | 3 | 3 | 3 | ✅ | +| BATCH-03 | takes a resource out the same way a delete does, then `name:"*parent*"` | parent | parent | parent | ✅ | +| BATCH-03 | takes a resource out the same way a delete does, then `name:"*child*"` | no match | no match | no match | ✅ | +| BATCH-04 | moves a resource the same way a move does, then `path:"./my/newname/child.pdf"` | child.pdf | child.pdf | child.pdf | ✅ | +| BATCH-04 | moves a resource the same way a move does, then `path:"./parent/child.pdf"` | no match | no match | no match | ✅ | + +## Response + +### entity + +Fixtures: + +- `parent`, ID = 1$1!2, folder +- `bar.pdf`, ID = 1$1!3, MimeType = application/pdf, Path = ./parent/bar.pdf, Size = 1234 +- `notes.txt`, ID = 1$1!4, Content = "foo bar baz" + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| ENTITY-01 | `name:"bar.pdf"` reads `Ref.Path` | ./parent/bar.pdf | ./parent/bar.pdf | ./parent/bar.pdf | ✅ | +| ENTITY-02 | `name:"bar.pdf"` reads `Name` | bar.pdf | bar.pdf | bar.pdf | ✅ | +| ENTITY-03 | `name:"bar.pdf"` reads `Id` | 1$1!3 | 1$1!3 | 1$1!3 | ✅ | +| ENTITY-04 | `name:"bar.pdf"` reads `ParentId` | 1$1!2 | 1$1!2 | 1$1!2 | ✅ | +| ENTITY-05 | `name:"bar.pdf"` reads `Ref.ResourceId` | 1$1!1 | 1$1!1 | 1$1!1 | ✅ | +| ENTITY-06 | `name:"bar.pdf"` reads `Size` | 1234 | 1234 | 1234 | ✅ | +| ENTITY-07 | `name:"bar.pdf"` reads `Type` | 1 | 1 | 1 | ✅ | +| ENTITY-08 | `name:"bar.pdf"` reads `MimeType` | application/pdf | application/pdf | application/pdf | ✅ | +| ENTITY-09 | `name:"bar.pdf"` reads `Deleted` | false | false | false | ✅ | +| ENTITY-10 | `name:"bar.pdf"` reads `Score` | above zero | above zero | above zero | ✅ | +| ENTITY-11 | `path:"./parent"` reads `TotalMatches` | 2 | 2 | 2 | ✅ | +| ENTITY-12 | `name:"*notes*"` reads `Highlights` | "" | "" | "" | ✅ | +| ENTITY-13 | `content:bar` reads `Highlights` | foo bar baz | foo bar baz | foo bar baz | ✅ | +| ENTITY-14 | moved to another parent, then `name:"newname"` reads `ParentId` | 1$1!9 | 1$1!9 | 1$1!9 | ✅ | +| ENTITY-15 | moved to another parent, then `name:"bar.pdf"` reads `ParentId` | 1$1!2 | 1$1!2 | 1$1!2 | ✅ | +| ENTITY-16 | moved to another parent, then `name:"bar.pdf"` reads `Ref.Path` | ./somewher.../bar.pdf | ./somewher.../bar.pdf | ./somewher.../bar.pdf | ✅ | + +### metadata + +Fixtures: + +- `some_song.mp3`, ID = 1$1!5, MimeType = audio/mpeg +- `team.jpg`, ID = 1$1!6, MimeType = image/jpeg + +| Case | Query | expected | bleve | OpenSearch | same? | +|---|---|---|---|---|---| +| METADATA-01 | `*song*` reads `Audio` | all 16 fields unchanged | all 16 fields unchanged | all 16 fields unchanged | ✅ | +| METADATA-02 | `*team*` reads `Location` | all 3 fields unchanged | all 3 fields unchanged | all 3 fields unchanged | ✅ | +| METADATA-03 | `*team*` reads `Audio` | none | none | none | ✅ | diff --git a/services/search/pkg/parity/engines_test.go b/services/search/pkg/parity/engines_test.go new file mode 100644 index 0000000000..b91fcedf07 --- /dev/null +++ b/services/search/pkg/parity/engines_test.go @@ -0,0 +1,127 @@ +package parity + +import ( + "context" + "fmt" + "testing" + + bleveSearch "github.com/blevesearch/bleve/v2" + sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/opencloud-eu/reva/v2/pkg/storagespace" + "github.com/stretchr/testify/require" + + "github.com/opencloud-eu/opencloud/pkg/log" + searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + bleveEngine "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" + "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" + bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +type testEngine struct { + name string + backend search.Engine + settle func() + unavailable string +} + +func newEngines(t *testing.T, index string, fixtures []search.Resource) []testEngine { + t.Helper() + + return []testEngine{ + newBleve(t, fixtures), + newOpenSearch(t, index, fixtures), + } +} + +func newBleve(t *testing.T, fixtures []search.Resource) testEngine { + t.Helper() + + mapping, err := bleveEngine.NewMapping() + require.NoError(t, err, "failed to build the bleve mapping") + + idx, err := bleveSearch.NewMemOnly(mapping) + require.NoError(t, err, "failed to create the bleve index") + t.Cleanup(func() { _ = idx.Close() }) + + backend := bleveEngine.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}) + for _, doc := range fixtures { + require.NoError(t, backend.Upsert(doc.ID, doc), "bleve upsert %s", doc.ID) + } + + return testEngine{name: "bleve", backend: backend, settle: func() {}} +} + +func newOpenSearch(t *testing.T, index string, fixtures []search.Resource) testEngine { + t.Helper() + + tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client) + versioned := opensearch.IndexName(index) + + if err := tc.IndicesReset(t.Context(), []string{versioned}); err != nil { + return testEngine{name: "opensearch", unavailable: err.Error()} + } + + backend, err := opensearch.NewBackend(index, tc.Client()) + if err != nil { + return testEngine{name: "opensearch", unavailable: err.Error()} + } + + t.Cleanup(func() { + require.NoError(t, tc.IndicesDelete(context.Background(), []string{versioned})) + }) + + settle := func() { tc.Require.IndicesRefresh([]string{versioned}, nil) } + for _, doc := range fixtures { + require.NoError(t, backend.Upsert(doc.ID, doc), "opensearch upsert %s", doc.ID) + } + settle() + + return testEngine{name: "opensearch", backend: backend, settle: settle} +} + +func badRequestAnswer(err error) []string { + if err == nil { + return []string{"no error"} + } + + return []string{"bad request"} +} + +func reads(read func(*searchMessage.Match) string) func(*searchService.SearchIndexResponse) []string { + return readsMany(func(m *searchMessage.Match) []string { return []string{read(m)} }) +} + +func readsMany(read func(*searchMessage.Match) []string) func(*searchService.SearchIndexResponse) []string { + return func(resp *searchService.SearchIndexResponse) []string { + if len(resp.Matches) != 1 { + return []string{fmt.Sprintf("%d matches", len(resp.Matches))} + } + + return read(resp.Matches[0]) + } +} + +func resourceID(id *searchMessage.ResourceID) string { + return storagespace.FormatResourceID(&sprovider.ResourceId{ + StorageId: id.GetStorageId(), + SpaceId: id.GetSpaceId(), + OpaqueId: id.GetOpaqueId(), + }) +} + +func hits(t *testing.T, e search.Engine, request *searchService.SearchIndexRequest) []string { + t.Helper() + + resp, err := e.Search(context.Background(), request) + require.NoError(t, err, "the query has to answer, an empty result is an answer") + + names := make([]string, 0, len(resp.Matches)) + for _, m := range resp.Matches { + names = append(names, m.Entity.GetName()) + } + + return names +} diff --git a/services/search/pkg/parity/fixtures_test.go b/services/search/pkg/parity/fixtures_test.go new file mode 100644 index 0000000000..4bfebd5e8c --- /dev/null +++ b/services/search/pkg/parity/fixtures_test.go @@ -0,0 +1,108 @@ +package parity + +import ( + "fmt" + "slices" + "strings" + "time" + + sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + libregraph "github.com/opencloud-eu/libre-graph-api-go" + + "github.com/opencloud-eu/opencloud/services/search/pkg/content" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +var fixtureLongName = strings.Repeat("a", 260) + "needle.txt" + +var fixtureNow = time.Now().UTC() + +type fixtureOption func(*search.Resource) + +func withPath(path string) fixtureOption { return func(r *search.Resource) { r.Path = path } } +func withMime(mime string) fixtureOption { return func(r *search.Resource) { r.MimeType = mime } } +func withTitle(t string) fixtureOption { return func(r *search.Resource) { r.Title = t } } +func withContent(c string) fixtureOption { return func(r *search.Resource) { r.Content = c } } +func withSize(s uint64) fixtureOption { return func(r *search.Resource) { r.Size = s } } +func withMtime(m string) fixtureOption { return func(r *search.Resource) { r.Mtime = m } } +func withID(id string) fixtureOption { return func(r *search.Resource) { r.ID = id } } +func withParent(id string) fixtureOption { return func(r *search.Resource) { r.ParentID = id } } +func withRoot(id string) fixtureOption { return func(r *search.Resource) { r.RootID = id } } +func isHidden() fixtureOption { return func(r *search.Resource) { r.Hidden = true } } +func isDeleted() fixtureOption { return func(r *search.Resource) { r.Deleted = true } } +func withTags(tags ...string) fixtureOption { + return func(r *search.Resource) { r.Tags = tags } +} + +func withFavorite(userID string) fixtureOption { + return func(r *search.Resource) { r.Favorites = []string{userID} } +} + +func withAudio(audio *libregraph.Audio) fixtureOption { + return func(r *search.Resource) { r.Audio = audio } +} + +func withLocation(location *libregraph.GeoCoordinates) fixtureOption { + return func(r *search.Resource) { r.Location = location } +} + +func fixtureDoc(name string, opts ...fixtureOption) search.Resource { + r := search.Resource{ + ID: "1$1!" + name, + RootID: "1$1!1", + ParentID: "1$1!1", + Path: "./" + name, + Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), + Document: content.Document{ + Name: name, + MimeType: "text/plain", + Mtime: fixtureNow.Format(time.RFC3339Nano), + Size: 1000, + }, + } + + for _, opt := range opts { + opt(&r) + } + + return r +} + +func fixtureFolder(name string, opts ...fixtureOption) search.Resource { + folder := []fixtureOption{withMime("httpd/unix-directory")} + r := fixtureDoc(name, append(folder, opts...)...) + r.Type = uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER) + + return r +} + +func fixtureBulk(count int) []search.Resource { + docs := make([]search.Resource, 0, count) + for i := range count { + docs = append(docs, fixtureDoc(fmt.Sprintf("bulk-%d.txt", i))) + } + + return docs +} + +func fixtureTree() (parent, child search.Resource) { + parent = fixtureFolder("parent", withID("1$1!2")) + child = fixtureDoc("child.pdf", withID("1$1!3"), withParent(parent.ID), withPath("./parent/child.pdf")) + + return parent, child +} + +func treeIsLeft(names ...string) []expectation { + left := func(name string) []string { + if slices.Contains(names, name) { + return []string{name} + } + + return nil + } + + return []expectation{ + {`name:"*parent*"`, left("parent")}, + {`name:"*child*"`, left("child.pdf")}, + } +} diff --git a/services/search/pkg/parity/lifecycle_batch_test.go b/services/search/pkg/parity/lifecycle_batch_test.go new file mode 100644 index 0000000000..5234e7758d --- /dev/null +++ b/services/search/pkg/parity/lifecycle_batch_test.go @@ -0,0 +1,83 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func batchLifecycle() lifecycleGroup { + parent, child := fixtureTree() + + added := fixtureDoc("added.pdf", withID("1$1!7"), withParent(parent.ID), withPath("./parent/added.pdf")) + + return lifecycleGroup{ + name: "batch", + fixtures: []search.Resource{parent, child}, + cases: []lifecycleCase{ + { + id: 1, title: "holds what it was given until it is pushed", + do: func(e search.Engine) error { + batch, err := e.NewBatch(100) + if err != nil { + return err + } + + return batch.Upsert(added.ID, added) + }, + expect: []expectation{{`name:"*added*"`, nil}}, + }, + { + id: 2, title: "writes what it was given once it is pushed", + do: func(e search.Engine) error { + batch, err := e.NewBatch(100) + if err != nil { + return err + } + + if err := batch.Upsert(added.ID, added); err != nil { + return err + } + + return batch.Push() + }, + expect: []expectation{{`name:"*added*"`, []string{"added.pdf"}}}, + wantDocCount: conversions.ToPointer(uint64(3)), + }, + { + id: 3, title: "takes a resource out the same way a delete does", + do: func(e search.Engine) error { + batch, err := e.NewBatch(100) + if err != nil { + return err + } + + if err := batch.Delete(child.ID); err != nil { + return err + } + + return batch.Push() + }, + expect: treeIsLeft("parent"), + }, + { + id: 4, title: "moves a resource the same way a move does", + do: func(e search.Engine) error { + batch, err := e.NewBatch(100) + if err != nil { + return err + } + + if err := batch.Move(parent.ID, parent.ParentID, "./my/newname"); err != nil { + return err + } + + return batch.Push() + }, + expect: []expectation{ + {`path:"./my/newname/child.pdf"`, []string{"child.pdf"}}, + {`path:"./parent/child.pdf"`, nil}, + }, + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_casepath_test.go b/services/search/pkg/parity/lifecycle_casepath_test.go new file mode 100644 index 0000000000..95b3b36208 --- /dev/null +++ b/services/search/pkg/parity/lifecycle_casepath_test.go @@ -0,0 +1,48 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func casePathLifecycle() lifecycleGroup { + folder := fixtureFolder("Documents", withID("1$1!2"), withPath("./Documents")) + picture := fixtureDoc("Picture.jpg", + withID("1$1!3"), + withParent(folder.ID), + withMime("image/jpeg"), + withPath("./Documents/Picture.jpg"), + ) + + left := func(names ...string) []expectation { + return []expectation{ + {`path:"./Documents"`, names}, + } + } + + return lifecycleGroup{ + name: "casepath", + fixtures: []search.Resource{folder, picture}, + cases: []lifecycleCase{ + { + id: 1, title: "takes the descendants along when deleting", + do: func(e search.Engine) error { return e.Delete(folder.ID) }, + expect: left(), + }, + { + id: 2, title: "takes the descendants along when moving", + do: func(e search.Engine) error { return e.Move(folder.ID, folder.ParentID, "./Other Documents") }, + expect: []expectation{ + {`path:"./Other Documents"`, []string{"Other Documents", "Picture.jpg"}}, + {`path:"./Documents"`, nil}, + }, + }, + { + id: 3, title: "reaches the descendants when purging", + do: func(e search.Engine) error { return e.Purge(folder.ID, false) }, + expect: left(), + wantDocCount: conversions.ToPointer(uint64(0)), + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_delete_test.go b/services/search/pkg/parity/lifecycle_delete_test.go new file mode 100644 index 0000000000..3a4c49b2c6 --- /dev/null +++ b/services/search/pkg/parity/lifecycle_delete_test.go @@ -0,0 +1,44 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func deleteLifecycle() lifecycleGroup { + parent, child := fixtureTree() + + return lifecycleGroup{ + name: "delete", + fixtures: []search.Resource{parent, child}, + cases: []lifecycleCase{ + { + id: 1, title: "takes the resource out of the results", + do: func(e search.Engine) error { return e.Delete(child.ID) }, + expect: treeIsLeft("parent"), + }, + { + id: 2, title: "takes the descendants along", + do: func(e search.Engine) error { return e.Delete(parent.ID) }, + expect: treeIsLeft(), + }, + { + id: 3, title: "leaves the resource in the index", + do: func(e search.Engine) error { return e.Delete(child.ID) }, + wantDocCount: conversions.ToPointer(uint64(2)), + }, + { + id: 4, title: "takes a resource out that was just written", + do: func(e search.Engine) error { + fresh := fixtureDoc("fresh.txt", withID("1$1!8")) + if err := e.Upsert(fresh.ID, fresh); err != nil { + return err + } + + return e.Delete(fresh.ID) + }, + expect: []expectation{{`name:"*fresh*"`, nil}}, + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_hidden_test.go b/services/search/pkg/parity/lifecycle_hidden_test.go new file mode 100644 index 0000000000..c703d6ce74 --- /dev/null +++ b/services/search/pkg/parity/lifecycle_hidden_test.go @@ -0,0 +1,48 @@ +package parity + +import ( + "path" + + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func hiddenLifecycle() lifecycleGroup { + group := lifecycleGroup{name: "hidden"} + + for _, move := range []struct { + id int + title string + from string + target string + hidden bool + }{ + {1, "into a dot folder", "./parent", "./.trash/parent", true}, + {2, "into a plain folder", "./parent", "./archive/parent", false}, + {3, "renamed with a leading dot", "./parent", "./.parent", true}, + {4, "out of a dot folder", "./.trash/parent", "./archive/parent", false}, + {5, "renamed without the leading dot", "./.parent", "./parent", false}, + {6, "within the same dot folder", "./.trash/parent", "./.trash/moved", true}, + } { + parent, child := fixtureTree() + parent.Path = move.from + parent.Hidden = search.IsHidden(move.from) + child.Path = move.from + "/child.pdf" + child.Hidden = parent.Hidden + + var hidden []string + if move.hidden { + hidden = []string{path.Base(move.target), "child.pdf"} + } + + group.cases = append(group.cases, lifecycleCase{ + id: move.id, title: "follows a move " + move.title, + fixtures: []search.Resource{parent, child}, + do: func(e search.Engine) error { + return e.Move(parent.ID, parent.ParentID, move.target) + }, + expect: []expectation{{`hidden:true`, hidden}}, + }) + } + + return group +} diff --git a/services/search/pkg/parity/lifecycle_idempotency_test.go b/services/search/pkg/parity/lifecycle_idempotency_test.go new file mode 100644 index 0000000000..73b32d86f3 --- /dev/null +++ b/services/search/pkg/parity/lifecycle_idempotency_test.go @@ -0,0 +1,76 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func idempotencyLifecycle() lifecycleGroup { + parent, child := fixtureTree() + missing := "1$1!gone" + + return lifecycleGroup{ + name: "idempotency", + fixtures: []search.Resource{parent, child}, + cases: []lifecycleCase{ + { + id: 1, title: "deleting the same resource twice is not an error", + do: func(e search.Engine) error { + if err := e.Delete(child.ID); err != nil { + return err + } + + return e.Delete(child.ID) + }, + expect: treeIsLeft("parent"), + }, + { + id: 2, title: "deleting a resource the index does not have reports it", + do: func(e search.Engine) error { return e.Delete(missing) }, + expect: treeIsLeft("parent", "child.pdf"), + wantErr: true, + }, + { + id: 3, title: "restoring a resource that was never deleted leaves it alone", + do: func(e search.Engine) error { return e.Restore(child.ID) }, + expect: treeIsLeft("parent", "child.pdf"), + }, + { + id: 4, title: "purging the same resource twice reports the second one", + do: func(e search.Engine) error { + if err := e.Purge(child.ID, false); err != nil { + return err + } + + return e.Purge(child.ID, false) + }, + expect: treeIsLeft("parent"), + wantDocCount: conversions.ToPointer(uint64(1)), + wantErr: true, + }, + { + id: 5, title: "purging a resource the index does not have reports it", + do: func(e search.Engine) error { return e.Purge(missing, false) }, + expect: treeIsLeft("parent", "child.pdf"), + wantDocCount: conversions.ToPointer(uint64(2)), + wantErr: true, + }, + { + id: 6, title: "moving a resource onto its own path leaves it where it is", + do: func(e search.Engine) error { + return e.Move(parent.ID, parent.ParentID, parent.Path) + }, + expect: []expectation{ + {`path:"./parent/child.pdf"`, []string{"child.pdf"}}, + {`name:"*parent*"`, []string{"parent"}}, + }, + }, + { + id: 7, title: "purging a whole space that holds nothing is not an error", + do: func(e search.Engine) error { return e.PurgeSpace("9$9!9") }, + expect: treeIsLeft("parent", "child.pdf"), + wantDocCount: conversions.ToPointer(uint64(2)), + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_move_test.go b/services/search/pkg/parity/lifecycle_move_test.go new file mode 100644 index 0000000000..eb54fd16b6 --- /dev/null +++ b/services/search/pkg/parity/lifecycle_move_test.go @@ -0,0 +1,37 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func moveLifecycle() lifecycleGroup { + parent, child := fixtureTree() + + return lifecycleGroup{ + name: "move", + fixtures: []search.Resource{parent, child}, + cases: []lifecycleCase{ + { + id: 1, title: "carries the descendants to the new path", + do: func(e search.Engine) error { + return e.Move(parent.ID, parent.ParentID, "./my/newname") + }, + expect: []expectation{ + {`path:"./my/newname/child.pdf"`, []string{"child.pdf"}}, + {`path:"./parent/child.pdf"`, nil}, + }, + }, + { + id: 2, title: "through the trash and back leaves the flag behind", + do: func(e search.Engine) error { + if err := e.Move(parent.ID, parent.ParentID, "./.trash/parent"); err != nil { + return err + } + + return e.Move(parent.ID, parent.ParentID, "./parent") + }, + expect: []expectation{{`hidden:true`, nil}}, + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_purge_test.go b/services/search/pkg/parity/lifecycle_purge_test.go new file mode 100644 index 0000000000..841d02c53d --- /dev/null +++ b/services/search/pkg/parity/lifecycle_purge_test.go @@ -0,0 +1,40 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func purgeLifecycle() lifecycleGroup { + parent, child := fixtureTree() + + return lifecycleGroup{ + name: "purge", + fixtures: []search.Resource{parent, child}, + cases: []lifecycleCase{ + { + id: 1, title: "removes one resource", + do: func(e search.Engine) error { return e.Purge(child.ID, false) }, + expect: treeIsLeft("parent"), + wantDocCount: conversions.ToPointer(uint64(1)), + }, + { + id: 2, title: "removes the tree", + do: func(e search.Engine) error { return e.Purge(parent.ID, false) }, + expect: treeIsLeft(), + wantDocCount: conversions.ToPointer(uint64(0)), + }, + { + id: 3, title: "takes only the deleted ones when it is told to", + do: func(e search.Engine) error { + if err := e.Delete(child.ID); err != nil { + return err + } + + return e.Purge(parent.ID, true) + }, + expect: treeIsLeft("parent"), + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_purgespace_test.go b/services/search/pkg/parity/lifecycle_purgespace_test.go new file mode 100644 index 0000000000..543de75b8d --- /dev/null +++ b/services/search/pkg/parity/lifecycle_purgespace_test.go @@ -0,0 +1,36 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func purgeSpaceLifecycle() lifecycleGroup { + inSpace := fixtureDoc("inSpace.txt", withID("1$1!4")) + elsewhere := fixtureDoc("elsewhere.txt", withID("2$2!4"), withRoot("2$2!1")) + + return lifecycleGroup{ + name: "purgespace", + fixtures: []search.Resource{inSpace, elsewhere}, + cases: []lifecycleCase{ + { + id: 1, title: "leaves the other space alone", + do: func(e search.Engine) error { return e.PurgeSpace(inSpace.RootID) }, + expect: []expectation{ + {`name:"*inSpace*"`, nil}, + {`name:"*elsewhere*"`, []string{"elsewhere.txt"}}, + }, + }, + { + id: 2, title: "takes a space out that holds more than one round", + fixtures: append(fixtureBulk(60), elsewhere), + do: func(e search.Engine) error { return e.PurgeSpace(inSpace.RootID) }, + expect: []expectation{ + {`name:"*bulk*"`, nil}, + {`name:"*elsewhere*"`, []string{"elsewhere.txt"}}, + }, + wantDocCount: conversions.ToPointer(uint64(1)), + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_restore_test.go b/services/search/pkg/parity/lifecycle_restore_test.go new file mode 100644 index 0000000000..c85c4c596a --- /dev/null +++ b/services/search/pkg/parity/lifecycle_restore_test.go @@ -0,0 +1,40 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func restoreLifecycle() lifecycleGroup { + parent, child := fixtureTree() + secret := fixtureDoc("file.txt", withID("1$1!5"), withPath("./.secret/file.txt"), isHidden()) + + return lifecycleGroup{ + name: "restore", + fixtures: []search.Resource{parent, child}, + cases: []lifecycleCase{ + { + id: 1, title: "brings the descendants back", + do: func(e search.Engine) error { + if err := e.Delete(parent.ID); err != nil { + return err + } + + return e.Restore(parent.ID) + }, + expect: treeIsLeft("parent", "child.pdf"), + }, + { + id: 2, title: "leaves the hidden flag alone", + fixtures: []search.Resource{secret}, + do: func(e search.Engine) error { + if err := e.Delete(secret.ID); err != nil { + return err + } + + return e.Restore(secret.ID) + }, + expect: []expectation{{`hidden:true`, []string{"file.txt"}}}, + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_rootscope_test.go b/services/search/pkg/parity/lifecycle_rootscope_test.go new file mode 100644 index 0000000000..163df7d51f --- /dev/null +++ b/services/search/pkg/parity/lifecycle_rootscope_test.go @@ -0,0 +1,50 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func rootScopeLifecycle() lifecycleGroup { + const shared = "./same/path.txt" + + target := fixtureDoc("target.txt", withID("1$1!3"), withPath(shared)) + twin := fixtureDoc("twin.txt", withID("2$2!3"), withRoot("2$2!1"), withParent("2$2!2"), withPath(shared)) + + deleted := func(r search.Resource) search.Resource { + r.Deleted = true + + return r + } + + return lifecycleGroup{ + name: "rootscope", + fixtures: []search.Resource{target, twin}, + cases: []lifecycleCase{ + { + id: 1, title: "deletes only the one in the target root", + do: func(e search.Engine) error { return e.Delete(target.ID) }, + expect: []expectation{ + {`name:"*target*"`, nil}, + {`name:"*twin*"`, []string{"twin.txt"}}, + }, + }, + { + id: 2, title: "restores only the one in the target root", + fixtures: []search.Resource{deleted(target), deleted(twin)}, + do: func(e search.Engine) error { return e.Restore(target.ID) }, + expect: []expectation{ + {`name:"*target*"`, []string{"target.txt"}}, + {`name:"*twin*"`, nil}, + }, + }, + { + id: 3, title: "moves only the one in the target root", + do: func(e search.Engine) error { return e.Move(target.ID, target.ParentID, "./moved.txt") }, + expect: []expectation{ + {`path:"./moved.txt"`, []string{"moved.txt"}}, + {`path:"` + shared + `"`, []string{"twin.txt"}}, + }, + }, + }, + } +} diff --git a/services/search/pkg/parity/lifecycle_upsert_test.go b/services/search/pkg/parity/lifecycle_upsert_test.go new file mode 100644 index 0000000000..d867d1dc1d --- /dev/null +++ b/services/search/pkg/parity/lifecycle_upsert_test.go @@ -0,0 +1,27 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func upsertLifecycle() lifecycleGroup { + parent, child := fixtureTree() + + renamed := child + renamed.Name = "renamed.pdf" + renamed.Path = "./parent/renamed.pdf" + + return lifecycleGroup{ + name: "upsert", + fixtures: []search.Resource{parent, child}, + cases: []lifecycleCase{ + { + id: 1, title: "replaces the resource it already knows", + do: func(e search.Engine) error { return e.Upsert(renamed.ID, renamed) }, + expect: append(treeIsLeft("parent"), expectation{`name:"*renamed*"`, []string{"renamed.pdf"}}), + wantDocCount: conversions.ToPointer(uint64(2)), + }, + }, + } +} diff --git a/services/search/pkg/parity/matrix_test.go b/services/search/pkg/parity/matrix_test.go new file mode 100644 index 0000000000..3ec33bd4cc --- /dev/null +++ b/services/search/pkg/parity/matrix_test.go @@ -0,0 +1,395 @@ +package parity + +import ( + "fmt" + "os" + "sort" + "strings" + "sync" + + sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/opencloud-eu/reva/v2/pkg/storagespace" + + searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +const matrixFile = "README.md" + +type matrixRow struct { + section string + wantCount *int + wantBadRequest bool + limit int32 + ref *searchMessage.Reference + group string + id string + query string + reads string + context string + want []string + answered map[string][]string + skipped map[string]bool + + groupAt, caseAt, queryAt int +} + +var ( + matrixMu sync.Mutex + matrixRows = map[string]*matrixRow{} +) + +func recordAnswer(row matrixRow, engine string, answer []string) { + matrixMu.Lock() + defer matrixMu.Unlock() + + key := fmt.Sprintf("%s/%s/%s", row.group, row.id, row.query) + known, ok := matrixRows[key] + if !ok { + row.answered = map[string][]string{} + row.skipped = map[string]bool{} + known = &row + matrixRows[key] = known + } + + if answer == nil { + known.skipped[engine] = true + return + } + + known.answered[engine] = answer +} + +func recordSkip(row matrixRow, engine string) { + recordAnswer(row, engine, nil) +} + +func writeMatrix() { + matrixMu.Lock() + defer matrixMu.Unlock() + + rows := make([]*matrixRow, 0, len(matrixRows)) + skipped, incomplete := false, false + for _, row := range matrixRows { + rows = append(rows, row) + skipped = skipped || len(row.skipped) > 0 + incomplete = incomplete || len(row.answered)+len(row.skipped) != 2 + } + + switch { + case len(rows) != matrixExpectedRows(): + fmt.Fprintf(os.Stderr, "%s left alone, %d of %d cases ran\n", matrixFile, len(rows), matrixExpectedRows()) + return + case skipped: + fmt.Fprintf(os.Stderr, "%s left alone, an engine was not reachable\n", matrixFile) + return + case incomplete: + fmt.Fprintf(os.Stderr, "%s left alone, an engine died before it answered\n", matrixFile) + return + } + + sort.Slice(rows, func(i, j int) bool { + a, b := rows[i], rows[j] + switch { + case a.groupAt != b.groupAt: + return a.groupAt < b.groupAt + case a.caseAt != b.caseAt: + return a.caseAt < b.caseAt + default: + return a.queryAt < b.queryAt + } + }) + + out := &strings.Builder{} + + group, section := "", "" + for _, row := range rows { + if row.section != section { + section = row.section + if out.Len() > 0 { + out.WriteString("\n") + } + + fmt.Fprintf(out, "## %s\n", section) + } + + if row.group != group { + group = row.group + fmt.Fprintf(out, "\n### %s\n\n%s\n\n", group, matrixFixtures(group)) + out.WriteString("| Case | Query | expected | bleve | OpenSearch | same? |\n") + out.WriteString("|---|---|---|---|---|---|\n") + } + + query := "`" + shortenQuery(row.query) + "`" + if row.ref != nil { + query += " " + matrixScope(row.ref) + } + + if row.limit != 0 { + query = fmt.Sprintf("%s with a limit of %d", query, row.limit) + } + + if row.reads != "" { + query += " reads `" + row.reads + "`" + } + + if row.context != "" { + query = row.context + ", then " + query + } + + expected := matrixNames(row.want) + switch { + case row.wantBadRequest: + expected = "bad request" + case row.wantCount != nil: + expected = fmt.Sprintf("%d items", *row.wantCount) + } + + fmt.Fprintf(out, "| %s | %s | %s | %s | %s | %s |\n", + row.id, query, expected, + matrixNames(row.answered["bleve"]), matrixNames(row.answered["opensearch"]), matrixVerdict(row)) + } + + if err := os.WriteFile(matrixFile, []byte(out.String()), 0o644); err != nil { + fmt.Fprintf(os.Stderr, "failed to write %s: %v\n", matrixFile, err) + } +} + +func matrixExpectedRows() int { + rows := 0 + for _, g := range queryGroups() { + rows += len(g.cases) + } + + for _, g := range lifecycleGroups() { + for _, c := range g.cases { + rows += len(c.expect) + if c.wantDocCount != nil { + rows++ + } + } + } + + for _, g := range responseGroups() { + rows += len(g.cases) + } + + return rows +} + +func matrixFixtures(group string) string { + var ( + fixtures []search.Resource + withIDs bool + ) + + for _, g := range queryGroups() { + if g.name != group { + continue + } + + fixtures = g.fixtures + for _, c := range g.cases { + withIDs = withIDs || strings.Contains(c.query, "id:") + } + } + + for _, g := range lifecycleGroups() { + if g.name != group { + continue + } + + withIDs = true + fixtures = g.fixtures + for _, c := range g.cases { + fixtures = append(fixtures, c.fixtures...) + } + } + + for _, g := range responseGroups() { + if g.name != group { + continue + } + + withIDs = true + fixtures = g.fixtures + } + + if len(fixtures) == 0 { + return "Fixtures: none" + } + + seen := map[string]bool{} + lines := []string{"Fixtures:", ""} + for _, f := range fixtures { + line := "- `" + shorten(f.Name) + "`" + if fields := fixtureFields(f, withIDs); fields != "" { + line += ", " + fields + } + + if seen[line] { + continue + } + + seen[line] = true + lines = append(lines, line) + } + + if listed := len(lines) - 2; listed > 12 { + lines = append(lines[:12], fmt.Sprintf("- ... and %d more of the same", listed-10)) + } + + return strings.Join(lines, "\n") +} + +func fixtureFields(f search.Resource, withID bool) string { + var fields []string + + add := func(format string, args ...any) { + fields = append(fields, fmt.Sprintf(format, args...)) + } + + if withID { + add("ID = %s", f.ID) + } + + if f.Type == uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER) { + add("folder") + } else if f.MimeType != "text/plain" { + add("MimeType = %s", f.MimeType) + } + + if f.Path != "./"+f.Name { + add("Path = %s", f.Path) + } + + if f.Title != "" { + add("Title = %q", f.Title) + } + + if f.Content != "" { + add("Content = %q", f.Content) + } + + if len(f.Tags) > 0 { + add("Tags = %s", strings.Join(f.Tags, ", ")) + } + + if len(f.Favorites) > 0 { + add("Favorites = %s", strings.Join(f.Favorites, ", ")) + } + + if f.Size != 1000 { + add("Size = %d", f.Size) + } + + if !strings.HasPrefix(f.Mtime, fixtureNow.Format("2006-01-02")) { + add("Mtime = %s", f.Mtime) + } + + if f.Hidden { + add("hidden") + } + + if f.Deleted { + add("deleted") + } + + return strings.Join(fields, ", ") +} + +func matrixScope(ref *searchMessage.Reference) string { + space := storagespace.FormatResourceID(&sprovider.ResourceId{ + StorageId: ref.GetResourceId().GetStorageId(), + SpaceId: ref.GetResourceId().GetSpaceId(), + OpaqueId: ref.GetResourceId().GetOpaqueId(), + }) + + if path := ref.GetPath(); path != "" { + return "in " + space + " under " + path + } + + return "in " + space +} + +func matrixNames(names []string) string { + if len(names) == 0 { + return "no match" + } + + shortened := make([]string, 0, len(names)) + for _, name := range names { + shortened = append(shortened, shorten(name)) + } + sort.Strings(shortened) + + return strings.Join(shortened, ", ") +} + +func shorten(name string) string { + if len(name) <= 24 { + return name + } + + return name[:10] + "..." + name[len(name)-8:] +} + +func shortenQuery(q string) string { + if len(q) <= 64 { + return q + } + + return q[:32] + "..." + q[len(q)-24:] +} + +func matrixVerdict(row *matrixRow) string { + var off []string + for _, engine := range []string{"bleve", "opensearch"} { + if row.wantBadRequest { + if matrixNames(row.answered[engine]) != "bad request" { + off = append(off, engine) + } + + continue + } + + if row.wantCount != nil { + if len(row.answered[engine]) != *row.wantCount { + off = append(off, engine) + } + + continue + } + + if matrixNames(row.answered[engine]) != matrixNames(row.want) { + off = append(off, engine) + } + } + + if len(off) == 0 { + return "✅" + } + + return "❌" +} + +func (c lifecycleCase) matrixRows(group string, groupAt, caseAt int) []matrixRow { + rows := make([]matrixRow, 0, len(c.expect)+1) + for i, expect := range c.expect { + rows = append(rows, matrixRow{ + section: "Operations", group: group, id: c.label(group), query: expect.query, context: c.title, want: expect.want, + groupAt: groupAt, caseAt: caseAt, queryAt: i, + }) + } + + if c.wantDocCount != nil { + rows = append(rows, matrixRow{ + section: "Operations", group: group, id: c.label(group), query: "DocCount()", context: c.title, + want: []string{fmt.Sprint(*c.wantDocCount)}, + groupAt: groupAt, caseAt: caseAt, queryAt: len(c.expect), + }) + } + + return rows +} diff --git a/services/search/pkg/parity/parity_test.go b/services/search/pkg/parity/parity_test.go new file mode 100644 index 0000000000..b5b149cb46 --- /dev/null +++ b/services/search/pkg/parity/parity_test.go @@ -0,0 +1,318 @@ +package parity + +import ( + "context" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/opencloud-eu/reva/v2/pkg/errtypes" + + "github.com/opencloud-eu/opencloud/pkg/kql" + searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/pkg/config" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +var defaultConfig *config.Config + +func TestMain(m *testing.M) { + // the fixtures are stamped with fixtureNow, holding the clock still keeps + // "today" on their side of midnight for the whole run + kql.PatchTimeNow(func() time.Time { return fixtureNow }) + + cfg, done, err := opensearchtest.SetupTests(context.Background()) + if err != nil { + _, _ = fmt.Fprintln(os.Stderr, "Failed to setup tests:", err) + os.Exit(1) + return + } + + defaultConfig = cfg + code := m.Run() + writeMatrix() + done() + os.Exit(code) +} + +type queryCase struct { + id int + query string + want []string + limit int32 + wantCount *int + ref *searchMessage.Reference + wantBadRequest bool +} + +func (c queryCase) label(group string) string { + return fmt.Sprintf("%s-%02d", strings.ToUpper(group), c.id) +} + +func (c queryCase) name(group string) string { + return c.label(group) + " " + c.query +} + +type queryGroup struct { + name string + fixtures []search.Resource + cases []queryCase +} + +func queryGroups() []queryGroup { + return []queryGroup{ + nameGroup(), + extensionGroup(), + tagsGroup(), + titleGroup(), + contentGroup(), + favoritesGroup(), + mediatypeGroup(), + pathGroup(), + fieldsGroup(), + deletedGroup(), + visibilityGroup(), + booleanGroup(), + samenameGroup(), + stressGroup(), + everythingGroup(), + rangeGroup(), + scopeGroup(), + invalidGroup(), + } +} + +type responseCase struct { + id int + do func(e search.Engine) error + after string + query string + reads string + read func(*searchService.SearchIndexResponse) []string + want []string +} + +func (c responseCase) label(group string) string { + return fmt.Sprintf("%s-%02d", strings.ToUpper(group), c.id) +} + +func (c responseCase) name(group string) string { + return c.label(group) + " " + c.reads +} + +type responseGroup struct { + name string + fixtures []search.Resource + cases []responseCase +} + +func responseGroups() []responseGroup { + return []responseGroup{ + entityGroup(), + metadataGroup(), + } +} + +type lifecycleCase struct { + id int + title string + do func(e search.Engine) error + expect []expectation + wantErr bool + + fixtures []search.Resource + + wantDocCount *uint64 +} + +type expectation struct { + query string + want []string +} + +func (c lifecycleCase) label(group string) string { + return fmt.Sprintf("%s-%02d", strings.ToUpper(group), c.id) +} + +func (c lifecycleCase) name(group string) string { + return c.label(group) + " " + c.title +} + +type lifecycleGroup struct { + name string + fixtures []search.Resource + cases []lifecycleCase +} + +func lifecycleGroups() []lifecycleGroup { + return []lifecycleGroup{ + deleteLifecycle(), + restoreLifecycle(), + purgeLifecycle(), + purgeSpaceLifecycle(), + moveLifecycle(), + rootScopeLifecycle(), + casePathLifecycle(), + hiddenLifecycle(), + upsertLifecycle(), + idempotencyLifecycle(), + batchLifecycle(), + } +} + +func TestEngineParity(t *testing.T) { + for groupAt, group := range queryGroups() { + t.Run(group.name, func(t *testing.T) { + t.Parallel() + + engines := newEngines(t, "opencloud-test-engine-parity-"+group.name, group.fixtures) + + for caseAt, c := range group.cases { + row := matrixRow{ + section: "Queries", group: group.name, id: c.label(group.name), query: c.query, want: c.want, wantCount: c.wantCount, wantBadRequest: c.wantBadRequest, limit: c.limit, ref: c.ref, + groupAt: groupAt, caseAt: caseAt, + } + + t.Run(c.name(group.name), func(t *testing.T) { + for _, e := range engines { + t.Run(e.name, func(t *testing.T) { + if e.unavailable != "" { + recordSkip(row, e.name) + t.Skip(e.unavailable) + } + + request := &searchService.SearchIndexRequest{Query: c.query, PageSize: c.limit, Ref: c.ref} + + if c.wantBadRequest { + _, err := e.backend.Search(context.Background(), request) + recordAnswer(row, e.name, badRequestAnswer(err)) + require.IsType(t, errtypes.BadRequest(""), err) + + return + } + + answer := hits(t, e.backend, request) + recordAnswer(row, e.name, answer) + + if c.wantCount != nil { + require.Len(t, answer, *c.wantCount) + return + } + + require.ElementsMatch(t, c.want, answer) + }) + } + }) + } + }) + } +} + +func TestEngineParityResponse(t *testing.T) { + offset := len(queryGroups()) + len(lifecycleGroups()) + + for groupAt, group := range responseGroups() { + t.Run(group.name, func(t *testing.T) { + t.Parallel() + + shared := newEngines(t, "opencloud-test-engine-parity-"+group.name, group.fixtures) + + for caseAt, c := range group.cases { + row := matrixRow{ + section: "Response", group: group.name, id: c.label(group.name), query: c.query, reads: c.reads, context: c.after, want: c.want, + groupAt: offset + groupAt, caseAt: caseAt, + } + + t.Run(c.name(group.name), func(t *testing.T) { + engines := shared + if c.do != nil { + engines = newEngines(t, fmt.Sprintf("opencloud-test-engine-parity-%s-%d", group.name, c.id), group.fixtures) + } + + for _, e := range engines { + t.Run(e.name, func(t *testing.T) { + if e.unavailable != "" { + recordSkip(row, e.name) + t.Skip(e.unavailable) + } + + if c.do != nil { + require.NoError(t, c.do(e.backend), "the operation under test failed") + e.settle() + } + + resp, err := e.backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: c.query}) + require.NoError(t, err, "the query has to answer, an empty result is an answer") + + answer := c.read(resp) + recordAnswer(row, e.name, answer) + require.Equal(t, c.want, answer, c.reads) + }) + } + }) + } + }) + } +} + +func TestEngineParityLifecycle(t *testing.T) { + for groupAt, group := range lifecycleGroups() { + t.Run(group.name, func(t *testing.T) { + t.Parallel() + + for caseAt, c := range group.cases { + t.Run(c.name(group.name), func(t *testing.T) { + t.Parallel() + + fixtures := c.fixtures + if fixtures == nil { + fixtures = group.fixtures + } + + index := fmt.Sprintf("opencloud-test-engine-parity-%s-%d", group.name, c.id) + rows := c.matrixRows(group.name, len(queryGroups())+groupAt, caseAt) + + for _, e := range newEngines(t, index, fixtures) { + t.Run(e.name, func(t *testing.T) { + if e.unavailable != "" { + for _, row := range rows { + recordSkip(row, e.name) + } + + t.Skip(e.unavailable) + } + + err := c.do(e.backend) + if c.wantErr { + require.Error(t, err, "the operation had to report that it did not find the resource") + } else { + require.NoError(t, err, "the operation under test failed") + } + + e.settle() + + for i, expect := range c.expect { + answer := hits(t, e.backend, &searchService.SearchIndexRequest{Query: expect.query}) + recordAnswer(rows[i], e.name, answer) + require.ElementsMatch(t, expect.want, answer, expect.query) + } + + if c.wantDocCount != nil { + count, err := e.backend.DocCount() + require.NoError(t, err) + recordAnswer(rows[len(rows)-1], e.name, []string{fmt.Sprint(count)}) + require.Equal(t, *c.wantDocCount, count, "documents left in the index") + } + }) + } + }) + } + }) + } +} diff --git a/services/search/pkg/parity/query_boolean_test.go b/services/search/pkg/parity/query_boolean_test.go new file mode 100644 index 0000000000..4f4349989d --- /dev/null +++ b/services/search/pkg/parity/query_boolean_test.go @@ -0,0 +1,24 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func booleanGroup() queryGroup { + return queryGroup{ + name: "boolean", + fixtures: []search.Resource{ + fixtureDoc("alpha.txt", withTags("red")), + fixtureDoc("beta.txt", withTags("blue")), + fixtureDoc("gamma.md", withMime("text/markdown")), + }, + cases: []queryCase{ + {id: 1, query: `name:"*alpha*" AND name:"*txt*"`, want: []string{"alpha.txt"}}, + {id: 2, query: `name:"*alpha*" OR name:"*beta*"`, want: []string{"alpha.txt", "beta.txt"}}, + {id: 3, query: `name:"*a*" AND NOT name:"*alpha*"`, want: []string{"beta.txt", "gamma.md"}}, + {id: 4, query: `name:"*a*" AND tag:("red")`, want: []string{"alpha.txt"}}, + {id: 5, query: `(name:"*alpha*" OR name:"*beta*") AND mediatype:text/plain`, want: []string{"alpha.txt", "beta.txt"}}, + {id: 6, query: `name:"*a*" AND mediatype:text/markdown`, want: []string{"gamma.md"}}, + }, + } +} diff --git a/services/search/pkg/parity/query_content_test.go b/services/search/pkg/parity/query_content_test.go new file mode 100644 index 0000000000..010b5545c3 --- /dev/null +++ b/services/search/pkg/parity/query_content_test.go @@ -0,0 +1,27 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func contentGroup() queryGroup { + return queryGroup{ + name: "content", + fixtures: []search.Resource{ + fixtureDoc("monthly.txt", withContent("the monthly reports are due")), + fixtureDoc("links.txt", withContent("see https://opencloud.example.com/help or write to alan@example.org")), + }, + cases: []queryCase{ + {id: 1, query: `Content:report`}, + {id: 2, query: `Content:REPORTS`, want: []string{"monthly.txt"}}, + {id: 3, query: `Content:"monthly reports"`, want: []string{"monthly.txt"}}, + {id: 4, query: `Content:"reports monthly"`}, + {id: 5, query: `Content:report*`, want: []string{"monthly.txt"}}, + {id: 6, query: `Content:*eport*`, want: []string{"monthly.txt"}}, + {id: 7, query: `Content:month*`, want: []string{"monthly.txt"}}, + {id: 8, query: `Content:"https://opencloud.example.com/help"`, want: []string{"links.txt"}}, + {id: 9, query: `Content:"alan@example.org"`, want: []string{"links.txt"}}, + {id: 10, query: `Content:opencloud`, want: []string{"links.txt"}}, + }, + } +} diff --git a/services/search/pkg/parity/query_deleted_test.go b/services/search/pkg/parity/query_deleted_test.go new file mode 100644 index 0000000000..1e048b1e9d --- /dev/null +++ b/services/search/pkg/parity/query_deleted_test.go @@ -0,0 +1,26 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func deletedGroup() queryGroup { + return queryGroup{ + name: "deleted", + fixtures: []search.Resource{ + fixtureDoc("trashed.txt", isDeleted()), + fixtureDoc("kept.txt"), + fixtureFolder("bin", isDeleted()), + fixtureDoc("receipt.txt", withParent("1$1!bin"), withPath("./bin/receipt.txt"), isDeleted()), + fixtureFolder("shelf"), + fixtureDoc("book.txt", withParent("1$1!shelf"), withPath("./shelf/book.txt")), + }, + cases: []queryCase{ + {id: 1, query: `name:"*trashed*"`}, + {id: 2, query: `name:"*.txt"`, want: []string{"kept.txt", "book.txt"}}, + {id: 3, query: `name:"*receipt*"`}, + {id: 4, query: `path:"./bin"`}, + {id: 5, query: `path:"./shelf"`, want: []string{"shelf", "book.txt"}}, + }, + } +} diff --git a/services/search/pkg/parity/query_everything_test.go b/services/search/pkg/parity/query_everything_test.go new file mode 100644 index 0000000000..449a93e8d9 --- /dev/null +++ b/services/search/pkg/parity/query_everything_test.go @@ -0,0 +1,23 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func everythingGroup() queryGroup { + return queryGroup{ + name: "everything", + fixtures: []search.Resource{ + fixtureDoc("alpha.txt"), + fixtureDoc("beta.txt"), + fixtureFolder("box"), + }, + cases: []queryCase{ + {id: 1, query: `*`, want: []string{"alpha.txt", "beta.txt", "box"}}, + {id: 2, query: `name:"*"`, want: []string{"alpha.txt", "beta.txt", "box"}}, + {id: 3, query: `*`, limit: 2, wantCount: conversions.ToPointer(2)}, + {id: 4, query: `*`, limit: -1, wantCount: conversions.ToPointer(3)}, + }, + } +} diff --git a/services/search/pkg/parity/query_extension_test.go b/services/search/pkg/parity/query_extension_test.go new file mode 100644 index 0000000000..a813640bcb --- /dev/null +++ b/services/search/pkg/parity/query_extension_test.go @@ -0,0 +1,22 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func extensionGroup() queryGroup { + return queryGroup{ + name: "extension", + fixtures: []search.Resource{ + fixtureDoc("report.txt"), + fixtureDoc("notes.md", withMime("text/markdown")), + fixtureFolder("archive"), + }, + cases: []queryCase{ + {id: 1, query: `txt`, want: []string{"report.txt"}}, + {id: 2, query: `md`, want: []string{"notes.md"}}, + {id: 3, query: `name:"*.txt"`, want: []string{"report.txt"}}, + {id: 4, query: `report`, want: []string{"report.txt"}}, + }, + } +} diff --git a/services/search/pkg/parity/query_favorites_test.go b/services/search/pkg/parity/query_favorites_test.go new file mode 100644 index 0000000000..5413f3eaf4 --- /dev/null +++ b/services/search/pkg/parity/query_favorites_test.go @@ -0,0 +1,22 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func favoritesGroup() queryGroup { + return queryGroup{ + name: "favorites", + fixtures: []search.Resource{ + fixtureDoc("starred.txt", withFavorite("A1B2-Upper")), + fixtureDoc("plain.txt"), + fixtureFolder("keepsakes", withFavorite("A1B2-Upper")), + fixtureDoc("photo.jpg", withParent("1$1!keepsakes"), withPath("./keepsakes/photo.jpg"), withMime("image/jpeg")), + }, + cases: []queryCase{ + {id: 1, query: `Favorites:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}}, + {id: 2, query: `favorite:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}}, + {id: 3, query: `Favorites:"somebody-else"`}, + }, + } +} diff --git a/services/search/pkg/parity/query_fields_test.go b/services/search/pkg/parity/query_fields_test.go new file mode 100644 index 0000000000..4e737c90a3 --- /dev/null +++ b/services/search/pkg/parity/query_fields_test.go @@ -0,0 +1,35 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func fieldsGroup() queryGroup { + return queryGroup{ + name: "fields", + fixtures: []search.Resource{ + fixtureDoc("small.txt", withSize(42)), + fixtureDoc("old.txt", withMtime("2020-01-01T00:00:00Z")), + fixtureDoc("known.txt", withID("1$1!23")), + fixtureDoc("cased.txt", withID("1$1!AB-23")), + fixtureDoc("hidden.txt", isHidden()), + fixtureDoc("plain.txt"), + fixtureFolder("box"), + fixtureDoc("boxed.txt", withParent("1$1!box"), withPath("./box/boxed.txt")), + }, + cases: []queryCase{ + {id: 1, query: `size:42`, want: []string{"small.txt"}}, + {id: 2, query: `mtime<"2021-01-01T00:00:00Z"`, want: []string{"old.txt"}}, + {id: 3, query: `id:"1$1!23"`, want: []string{"known.txt"}}, + {id: 4, query: `hidden:true`, want: []string{"hidden.txt"}}, + {id: 5, query: `type:file`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt"}}, + {id: 6, query: `type:folder`, want: []string{"box"}}, + {id: 7, query: `unknown:field`}, + {id: 8, query: `type:File`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt"}}, + {id: 9, query: `type:FOLDER`, want: []string{"box"}}, + {id: 10, query: `hidden:TRUE`, want: []string{"hidden.txt"}}, + {id: 11, query: `id:"1$1!AB-23"`, want: []string{"cased.txt"}}, + {id: 12, query: `id:"1$1!ab-23"`}, + }, + } +} diff --git a/services/search/pkg/parity/query_invalid_test.go b/services/search/pkg/parity/query_invalid_test.go new file mode 100644 index 0000000000..87da4f0592 --- /dev/null +++ b/services/search/pkg/parity/query_invalid_test.go @@ -0,0 +1,18 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func invalidGroup() queryGroup { + return queryGroup{ + name: "invalid", + fixtures: []search.Resource{ + fixtureDoc("alpha.txt"), + }, + cases: []queryCase{ + {id: 1, query: `AND mediatype:document`, wantBadRequest: true}, + {id: 2, query: `mediatype:document AND`, want: []string{"alpha.txt"}}, + }, + } +} diff --git a/services/search/pkg/parity/query_mediatype_test.go b/services/search/pkg/parity/query_mediatype_test.go new file mode 100644 index 0000000000..cd43ff89d5 --- /dev/null +++ b/services/search/pkg/parity/query_mediatype_test.go @@ -0,0 +1,25 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func mediatypeGroup() queryGroup { + return queryGroup{ + name: "mediatype", + fixtures: []search.Resource{ + fixtureDoc("notes.md", withMime("text/markdown")), + fixtureDoc("photo.jpg", withMime("image/jpeg")), + fixtureFolder("albums"), + fixtureFolder("drafts"), + }, + cases: []queryCase{ + {id: 1, query: `mediatype:text/markdown`, want: []string{"notes.md"}}, + {id: 2, query: `mediatype:TEXT/MARKDOWN`, want: []string{"notes.md"}}, + {id: 3, query: `mediatype:image/jpeg`, want: []string{"photo.jpg"}}, + {id: 4, query: `mediatype:*jpeg`, want: []string{"photo.jpg"}}, + {id: 5, query: `mediatype:image`, want: []string{"photo.jpg"}}, + {id: 6, query: `mediatype:folder`, want: []string{"albums", "drafts"}}, + }, + } +} diff --git a/services/search/pkg/parity/query_name_test.go b/services/search/pkg/parity/query_name_test.go new file mode 100644 index 0000000000..ee7b8dc177 --- /dev/null +++ b/services/search/pkg/parity/query_name_test.go @@ -0,0 +1,66 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func nameGroup() queryGroup { + return queryGroup{ + name: "name", + fixtures: []search.Resource{ + fixtureFolder("new-folder"), + fixtureDoc("quarterly notes.txt"), + fixtureDoc("Report.txt"), + fixtureDoc("Übung.txt"), + fixtureDoc("a+b.txt"), + fixtureDoc("c(d).txt"), + fixtureDoc("e&f.txt"), + fixtureDoc("v1.2.3.txt"), + fixtureDoc("foo bar.txt"), + fixtureDoc(fixtureLongName), + }, + cases: []queryCase{ + {id: 1, query: `new`, want: []string{"new-folder"}}, + {id: 2, query: `quarterly`, want: []string{"quarterly notes.txt"}}, + {id: 3, query: `report`, want: []string{"Report.txt"}}, + {id: 4, query: `name:"*new-folder*"`, want: []string{"new-folder"}}, + {id: 5, query: `name:"*w-fol*"`, want: []string{"new-folder"}}, + {id: 6, query: `name:"*oo ba*"`, want: []string{"foo bar.txt"}}, + {id: 7, query: `name:"*REPORT*"`, want: []string{"Report.txt"}}, + {id: 8, query: `name:"*übung*"`, want: []string{"Übung.txt"}}, + {id: 9, query: `name:"*ÜBUNG*"`, want: []string{"Übung.txt"}}, + {id: 10, query: `name:"*a+b*"`, want: []string{"a+b.txt"}}, + {id: 11, query: `name:"*c(d)*"`, want: []string{"c(d).txt"}}, + {id: 12, query: `name:"*e&f*"`, want: []string{"e&f.txt"}}, + {id: 13, query: `name:"*v1.2*"`, want: []string{"v1.2.3.txt"}}, + {id: 14, query: `new-folder`, want: []string{"new-folder"}}, + {id: 15, query: `*folder*`, want: []string{"new-folder"}}, + {id: 16, query: `name:"*foo bar*"`, want: []string{"foo bar.txt"}}, + {id: 17, query: `name:"foo bar.txt"`, want: []string{"foo bar.txt"}}, + {id: 18, query: `name:"*needle*"`, want: []string{fixtureLongName}}, + {id: 19, query: `name:"report*"`, want: []string{"Report.txt"}}, + {id: 20, query: `name:"*report"`, want: []string{"Report.txt"}}, + {id: 21, query: `name:"Rep*rt.txt"`, want: []string{"Report.txt"}}, + {id: 22, query: `Name:"*report*"`, want: []string{"Report.txt"}}, + {id: 23, query: `NAME:"*report*"`, want: []string{"Report.txt"}}, + {id: 24, query: `name:Rep?rt.txt`, want: []string{"Report.txt"}}, + {id: 25, query: `name:"*eport"`, want: []string{"Report.txt"}}, + {id: 26, query: `name:"repor*"`, want: []string{"Report.txt"}}, + {id: 27, query: `REPORT`, want: []string{"Report.txt"}}, + {id: 28, query: `name:REPORT`, want: []string{"Report.txt"}}, + {id: 29, query: `name:"REPORT.TXT"`, want: []string{"Report.txt"}}, + {id: 30, query: `name:"FOO BAR.TXT"`, want: []string{"foo bar.txt"}}, + {id: 31, query: `name:"ÜBUNG.TXT"`, want: []string{"Übung.txt"}}, + {id: 32, query: `name:"folder*"`}, + {id: 33, query: `name:"*new"`}, + {id: 34, query: `name:new`, want: []string{"new-folder"}}, + {id: 35, query: `name:"new"`}, + {id: 36, query: `name:"new-folder"`, want: []string{"new-folder"}}, + {id: 37, query: `name:"new-*"`, want: []string{"new-folder"}}, + {id: 38, query: `name:"new*"`, want: []string{"new-folder"}}, + {id: 39, query: `name:new-*`, want: []string{"new-folder"}}, + {id: 40, query: `name:"*-folder"`, want: []string{"new-folder"}}, + {id: 41, query: `name:"Rep?rt.txt"`, want: []string{"Report.txt"}}, + }, + } +} diff --git a/services/search/pkg/parity/query_path_test.go b/services/search/pkg/parity/query_path_test.go new file mode 100644 index 0000000000..e97e8e1596 --- /dev/null +++ b/services/search/pkg/parity/query_path_test.go @@ -0,0 +1,28 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func pathGroup() queryGroup { + return queryGroup{ + name: "path", + fixtures: []search.Resource{ + fixtureFolder("parent"), + fixtureDoc("child.jpg", withMime("image/jpeg"), withPath("./parent/child.jpg")), + fixtureFolder("docs-lower", withPath("./documents")), + fixtureFolder("docs-upper", withPath("./DOCUMENTS")), + fixtureFolder("docs-mixed", withPath("./Documents")), + }, + cases: []queryCase{ + {id: 1, query: `path:"./parent"`, want: []string{"parent", "child.jpg"}}, + {id: 2, query: `path:"./parent/child.jpg"`, want: []string{"child.jpg"}}, + {id: 3, query: `path:"./Parent"`}, + {id: 4, query: `path:"*child*"`, want: []string{"child.jpg"}}, + {id: 5, query: `path:"./documents"`, want: []string{"docs-lower"}}, + {id: 6, query: `path:"./DOCUMENTS"`, want: []string{"docs-upper"}}, + {id: 7, query: `path:"./Documents"`, want: []string{"docs-mixed"}}, + {id: 8, query: `path:"./parent/"`, want: []string{"parent", "child.jpg"}}, + }, + } +} diff --git a/services/search/pkg/parity/query_range_test.go b/services/search/pkg/parity/query_range_test.go new file mode 100644 index 0000000000..f5c7033012 --- /dev/null +++ b/services/search/pkg/parity/query_range_test.go @@ -0,0 +1,26 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func rangeGroup() queryGroup { + return queryGroup{ + name: "range", + fixtures: []search.Resource{ + fixtureDoc("small.txt", withSize(50)), + fixtureDoc("big.txt", withSize(500)), + fixtureDoc("ancient.txt", withSize(10), withMtime("2020-01-01T00:00:00Z")), + }, + cases: []queryCase{ + {id: 1, query: `size>100`, want: []string{"big.txt"}}, + {id: 2, query: `size<100`, want: []string{"small.txt", "ancient.txt"}}, + {id: 3, query: `mtime>"2021-01-01T00:00:00Z"`, want: []string{"small.txt", "big.txt"}}, + {id: 4, query: `mtime<"2021-01-01T00:00:00Z"`, want: []string{"ancient.txt"}}, + {id: 5, query: `Mtime:"today"`, want: []string{"small.txt", "big.txt"}}, + {id: 6, query: `Mtime:"yesterday"`}, + {id: 7, query: `mtime>2021`}, + {id: 8, query: `name>100`}, + }, + } +} diff --git a/services/search/pkg/parity/query_samename_test.go b/services/search/pkg/parity/query_samename_test.go new file mode 100644 index 0000000000..46b07597b1 --- /dev/null +++ b/services/search/pkg/parity/query_samename_test.go @@ -0,0 +1,34 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/pkg/conversions" + searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func samenameGroup() queryGroup { + folder := fixtureFolder("doc", withID("1$1!2")) + under := func(path string) *searchMessage.Reference { + return &searchMessage.Reference{ + ResourceId: &searchMessage.ResourceID{StorageId: "1", SpaceId: "1", OpaqueId: "1"}, + Path: path, + } + } + + return queryGroup{ + name: "samename", + fixtures: []search.Resource{ + folder, + fixtureDoc("doc.pdf", withID("1$1!3")), + fixtureDoc("file.pdf", withID("1$1!4")), + fixtureDoc("doc.pdf", withID("1$1!5"), withParent(folder.ID), withPath("./doc/doc.pdf")), + fixtureDoc("file.pdf", withID("1$1!6"), withParent(folder.ID), withPath("./doc/file.pdf")), + }, + cases: []queryCase{ + {id: 1, query: `name:"*doc*"`, wantCount: conversions.ToPointer(3)}, + {id: 2, query: `name:"*doc*"`, ref: under("./doc"), wantCount: conversions.ToPointer(2)}, + {id: 3, query: `name:"*file*"`, wantCount: conversions.ToPointer(2)}, + {id: 4, query: `name:"*file*"`, ref: under("./doc"), wantCount: conversions.ToPointer(1)}, + }, + } +} diff --git a/services/search/pkg/parity/query_scope_test.go b/services/search/pkg/parity/query_scope_test.go new file mode 100644 index 0000000000..9bb01341f2 --- /dev/null +++ b/services/search/pkg/parity/query_scope_test.go @@ -0,0 +1,32 @@ +package parity + +import ( + searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func scopeGroup() queryGroup { + space := func(path string) *searchMessage.Reference { + return &searchMessage.Reference{ + ResourceId: &searchMessage.ResourceID{StorageId: "1", SpaceId: "1", OpaqueId: "1"}, + Path: path, + } + } + + return queryGroup{ + name: "scope", + fixtures: []search.Resource{ + fixtureFolder("parent"), + fixtureDoc("child.pdf", withPath("./parent/child.pdf")), + fixtureDoc("outside.txt"), + fixtureDoc("elsewhere.txt", withRoot("2$2!1")), + }, + cases: []queryCase{ + {id: 1, query: `*`, want: []string{"parent", "child.pdf", "outside.txt", "elsewhere.txt"}}, + {id: 2, query: `*`, ref: space(""), want: []string{"parent", "child.pdf", "outside.txt"}}, + {id: 3, query: `*`, ref: space("./parent"), want: []string{"parent", "child.pdf"}}, + {id: 4, query: `*`, ref: space("./parent/child.pdf"), want: []string{"child.pdf"}}, + {id: 5, query: `name:"*elsewhere*"`, ref: space(""), want: nil}, + }, + } +} diff --git a/services/search/pkg/parity/query_stress_test.go b/services/search/pkg/parity/query_stress_test.go new file mode 100644 index 0000000000..36dc5acafb --- /dev/null +++ b/services/search/pkg/parity/query_stress_test.go @@ -0,0 +1,38 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func stressGroup() queryGroup { + return queryGroup{ + name: "stress", + fixtures: []search.Resource{ + fixtureDoc("quarterly report.docx", + withMime("application/vnd.openxmlformats-officedocument.wordprocessingml.document"), + withTags("final"), withSize(2000)), + fixtureDoc("draft report.txt", withTags("draft"), withSize(50), withMtime("2020-01-01T00:00:00Z")), + fixtureDoc("photo.jpg", withMime("image/jpeg"), withTags("final")), + fixtureDoc("notes.md", withMime("text/markdown"), isHidden()), + fixtureFolder("archive"), + }, + cases: []queryCase{ + {id: 1, query: `name:"*report*" AND mediatype:document`, want: []string{"quarterly report.docx", "draft report.txt"}}, + {id: 2, query: `name:"*report*" AND NOT tag:("draft")`, want: []string{"quarterly report.docx"}}, + {id: 3, query: `(tag:("final") OR tag:("draft")) AND mediatype:image`, want: []string{"photo.jpg"}}, + {id: 4, query: `mediatype:document AND mtime>"2021-01-01T00:00:00Z"`, want: []string{"quarterly report.docx", "notes.md"}}, + {id: 5, query: `name:"*report*" AND size>100`, want: []string{"quarterly report.docx"}}, + {id: 6, query: `tag:("final") AND NOT mediatype:folder`, want: []string{"quarterly report.docx", "photo.jpg"}}, + {id: 7, query: `hidden:true AND name:"*notes*"`, want: []string{"notes.md"}}, + {id: 8, query: `name:quarterly report`, want: []string{"quarterly report.docx"}}, + {id: 9, query: `name:"quarterly report"`}, + {id: 10, query: `name:"quarterly report.docx"`, want: []string{"quarterly report.docx"}}, + {id: 11, query: `NOT tag:("draft")`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md", "archive"}}, + {id: 12, query: `tag:("final") OR hidden:true`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md"}}, + {id: 13, query: `(name:"*report*" OR name:"*notes*") AND NOT (tag:("draft") OR hidden:true)`, want: []string{"quarterly report.docx"}}, + {id: 14, query: `mediatype:image OR (mediatype:document AND tag:("draft"))`, want: []string{"photo.jpg", "draft report.txt"}}, + {id: 15, query: `NOT (mediatype:folder OR hidden:true)`, want: []string{"quarterly report.docx", "draft report.txt", "photo.jpg"}}, + {id: 16, query: `name:"*report*" AND (size>100 OR tag:("draft"))`, want: []string{"quarterly report.docx", "draft report.txt"}}, + }, + } +} diff --git a/services/search/pkg/parity/query_tags_test.go b/services/search/pkg/parity/query_tags_test.go new file mode 100644 index 0000000000..c8505074d3 --- /dev/null +++ b/services/search/pkg/parity/query_tags_test.go @@ -0,0 +1,34 @@ +package parity + +import ( + "strings" + + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func tagsGroup() queryGroup { + longTag := strings.Repeat("z", 300) + "needle" + + return queryGroup{ + name: "tags", + fixtures: []search.Resource{ + fixtureDoc("invoice.txt", withTags("foo-bar")), + fixtureDoc("memo.txt", withTags("foo")), + fixtureDoc("spaced.txt", withTags("spaced tag")), + fixtureFolder("project", withTags("work")), + fixtureDoc("draft.txt", withParent("1$1!project"), withPath("./project/draft.txt")), + fixtureDoc("longtag.txt", withTags(longTag)), + }, + cases: []queryCase{ + {id: 1, query: `name:"*foo-bar*"`}, + {id: 2, query: `tag:("foo-bar")`, want: []string{"invoice.txt"}}, + {id: 3, query: `tag:("foo")`, want: []string{"memo.txt"}}, + {id: 4, query: `tag:("FOO-BAR")`, want: []string{"invoice.txt"}}, + {id: 5, query: `tag:("*foo*")`, want: []string{"invoice.txt", "memo.txt"}}, + {id: 6, query: `tag:("spaced tag")`, want: []string{"spaced.txt"}}, + {id: 7, query: `tag:("*paced ta*")`, want: []string{"spaced.txt"}}, + {id: 8, query: `tag:("work")`, want: []string{"project"}}, + {id: 9, query: `tag:("` + longTag + `")`, want: []string{"longtag.txt"}}, + }, + } +} diff --git a/services/search/pkg/parity/query_title_test.go b/services/search/pkg/parity/query_title_test.go new file mode 100644 index 0000000000..26786a0050 --- /dev/null +++ b/services/search/pkg/parity/query_title_test.go @@ -0,0 +1,23 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func titleGroup() queryGroup { + return queryGroup{ + name: "title", + fixtures: []search.Resource{ + fixtureDoc("q1.html", withMime("text/html"), withTitle("quarterly report")), + }, + cases: []queryCase{ + {id: 1, query: `Title:"quarterly report"`, want: []string{"q1.html"}}, + {id: 2, query: `Title:quarterly`, want: []string{"q1.html"}}, + {id: 3, query: `Title:QUARTERLY`, want: []string{"q1.html"}}, + {id: 4, query: `Title:quarterl*`, want: []string{"q1.html"}}, + {id: 5, query: `Title:"*ly rep*"`, want: []string{"q1.html"}}, + {id: 6, query: `title:quarterly`, want: []string{"q1.html"}}, + {id: 7, query: `Title:"QUARTERLY REPORT"`, want: []string{"q1.html"}}, + }, + } +} diff --git a/services/search/pkg/parity/query_visibility_test.go b/services/search/pkg/parity/query_visibility_test.go new file mode 100644 index 0000000000..5a3d879946 --- /dev/null +++ b/services/search/pkg/parity/query_visibility_test.go @@ -0,0 +1,26 @@ +package parity + +import ( + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func visibilityGroup() queryGroup { + return queryGroup{ + name: "visibility", + fixtures: []search.Resource{ + fixtureDoc("visible.txt"), + fixtureDoc("dotfile.txt", isHidden()), + fixtureFolder(".private", isHidden()), + fixtureDoc("secret.txt", withParent("1$1!.private"), withPath("./.private/secret.txt"), isHidden()), + }, + cases: []queryCase{ + {id: 1, query: `hidden:true`, want: []string{"dotfile.txt", ".private", "secret.txt"}}, + {id: 2, query: `hidden:TRUE`, want: []string{"dotfile.txt", ".private", "secret.txt"}}, + {id: 3, query: `hidden:false`, want: []string{"visible.txt"}}, + {id: 4, query: `name:"*secret*"`, want: []string{"secret.txt"}}, + {id: 5, query: `path:"./.private"`, want: []string{".private", "secret.txt"}}, + {id: 6, query: `hidden:banana`}, + {id: 7, query: `hidden:"true"`, want: []string{"dotfile.txt", ".private", "secret.txt"}}, + }, + } +} diff --git a/services/search/pkg/parity/response_entity_test.go b/services/search/pkg/parity/response_entity_test.go new file mode 100644 index 0000000000..c32ecbf5df --- /dev/null +++ b/services/search/pkg/parity/response_entity_test.go @@ -0,0 +1,102 @@ +package parity + +import ( + "fmt" + + searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func entityGroup() responseGroup { + parent := fixtureFolder("parent", withID("1$1!2")) + doc := fixtureDoc("bar.pdf", + withID("1$1!3"), + withParent(parent.ID), + withPath("./parent/bar.pdf"), + withMime("application/pdf"), + withSize(1234), + ) + notes := fixtureDoc("notes.txt", withID("1$1!4"), withContent("foo bar baz")) + + moved := func(e search.Engine) error { return e.Move(parent.ID, "1$1!9", "./somewhere/newname") } + + return responseGroup{ + name: "entity", + fixtures: []search.Resource{parent, doc, notes}, + cases: []responseCase{ + { + id: 1, query: `name:"bar.pdf"`, reads: "Ref.Path", want: []string{"./parent/bar.pdf"}, + read: reads(func(m *searchMessage.Match) string { return m.GetEntity().GetRef().GetPath() }), + }, + { + id: 2, query: `name:"bar.pdf"`, reads: "Name", want: []string{"bar.pdf"}, + read: reads(func(m *searchMessage.Match) string { return m.GetEntity().GetName() }), + }, + { + id: 3, query: `name:"bar.pdf"`, reads: "Id", want: []string{"1$1!3"}, + read: reads(func(m *searchMessage.Match) string { return resourceID(m.GetEntity().GetId()) }), + }, + { + id: 4, query: `name:"bar.pdf"`, reads: "ParentId", want: []string{"1$1!2"}, + read: reads(func(m *searchMessage.Match) string { return resourceID(m.GetEntity().GetParentId()) }), + }, + { + id: 5, query: `name:"bar.pdf"`, reads: "Ref.ResourceId", want: []string{"1$1!1"}, + read: reads(func(m *searchMessage.Match) string { return resourceID(m.GetEntity().GetRef().GetResourceId()) }), + }, + { + id: 6, query: `name:"bar.pdf"`, reads: "Size", want: []string{"1234"}, + read: reads(func(m *searchMessage.Match) string { return fmt.Sprint(m.GetEntity().GetSize()) }), + }, + { + id: 7, query: `name:"bar.pdf"`, reads: "Type", want: []string{"1"}, + read: reads(func(m *searchMessage.Match) string { return fmt.Sprint(m.GetEntity().GetType()) }), + }, + { + id: 8, query: `name:"bar.pdf"`, reads: "MimeType", want: []string{"application/pdf"}, + read: reads(func(m *searchMessage.Match) string { return m.GetEntity().GetMimeType() }), + }, + { + id: 9, query: `name:"bar.pdf"`, reads: "Deleted", want: []string{"false"}, + read: reads(func(m *searchMessage.Match) string { return fmt.Sprint(m.GetEntity().GetDeleted()) }), + }, + { + id: 10, query: `name:"bar.pdf"`, reads: "Score", want: []string{"above zero"}, + read: reads(func(m *searchMessage.Match) string { + if m.GetScore() > 0 { + return "above zero" + } + + return fmt.Sprint(m.GetScore()) + }), + }, + { + id: 11, query: `path:"./parent"`, reads: "TotalMatches", want: []string{"2"}, + read: func(resp *searchService.SearchIndexResponse) []string { + return []string{fmt.Sprint(resp.GetTotalMatches())} + }, + }, + { + id: 12, query: `name:"*notes*"`, reads: "Highlights", want: []string{`""`}, + read: reads(func(m *searchMessage.Match) string { return fmt.Sprintf("%q", m.GetEntity().GetHighlights()) }), + }, + { + id: 13, query: `content:bar`, reads: "Highlights", want: []string{"foo bar baz"}, + read: reads(func(m *searchMessage.Match) string { return m.GetEntity().GetHighlights() }), + }, + { + id: 14, do: moved, after: "moved to another parent", query: `name:"newname"`, reads: "ParentId", want: []string{"1$1!9"}, + read: reads(func(m *searchMessage.Match) string { return resourceID(m.GetEntity().GetParentId()) }), + }, + { + id: 15, do: moved, after: "moved to another parent", query: `name:"bar.pdf"`, reads: "ParentId", want: []string{"1$1!2"}, + read: reads(func(m *searchMessage.Match) string { return resourceID(m.GetEntity().GetParentId()) }), + }, + { + id: 16, do: moved, after: "moved to another parent", query: `name:"bar.pdf"`, reads: "Ref.Path", want: []string{"./somewhere/newname/bar.pdf"}, + read: reads(func(m *searchMessage.Match) string { return m.GetEntity().GetRef().GetPath() }), + }, + }, + } +} diff --git a/services/search/pkg/parity/response_metadata_test.go b/services/search/pkg/parity/response_metadata_test.go new file mode 100644 index 0000000000..e4bbca5e34 --- /dev/null +++ b/services/search/pkg/parity/response_metadata_test.go @@ -0,0 +1,143 @@ +package parity + +import ( + "fmt" + + libregraph "github.com/opencloud-eu/libre-graph-api-go" + + searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" +) + +func metadataGroup() responseGroup { + song := fixtureDoc("some_song.mp3", + withID("1$1!5"), + withMime("audio/mpeg"), + withAudio(&libregraph.Audio{ + Album: libregraph.PtrString("Some Album"), + AlbumArtist: libregraph.PtrString("Some AlbumArtist"), + Artist: libregraph.PtrString("Some Artist"), + Bitrate: libregraph.PtrInt64(192), + Composers: libregraph.PtrString("Some Composers"), + Copyright: libregraph.PtrString(""), + Disc: libregraph.PtrInt32(2), + DiscCount: libregraph.PtrInt32(5), + Duration: libregraph.PtrInt64(225000), + Genre: libregraph.PtrString("Some Genre"), + HasDrm: libregraph.PtrBool(false), + IsVariableBitrate: libregraph.PtrBool(true), + Title: libregraph.PtrString("Some Title"), + Track: libregraph.PtrInt32(34), + TrackCount: libregraph.PtrInt32(99), + Year: libregraph.PtrInt32(2004), + }), + ) + + team := fixtureDoc("team.jpg", + withID("1$1!6"), + withMime("image/jpeg"), + withLocation(&libregraph.GeoCoordinates{ + Altitude: libregraph.PtrFloat64(1047.7), + Latitude: libregraph.PtrFloat64(49.48675890884328), + Longitude: libregraph.PtrFloat64(11.103870357204285), + }), + ) + + indexed := []string{ + "Album=Some Album", + "AlbumArtist=Some AlbumArtist", + "Artist=Some Artist", + "Bitrate=192", + "Composers=Some Composers", + "Copyright=", + "Disc=2", + "DiscCount=5", + "Duration=225000", + "Genre=Some Genre", + "HasDrm=false", + "IsVariableBitrate=true", + "Title=Some Title", + "Track=34", + "TrackCount=99", + "Year=2004", + } + + located := []string{ + "Altitude=1047.7", + "Latitude=49.48675890884328", + "Longitude=11.103870357204285", + } + + return responseGroup{ + name: "metadata", + fixtures: []search.Resource{song, team}, + cases: []responseCase{ + { + id: 1, query: `*song*`, reads: "Audio", want: unchanged(indexed), + read: readsMany(func(m *searchMessage.Match) []string { + audio := m.GetEntity().GetAudio() + + return changes(indexed, []string{ + fmt.Sprintf("Album=%s", audio.GetAlbum()), + fmt.Sprintf("AlbumArtist=%s", audio.GetAlbumArtist()), + fmt.Sprintf("Artist=%s", audio.GetArtist()), + fmt.Sprintf("Bitrate=%d", audio.GetBitrate()), + fmt.Sprintf("Composers=%s", audio.GetComposers()), + fmt.Sprintf("Copyright=%s", audio.GetCopyright()), + fmt.Sprintf("Disc=%d", audio.GetDisc()), + fmt.Sprintf("DiscCount=%d", audio.GetDiscCount()), + fmt.Sprintf("Duration=%d", audio.GetDuration()), + fmt.Sprintf("Genre=%s", audio.GetGenre()), + fmt.Sprintf("HasDrm=%t", audio.GetHasDrm()), + fmt.Sprintf("IsVariableBitrate=%t", audio.GetIsVariableBitrate()), + fmt.Sprintf("Title=%s", audio.GetTitle()), + fmt.Sprintf("Track=%d", audio.GetTrack()), + fmt.Sprintf("TrackCount=%d", audio.GetTrackCount()), + fmt.Sprintf("Year=%d", audio.GetYear()), + }) + }), + }, + { + id: 2, query: `*team*`, reads: "Location", want: unchanged(located), + read: readsMany(func(m *searchMessage.Match) []string { + location := m.GetEntity().GetLocation() + + return changes(located, []string{ + fmt.Sprintf("Altitude=%v", location.GetAltitude()), + fmt.Sprintf("Latitude=%v", location.GetLatitude()), + fmt.Sprintf("Longitude=%v", location.GetLongitude()), + }) + }), + }, + { + id: 3, query: `*team*`, reads: "Audio", want: []string{"none"}, + read: reads(func(m *searchMessage.Match) string { + if m.GetEntity().GetAudio() == nil { + return "none" + } + + return "set" + }), + }, + }, + } +} + +func unchanged(indexed []string) []string { + return []string{fmt.Sprintf("all %d fields unchanged", len(indexed))} +} + +func changes(indexed, got []string) []string { + var off []string + for i, field := range indexed { + if got[i] != field { + off = append(off, fmt.Sprintf("%s instead of %s", got[i], field)) + } + } + + if len(off) == 0 { + return unchanged(indexed) + } + + return off +} From a3e40a74714f97e4dcf4bb7f0779931823855cdf Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Fri, 28 Aug 2026 18:43:14 +0200 Subject: [PATCH 2/9] test(search): run the parity suite against the current main Move the OpenSearch test helpers to services/search/internal/opensearchtest so the parity package can import them, and drop the index generation and clock patch the suite was written against, neither exists on main yet. --- .../internal/test => internal/opensearchtest}/helper.go | 0 .../internal/test => internal/opensearchtest}/os.go | 0 .../internal/test => internal/opensearchtest}/suite.go | 0 .../internal/test => internal/opensearchtest}/test.go | 0 .../internal/test => internal/opensearchtest}/testdata.go | 0 .../opensearchtest}/testdata/resource_file.json | 0 .../opensearchtest}/testdata/resource_folder.json | 0 .../opensearchtest}/testdata/resource_root.json | 0 services/search/pkg/opensearch/backend_test.go | 2 +- services/search/pkg/opensearch/index_test.go | 2 +- .../pkg/opensearch/internal/convert/kql_expand_test.go | 2 +- .../pkg/opensearch/internal/convert/kql_transpile_test.go | 2 +- .../pkg/opensearch/internal/convert/opensearch_test.go | 2 +- .../search/pkg/opensearch/internal/osu/query_bool_test.go | 2 +- .../internal/osu/query_full_text_match_phrase_test.go | 2 +- .../opensearch/internal/osu/query_term_level_ids_test.go | 2 +- .../opensearch/internal/osu/query_term_level_range_test.go | 2 +- .../opensearch/internal/osu/query_term_level_term_test.go | 2 +- .../internal/osu/query_term_level_wildcard_test.go | 2 +- .../search/pkg/opensearch/internal/osu/request_test.go | 2 +- services/search/pkg/opensearch/opensearch_test.go | 2 +- services/search/pkg/parity/engines_test.go | 3 ++- services/search/pkg/parity/parity_test.go | 7 ++----- 23 files changed, 17 insertions(+), 19 deletions(-) rename services/search/{pkg/opensearch/internal/test => internal/opensearchtest}/helper.go (100%) rename services/search/{pkg/opensearch/internal/test => internal/opensearchtest}/os.go (100%) rename services/search/{pkg/opensearch/internal/test => internal/opensearchtest}/suite.go (100%) rename services/search/{pkg/opensearch/internal/test => internal/opensearchtest}/test.go (100%) rename services/search/{pkg/opensearch/internal/test => internal/opensearchtest}/testdata.go (100%) rename services/search/{pkg/opensearch/internal/test => internal/opensearchtest}/testdata/resource_file.json (100%) rename services/search/{pkg/opensearch/internal/test => internal/opensearchtest}/testdata/resource_folder.json (100%) rename services/search/{pkg/opensearch/internal/test => internal/opensearchtest}/testdata/resource_root.json (100%) diff --git a/services/search/pkg/opensearch/internal/test/helper.go b/services/search/internal/opensearchtest/helper.go similarity index 100% rename from services/search/pkg/opensearch/internal/test/helper.go rename to services/search/internal/opensearchtest/helper.go diff --git a/services/search/pkg/opensearch/internal/test/os.go b/services/search/internal/opensearchtest/os.go similarity index 100% rename from services/search/pkg/opensearch/internal/test/os.go rename to services/search/internal/opensearchtest/os.go diff --git a/services/search/pkg/opensearch/internal/test/suite.go b/services/search/internal/opensearchtest/suite.go similarity index 100% rename from services/search/pkg/opensearch/internal/test/suite.go rename to services/search/internal/opensearchtest/suite.go diff --git a/services/search/pkg/opensearch/internal/test/test.go b/services/search/internal/opensearchtest/test.go similarity index 100% rename from services/search/pkg/opensearch/internal/test/test.go rename to services/search/internal/opensearchtest/test.go diff --git a/services/search/pkg/opensearch/internal/test/testdata.go b/services/search/internal/opensearchtest/testdata.go similarity index 100% rename from services/search/pkg/opensearch/internal/test/testdata.go rename to services/search/internal/opensearchtest/testdata.go diff --git a/services/search/pkg/opensearch/internal/test/testdata/resource_file.json b/services/search/internal/opensearchtest/testdata/resource_file.json similarity index 100% rename from services/search/pkg/opensearch/internal/test/testdata/resource_file.json rename to services/search/internal/opensearchtest/testdata/resource_file.json diff --git a/services/search/pkg/opensearch/internal/test/testdata/resource_folder.json b/services/search/internal/opensearchtest/testdata/resource_folder.json similarity index 100% rename from services/search/pkg/opensearch/internal/test/testdata/resource_folder.json rename to services/search/internal/opensearchtest/testdata/resource_folder.json diff --git a/services/search/pkg/opensearch/internal/test/testdata/resource_root.json b/services/search/internal/opensearchtest/testdata/resource_root.json similarity index 100% rename from services/search/pkg/opensearch/internal/test/testdata/resource_root.json rename to services/search/internal/opensearchtest/testdata/resource_root.json diff --git a/services/search/pkg/opensearch/backend_test.go b/services/search/pkg/opensearch/backend_test.go index 29859f751a..850b9a7bfa 100644 --- a/services/search/pkg/opensearch/backend_test.go +++ b/services/search/pkg/opensearch/backend_test.go @@ -14,8 +14,8 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/errtypes" searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" + opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) diff --git a/services/search/pkg/opensearch/index_test.go b/services/search/pkg/opensearch/index_test.go index f4b5451a9d..6a813572a2 100644 --- a/services/search/pkg/opensearch/index_test.go +++ b/services/search/pkg/opensearch/index_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/require" "github.com/tidwall/sjson" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestIndexManager(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go index 49218d5aa5..1b3cb0b914 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_expand_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_expand_test.go @@ -9,7 +9,7 @@ import ( "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" "github.com/opencloud-eu/opencloud/pkg/ast" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" ) func TestExpandKQLAST(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go index 9dbf7f2b9a..1b627c8133 100644 --- a/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go +++ b/services/search/pkg/opensearch/internal/convert/kql_transpile_test.go @@ -7,9 +7,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/opencloud-eu/opencloud/pkg/ast" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestTranspileKQLToOpenSearch(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/convert/opensearch_test.go b/services/search/pkg/opensearch/internal/convert/opensearch_test.go index bf80b7058f..d2fba09c70 100644 --- a/services/search/pkg/opensearch/internal/convert/opensearch_test.go +++ b/services/search/pkg/opensearch/internal/convert/opensearch_test.go @@ -10,8 +10,8 @@ import ( "github.com/opencloud-eu/opencloud/pkg/conversions" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" + opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/convert" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) diff --git a/services/search/pkg/opensearch/internal/osu/query_bool_test.go b/services/search/pkg/opensearch/internal/osu/query_bool_test.go index 3f33d4c1f4..d101cc5b7e 100644 --- a/services/search/pkg/opensearch/internal/osu/query_bool_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_bool_test.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/assert" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestBoolQuery(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase_test.go b/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase_test.go index 2fa712d039..b2c4b36a19 100644 --- a/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_full_text_match_phrase_test.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/assert" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestNewMatchPhraseQuery(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/osu/query_term_level_ids_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_ids_test.go index 1a15777649..ecfca2fff4 100644 --- a/services/search/pkg/opensearch/internal/osu/query_term_level_ids_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_ids_test.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/assert" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestIDsQuery(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/osu/query_term_level_range_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_range_test.go index 0bb65fca73..43b6241768 100644 --- a/services/search/pkg/opensearch/internal/osu/query_term_level_range_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_range_test.go @@ -8,8 +8,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestRangeQuery(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/osu/query_term_level_term_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_term_test.go index c4e9e8f325..e4244bf0a8 100644 --- a/services/search/pkg/opensearch/internal/osu/query_term_level_term_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_term_test.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/assert" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestTermQuery(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard_test.go b/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard_test.go index afc810e36d..e1a5f37088 100644 --- a/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard_test.go +++ b/services/search/pkg/opensearch/internal/osu/query_term_level_wildcard_test.go @@ -5,8 +5,8 @@ import ( "github.com/stretchr/testify/assert" + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestWildcardQuery(t *testing.T) { diff --git a/services/search/pkg/opensearch/internal/osu/request_test.go b/services/search/pkg/opensearch/internal/osu/request_test.go index 535f05bd82..f8e0010258 100644 --- a/services/search/pkg/opensearch/internal/osu/request_test.go +++ b/services/search/pkg/opensearch/internal/osu/request_test.go @@ -8,8 +8,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/osu" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) func TestRequestBody(t *testing.T) { diff --git a/services/search/pkg/opensearch/opensearch_test.go b/services/search/pkg/opensearch/opensearch_test.go index 6278ce6d66..cd5fb6fdbc 100644 --- a/services/search/pkg/opensearch/opensearch_test.go +++ b/services/search/pkg/opensearch/opensearch_test.go @@ -6,8 +6,8 @@ import ( "os" "testing" + opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/config" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test" ) var defaultConfig *config.Config diff --git a/services/search/pkg/parity/engines_test.go b/services/search/pkg/parity/engines_test.go index b91fcedf07..74091a3acf 100644 --- a/services/search/pkg/parity/engines_test.go +++ b/services/search/pkg/parity/engines_test.go @@ -58,7 +58,8 @@ func newOpenSearch(t *testing.T, index string, fixtures []search.Resource) testE t.Helper() tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client) - versioned := opensearch.IndexName(index) + // main has no index generations yet, the index name is used as given + versioned := index if err := tc.IndicesReset(t.Context(), []string{versioned}); err != nil { return testEngine{name: "opensearch", unavailable: err.Error()} diff --git a/services/search/pkg/parity/parity_test.go b/services/search/pkg/parity/parity_test.go index b5b149cb46..d9238dbde4 100644 --- a/services/search/pkg/parity/parity_test.go +++ b/services/search/pkg/parity/parity_test.go @@ -6,13 +6,11 @@ import ( "os" "strings" "testing" - "time" "github.com/stretchr/testify/require" "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/opencloud/pkg/kql" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" @@ -23,9 +21,8 @@ import ( var defaultConfig *config.Config func TestMain(m *testing.M) { - // the fixtures are stamped with fixtureNow, holding the clock still keeps - // "today" on their side of midnight for the whole run - kql.PatchTimeNow(func() time.Time { return fixtureNow }) + // the fixtures are stamped with fixtureNow; a run crossing midnight would + // see "today" move away from them (RANGE-05/06) cfg, done, err := opensearchtest.SetupTests(context.Background()) if err != nil { From a03a294a12aea9ba3dbf9193f8e3da7d081423e1 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Fri, 28 Aug 2026 18:44:47 +0200 Subject: [PATCH 3/9] test(search): make the OpenSearch testcontainer start reliably Cold boots take well over the 5s startup timeout, and a full host disk tripped the flood-stage create-index block mid-run; test indexes are tiny. --- services/search/internal/opensearchtest/suite.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/services/search/internal/opensearchtest/suite.go b/services/search/internal/opensearchtest/suite.go index e565a399e7..ce84776345 100644 --- a/services/search/internal/opensearchtest/suite.go +++ b/services/search/internal/opensearchtest/suite.go @@ -72,9 +72,15 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func opensearch.WithPassword(cfg.Engine.OpenSearch.Client.Password), testcontainers.WithName(containerName), testcontainers.WithReuseByName(containerName), + // test indexes are tiny; don't let a full host disk trip the flood-stage + // create-index / read-only blocks mid-run + testcontainers.WithEnv(map[string]string{ + "cluster.routing.allocation.disk.threshold_enabled": "false", + }), testcontainers.WithWaitStrategy( wait.ForLog("ML configuration initialized successfully"). - WithStartupTimeout(5*time.Second), + // a cold OpenSearch boot takes well over the previous 5s + WithStartupTimeout(2*time.Minute), ), ) if err != nil { From c24bfb2e411e8fedeeec83d03ebba775d16d2948 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Fri, 28 Aug 2026 19:15:50 +0200 Subject: [PATCH 4/9] test(search): keep the OpenSearch testcontainer between runs on request KEEP_TEST_CONTAINER=true skips the terminate; the next run picks the container up by name instead of booting a fresh one. --- .../search/internal/opensearchtest/suite.go | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/services/search/internal/opensearchtest/suite.go b/services/search/internal/opensearchtest/suite.go index ce84776345..433907292a 100644 --- a/services/search/internal/opensearchtest/suite.go +++ b/services/search/internal/opensearchtest/suite.go @@ -3,6 +3,7 @@ package opensearchtest import ( "context" "fmt" + "net/http" "os" "slices" "strings" @@ -18,7 +19,9 @@ import ( ) const ( - openSearchImage = "opensearchproject/opensearch:2" + openSearchImage = "opensearchproject/opensearch:2" + openSearchPort = "9200" + openSearchStartupTimeout = 3 * time.Minute ) func SetupTests(ctx context.Context) (*config.Config, func(), error) { @@ -62,6 +65,14 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func return func() {}, nil } + keepContainer := os.Getenv("KEEP_TEST_CONTAINER") == "true" + if keepContainer { + // the reaper would take the kept container down with the session + if err := os.Setenv("TESTCONTAINERS_RYUK_DISABLED", "true"); err != nil { + return nil, fmt.Errorf("failed to disable the testcontainers reaper: %w", err) + } + } + containerName := fmt.Sprintf("opencloud/test/%s", openSearchImage) containerName = strings.Replace(containerName, "/", "__", -1) containerName = strings.Replace(containerName, ":", "_", -1) @@ -77,10 +88,16 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func testcontainers.WithEnv(map[string]string{ "cluster.routing.allocation.disk.threshold_enabled": "false", }), + // a health probe answers at once on a reused container, a log wait + // would sit out the log timeout on it; a cold boot takes well over the + // previous 5s testcontainers.WithWaitStrategy( - wait.ForLog("ML configuration initialized successfully"). - // a cold OpenSearch boot takes well over the previous 5s - WithStartupTimeout(2*time.Minute), + wait.ForHTTP("/_cluster/health?wait_for_status=yellow&timeout=1s"). + WithPort(openSearchPort). + WithTLS(false). + WithBasicAuth(cfg.Engine.OpenSearch.Client.Username, cfg.Engine.OpenSearch.Client.Password). + WithStatusCodeMatcher(func(status int) bool { return status == http.StatusOK }). + WithStartupTimeout(openSearchStartupTimeout), ), ) if err != nil { @@ -97,6 +114,13 @@ func setupOpenSearchTestContainer(ctx context.Context, cfg *config.Config) (func cfg.Engine.OpenSearch.Client.Addresses = []string{address} return func() { + // KEEP_TEST_CONTAINER=true leaves the container up for the next run, + // which picks it up again by name instead of booting a fresh one + if keepContainer { + _, _ = fmt.Fprintf(os.Stderr, "keeping OpenSearch container %s\n", containerName) + return + } + err := container.Terminate(ctx) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "failed to terminate OpenSearch container: %v\n", err) From 1668e23f806f58c39fe7b37dd6e63391a928bda4 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Fri, 28 Aug 2026 19:37:45 +0200 Subject: [PATCH 5/9] test(search): port the parity suite to ginkgo Every case is one spec per engine, so one engine failing leaves the other's answer in the matrix. Query groups share their engines through an Ordered container, operations build their own. The README is assembled from report entries in ReportAfterSuite, which also works across ginkgo -p processes, and is only written once every planned row got an answer from every engine. A case can carry, per engine, what that engine answers today where it still differs from the expectation (engineOverrides): the spec asserts the override, the README marks the row as known, and the override fails once the engine answers as expected. --- services/search/pkg/parity/engines_test.go | 128 ++++-- services/search/pkg/parity/matrix_test.go | 320 +++++++------ .../search/pkg/parity/parity_suite_test.go | 48 ++ services/search/pkg/parity/parity_test.go | 431 ++++++++++++------ 4 files changed, 627 insertions(+), 300 deletions(-) create mode 100644 services/search/pkg/parity/parity_suite_test.go diff --git a/services/search/pkg/parity/engines_test.go b/services/search/pkg/parity/engines_test.go index 74091a3acf..a77fb61008 100644 --- a/services/search/pkg/parity/engines_test.go +++ b/services/search/pkg/parity/engines_test.go @@ -3,12 +3,15 @@ package parity import ( "context" "fmt" - "testing" + "strings" bleveSearch "github.com/blevesearch/bleve/v2" sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/types" "github.com/opencloud-eu/reva/v2/pkg/storagespace" - "github.com/stretchr/testify/require" + opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" "github.com/opencloud-eu/opencloud/pkg/log" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" @@ -20,6 +23,8 @@ import ( "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) +var engineNames = []string{"bleve", "opensearch"} + type testEngine struct { name string backend search.Engine @@ -27,41 +32,68 @@ type testEngine struct { unavailable string } -func newEngines(t *testing.T, index string, fixtures []search.Resource) []testEngine { - t.Helper() +// newEngines builds one backend per engine over the same fixtures. Call it +// from a BeforeAll: the cleanups it defers run when the container ends. +func newEngines(index string, fixtures []search.Resource) []testEngine { + GinkgoHelper() return []testEngine{ - newBleve(t, fixtures), - newOpenSearch(t, index, fixtures), + newBleve(fixtures), + newOpenSearch(index, fixtures), } } -func newBleve(t *testing.T, fixtures []search.Resource) testEngine { - t.Helper() +// newEngine builds just the one engine a spec is about; its cleanup runs when +// that spec ends. +func newEngine(name, index string, fixtures []search.Resource) testEngine { + GinkgoHelper() + + switch name { + case "bleve": + return newBleve(fixtures) + case "opensearch": + return newOpenSearch(index, fixtures) + } + + Fail("no engine named " + name) + return testEngine{} +} + +func engineNamed(engines []testEngine, name string) testEngine { + GinkgoHelper() + + for _, e := range engines { + if e.name == name { + return e + } + } + + Fail("no engine named " + name) + return testEngine{} +} + +func newBleve(fixtures []search.Resource) testEngine { + GinkgoHelper() mapping, err := bleveEngine.NewMapping() - require.NoError(t, err, "failed to build the bleve mapping") + Expect(err).NotTo(HaveOccurred(), "failed to build the bleve mapping") idx, err := bleveSearch.NewMemOnly(mapping) - require.NoError(t, err, "failed to create the bleve index") - t.Cleanup(func() { _ = idx.Close() }) + Expect(err).NotTo(HaveOccurred(), "failed to create the bleve index") + DeferCleanup(func() { _ = idx.Close() }) backend := bleveEngine.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}) - for _, doc := range fixtures { - require.NoError(t, backend.Upsert(doc.ID, doc), "bleve upsert %s", doc.ID) - } + load(backend, fixtures) return testEngine{name: "bleve", backend: backend, settle: func() {}} } -func newOpenSearch(t *testing.T, index string, fixtures []search.Resource) testEngine { - t.Helper() +func newOpenSearch(index string, fixtures []search.Resource) testEngine { + GinkgoHelper() - tc := opensearchtest.NewDefaultTestClient(t, defaultConfig.Engine.OpenSearch.Client) - // main has no index generations yet, the index name is used as given - versioned := index + tc := opensearchtest.NewDefaultTestClient(GinkgoTB(), openSearchClient) - if err := tc.IndicesReset(t.Context(), []string{versioned}); err != nil { + if err := tc.IndicesReset(context.Background(), []string{index}); err != nil { return testEngine{name: "opensearch", unavailable: err.Error()} } @@ -70,19 +102,41 @@ func newOpenSearch(t *testing.T, index string, fixtures []search.Resource) testE return testEngine{name: "opensearch", unavailable: err.Error()} } - t.Cleanup(func() { - require.NoError(t, tc.IndicesDelete(context.Background(), []string{versioned})) + DeferCleanup(func() { + Expect(tc.IndicesDelete(context.Background(), []string{index})).To(Succeed()) }) - settle := func() { tc.Require.IndicesRefresh([]string{versioned}, nil) } - for _, doc := range fixtures { - require.NoError(t, backend.Upsert(doc.ID, doc), "opensearch upsert %s", doc.ID) - } + // every write waits for the next refresh (Refresh: wait_for), which is a + // second apart by default; the test index can refresh far more often + _, err = tc.Client().Indices.Settings.Put(context.Background(), opensearchgoAPI.SettingsPutReq{ + Indices: []string{index}, + Body: strings.NewReader(`{"index": {"refresh_interval": "50ms"}}`), + }) + Expect(err).NotTo(HaveOccurred(), "failed to speed up the refresh of %s", index) + + settle := func() { tc.Require.IndicesRefresh([]string{index}, nil) } + load(backend, fixtures) settle() return testEngine{name: "opensearch", backend: backend, settle: settle} } +// load writes the fixtures the way the service does, in one batch. +func load(backend search.Engine, fixtures []search.Resource) { + GinkgoHelper() + + if len(fixtures) == 0 { + return + } + + batch, err := backend.NewBatch(len(fixtures) + 1) + Expect(err).NotTo(HaveOccurred(), "failed to open a batch for the fixtures") + for _, doc := range fixtures { + Expect(batch.Upsert(doc.ID, doc)).To(Succeed(), "upsert %s", doc.ID) + } + Expect(batch.Push()).To(Succeed(), "failed to write the fixtures") +} + func badRequestAnswer(err error) []string { if err == nil { return []string{"no error"} @@ -113,16 +167,28 @@ func resourceID(id *searchMessage.ResourceID) string { }) } -func hits(t *testing.T, e search.Engine, request *searchService.SearchIndexRequest) []string { - t.Helper() - +// ask runs a query and returns the matched names; an engine error is the +// answer "error", so a query the engine rejects still shows in the matrix. +func ask(e search.Engine, request *searchService.SearchIndexRequest) ([]string, error) { resp, err := e.Search(context.Background(), request) - require.NoError(t, err, "the query has to answer, an empty result is an answer") + if err != nil { + return []string{"error"}, err + } names := make([]string, 0, len(resp.Matches)) for _, m := range resp.Matches { names = append(names, m.Entity.GetName()) } - return names + return names, nil +} + +// matchNames is ConsistOf for a wanted list, spelled out for the empty case so +// a "no match" expectation reads as such in the failure. +func matchNames(want []string) types.GomegaMatcher { + if len(want) == 0 { + return BeEmpty() + } + + return ConsistOf(want) } diff --git a/services/search/pkg/parity/matrix_test.go b/services/search/pkg/parity/matrix_test.go index 3ec33bd4cc..4b4d65fca2 100644 --- a/services/search/pkg/parity/matrix_test.go +++ b/services/search/pkg/parity/matrix_test.go @@ -1,112 +1,184 @@ package parity import ( + "encoding/json" "fmt" "os" "sort" "strings" - "sync" sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + . "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/types" "github.com/opencloud-eu/reva/v2/pkg/storagespace" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" - "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) -const matrixFile = "README.md" - -type matrixRow struct { - section string - wantCount *int - wantBadRequest bool - limit int32 - ref *searchMessage.Reference - group string - id string - query string - reads string - context string - want []string - answered map[string][]string - skipped map[string]bool - - groupAt, caseAt, queryAt int -} - -var ( - matrixMu sync.Mutex - matrixRows = map[string]*matrixRow{} +const ( + matrixFile = "README.md" + matrixEntry = "engine parity" ) +// matrixRow is one line of the README. It travels in a report entry, which +// crosses processes as JSON under ginkgo -p, hence the exported fields and +// the scope already rendered to a string. +type matrixRow struct { + Section string + Group string + ID string + Query string + Scope string + Limit int32 + Reads string + Context string + Want []string + WantCount *int + WantBadRequest bool + // Overrides is what an engine is held to instead, rendered like expected + Overrides map[string]string + + GroupAt, CaseAt, QueryAt int +} + +func (r matrixRow) key() string { + return fmt.Sprintf("%s/%s/%s/%d", r.Group, r.ID, r.Query, r.QueryAt) +} + +// matrixAnswer is what a spec attaches to its report: one engine's answer to +// one row. A skipped engine leaves the README alone. +type matrixAnswer struct { + Row matrixRow + Engine string + Answer []string + Skipped bool +} + func recordAnswer(row matrixRow, engine string, answer []string) { - matrixMu.Lock() - defer matrixMu.Unlock() - - key := fmt.Sprintf("%s/%s/%s", row.group, row.id, row.query) - known, ok := matrixRows[key] - if !ok { - row.answered = map[string][]string{} - row.skipped = map[string]bool{} - known = &row - matrixRows[key] = known - } - - if answer == nil { - known.skipped[engine] = true - return - } - - known.answered[engine] = answer + AddReportEntry(matrixEntry, matrixAnswer{Row: row, Engine: engine, Answer: answer}, ReportEntryVisibilityNever) } func recordSkip(row matrixRow, engine string) { - recordAnswer(row, engine, nil) + AddReportEntry(matrixEntry, matrixAnswer{Row: row, Engine: engine, Skipped: true}, ReportEntryVisibilityNever) } -func writeMatrix() { - matrixMu.Lock() - defer matrixMu.Unlock() +// matrixPlanned is every row the spec tree announced while it was built, so +// the README is only written when each of them got an answer from every engine. +var matrixPlanned []matrixRow - rows := make([]*matrixRow, 0, len(matrixRows)) - skipped, incomplete := false, false - for _, row := range matrixRows { - rows = append(rows, row) - skipped = skipped || len(row.skipped) > 0 - incomplete = incomplete || len(row.answered)+len(row.skipped) != 2 +func planRow(rows ...matrixRow) { + matrixPlanned = append(matrixPlanned, rows...) +} + +type matrixResult struct { + matrixRow + answered map[string][]string + skipped map[string]bool +} + +// matrixAnswerOf reads an entry back: in-process it still carries the value, +// from another process only its JSON. +func matrixAnswerOf(entry types.ReportEntry) (matrixAnswer, bool) { + if answer, ok := entry.GetRawValue().(matrixAnswer); ok { + return answer, true } - switch { - case len(rows) != matrixExpectedRows(): - fmt.Fprintf(os.Stderr, "%s left alone, %d of %d cases ran\n", matrixFile, len(rows), matrixExpectedRows()) - return - case skipped: - fmt.Fprintf(os.Stderr, "%s left alone, an engine was not reachable\n", matrixFile) - return - case incomplete: - fmt.Fprintf(os.Stderr, "%s left alone, an engine died before it answered\n", matrixFile) + var answer matrixAnswer + if err := json.Unmarshal([]byte(entry.Value.AsJSON), &answer); err != nil { + return matrixAnswer{}, false + } + + return answer, true +} + +func collectMatrix(report types.Report) []*matrixResult { + results := map[string]*matrixResult{} + for _, spec := range report.SpecReports { + for _, entry := range spec.ReportEntries { + if entry.Name != matrixEntry { + continue + } + + answer, ok := matrixAnswerOf(entry) + if !ok { + continue + } + + key := answer.Row.key() + result, known := results[key] + if !known { + result = &matrixResult{matrixRow: answer.Row, answered: map[string][]string{}, skipped: map[string]bool{}} + results[key] = result + } + + if answer.Skipped { + result.skipped[answer.Engine] = true + continue + } + + result.answered[answer.Engine] = answer.Answer + } + } + + rows := make([]*matrixResult, 0, len(results)) + for _, result := range results { + rows = append(rows, result) + } + + return rows +} + +func writeMatrix(report types.Report) { + rows := collectMatrix(report) + + answered := map[string]*matrixResult{} + for _, row := range rows { + answered[row.key()] = row + } + + var missing []string + for _, planned := range matrixPlanned { + result, ok := answered[planned.key()] + switch { + case !ok: + missing = append(missing, planned.ID+" "+planned.Query) + case len(result.skipped) > 0: + fmt.Fprintf(os.Stderr, "%s left alone, an engine was not reachable\n", matrixFile) + return + case len(result.answered) != len(engineNames): + missing = append(missing, planned.ID+" "+planned.Query) + } + } + + if len(missing) > 0 { + fmt.Fprintf(os.Stderr, "%s left alone, no answer from every engine for: %s\n", matrixFile, strings.Join(missing, "; ")) return } sort.Slice(rows, func(i, j int) bool { a, b := rows[i], rows[j] switch { - case a.groupAt != b.groupAt: - return a.groupAt < b.groupAt - case a.caseAt != b.caseAt: - return a.caseAt < b.caseAt + case a.GroupAt != b.GroupAt: + return a.GroupAt < b.GroupAt + case a.CaseAt != b.CaseAt: + return a.CaseAt < b.CaseAt default: - return a.queryAt < b.queryAt + return a.QueryAt < b.QueryAt } }) out := &strings.Builder{} + out.WriteString("# Engine parity\n\n") + out.WriteString("Written by the parity suite (`go test ./services/search/pkg/parity/`), do not edit.\n") + out.WriteString("Every case runs against bleve and OpenSearch. `same?` is ✅ when both answer as\n") + out.WriteString("expected, `❌ known` when an engine's divergence is documented in the case\n") + out.WriteString("(`engineOverrides`), `❌` when it is not.\n") group, section := "", "" for _, row := range rows { - if row.section != section { - section = row.section + if row.Section != section { + section = row.Section if out.Len() > 0 { out.WriteString("\n") } @@ -114,40 +186,40 @@ func writeMatrix() { fmt.Fprintf(out, "## %s\n", section) } - if row.group != group { - group = row.group + if row.Group != group { + group = row.Group fmt.Fprintf(out, "\n### %s\n\n%s\n\n", group, matrixFixtures(group)) out.WriteString("| Case | Query | expected | bleve | OpenSearch | same? |\n") out.WriteString("|---|---|---|---|---|---|\n") } - query := "`" + shortenQuery(row.query) + "`" - if row.ref != nil { - query += " " + matrixScope(row.ref) + query := "`" + shortenQuery(row.Query) + "`" + if row.Scope != "" { + query += " " + row.Scope } - if row.limit != 0 { - query = fmt.Sprintf("%s with a limit of %d", query, row.limit) + if row.Limit != 0 { + query = fmt.Sprintf("%s with a limit of %d", query, row.Limit) } - if row.reads != "" { - query += " reads `" + row.reads + "`" + if row.Reads != "" { + query += " reads `" + row.Reads + "`" } - if row.context != "" { - query = row.context + ", then " + query + if row.Context != "" { + query = row.Context + ", then " + query } - expected := matrixNames(row.want) + expected := matrixNames(row.Want) switch { - case row.wantBadRequest: + case row.WantBadRequest: expected = "bad request" - case row.wantCount != nil: - expected = fmt.Sprintf("%d items", *row.wantCount) + case row.WantCount != nil: + expected = fmt.Sprintf("%d items", *row.WantCount) } fmt.Fprintf(out, "| %s | %s | %s | %s | %s | %s |\n", - row.id, query, expected, + row.ID, query, expected, matrixNames(row.answered["bleve"]), matrixNames(row.answered["opensearch"]), matrixVerdict(row)) } @@ -156,28 +228,6 @@ func writeMatrix() { } } -func matrixExpectedRows() int { - rows := 0 - for _, g := range queryGroups() { - rows += len(g.cases) - } - - for _, g := range lifecycleGroups() { - for _, c := range g.cases { - rows += len(c.expect) - if c.wantDocCount != nil { - rows++ - } - } - } - - for _, g := range responseGroups() { - rows += len(g.cases) - } - - return rows -} - func matrixFixtures(group string) string { var ( fixtures []search.Resource @@ -300,6 +350,10 @@ func fixtureFields(f search.Resource, withID bool) string { } func matrixScope(ref *searchMessage.Reference) string { + if ref == nil { + return "" + } + space := storagespace.FormatResourceID(&sprovider.ResourceId{ StorageId: ref.GetResourceId().GetStorageId(), SpaceId: ref.GetResourceId().GetSpaceId(), @@ -343,51 +397,59 @@ func shortenQuery(q string) string { return q[:32] + "..." + q[len(q)-24:] } -func matrixVerdict(row *matrixRow) string { +func matrixVerdict(row *matrixResult) string { var off []string - for _, engine := range []string{"bleve", "opensearch"} { - if row.wantBadRequest { - if matrixNames(row.answered[engine]) != "bad request" { - off = append(off, engine) - } + known := true + for _, engine := range engineNames { + answer := matrixNames(row.answered[engine]) + agrees := answer == matrixNames(row.Want) + switch { + case row.WantBadRequest: + agrees = answer == "bad request" + case row.WantCount != nil: + agrees = len(row.answered[engine]) == *row.WantCount + } + + if agrees { continue } - if row.wantCount != nil { - if len(row.answered[engine]) != *row.wantCount { - off = append(off, engine) - } - - continue + off = append(off, engine) + if row.WantCount != nil { + answer = fmt.Sprintf("%d items", len(row.answered[engine])) } - if matrixNames(row.answered[engine]) != matrixNames(row.want) { - off = append(off, engine) + if expected, ok := row.Overrides[engine]; !ok || expected != answer { + known = false } } - if len(off) == 0 { + switch { + case len(off) == 0: return "✅" + case known: + return "❌ known" + default: + return "❌" } - - return "❌" } func (c lifecycleCase) matrixRows(group string, groupAt, caseAt int) []matrixRow { rows := make([]matrixRow, 0, len(c.expect)+1) for i, expect := range c.expect { rows = append(rows, matrixRow{ - section: "Operations", group: group, id: c.label(group), query: expect.query, context: c.title, want: expect.want, - groupAt: groupAt, caseAt: caseAt, queryAt: i, + Section: "Operations", Group: group, ID: c.label(group), Query: expect.query, Context: c.title, Want: expect.want, Overrides: renderOverrides(c.overridesFor(expect.query)), + GroupAt: groupAt, CaseAt: caseAt, QueryAt: i, }) } if c.wantDocCount != nil { rows = append(rows, matrixRow{ - section: "Operations", group: group, id: c.label(group), query: "DocCount()", context: c.title, - want: []string{fmt.Sprint(*c.wantDocCount)}, - groupAt: groupAt, caseAt: caseAt, queryAt: len(c.expect), + Section: "Operations", Group: group, ID: c.label(group), Query: "DocCount()", Context: c.title, + Want: []string{fmt.Sprint(*c.wantDocCount)}, + Overrides: renderOverrides(c.docCountOverrides()), + GroupAt: groupAt, CaseAt: caseAt, QueryAt: len(c.expect), }) } diff --git a/services/search/pkg/parity/parity_suite_test.go b/services/search/pkg/parity/parity_suite_test.go new file mode 100644 index 0000000000..70baa137f3 --- /dev/null +++ b/services/search/pkg/parity/parity_suite_test.go @@ -0,0 +1,48 @@ +package parity + +import ( + "context" + "encoding/json" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" + "github.com/opencloud-eu/opencloud/services/search/pkg/config" +) + +var ( + // openSearchClient points every process at the one OpenSearch started by + // process 1; the container handle stays there. + openSearchClient config.EngineOpenSearchClient + stopOpenSearch func() +) + +func TestEngineParity(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Search Engine Parity Suite") +} + +var _ = SynchronizedBeforeSuite(func() []byte { + cfg, done, err := opensearchtest.SetupTests(context.Background()) + Expect(err).NotTo(HaveOccurred(), "failed to set up the OpenSearch test container") + stopOpenSearch = done + + client, err := json.Marshal(cfg.Engine.OpenSearch.Client) + Expect(err).NotTo(HaveOccurred()) + + return client +}, func(client []byte) { + Expect(json.Unmarshal(client, &openSearchClient)).To(Succeed()) +}) + +var _ = SynchronizedAfterSuite(func() {}, func() { + if stopOpenSearch != nil { + stopOpenSearch() + } +}) + +var _ = ReportAfterSuite("engine parity matrix", func(report Report) { + writeMatrix(report) +}) diff --git a/services/search/pkg/parity/parity_test.go b/services/search/pkg/parity/parity_test.go index d9238dbde4..3bacdd762b 100644 --- a/services/search/pkg/parity/parity_test.go +++ b/services/search/pkg/parity/parity_test.go @@ -3,41 +3,19 @@ package parity import ( "context" "fmt" - "os" "strings" - "testing" - "github.com/stretchr/testify/require" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/types" "github.com/opencloud-eu/reva/v2/pkg/errtypes" searchMessage "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" - "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" - "github.com/opencloud-eu/opencloud/services/search/pkg/config" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) -var defaultConfig *config.Config - -func TestMain(m *testing.M) { - // the fixtures are stamped with fixtureNow; a run crossing midnight would - // see "today" move away from them (RANGE-05/06) - - cfg, done, err := opensearchtest.SetupTests(context.Background()) - if err != nil { - _, _ = fmt.Fprintln(os.Stderr, "Failed to setup tests:", err) - os.Exit(1) - return - } - - defaultConfig = cfg - code := m.Run() - writeMatrix() - done() - os.Exit(code) -} - type queryCase struct { id int query string @@ -46,6 +24,56 @@ type queryCase struct { wantCount *int ref *searchMessage.Reference wantBadRequest bool + + // engineOverrides holds, per engine, what that engine answers today where + // it still differs from the expectation above. The spec asserts the + // override and the README marks the row as a known divergence. Once the + // engine answers as expected the override fails and has to go. + engineOverrides map[string]override +} + +// override is one engine's current answer to a case, in the expectation's +// own terms. +type override struct { + want []string + wantCount *int + wantBadRequest bool +} + +func (o override) matcher() types.GomegaMatcher { + switch { + case o.wantBadRequest: + return ConsistOf("bad request") + case o.wantCount != nil: + return HaveLen(*o.wantCount) + default: + return matchNames(o.want) + } +} + +// rendered spells the override the way the README spells an expectation. +func (o override) rendered() string { + switch { + case o.wantBadRequest: + return "bad request" + case o.wantCount != nil: + return fmt.Sprintf("%d items", *o.wantCount) + default: + return matrixNames(o.want) + } +} + +func renderOverrides(overrides map[string]override) map[string]string { + if len(overrides) == 0 { + return nil + } + + out := make(map[string]string, len(overrides)) + for engine, o := range overrides { + out[engine] = o.rendered() + } + + return out } func (c queryCase) label(group string) string { @@ -93,6 +121,8 @@ type responseCase struct { reads string read func(*searchService.SearchIndexResponse) []string want []string + + engineOverrides map[string]override } func (c responseCase) label(group string) string { @@ -126,6 +156,48 @@ type lifecycleCase struct { fixtures []search.Resource wantDocCount *uint64 + + // engineOverrides is the lifecycle spelling of queryCase.engineOverrides: + // a case has several expectations, so an override answers them by query. + engineOverrides map[string]lifecycleOverride +} + +// lifecycleOverride is one engine's current answers to a lifecycle case. +type lifecycleOverride struct { + expect map[string][]string + wantDocCount *uint64 +} + +// overridesFor picks one expectation's overrides out of the case's. +func (c lifecycleCase) overridesFor(query string) map[string]override { + var overrides map[string]override + for engine, o := range c.engineOverrides { + if answer, ok := o.expect[query]; ok { + if overrides == nil { + overrides = map[string]override{} + } + + overrides[engine] = override{want: answer} + } + } + + return overrides +} + +// docCountOverrides renders the DocCount overrides as a query override. +func (c lifecycleCase) docCountOverrides() map[string]override { + var overrides map[string]override + for engine, o := range c.engineOverrides { + if o.wantDocCount != nil { + if overrides == nil { + overrides = map[string]override{} + } + + overrides[engine] = override{want: []string{fmt.Sprint(*o.wantDocCount)}} + } + } + + return overrides } type expectation struct { @@ -163,148 +235,158 @@ func lifecycleGroups() []lifecycleGroup { } } -func TestEngineParity(t *testing.T) { - for groupAt, group := range queryGroups() { - t.Run(group.name, func(t *testing.T) { - t.Parallel() +// expectAnswer holds an engine to its override when it has one, to the +// expectation otherwise. An override that no longer holds fails on purpose: +// the engine answers as expected now, the override has to go. +func expectAnswer(engine string, answer []string, expected override, overrides map[string]override) { + GinkgoHelper() - engines := newEngines(t, "opencloud-test-engine-parity-"+group.name, group.fixtures) + if o, ok := overrides[engine]; ok { + Expect(o.rendered()).NotTo(Equal(expected.rendered()), "an override that equals the expectation documents nothing, remove it") + Expect(answer).To(o.matcher(), "the override for %s no longer holds, remove it", engine) + + return + } + + Expect(answer).To(expected.matcher()) +} + +// Every case runs once per engine as its own spec, so one engine failing +// leaves the other's answer in the matrix. The groups are Ordered containers: +// their engines are built once in a BeforeAll and shared by the specs inside, +// and they carry on after a failure so every engine gets to answer. + +var _ = Describe("Queries", func() { + for groupAt, group := range queryGroups() { + Describe(group.name, Ordered, ContinueOnFailure, func() { + var engines []testEngine + + BeforeAll(func() { + engines = newEngines("opencloud-test-engine-parity-"+group.name, group.fixtures) + }) for caseAt, c := range group.cases { row := matrixRow{ - section: "Queries", group: group.name, id: c.label(group.name), query: c.query, want: c.want, wantCount: c.wantCount, wantBadRequest: c.wantBadRequest, limit: c.limit, ref: c.ref, - groupAt: groupAt, caseAt: caseAt, + Section: "Queries", Group: group.name, ID: c.label(group.name), + Query: c.query, Scope: matrixScope(c.ref), Limit: c.limit, + Want: c.want, WantCount: c.wantCount, WantBadRequest: c.wantBadRequest, Overrides: renderOverrides(c.engineOverrides), + GroupAt: groupAt, CaseAt: caseAt, } + planRow(row) - t.Run(c.name(group.name), func(t *testing.T) { - for _, e := range engines { - t.Run(e.name, func(t *testing.T) { + Describe(c.name(group.name), func() { + for _, name := range engineNames { + It("on "+name, func() { + e := engineNamed(engines, name) if e.unavailable != "" { - recordSkip(row, e.name) - t.Skip(e.unavailable) + recordSkip(row, name) + Skip(e.unavailable) } request := &searchService.SearchIndexRequest{Query: c.query, PageSize: c.limit, Ref: c.ref} + expected := override{want: c.want, wantCount: c.wantCount, wantBadRequest: c.wantBadRequest} + _, overridden := c.engineOverrides[name] + if c.wantBadRequest { _, err := e.backend.Search(context.Background(), request) - recordAnswer(row, e.name, badRequestAnswer(err)) - require.IsType(t, errtypes.BadRequest(""), err) - - return - } - - answer := hits(t, e.backend, request) - recordAnswer(row, e.name, answer) - - if c.wantCount != nil { - require.Len(t, answer, *c.wantCount) - return - } - - require.ElementsMatch(t, c.want, answer) - }) - } - }) - } - }) - } -} - -func TestEngineParityResponse(t *testing.T) { - offset := len(queryGroups()) + len(lifecycleGroups()) - - for groupAt, group := range responseGroups() { - t.Run(group.name, func(t *testing.T) { - t.Parallel() - - shared := newEngines(t, "opencloud-test-engine-parity-"+group.name, group.fixtures) - - for caseAt, c := range group.cases { - row := matrixRow{ - section: "Response", group: group.name, id: c.label(group.name), query: c.query, reads: c.reads, context: c.after, want: c.want, - groupAt: offset + groupAt, caseAt: caseAt, - } - - t.Run(c.name(group.name), func(t *testing.T) { - engines := shared - if c.do != nil { - engines = newEngines(t, fmt.Sprintf("opencloud-test-engine-parity-%s-%d", group.name, c.id), group.fixtures) - } - - for _, e := range engines { - t.Run(e.name, func(t *testing.T) { - if e.unavailable != "" { - recordSkip(row, e.name) - t.Skip(e.unavailable) - } - - if c.do != nil { - require.NoError(t, c.do(e.backend), "the operation under test failed") - e.settle() - } - - resp, err := e.backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: c.query}) - require.NoError(t, err, "the query has to answer, an empty result is an answer") - - answer := c.read(resp) - recordAnswer(row, e.name, answer) - require.Equal(t, c.want, answer, c.reads) - }) - } - }) - } - }) - } -} - -func TestEngineParityLifecycle(t *testing.T) { - for groupAt, group := range lifecycleGroups() { - t.Run(group.name, func(t *testing.T) { - t.Parallel() - - for caseAt, c := range group.cases { - t.Run(c.name(group.name), func(t *testing.T) { - t.Parallel() - - fixtures := c.fixtures - if fixtures == nil { - fixtures = group.fixtures - } - - index := fmt.Sprintf("opencloud-test-engine-parity-%s-%d", group.name, c.id) - rows := c.matrixRows(group.name, len(queryGroups())+groupAt, caseAt) - - for _, e := range newEngines(t, index, fixtures) { - t.Run(e.name, func(t *testing.T) { - if e.unavailable != "" { - for _, row := range rows { - recordSkip(row, e.name) + answer := badRequestAnswer(err) + recordAnswer(row, name, answer) + if !overridden { + Expect(err).To(BeAssignableToTypeOf(errtypes.BadRequest(""))) } - t.Skip(e.unavailable) + expectAnswer(name, answer, expected, c.engineOverrides) + + return } + answer, err := ask(e.backend, request) + recordAnswer(row, name, answer) + if !overridden { + Expect(err).NotTo(HaveOccurred(), "the query has to answer, an empty result is an answer") + } + + expectAnswer(name, answer, expected, c.engineOverrides) + }) + } + }) + } + }) + } +}) + +var _ = Describe("Operations", func() { + offset := len(queryGroups()) + + for groupAt, group := range lifecycleGroups() { + Describe(group.name, func() { + for caseAt, c := range group.cases { + fixtures := c.fixtures + if fixtures == nil { + fixtures = group.fixtures + } + + index := fmt.Sprintf("opencloud-test-engine-parity-%s-%d", group.name, c.id) + rows := c.matrixRows(group.name, offset+groupAt, caseAt) + planRow(rows...) + + // an operation changes its index, so every spec builds its own engine + Describe(c.name(group.name), func() { + for _, name := range engineNames { + It("on "+name, func() { + e := newEngine(name, index, fixtures) + if e.unavailable != "" { + for _, row := range rows { + recordSkip(row, name) + } + + Skip(e.unavailable) + } + + // every row gets its answer before anything is asserted, a + // failed assertion must not leave the later rows unanswered err := c.do(e.backend) + if failed := err != nil; failed != c.wantErr { + for _, row := range rows { + recordAnswer(row, name, badRequestAnswer(err)) + } + } + if c.wantErr { - require.Error(t, err, "the operation had to report that it did not find the resource") + Expect(err).To(HaveOccurred(), "the operation had to report that it did not find the resource") } else { - require.NoError(t, err, "the operation under test failed") + Expect(err).NotTo(HaveOccurred(), "the operation under test failed") } e.settle() + answers := make([][]string, len(c.expect)) + errs := make([]error, len(c.expect)) for i, expect := range c.expect { - answer := hits(t, e.backend, &searchService.SearchIndexRequest{Query: expect.query}) - recordAnswer(rows[i], e.name, answer) - require.ElementsMatch(t, expect.want, answer, expect.query) + answers[i], errs[i] = ask(e.backend, &searchService.SearchIndexRequest{Query: expect.query}) + recordAnswer(rows[i], name, answers[i]) + } + + var count uint64 + if c.wantDocCount != nil { + count, err = e.backend.DocCount() + Expect(err).NotTo(HaveOccurred()) + recordAnswer(rows[len(rows)-1], name, []string{fmt.Sprint(count)}) + } + + for i, expect := range c.expect { + overrides := c.overridesFor(expect.query) + if _, overridden := overrides[name]; !overridden { + Expect(errs[i]).NotTo(HaveOccurred(), "the query has to answer, an empty result is an answer") + } + + expectAnswer(name, answers[i], override{want: expect.want}, overrides) } if c.wantDocCount != nil { - count, err := e.backend.DocCount() - require.NoError(t, err) - recordAnswer(rows[len(rows)-1], e.name, []string{fmt.Sprint(count)}) - require.Equal(t, *c.wantDocCount, count, "documents left in the index") + expected := override{want: []string{fmt.Sprint(*c.wantDocCount)}} + expectAnswer(name, []string{fmt.Sprint(count)}, expected, c.docCountOverrides()) } }) } @@ -312,4 +394,73 @@ func TestEngineParityLifecycle(t *testing.T) { } }) } -} +}) + +var _ = Describe("Response", func() { + offset := len(queryGroups()) + len(lifecycleGroups()) + + for groupAt, group := range responseGroups() { + Describe(group.name, Ordered, ContinueOnFailure, func() { + var shared []testEngine + + BeforeAll(func() { + shared = newEngines("opencloud-test-engine-parity-"+group.name, group.fixtures) + }) + + for caseAt, c := range group.cases { + row := matrixRow{ + Section: "Response", Group: group.name, ID: c.label(group.name), + Query: c.query, Reads: c.reads, Context: c.after, Want: c.want, Overrides: renderOverrides(c.engineOverrides), + GroupAt: offset + groupAt, CaseAt: caseAt, + } + planRow(row) + + index := fmt.Sprintf("opencloud-test-engine-parity-%s-%d", group.name, c.id) + + Describe(c.name(group.name), func() { + for _, name := range engineNames { + It("on "+name, func() { + // a case with an operation changes its index, so it gets its own + e := engineNamed(shared, name) + if c.do != nil { + e = newEngine(name, index, group.fixtures) + } + + if e.unavailable != "" { + recordSkip(row, name) + Skip(e.unavailable) + } + + if c.do != nil { + err := c.do(e.backend) + if err != nil { + recordAnswer(row, name, []string{"error"}) + } + + Expect(err).NotTo(HaveOccurred(), "the operation under test failed") + e.settle() + } + + resp, err := e.backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: c.query}) + if err != nil { + recordAnswer(row, name, []string{"error"}) + } + Expect(err).NotTo(HaveOccurred(), "the query has to answer, an empty result is an answer") + + answer := c.read(resp) + recordAnswer(row, name, answer) + if o, overridden := c.engineOverrides[name]; overridden { + Expect(o.want).NotTo(Equal(c.want), "an override that equals the expectation documents nothing, remove it") + Expect(answer).To(Equal(o.want), "the override for %s no longer holds, remove it", name) + + return + } + + Expect(answer).To(Equal(c.want), c.reads) + }) + } + }) + } + }) + } +}) From 889483d69c0b8b213cafcbe23ae36c83259013b5 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Fri, 28 Aug 2026 19:38:01 +0200 Subject: [PATCH 6/9] test(search): document where the engines still differ from the parity suite The expectations are the desired state; where bleve or OpenSearch answer differently on main today the case records that answer as an engine override, so the suite is green and the README lists every divergence as known. --- services/search/pkg/parity/README.md | 149 +++++++++--------- .../pkg/parity/lifecycle_casepath_test.go | 13 ++ .../pkg/parity/lifecycle_delete_test.go | 3 + .../pkg/parity/lifecycle_hidden_test.go | 10 +- .../pkg/parity/lifecycle_restore_test.go | 3 +- .../search/pkg/parity/query_content_test.go | 6 +- .../search/pkg/parity/query_deleted_test.go | 2 +- .../search/pkg/parity/query_extension_test.go | 6 +- .../search/pkg/parity/query_favorites_test.go | 4 +- .../search/pkg/parity/query_fields_test.go | 14 +- .../search/pkg/parity/query_mediatype_test.go | 2 +- services/search/pkg/parity/query_name_test.go | 30 ++-- services/search/pkg/parity/query_path_test.go | 12 +- .../search/pkg/parity/query_range_test.go | 4 +- .../search/pkg/parity/query_stress_test.go | 18 +-- services/search/pkg/parity/query_tags_test.go | 2 +- .../search/pkg/parity/query_title_test.go | 8 +- .../pkg/parity/query_visibility_test.go | 12 +- .../search/pkg/parity/response_entity_test.go | 2 +- 19 files changed, 165 insertions(+), 135 deletions(-) diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index 39fb5589dc..ad880695b4 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -1,3 +1,10 @@ +# Engine parity + +Written by the parity suite (`go test ./services/search/pkg/parity/`), do not edit. +Every case runs against bleve and OpenSearch. `same?` is ✅ when both answer as +expected, `❌ known` when an engine's divergence is documented in the case +(`engineOverrides`), `❌` when it is not. + ## Queries ### name @@ -17,15 +24,15 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| NAME-01 | `new` | new-folder | new-folder | new-folder | ✅ | -| NAME-02 | `quarterly` | quarterly notes.txt | quarterly notes.txt | quarterly notes.txt | ✅ | -| NAME-03 | `report` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-01 | `new` | new-folder | no match | new-folder | ❌ known | +| NAME-02 | `quarterly` | quarterly notes.txt | no match | quarterly notes.txt | ❌ known | +| NAME-03 | `report` | Report.txt | no match | no match | ❌ known | | NAME-04 | `name:"*new-folder*"` | new-folder | new-folder | new-folder | ✅ | | NAME-05 | `name:"*w-fol*"` | new-folder | new-folder | new-folder | ✅ | | NAME-06 | `name:"*oo ba*"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | | NAME-07 | `name:"*REPORT*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-08 | `name:"*übung*"` | Übung.txt | Übung.txt | Übung.txt | ✅ | -| NAME-09 | `name:"*ÜBUNG*"` | Übung.txt | Übung.txt | Übung.txt | ✅ | +| NAME-08 | `name:"*übung*"` | Übung.txt | Übung.txt | no match | ❌ known | +| NAME-09 | `name:"*ÜBUNG*"` | Übung.txt | Übung.txt | no match | ❌ known | | NAME-10 | `name:"*a+b*"` | a+b.txt | a+b.txt | a+b.txt | ✅ | | NAME-11 | `name:"*c(d)*"` | c(d).txt | c(d).txt | c(d).txt | ✅ | | NAME-12 | `name:"*e&f*"` | e&f.txt | e&f.txt | e&f.txt | ✅ | @@ -34,30 +41,30 @@ Fixtures: | NAME-15 | `*folder*` | new-folder | new-folder | new-folder | ✅ | | NAME-16 | `name:"*foo bar*"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | | NAME-17 | `name:"foo bar.txt"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | -| NAME-18 | `name:"*needle*"` | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | ✅ | +| NAME-18 | `name:"*needle*"` | aaaaaaaaaa...edle.txt | aaaaaaaaaa...edle.txt | no match | ❌ known | | NAME-19 | `name:"report*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-20 | `name:"*report"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-20 | `name:"*report"` | Report.txt | no match | no match | ❌ known | | NAME-21 | `name:"Rep*rt.txt"` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-22 | `Name:"*report*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-23 | `NAME:"*report*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-24 | `name:Rep?rt.txt` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-25 | `name:"*eport"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-23 | `NAME:"*report*"` | Report.txt | Report.txt | no match | ❌ known | +| NAME-24 | `name:Rep?rt.txt` | Report.txt | Report.txt | no match | ❌ known | +| NAME-25 | `name:"*eport"` | Report.txt | no match | no match | ❌ known | | NAME-26 | `name:"repor*"` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-27 | `REPORT` | Report.txt | Report.txt | Report.txt | ✅ | -| NAME-28 | `name:REPORT` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-27 | `REPORT` | Report.txt | no match | no match | ❌ known | +| NAME-28 | `name:REPORT` | Report.txt | no match | no match | ❌ known | | NAME-29 | `name:"REPORT.TXT"` | Report.txt | Report.txt | Report.txt | ✅ | | NAME-30 | `name:"FOO BAR.TXT"` | foo bar.txt | foo bar.txt | foo bar.txt | ✅ | | NAME-31 | `name:"ÜBUNG.TXT"` | Übung.txt | Übung.txt | Übung.txt | ✅ | | NAME-32 | `name:"folder*"` | no match | no match | no match | ✅ | | NAME-33 | `name:"*new"` | no match | no match | no match | ✅ | -| NAME-34 | `name:new` | new-folder | new-folder | new-folder | ✅ | -| NAME-35 | `name:"new"` | no match | no match | no match | ✅ | +| NAME-34 | `name:new` | new-folder | no match | new-folder | ❌ known | +| NAME-35 | `name:"new"` | no match | no match | new-folder | ❌ known | | NAME-36 | `name:"new-folder"` | new-folder | new-folder | new-folder | ✅ | | NAME-37 | `name:"new-*"` | new-folder | new-folder | new-folder | ✅ | | NAME-38 | `name:"new*"` | new-folder | new-folder | new-folder | ✅ | | NAME-39 | `name:new-*` | new-folder | new-folder | new-folder | ✅ | | NAME-40 | `name:"*-folder"` | new-folder | new-folder | new-folder | ✅ | -| NAME-41 | `name:"Rep?rt.txt"` | Report.txt | Report.txt | Report.txt | ✅ | +| NAME-41 | `name:"Rep?rt.txt"` | Report.txt | Report.txt | no match | ❌ known | ### extension @@ -69,10 +76,10 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| EXTENSION-01 | `txt` | report.txt | report.txt | report.txt | ✅ | -| EXTENSION-02 | `md` | notes.md | notes.md | notes.md | ✅ | +| EXTENSION-01 | `txt` | report.txt | no match | no match | ❌ known | +| EXTENSION-02 | `md` | notes.md | no match | no match | ❌ known | | EXTENSION-03 | `name:"*.txt"` | report.txt | report.txt | report.txt | ✅ | -| EXTENSION-04 | `report` | report.txt | report.txt | report.txt | ✅ | +| EXTENSION-04 | `report` | report.txt | no match | no match | ❌ known | ### tags @@ -95,7 +102,7 @@ Fixtures: | TAGS-06 | `tag:("spaced tag")` | spaced.txt | spaced.txt | spaced.txt | ✅ | | TAGS-07 | `tag:("*paced ta*")` | spaced.txt | spaced.txt | spaced.txt | ✅ | | TAGS-08 | `tag:("work")` | project | project | project | ✅ | -| TAGS-09 | `tag:("zzzzzzzzzzzzzzzzzzzzzzzzzz...zzzzzzzzzzzzzzzzneedle")` | longtag.txt | longtag.txt | longtag.txt | ✅ | +| TAGS-09 | `tag:("zzzzzzzzzzzzzzzzzzzzzzzzzz...zzzzzzzzzzzzzzzzneedle")` | longtag.txt | longtag.txt | no match | ❌ known | ### title @@ -106,12 +113,12 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| | TITLE-01 | `Title:"quarterly report"` | q1.html | q1.html | q1.html | ✅ | -| TITLE-02 | `Title:quarterly` | q1.html | q1.html | q1.html | ✅ | -| TITLE-03 | `Title:QUARTERLY` | q1.html | q1.html | q1.html | ✅ | +| TITLE-02 | `Title:quarterly` | q1.html | no match | q1.html | ❌ known | +| TITLE-03 | `Title:QUARTERLY` | q1.html | no match | q1.html | ❌ known | | TITLE-04 | `Title:quarterl*` | q1.html | q1.html | q1.html | ✅ | | TITLE-05 | `Title:"*ly rep*"` | q1.html | q1.html | q1.html | ✅ | -| TITLE-06 | `title:quarterly` | q1.html | q1.html | q1.html | ✅ | -| TITLE-07 | `Title:"QUARTERLY REPORT"` | q1.html | q1.html | q1.html | ✅ | +| TITLE-06 | `title:quarterly` | q1.html | no match | no match | ❌ known | +| TITLE-07 | `Title:"QUARTERLY REPORT"` | q1.html | no match | q1.html | ❌ known | ### content @@ -122,16 +129,16 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| CONTENT-01 | `Content:report` | no match | no match | no match | ✅ | +| CONTENT-01 | `Content:report` | no match | monthly.txt | no match | ❌ known | | CONTENT-02 | `Content:REPORTS` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-03 | `Content:"monthly reports"` | monthly.txt | monthly.txt | monthly.txt | ✅ | -| CONTENT-04 | `Content:"reports monthly"` | no match | no match | no match | ✅ | +| CONTENT-04 | `Content:"reports monthly"` | no match | monthly.txt | no match | ❌ known | | CONTENT-05 | `Content:report*` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-06 | `Content:*eport*` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-07 | `Content:month*` | monthly.txt | monthly.txt | monthly.txt | ✅ | | CONTENT-08 | `Content:"https://opencloud.example.com/help"` | links.txt | links.txt | links.txt | ✅ | | CONTENT-09 | `Content:"alan@example.org"` | links.txt | links.txt | links.txt | ✅ | -| CONTENT-10 | `Content:opencloud` | links.txt | links.txt | links.txt | ✅ | +| CONTENT-10 | `Content:opencloud` | links.txt | no match | no match | ❌ known | ### favorites @@ -144,8 +151,8 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| FAVORITES-01 | `Favorites:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | keepsakes, starred.txt | ✅ | -| FAVORITES-02 | `favorite:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | keepsakes, starred.txt | ✅ | +| FAVORITES-01 | `Favorites:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | no match | ❌ known | +| FAVORITES-02 | `favorite:"A1B2-Upper"` | keepsakes, starred.txt | keepsakes, starred.txt | no match | ❌ known | | FAVORITES-03 | `Favorites:"somebody-else"` | no match | no match | no match | ✅ | ### mediatype @@ -160,7 +167,7 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| | MEDIATYPE-01 | `mediatype:text/markdown` | notes.md | notes.md | notes.md | ✅ | -| MEDIATYPE-02 | `mediatype:TEXT/MARKDOWN` | notes.md | notes.md | notes.md | ✅ | +| MEDIATYPE-02 | `mediatype:TEXT/MARKDOWN` | notes.md | no match | notes.md | ❌ known | | MEDIATYPE-03 | `mediatype:image/jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ | | MEDIATYPE-04 | `mediatype:*jpeg` | photo.jpg | photo.jpg | photo.jpg | ✅ | | MEDIATYPE-05 | `mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ | @@ -178,14 +185,14 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| PATH-01 | `path:"./parent"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ | +| PATH-01 | `path:"./parent"` | child.jpg, parent | parent | child.jpg, parent | ❌ known | | PATH-02 | `path:"./parent/child.jpg"` | child.jpg | child.jpg | child.jpg | ✅ | -| PATH-03 | `path:"./Parent"` | no match | no match | no match | ✅ | +| PATH-03 | `path:"./Parent"` | no match | no match | child.jpg, parent | ❌ known | | PATH-04 | `path:"*child*"` | child.jpg | child.jpg | child.jpg | ✅ | -| PATH-05 | `path:"./documents"` | docs-lower | docs-lower | docs-lower | ✅ | -| PATH-06 | `path:"./DOCUMENTS"` | docs-upper | docs-upper | docs-upper | ✅ | -| PATH-07 | `path:"./Documents"` | docs-mixed | docs-mixed | docs-mixed | ✅ | -| PATH-08 | `path:"./parent/"` | child.jpg, parent | child.jpg, parent | child.jpg, parent | ✅ | +| PATH-05 | `path:"./documents"` | docs-lower | docs-lower | docs-lower, docs-mixed, docs-upper | ❌ known | +| PATH-06 | `path:"./DOCUMENTS"` | docs-upper | docs-upper | docs-lower, docs-mixed, docs-upper | ❌ known | +| PATH-07 | `path:"./Documents"` | docs-mixed | docs-mixed | docs-lower, docs-mixed, docs-upper | ❌ known | +| PATH-08 | `path:"./parent/"` | child.jpg, parent | no match | no match | ❌ known | ### fields @@ -205,14 +212,14 @@ Fixtures: | FIELDS-01 | `size:42` | small.txt | small.txt | small.txt | ✅ | | FIELDS-02 | `mtime<"2021-01-01T00:00:00Z"` | old.txt | old.txt | old.txt | ✅ | | FIELDS-03 | `id:"1$1!23"` | known.txt | known.txt | known.txt | ✅ | -| FIELDS-04 | `hidden:true` | hidden.txt | hidden.txt | hidden.txt | ✅ | -| FIELDS-05 | `type:file` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | ✅ | -| FIELDS-06 | `type:folder` | box | box | box | ✅ | +| FIELDS-04 | `hidden:true` | hidden.txt | no match | hidden.txt | ❌ known | +| FIELDS-05 | `type:file` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | no match | error | ❌ known | +| FIELDS-06 | `type:folder` | box | no match | error | ❌ known | | FIELDS-07 | `unknown:field` | no match | no match | no match | ✅ | -| FIELDS-08 | `type:File` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | ✅ | -| FIELDS-09 | `type:FOLDER` | box | box | box | ✅ | -| FIELDS-10 | `hidden:TRUE` | hidden.txt | hidden.txt | hidden.txt | ✅ | -| FIELDS-11 | `id:"1$1!AB-23"` | cased.txt | cased.txt | cased.txt | ✅ | +| FIELDS-08 | `type:File` | boxed.txt, cased.txt, hidden.txt, known.txt, old.txt, plain.txt, small.txt | no match | error | ❌ known | +| FIELDS-09 | `type:FOLDER` | box | no match | error | ❌ known | +| FIELDS-10 | `hidden:TRUE` | hidden.txt | no match | error | ❌ known | +| FIELDS-11 | `id:"1$1!AB-23"` | cased.txt | cased.txt | no match | ❌ known | | FIELDS-12 | `id:"1$1!ab-23"` | no match | no match | no match | ✅ | ### deleted @@ -232,7 +239,7 @@ Fixtures: | DELETED-02 | `name:"*.txt"` | book.txt, kept.txt | book.txt, kept.txt | book.txt, kept.txt | ✅ | | DELETED-03 | `name:"*receipt*"` | no match | no match | no match | ✅ | | DELETED-04 | `path:"./bin"` | no match | no match | no match | ✅ | -| DELETED-05 | `path:"./shelf"` | book.txt, shelf | book.txt, shelf | book.txt, shelf | ✅ | +| DELETED-05 | `path:"./shelf"` | book.txt, shelf | shelf | book.txt, shelf | ❌ known | ### visibility @@ -245,13 +252,13 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| VISIBILITY-01 | `hidden:true` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ | -| VISIBILITY-02 | `hidden:TRUE` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ | -| VISIBILITY-03 | `hidden:false` | visible.txt | visible.txt | visible.txt | ✅ | +| VISIBILITY-01 | `hidden:true` | .private, dotfile.txt, secret.txt | no match | .private, dotfile.txt, secret.txt | ❌ known | +| VISIBILITY-02 | `hidden:TRUE` | .private, dotfile.txt, secret.txt | no match | error | ❌ known | +| VISIBILITY-03 | `hidden:false` | visible.txt | no match | visible.txt | ❌ known | | VISIBILITY-04 | `name:"*secret*"` | secret.txt | secret.txt | secret.txt | ✅ | -| VISIBILITY-05 | `path:"./.private"` | .private, secret.txt | .private, secret.txt | .private, secret.txt | ✅ | -| VISIBILITY-06 | `hidden:banana` | no match | no match | no match | ✅ | -| VISIBILITY-07 | `hidden:"true"` | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | .private, dotfile.txt, secret.txt | ✅ | +| VISIBILITY-05 | `path:"./.private"` | .private, secret.txt | .private | .private, secret.txt | ❌ known | +| VISIBILITY-06 | `hidden:banana` | no match | no match | error | ❌ known | +| VISIBILITY-07 | `hidden:"true"` | .private, dotfile.txt, secret.txt | no match | .private, dotfile.txt, secret.txt | ❌ known | ### boolean @@ -300,21 +307,21 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| | STRESS-01 | `name:"*report*" AND mediatype:document` | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | ✅ | -| STRESS-02 | `name:"*report*" AND NOT tag:("draft")` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | +| STRESS-02 | `name:"*report*" AND NOT tag:("draft")` | quarterly report.docx | draft report.txt, quarterly report.docx | quarterly report.docx | ❌ known | | STRESS-03 | `(tag:("final") OR tag:("draft")) AND mediatype:image` | photo.jpg | photo.jpg | photo.jpg | ✅ | | STRESS-04 | `mediatype:document AND mtime>"2021-01-01T00:00:00Z"` | notes.md, quarterly report.docx | notes.md, quarterly report.docx | notes.md, quarterly report.docx | ✅ | -| STRESS-05 | `name:"*report*" AND size>100` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | +| STRESS-05 | `name:"*report*" AND size>100` | quarterly report.docx | no match | no match | ❌ known | | STRESS-06 | `tag:("final") AND NOT mediatype:folder` | photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | ✅ | -| STRESS-07 | `hidden:true AND name:"*notes*"` | notes.md | notes.md | notes.md | ✅ | -| STRESS-08 | `name:quarterly report` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | +| STRESS-07 | `hidden:true AND name:"*notes*"` | notes.md | no match | notes.md | ❌ known | +| STRESS-08 | `name:quarterly report` | quarterly report.docx | no match | no match | ❌ known | | STRESS-09 | `name:"quarterly report"` | no match | no match | no match | ✅ | | STRESS-10 | `name:"quarterly report.docx"` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | -| STRESS-11 | `NOT tag:("draft")` | archive, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | ✅ | -| STRESS-12 | `tag:("final") OR hidden:true` | notes.md, photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | ✅ | -| STRESS-13 | `(name:"*report*" OR name:"*notes..."draft") OR hidden:true)` | quarterly report.docx | quarterly report.docx | quarterly report.docx | ✅ | +| STRESS-11 | `NOT tag:("draft")` | archive, notes.md, photo.jpg, quarterly report.docx | archive, draft report.txt, notes.md, photo.jpg, quarterly report.docx | archive, notes.md, photo.jpg, quarterly report.docx | ❌ known | +| STRESS-12 | `tag:("final") OR hidden:true` | notes.md, photo.jpg, quarterly report.docx | photo.jpg, quarterly report.docx | notes.md, photo.jpg, quarterly report.docx | ❌ known | +| STRESS-13 | `(name:"*report*" OR name:"*notes..."draft") OR hidden:true)` | quarterly report.docx | notes.md, quarterly report.docx | quarterly report.docx | ❌ known | | STRESS-14 | `mediatype:image OR (mediatype:document AND tag:("draft"))` | draft report.txt, photo.jpg | draft report.txt, photo.jpg | draft report.txt, photo.jpg | ✅ | -| STRESS-15 | `NOT (mediatype:folder OR hidden:true)` | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | ✅ | -| STRESS-16 | `name:"*report*" AND (size>100 OR tag:("draft"))` | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | draft report.txt, quarterly report.docx | ✅ | +| STRESS-15 | `NOT (mediatype:folder OR hidden:true)` | draft report.txt, photo.jpg, quarterly report.docx | draft report.txt, notes.md, photo.jpg, quarterly report.docx | draft report.txt, photo.jpg, quarterly report.docx | ❌ known | +| STRESS-16 | `name:"*report*" AND (size>100 OR tag:("draft"))` | draft report.txt, quarterly report.docx | draft report.txt | draft report.txt | ❌ known | ### everything @@ -341,8 +348,8 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| RANGE-01 | `size>100` | big.txt | big.txt | big.txt | ✅ | -| RANGE-02 | `size<100` | ancient.txt, small.txt | ancient.txt, small.txt | ancient.txt, small.txt | ✅ | +| RANGE-01 | `size>100` | big.txt | no match | no match | ❌ known | +| RANGE-02 | `size<100` | ancient.txt, small.txt | no match | no match | ❌ known | | RANGE-03 | `mtime>"2021-01-01T00:00:00Z"` | big.txt, small.txt | big.txt, small.txt | big.txt, small.txt | ✅ | | RANGE-04 | `mtime<"2021-01-01T00:00:00Z"` | ancient.txt | ancient.txt | ancient.txt | ✅ | | RANGE-05 | `Mtime:"today"` | big.txt, small.txt | big.txt, small.txt | big.txt, small.txt | ✅ | @@ -393,7 +400,7 @@ Fixtures: | DELETE-01 | takes the resource out of the results, then `name:"*child*"` | no match | no match | no match | ✅ | | DELETE-02 | takes the descendants along, then `name:"*parent*"` | no match | no match | no match | ✅ | | DELETE-02 | takes the descendants along, then `name:"*child*"` | no match | no match | no match | ✅ | -| DELETE-03 | leaves the resource in the index, then `DocCount()` | 2 | 2 | 2 | ✅ | +| DELETE-03 | leaves the resource in the index, then `DocCount()` | 2 | 2 | 1 | ❌ known | | DELETE-04 | takes a resource out that was just written, then `name:"*fresh*"` | no match | no match | no match | ✅ | ### restore @@ -408,7 +415,7 @@ Fixtures: |---|---|---|---|---|---| | RESTORE-01 | brings the descendants back, then `name:"*parent*"` | parent | parent | parent | ✅ | | RESTORE-01 | brings the descendants back, then `name:"*child*"` | child.pdf | child.pdf | child.pdf | ✅ | -| RESTORE-02 | leaves the hidden flag alone, then `hidden:true` | file.txt | file.txt | file.txt | ✅ | +| RESTORE-02 | leaves the hidden flag alone, then `hidden:true` | file.txt | no match | file.txt | ❌ known | ### purge @@ -492,11 +499,11 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| CASEPATH-01 | takes the descendants along when deleting, then `path:"./Documents"` | no match | no match | no match | ✅ | -| CASEPATH-02 | takes the descendants along when moving, then `path:"./Other Documents"` | Other Documents, Picture.jpg | Other Documents, Picture.jpg | Other Documents, Picture.jpg | ✅ | -| CASEPATH-02 | takes the descendants along when moving, then `path:"./Documents"` | no match | no match | no match | ✅ | -| CASEPATH-03 | reaches the descendants when purging, then `path:"./Documents"` | no match | no match | no match | ✅ | -| CASEPATH-03 | reaches the descendants when purging, then `DocCount()` | 0 | 0 | 0 | ✅ | +| CASEPATH-01 | takes the descendants along when deleting, then `path:"./Documents"` | no match | no match | Documents, Picture.jpg | ❌ known | +| CASEPATH-02 | takes the descendants along when moving, then `path:"./Other Documents"` | Other Documents, Picture.jpg | Other Documents | no match | ❌ known | +| CASEPATH-02 | takes the descendants along when moving, then `path:"./Documents"` | no match | no match | Documents, Picture.jpg | ❌ known | +| CASEPATH-03 | reaches the descendants when purging, then `path:"./Documents"` | no match | no match | Documents, Picture.jpg | ❌ known | +| CASEPATH-03 | reaches the descendants when purging, then `DocCount()` | 0 | 0 | 2 | ❌ known | ### hidden @@ -511,12 +518,12 @@ Fixtures: | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| -| HIDDEN-01 | follows a move into a dot folder, then `hidden:true` | child.pdf, parent | child.pdf, parent | child.pdf, parent | ✅ | +| HIDDEN-01 | follows a move into a dot folder, then `hidden:true` | child.pdf, parent | no match | child.pdf, parent | ❌ known | | HIDDEN-02 | follows a move into a plain folder, then `hidden:true` | no match | no match | no match | ✅ | -| HIDDEN-03 | follows a move renamed with a leading dot, then `hidden:true` | .parent, child.pdf | .parent, child.pdf | .parent, child.pdf | ✅ | +| HIDDEN-03 | follows a move renamed with a leading dot, then `hidden:true` | .parent, child.pdf | no match | .parent, child.pdf | ❌ known | | HIDDEN-04 | follows a move out of a dot folder, then `hidden:true` | no match | no match | no match | ✅ | | HIDDEN-05 | follows a move renamed without the leading dot, then `hidden:true` | no match | no match | no match | ✅ | -| HIDDEN-06 | follows a move within the same dot folder, then `hidden:true` | child.pdf, moved | child.pdf, moved | child.pdf, moved | ✅ | +| HIDDEN-06 | follows a move within the same dot folder, then `hidden:true` | child.pdf, moved | no match | child.pdf, moved | ❌ known | ### upsert @@ -598,7 +605,7 @@ Fixtures: | ENTITY-08 | `name:"bar.pdf"` reads `MimeType` | application/pdf | application/pdf | application/pdf | ✅ | | ENTITY-09 | `name:"bar.pdf"` reads `Deleted` | false | false | false | ✅ | | ENTITY-10 | `name:"bar.pdf"` reads `Score` | above zero | above zero | above zero | ✅ | -| ENTITY-11 | `path:"./parent"` reads `TotalMatches` | 2 | 2 | 2 | ✅ | +| ENTITY-11 | `path:"./parent"` reads `TotalMatches` | 2 | 1 | 2 | ❌ known | | ENTITY-12 | `name:"*notes*"` reads `Highlights` | "" | "" | "" | ✅ | | ENTITY-13 | `content:bar` reads `Highlights` | foo bar baz | foo bar baz | foo bar baz | ✅ | | ENTITY-14 | moved to another parent, then `name:"newname"` reads `ParentId` | 1$1!9 | 1$1!9 | 1$1!9 | ✅ | diff --git a/services/search/pkg/parity/lifecycle_casepath_test.go b/services/search/pkg/parity/lifecycle_casepath_test.go index 95b3b36208..8347fcf318 100644 --- a/services/search/pkg/parity/lifecycle_casepath_test.go +++ b/services/search/pkg/parity/lifecycle_casepath_test.go @@ -28,6 +28,9 @@ func casePathLifecycle() lifecycleGroup { id: 1, title: "takes the descendants along when deleting", do: func(e search.Engine) error { return e.Delete(folder.ID) }, expect: left(), + engineOverrides: map[string]lifecycleOverride{ + "opensearch": {expect: map[string][]string{`path:"./Documents"`: {"Documents", "Picture.jpg"}}}, + }, }, { id: 2, title: "takes the descendants along when moving", @@ -36,12 +39,22 @@ func casePathLifecycle() lifecycleGroup { {`path:"./Other Documents"`, []string{"Other Documents", "Picture.jpg"}}, {`path:"./Documents"`, nil}, }, + engineOverrides: map[string]lifecycleOverride{ + "bleve": {expect: map[string][]string{`path:"./Other Documents"`: {"Other Documents"}}}, + "opensearch": {expect: map[string][]string{`path:"./Other Documents"`: {}, `path:"./Documents"`: {"Documents", "Picture.jpg"}}}, + }, }, { id: 3, title: "reaches the descendants when purging", do: func(e search.Engine) error { return e.Purge(folder.ID, false) }, expect: left(), wantDocCount: conversions.ToPointer(uint64(0)), + engineOverrides: map[string]lifecycleOverride{ + "opensearch": { + expect: map[string][]string{`path:"./Documents"`: {"Documents", "Picture.jpg"}}, + wantDocCount: conversions.ToPointer(uint64(2)), + }, + }, }, }, } diff --git a/services/search/pkg/parity/lifecycle_delete_test.go b/services/search/pkg/parity/lifecycle_delete_test.go index 3a4c49b2c6..c483deb901 100644 --- a/services/search/pkg/parity/lifecycle_delete_test.go +++ b/services/search/pkg/parity/lifecycle_delete_test.go @@ -26,6 +26,9 @@ func deleteLifecycle() lifecycleGroup { id: 3, title: "leaves the resource in the index", do: func(e search.Engine) error { return e.Delete(child.ID) }, wantDocCount: conversions.ToPointer(uint64(2)), + engineOverrides: map[string]lifecycleOverride{ + "opensearch": {wantDocCount: conversions.ToPointer(uint64(1))}, + }, }, { id: 4, title: "takes a resource out that was just written", diff --git a/services/search/pkg/parity/lifecycle_hidden_test.go b/services/search/pkg/parity/lifecycle_hidden_test.go index c703d6ce74..59ec242088 100644 --- a/services/search/pkg/parity/lifecycle_hidden_test.go +++ b/services/search/pkg/parity/lifecycle_hidden_test.go @@ -29,9 +29,14 @@ func hiddenLifecycle() lifecycleGroup { child.Path = move.from + "/child.pdf" child.Hidden = parent.Hidden - var hidden []string + var ( + hidden []string + overrides map[string]lifecycleOverride + ) if move.hidden { hidden = []string{path.Base(move.target), "child.pdf"} + // bleve does not answer hidden:true at all today + overrides = map[string]lifecycleOverride{"bleve": {expect: map[string][]string{`hidden:true`: {}}}} } group.cases = append(group.cases, lifecycleCase{ @@ -40,7 +45,8 @@ func hiddenLifecycle() lifecycleGroup { do: func(e search.Engine) error { return e.Move(parent.ID, parent.ParentID, move.target) }, - expect: []expectation{{`hidden:true`, hidden}}, + expect: []expectation{{`hidden:true`, hidden}}, + engineOverrides: overrides, }) } diff --git a/services/search/pkg/parity/lifecycle_restore_test.go b/services/search/pkg/parity/lifecycle_restore_test.go index c85c4c596a..79dd3e3f02 100644 --- a/services/search/pkg/parity/lifecycle_restore_test.go +++ b/services/search/pkg/parity/lifecycle_restore_test.go @@ -33,7 +33,8 @@ func restoreLifecycle() lifecycleGroup { return e.Restore(secret.ID) }, - expect: []expectation{{`hidden:true`, []string{"file.txt"}}}, + expect: []expectation{{`hidden:true`, []string{"file.txt"}}}, + engineOverrides: map[string]lifecycleOverride{"bleve": {expect: map[string][]string{`hidden:true`: {}}}}, }, }, } diff --git a/services/search/pkg/parity/query_content_test.go b/services/search/pkg/parity/query_content_test.go index 010b5545c3..d51d36d53b 100644 --- a/services/search/pkg/parity/query_content_test.go +++ b/services/search/pkg/parity/query_content_test.go @@ -12,16 +12,16 @@ func contentGroup() queryGroup { fixtureDoc("links.txt", withContent("see https://opencloud.example.com/help or write to alan@example.org")), }, cases: []queryCase{ - {id: 1, query: `Content:report`}, + {id: 1, query: `Content:report`, engineOverrides: map[string]override{"bleve": override{want: []string{"monthly.txt"}}}}, {id: 2, query: `Content:REPORTS`, want: []string{"monthly.txt"}}, {id: 3, query: `Content:"monthly reports"`, want: []string{"monthly.txt"}}, - {id: 4, query: `Content:"reports monthly"`}, + {id: 4, query: `Content:"reports monthly"`, engineOverrides: map[string]override{"bleve": override{want: []string{"monthly.txt"}}}}, {id: 5, query: `Content:report*`, want: []string{"monthly.txt"}}, {id: 6, query: `Content:*eport*`, want: []string{"monthly.txt"}}, {id: 7, query: `Content:month*`, want: []string{"monthly.txt"}}, {id: 8, query: `Content:"https://opencloud.example.com/help"`, want: []string{"links.txt"}}, {id: 9, query: `Content:"alan@example.org"`, want: []string{"links.txt"}}, - {id: 10, query: `Content:opencloud`, want: []string{"links.txt"}}, + {id: 10, query: `Content:opencloud`, want: []string{"links.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, }, } } diff --git a/services/search/pkg/parity/query_deleted_test.go b/services/search/pkg/parity/query_deleted_test.go index 1e048b1e9d..905725bd8b 100644 --- a/services/search/pkg/parity/query_deleted_test.go +++ b/services/search/pkg/parity/query_deleted_test.go @@ -20,7 +20,7 @@ func deletedGroup() queryGroup { {id: 2, query: `name:"*.txt"`, want: []string{"kept.txt", "book.txt"}}, {id: 3, query: `name:"*receipt*"`}, {id: 4, query: `path:"./bin"`}, - {id: 5, query: `path:"./shelf"`, want: []string{"shelf", "book.txt"}}, + {id: 5, query: `path:"./shelf"`, want: []string{"shelf", "book.txt"}, engineOverrides: map[string]override{"bleve": override{want: []string{"shelf"}}}}, }, } } diff --git a/services/search/pkg/parity/query_extension_test.go b/services/search/pkg/parity/query_extension_test.go index a813640bcb..e02b5e30fa 100644 --- a/services/search/pkg/parity/query_extension_test.go +++ b/services/search/pkg/parity/query_extension_test.go @@ -13,10 +13,10 @@ func extensionGroup() queryGroup { fixtureFolder("archive"), }, cases: []queryCase{ - {id: 1, query: `txt`, want: []string{"report.txt"}}, - {id: 2, query: `md`, want: []string{"notes.md"}}, + {id: 1, query: `txt`, want: []string{"report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 2, query: `md`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, {id: 3, query: `name:"*.txt"`, want: []string{"report.txt"}}, - {id: 4, query: `report`, want: []string{"report.txt"}}, + {id: 4, query: `report`, want: []string{"report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, }, } } diff --git a/services/search/pkg/parity/query_favorites_test.go b/services/search/pkg/parity/query_favorites_test.go index 5413f3eaf4..ba1f4d7d1c 100644 --- a/services/search/pkg/parity/query_favorites_test.go +++ b/services/search/pkg/parity/query_favorites_test.go @@ -14,8 +14,8 @@ func favoritesGroup() queryGroup { fixtureDoc("photo.jpg", withParent("1$1!keepsakes"), withPath("./keepsakes/photo.jpg"), withMime("image/jpeg")), }, cases: []queryCase{ - {id: 1, query: `Favorites:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}}, - {id: 2, query: `favorite:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}}, + {id: 1, query: `Favorites:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 2, query: `favorite:"A1B2-Upper"`, want: []string{"starred.txt", "keepsakes"}, engineOverrides: map[string]override{"opensearch": override{}}}, {id: 3, query: `Favorites:"somebody-else"`}, }, } diff --git a/services/search/pkg/parity/query_fields_test.go b/services/search/pkg/parity/query_fields_test.go index 4e737c90a3..265a7aecd5 100644 --- a/services/search/pkg/parity/query_fields_test.go +++ b/services/search/pkg/parity/query_fields_test.go @@ -21,14 +21,14 @@ func fieldsGroup() queryGroup { {id: 1, query: `size:42`, want: []string{"small.txt"}}, {id: 2, query: `mtime<"2021-01-01T00:00:00Z"`, want: []string{"old.txt"}}, {id: 3, query: `id:"1$1!23"`, want: []string{"known.txt"}}, - {id: 4, query: `hidden:true`, want: []string{"hidden.txt"}}, - {id: 5, query: `type:file`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt"}}, - {id: 6, query: `type:folder`, want: []string{"box"}}, + {id: 4, query: `hidden:true`, want: []string{"hidden.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 5, query: `type:file`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, + {id: 6, query: `type:folder`, want: []string{"box"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, {id: 7, query: `unknown:field`}, - {id: 8, query: `type:File`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt"}}, - {id: 9, query: `type:FOLDER`, want: []string{"box"}}, - {id: 10, query: `hidden:TRUE`, want: []string{"hidden.txt"}}, - {id: 11, query: `id:"1$1!AB-23"`, want: []string{"cased.txt"}}, + {id: 8, query: `type:File`, want: []string{"small.txt", "old.txt", "known.txt", "cased.txt", "hidden.txt", "plain.txt", "boxed.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, + {id: 9, query: `type:FOLDER`, want: []string{"box"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, + {id: 10, query: `hidden:TRUE`, want: []string{"hidden.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, + {id: 11, query: `id:"1$1!AB-23"`, want: []string{"cased.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, {id: 12, query: `id:"1$1!ab-23"`}, }, } diff --git a/services/search/pkg/parity/query_mediatype_test.go b/services/search/pkg/parity/query_mediatype_test.go index cd43ff89d5..11a5065d99 100644 --- a/services/search/pkg/parity/query_mediatype_test.go +++ b/services/search/pkg/parity/query_mediatype_test.go @@ -15,7 +15,7 @@ func mediatypeGroup() queryGroup { }, cases: []queryCase{ {id: 1, query: `mediatype:text/markdown`, want: []string{"notes.md"}}, - {id: 2, query: `mediatype:TEXT/MARKDOWN`, want: []string{"notes.md"}}, + {id: 2, query: `mediatype:TEXT/MARKDOWN`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}}}, {id: 3, query: `mediatype:image/jpeg`, want: []string{"photo.jpg"}}, {id: 4, query: `mediatype:*jpeg`, want: []string{"photo.jpg"}}, {id: 5, query: `mediatype:image`, want: []string{"photo.jpg"}}, diff --git a/services/search/pkg/parity/query_name_test.go b/services/search/pkg/parity/query_name_test.go index ee7b8dc177..807041166d 100644 --- a/services/search/pkg/parity/query_name_test.go +++ b/services/search/pkg/parity/query_name_test.go @@ -20,15 +20,15 @@ func nameGroup() queryGroup { fixtureDoc(fixtureLongName), }, cases: []queryCase{ - {id: 1, query: `new`, want: []string{"new-folder"}}, - {id: 2, query: `quarterly`, want: []string{"quarterly notes.txt"}}, - {id: 3, query: `report`, want: []string{"Report.txt"}}, + {id: 1, query: `new`, want: []string{"new-folder"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 2, query: `quarterly`, want: []string{"quarterly notes.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 3, query: `report`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, {id: 4, query: `name:"*new-folder*"`, want: []string{"new-folder"}}, {id: 5, query: `name:"*w-fol*"`, want: []string{"new-folder"}}, {id: 6, query: `name:"*oo ba*"`, want: []string{"foo bar.txt"}}, {id: 7, query: `name:"*REPORT*"`, want: []string{"Report.txt"}}, - {id: 8, query: `name:"*übung*"`, want: []string{"Übung.txt"}}, - {id: 9, query: `name:"*ÜBUNG*"`, want: []string{"Übung.txt"}}, + {id: 8, query: `name:"*übung*"`, want: []string{"Übung.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 9, query: `name:"*ÜBUNG*"`, want: []string{"Übung.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, {id: 10, query: `name:"*a+b*"`, want: []string{"a+b.txt"}}, {id: 11, query: `name:"*c(d)*"`, want: []string{"c(d).txt"}}, {id: 12, query: `name:"*e&f*"`, want: []string{"e&f.txt"}}, @@ -37,30 +37,30 @@ func nameGroup() queryGroup { {id: 15, query: `*folder*`, want: []string{"new-folder"}}, {id: 16, query: `name:"*foo bar*"`, want: []string{"foo bar.txt"}}, {id: 17, query: `name:"foo bar.txt"`, want: []string{"foo bar.txt"}}, - {id: 18, query: `name:"*needle*"`, want: []string{fixtureLongName}}, + {id: 18, query: `name:"*needle*"`, want: []string{fixtureLongName}, engineOverrides: map[string]override{"opensearch": override{}}}, {id: 19, query: `name:"report*"`, want: []string{"Report.txt"}}, - {id: 20, query: `name:"*report"`, want: []string{"Report.txt"}}, + {id: 20, query: `name:"*report"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, {id: 21, query: `name:"Rep*rt.txt"`, want: []string{"Report.txt"}}, {id: 22, query: `Name:"*report*"`, want: []string{"Report.txt"}}, - {id: 23, query: `NAME:"*report*"`, want: []string{"Report.txt"}}, - {id: 24, query: `name:Rep?rt.txt`, want: []string{"Report.txt"}}, - {id: 25, query: `name:"*eport"`, want: []string{"Report.txt"}}, + {id: 23, query: `NAME:"*report*"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 24, query: `name:Rep?rt.txt`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 25, query: `name:"*eport"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, {id: 26, query: `name:"repor*"`, want: []string{"Report.txt"}}, - {id: 27, query: `REPORT`, want: []string{"Report.txt"}}, - {id: 28, query: `name:REPORT`, want: []string{"Report.txt"}}, + {id: 27, query: `REPORT`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 28, query: `name:REPORT`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, {id: 29, query: `name:"REPORT.TXT"`, want: []string{"Report.txt"}}, {id: 30, query: `name:"FOO BAR.TXT"`, want: []string{"foo bar.txt"}}, {id: 31, query: `name:"ÜBUNG.TXT"`, want: []string{"Übung.txt"}}, {id: 32, query: `name:"folder*"`}, {id: 33, query: `name:"*new"`}, - {id: 34, query: `name:new`, want: []string{"new-folder"}}, - {id: 35, query: `name:"new"`}, + {id: 34, query: `name:new`, want: []string{"new-folder"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 35, query: `name:"new"`, engineOverrides: map[string]override{"opensearch": override{want: []string{"new-folder"}}}}, {id: 36, query: `name:"new-folder"`, want: []string{"new-folder"}}, {id: 37, query: `name:"new-*"`, want: []string{"new-folder"}}, {id: 38, query: `name:"new*"`, want: []string{"new-folder"}}, {id: 39, query: `name:new-*`, want: []string{"new-folder"}}, {id: 40, query: `name:"*-folder"`, want: []string{"new-folder"}}, - {id: 41, query: `name:"Rep?rt.txt"`, want: []string{"Report.txt"}}, + {id: 41, query: `name:"Rep?rt.txt"`, want: []string{"Report.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, }, } } diff --git a/services/search/pkg/parity/query_path_test.go b/services/search/pkg/parity/query_path_test.go index e97e8e1596..828d298951 100644 --- a/services/search/pkg/parity/query_path_test.go +++ b/services/search/pkg/parity/query_path_test.go @@ -15,14 +15,14 @@ func pathGroup() queryGroup { fixtureFolder("docs-mixed", withPath("./Documents")), }, cases: []queryCase{ - {id: 1, query: `path:"./parent"`, want: []string{"parent", "child.jpg"}}, + {id: 1, query: `path:"./parent"`, want: []string{"parent", "child.jpg"}, engineOverrides: map[string]override{"bleve": override{want: []string{"parent"}}}}, {id: 2, query: `path:"./parent/child.jpg"`, want: []string{"child.jpg"}}, - {id: 3, query: `path:"./Parent"`}, + {id: 3, query: `path:"./Parent"`, engineOverrides: map[string]override{"opensearch": override{want: []string{"child.jpg", "parent"}}}}, {id: 4, query: `path:"*child*"`, want: []string{"child.jpg"}}, - {id: 5, query: `path:"./documents"`, want: []string{"docs-lower"}}, - {id: 6, query: `path:"./DOCUMENTS"`, want: []string{"docs-upper"}}, - {id: 7, query: `path:"./Documents"`, want: []string{"docs-mixed"}}, - {id: 8, query: `path:"./parent/"`, want: []string{"parent", "child.jpg"}}, + {id: 5, query: `path:"./documents"`, want: []string{"docs-lower"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"docs-lower", "docs-mixed", "docs-upper"}}}}, + {id: 6, query: `path:"./DOCUMENTS"`, want: []string{"docs-upper"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"docs-lower", "docs-mixed", "docs-upper"}}}}, + {id: 7, query: `path:"./Documents"`, want: []string{"docs-mixed"}, engineOverrides: map[string]override{"opensearch": override{want: []string{"docs-lower", "docs-mixed", "docs-upper"}}}}, + {id: 8, query: `path:"./parent/"`, want: []string{"parent", "child.jpg"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, }, } } diff --git a/services/search/pkg/parity/query_range_test.go b/services/search/pkg/parity/query_range_test.go index f5c7033012..59a1618272 100644 --- a/services/search/pkg/parity/query_range_test.go +++ b/services/search/pkg/parity/query_range_test.go @@ -13,8 +13,8 @@ func rangeGroup() queryGroup { fixtureDoc("ancient.txt", withSize(10), withMtime("2020-01-01T00:00:00Z")), }, cases: []queryCase{ - {id: 1, query: `size>100`, want: []string{"big.txt"}}, - {id: 2, query: `size<100`, want: []string{"small.txt", "ancient.txt"}}, + {id: 1, query: `size>100`, want: []string{"big.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 2, query: `size<100`, want: []string{"small.txt", "ancient.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, {id: 3, query: `mtime>"2021-01-01T00:00:00Z"`, want: []string{"small.txt", "big.txt"}}, {id: 4, query: `mtime<"2021-01-01T00:00:00Z"`, want: []string{"ancient.txt"}}, {id: 5, query: `Mtime:"today"`, want: []string{"small.txt", "big.txt"}}, diff --git a/services/search/pkg/parity/query_stress_test.go b/services/search/pkg/parity/query_stress_test.go index 36dc5acafb..a739800ad2 100644 --- a/services/search/pkg/parity/query_stress_test.go +++ b/services/search/pkg/parity/query_stress_test.go @@ -18,21 +18,21 @@ func stressGroup() queryGroup { }, cases: []queryCase{ {id: 1, query: `name:"*report*" AND mediatype:document`, want: []string{"quarterly report.docx", "draft report.txt"}}, - {id: 2, query: `name:"*report*" AND NOT tag:("draft")`, want: []string{"quarterly report.docx"}}, + {id: 2, query: `name:"*report*" AND NOT tag:("draft")`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{want: []string{"draft report.txt", "quarterly report.docx"}}}}, {id: 3, query: `(tag:("final") OR tag:("draft")) AND mediatype:image`, want: []string{"photo.jpg"}}, {id: 4, query: `mediatype:document AND mtime>"2021-01-01T00:00:00Z"`, want: []string{"quarterly report.docx", "notes.md"}}, - {id: 5, query: `name:"*report*" AND size>100`, want: []string{"quarterly report.docx"}}, + {id: 5, query: `name:"*report*" AND size>100`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, {id: 6, query: `tag:("final") AND NOT mediatype:folder`, want: []string{"quarterly report.docx", "photo.jpg"}}, - {id: 7, query: `hidden:true AND name:"*notes*"`, want: []string{"notes.md"}}, - {id: 8, query: `name:quarterly report`, want: []string{"quarterly report.docx"}}, + {id: 7, query: `hidden:true AND name:"*notes*"`, want: []string{"notes.md"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 8, query: `name:quarterly report`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, {id: 9, query: `name:"quarterly report"`}, {id: 10, query: `name:"quarterly report.docx"`, want: []string{"quarterly report.docx"}}, - {id: 11, query: `NOT tag:("draft")`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md", "archive"}}, - {id: 12, query: `tag:("final") OR hidden:true`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md"}}, - {id: 13, query: `(name:"*report*" OR name:"*notes*") AND NOT (tag:("draft") OR hidden:true)`, want: []string{"quarterly report.docx"}}, + {id: 11, query: `NOT tag:("draft")`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md", "archive"}, engineOverrides: map[string]override{"bleve": override{want: []string{"archive", "draft report.txt", "notes.md", "photo.jpg", "quarterly report.docx"}}}}, + {id: 12, query: `tag:("final") OR hidden:true`, want: []string{"quarterly report.docx", "photo.jpg", "notes.md"}, engineOverrides: map[string]override{"bleve": override{want: []string{"photo.jpg", "quarterly report.docx"}}}}, + {id: 13, query: `(name:"*report*" OR name:"*notes*") AND NOT (tag:("draft") OR hidden:true)`, want: []string{"quarterly report.docx"}, engineOverrides: map[string]override{"bleve": override{want: []string{"notes.md", "quarterly report.docx"}}}}, {id: 14, query: `mediatype:image OR (mediatype:document AND tag:("draft"))`, want: []string{"photo.jpg", "draft report.txt"}}, - {id: 15, query: `NOT (mediatype:folder OR hidden:true)`, want: []string{"quarterly report.docx", "draft report.txt", "photo.jpg"}}, - {id: 16, query: `name:"*report*" AND (size>100 OR tag:("draft"))`, want: []string{"quarterly report.docx", "draft report.txt"}}, + {id: 15, query: `NOT (mediatype:folder OR hidden:true)`, want: []string{"quarterly report.docx", "draft report.txt", "photo.jpg"}, engineOverrides: map[string]override{"bleve": override{want: []string{"draft report.txt", "notes.md", "photo.jpg", "quarterly report.docx"}}}}, + {id: 16, query: `name:"*report*" AND (size>100 OR tag:("draft"))`, want: []string{"quarterly report.docx", "draft report.txt"}, engineOverrides: map[string]override{"bleve": override{want: []string{"draft report.txt"}}, "opensearch": override{want: []string{"draft report.txt"}}}}, }, } } diff --git a/services/search/pkg/parity/query_tags_test.go b/services/search/pkg/parity/query_tags_test.go index c8505074d3..74cc3bc716 100644 --- a/services/search/pkg/parity/query_tags_test.go +++ b/services/search/pkg/parity/query_tags_test.go @@ -28,7 +28,7 @@ func tagsGroup() queryGroup { {id: 6, query: `tag:("spaced tag")`, want: []string{"spaced.txt"}}, {id: 7, query: `tag:("*paced ta*")`, want: []string{"spaced.txt"}}, {id: 8, query: `tag:("work")`, want: []string{"project"}}, - {id: 9, query: `tag:("` + longTag + `")`, want: []string{"longtag.txt"}}, + {id: 9, query: `tag:("` + longTag + `")`, want: []string{"longtag.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, }, } } diff --git a/services/search/pkg/parity/query_title_test.go b/services/search/pkg/parity/query_title_test.go index 26786a0050..8251c668db 100644 --- a/services/search/pkg/parity/query_title_test.go +++ b/services/search/pkg/parity/query_title_test.go @@ -12,12 +12,12 @@ func titleGroup() queryGroup { }, cases: []queryCase{ {id: 1, query: `Title:"quarterly report"`, want: []string{"q1.html"}}, - {id: 2, query: `Title:quarterly`, want: []string{"q1.html"}}, - {id: 3, query: `Title:QUARTERLY`, want: []string{"q1.html"}}, + {id: 2, query: `Title:quarterly`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 3, query: `Title:QUARTERLY`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}}, {id: 4, query: `Title:quarterl*`, want: []string{"q1.html"}}, {id: 5, query: `Title:"*ly rep*"`, want: []string{"q1.html"}}, - {id: 6, query: `title:quarterly`, want: []string{"q1.html"}}, - {id: 7, query: `Title:"QUARTERLY REPORT"`, want: []string{"q1.html"}}, + {id: 6, query: `title:quarterly`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{}}}, + {id: 7, query: `Title:"QUARTERLY REPORT"`, want: []string{"q1.html"}, engineOverrides: map[string]override{"bleve": override{}}}, }, } } diff --git a/services/search/pkg/parity/query_visibility_test.go b/services/search/pkg/parity/query_visibility_test.go index 5a3d879946..12939126c2 100644 --- a/services/search/pkg/parity/query_visibility_test.go +++ b/services/search/pkg/parity/query_visibility_test.go @@ -14,13 +14,13 @@ func visibilityGroup() queryGroup { fixtureDoc("secret.txt", withParent("1$1!.private"), withPath("./.private/secret.txt"), isHidden()), }, cases: []queryCase{ - {id: 1, query: `hidden:true`, want: []string{"dotfile.txt", ".private", "secret.txt"}}, - {id: 2, query: `hidden:TRUE`, want: []string{"dotfile.txt", ".private", "secret.txt"}}, - {id: 3, query: `hidden:false`, want: []string{"visible.txt"}}, + {id: 1, query: `hidden:true`, want: []string{"dotfile.txt", ".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, + {id: 2, query: `hidden:TRUE`, want: []string{"dotfile.txt", ".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, + {id: 3, query: `hidden:false`, want: []string{"visible.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, {id: 4, query: `name:"*secret*"`, want: []string{"secret.txt"}}, - {id: 5, query: `path:"./.private"`, want: []string{".private", "secret.txt"}}, - {id: 6, query: `hidden:banana`}, - {id: 7, query: `hidden:"true"`, want: []string{"dotfile.txt", ".private", "secret.txt"}}, + {id: 5, query: `path:"./.private"`, want: []string{".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{want: []string{".private"}}}}, + {id: 6, query: `hidden:banana`, engineOverrides: map[string]override{"opensearch": override{want: []string{"error"}}}}, + {id: 7, query: `hidden:"true"`, want: []string{"dotfile.txt", ".private", "secret.txt"}, engineOverrides: map[string]override{"bleve": override{}}}, }, } } diff --git a/services/search/pkg/parity/response_entity_test.go b/services/search/pkg/parity/response_entity_test.go index c32ecbf5df..29ecffa8df 100644 --- a/services/search/pkg/parity/response_entity_test.go +++ b/services/search/pkg/parity/response_entity_test.go @@ -72,7 +72,7 @@ func entityGroup() responseGroup { }), }, { - id: 11, query: `path:"./parent"`, reads: "TotalMatches", want: []string{"2"}, + id: 11, query: `path:"./parent"`, reads: "TotalMatches", want: []string{"2"}, engineOverrides: map[string]override{"bleve": override{want: []string{"1"}}}, read: func(resp *searchService.SearchIndexResponse) []string { return []string{fmt.Sprint(resp.GetTotalMatches())} }, From 545aa114e89b1c17bc4476f80e6088f2bf026f0f Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Fri, 28 Aug 2026 19:38:01 +0200 Subject: [PATCH 7/9] test(search): let the parity suite replace the per-engine backend suites Everything the bleve and OpenSearch backend suites checked now runs against both engines in services/search/pkg/parity. The two cases without a twin there join it (a facet value keeps its case, batches stay apart), the one thing only OpenSearch can do, refuse an unhealthy cluster, stays in its package. --- services/search/pkg/bleve/backend_test.go | 808 ------------------ .../search/pkg/opensearch/backend_test.go | 662 +------------- services/search/pkg/parity/README.md | 5 + .../search/pkg/parity/lifecycle_batch_test.go | 29 + .../search/pkg/parity/query_fields_test.go | 6 + 5 files changed, 42 insertions(+), 1468 deletions(-) delete mode 100644 services/search/pkg/bleve/backend_test.go diff --git a/services/search/pkg/bleve/backend_test.go b/services/search/pkg/bleve/backend_test.go deleted file mode 100644 index b9feef8e8d..0000000000 --- a/services/search/pkg/bleve/backend_test.go +++ /dev/null @@ -1,808 +0,0 @@ -package bleve_test - -import ( - "context" - "fmt" - - bleveSearch "github.com/blevesearch/bleve/v2" - sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - libregraph "github.com/opencloud-eu/libre-graph-api-go" - "github.com/opencloud-eu/reva/v2/pkg/storagespace" - - "github.com/opencloud-eu/opencloud/pkg/log" - searchmsg "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/messages/search/v0" - searchsvc "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" - "github.com/opencloud-eu/opencloud/services/search/pkg/bleve" - "github.com/opencloud-eu/opencloud/services/search/pkg/content" - bleveQuery "github.com/opencloud-eu/opencloud/services/search/pkg/query/bleve" - "github.com/opencloud-eu/opencloud/services/search/pkg/search" -) - -func hiddenByID(idx bleveSearch.Index, id string) bool { - GinkgoHelper() - - req := bleveSearch.NewSearchRequest(bleveSearch.NewDocIDQuery([]string{id})) - req.Fields = []string{"Hidden"} - - res, err := idx.Search(req) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Hits).To(HaveLen(1), "no record for %s", id) - - hidden, _ := res.Hits[0].Fields["Hidden"].(bool) - return hidden -} - -var _ = Describe("Bleve", func() { - var ( - eng *bleve.Backend - idx bleveSearch.Index - - doSearch = func(id string, query, path string) (*searchsvc.SearchIndexResponse, error) { - rID, err := storagespace.ParseID(id) - if err != nil { - return nil, err - } - - return eng.Search(context.Background(), &searchsvc.SearchIndexRequest{ - Query: query, - Ref: &searchmsg.Reference{ - ResourceId: &searchmsg.ResourceID{ - StorageId: rID.StorageId, - SpaceId: rID.SpaceId, - OpaqueId: rID.OpaqueId, - }, - Path: path, - }, - }) - } - - assertDocCount = func(id string, query string, expectedCount int) []*searchmsg.Match { - res, err := doSearch(id, query, "") - - ExpectWithOffset(1, err).ToNot(HaveOccurred()) - ExpectWithOffset(1, len(res.Matches)).To(Equal(expectedCount), "query returned unexpected number of results: "+query) - return res.Matches - } - - rootResource search.Resource - parentResource search.Resource - childResource search.Resource - childResource2 search.Resource - ) - - BeforeEach(func() { - mapping, err := bleve.NewMapping() - Expect(err).ToNot(HaveOccurred()) - - idx, err = bleveSearch.NewMemOnly(mapping) - Expect(err).ToNot(HaveOccurred()) - - eng = bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}) - Expect(err).ToNot(HaveOccurred()) - - rootResource = search.Resource{ - ID: "1$2!2", - RootID: "1$2!2", - Path: ".", - Document: content.Document{}, - } - - parentResource = search.Resource{ - ID: "1$2!3", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./parent d!r", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER), - Document: content.Document{Name: "parent d!r"}, - } - - childResource = search.Resource{ - ID: "1$2!4", - ParentID: parentResource.ID, - RootID: rootResource.ID, - Path: "./parent d!r/child.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "child.pdf"}, - } - - childResource2 = search.Resource{ - ID: "1$2!5", - ParentID: parentResource.ID, - RootID: rootResource.ID, - Path: "./parent d!r/child2.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "child2.pdf"}, - } - }) - - Describe("PurgeSpace", func() { - It("takes every record of that space out of the index", func() { - otherSpace := search.Resource{ - ID: "1$9!9", - RootID: "1$9!9", - Path: ".", - Document: content.Document{Name: "other"}, - } - for _, resource := range []search.Resource{rootResource, parentResource, childResource, otherSpace} { - Expect(eng.Upsert(resource.ID, resource)).To(Succeed()) - } - - Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1)), "only the records of that space are gone") - }) - - It("takes a space out that holds more records than one round", func() { - otherSpace := search.Resource{ - ID: "1$9!9", - RootID: "1$9!9", - Path: ".", - Document: content.Document{Name: "other"}, - } - Expect(eng.Upsert(otherSpace.ID, otherSpace)).To(Succeed()) - - for i := range 120 { - resource := search.Resource{ - ID: fmt.Sprintf("%s!file-%d", rootResource.RootID, i), - RootID: rootResource.RootID, - Path: fmt.Sprintf("./file-%d", i), - Document: content.Document{Name: fmt.Sprintf("file-%d", i)}, - } - Expect(eng.Upsert(resource.ID, resource)).To(Succeed()) - } - - Expect(eng.PurgeSpace(rootResource.RootID)).To(Succeed()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1)), "only the record of the other space is left") - }) - }) - - Describe("New", func() { - It("returns a new index instance", func() { - b := bleve.NewBackend(idx, bleveQuery.DefaultCreator, log.Logger{}) - Expect(b).ToNot(BeNil()) - }) - }) - - Describe("Search", func() { - Context("by other fields than filename", func() { - It("finds files by tags", func() { - parentResource.Document.Tags = []string{"foo", "bar"} - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Tags:foo", 1) - assertDocCount(rootResource.ID, "Tags:bar", 1) - assertDocCount(rootResource.ID, "Tags:foo Tags:bar", 1) - assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1) - assertDocCount(rootResource.ID, "Tags:foo Tags:bar Tags:baz", 1) - assertDocCount(rootResource.ID, "Tags:baz", 0) - }) - - It("finds files by size", func() { - parentResource.Document.Size = 12345 - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Size:12345", 1) - assertDocCount(rootResource.ID, "Size:>1000", 1) - assertDocCount(rootResource.ID, "Size:<100000", 1) - assertDocCount(rootResource.ID, "Size:12344", 0) - assertDocCount(rootResource.ID, "Size:<1000", 0) - assertDocCount(rootResource.ID, "Size:>100000", 0) - }) - - It("preserves value case for fields not explicitly marked lowercase", func() { - parentResource.Document.Audio = &libregraph.Audio{ - Artist: libregraph.PtrString("Some Artist"), - } - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `audio.artist:"Some Artist"`, 1) - assertDocCount(rootResource.ID, `audio.artist:"some artist"`, 0) - }) - }) - - Context("by filename", func() { - It("finds files with spaces in the filename", func() { - parentResource.Document.Name = "Foo oo.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `name:"foo o*"`, 1) - }) - - It("finds files by digits in the filename", func() { - parentResource.Document.Name = "12345.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:1234*", 1) - }) - - It("filters hidden files", func() { - childResource.Hidden = true - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Hidden:T", 1) - assertDocCount(rootResource.ID, "Hidden:F", 0) - }) - - Context("with a file in the root of the space", func() { - It("scopes the search to the specified space", func() { - parentResource.Document.Name = "foo.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:foo.pdf", 1) - assertDocCount("9$8!7", "Name:foo.pdf", 0) - }) - }) - - It("limits the search to the specified fields", func() { - parentResource.Document.Name = "bar.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:bar.pdf", 1) - assertDocCount(rootResource.ID, "Unknown:field", 0) - }) - - It("returns the total number of hits", func() { - parentResource.Document.Name = "bar.pdf" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - res, err := doSearch(rootResource.ID, "Name:bar*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - }) - - It("returns all desired fields", func() { - parentResource.Document.Name = "bar.pdf" - parentResource.Type = 3 - parentResource.MimeType = "application/pdf" - - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - matches := assertDocCount(rootResource.ID, fmt.Sprintf("Name:%s", parentResource.Name), 1) - match := matches[0] - Expect(match.Entity.Ref.Path).To(Equal(parentResource.Path)) - Expect(match.Entity.Name).To(Equal(parentResource.Name)) - Expect(match.Entity.Size).To(Equal(parentResource.Size)) - Expect(match.Entity.Type).To(Equal(parentResource.Type)) - Expect(match.Entity.MimeType).To(Equal(parentResource.MimeType)) - Expect(match.Entity.Deleted).To(BeFalse()) - Expect(match.Score > 0).To(BeTrue()) - }) - - It("finds files by name, prefix or substring match", func() { - parentResource.Document.Name = "foo.pdf" - - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - queries := []string{"foo.pdf", "foo*", "*oo.p*"} - for _, query := range queries { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, query, 1) - } - }) - - It("does a case-insensitive search", func() { - parentResource.Document.Name = "foo.pdf" - - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:foo*", 1) - assertDocCount(rootResource.ID, "Name:Foo*", 1) - }) - - Context("and an additional file in a subdirectory", func() { - BeforeEach(func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - }) - - It("finds files living deeper in the tree by filename, prefix or substring match", func() { - queries := []string{"child.pdf", "child*", "*ld.*"} - for _, query := range queries { - assertDocCount(rootResource.ID, query, 1) - } - }) - }) - }) - - Context("Highlights", func() { - - It("highlights only for content searches", func() { - parentResource.Document.Name = "baz.pdf" - parentResource.Document.Content = "foo bar baz" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - res, err := doSearch(rootResource.ID, "Name:baz*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - Expect(res.Matches[0].Entity.Highlights).To(Equal("")) - }) - - It("highlights search terms", func() { - parentResource.Document.Name = "baz.pdf" - parentResource.Document.Content = "foo bar baz" - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - res, err := doSearch(rootResource.ID, "Content:bar", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - Expect(res.Matches[0].Entity.Highlights).To(Equal("foo bar baz")) - }) - - }) - - Context("with a file in the root of the space and folder with a file. all of them have the same name", func() { - BeforeEach(func() { - parentResource := search.Resource{ - ID: "1$2!3", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./doc", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_CONTAINER), - Document: content.Document{Name: "doc"}, - } - - childResource := search.Resource{ - ID: "1$2!4", - ParentID: parentResource.ID, - RootID: rootResource.ID, - Path: "./doc/doc.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "doc.pdf"}, - } - - childResource2 := search.Resource{ - ID: "1$2!7", - ParentID: parentResource.ID, - RootID: rootResource.ID, - Path: "./doc/file.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "file.pdf"}, - } - - rootChildResource := search.Resource{ - ID: "1$2!5", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./doc.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "doc.pdf"}, - } - - rootChildResource2 := search.Resource{ - ID: "1$2!6", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./file.pdf", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{Name: "file.pdf"}, - } - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(rootChildResource.ID, rootChildResource) - Expect(err).ToNot(HaveOccurred()) - err = eng.Upsert(rootChildResource2.ID, rootChildResource2) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - err = eng.Upsert(childResource2.ID, childResource2) - Expect(err).ToNot(HaveOccurred()) - }) - It("search *doc* in a root", func() { - res, err := doSearch(rootResource.ID, "Name:*doc*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(3))) - }) - It("search *doc* in a subfolder", func() { - res, err := doSearch(rootResource.ID, "Name:*doc*", "./doc") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(2))) - }) - It("search *file* in a root", func() { - res, err := doSearch(rootResource.ID, "Name:*file*", "") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(2))) - }) - It("search *file* in a subfolder", func() { - res, err := doSearch(rootResource.ID, "Name:*file*", "./doc") - Expect(err).ToNot(HaveOccurred()) - Expect(res.TotalMatches).To(Equal(int32(1))) - }) - }) - - }) - - Describe("Upsert", func() { - It("adds a resourceInfo to the index", func() { - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - query := bleveSearch.NewMatchQuery("child.pdf") - res, err := idx.Search(bleveSearch.NewSearchRequest(query)) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Hits.Len()).To(Equal(1)) - }) - - It("updates an existing resource in the index", func() { - - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - countA, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(countA).To(Equal(uint64(1))) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - countB, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(countB).To(Equal(uint64(1))) - }) - }) - - Describe("Delete", func() { - It("marks a resource as deleted", func() { - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:*child*", 1) - - err = eng.Delete(childResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, "Name:*child*", 0) - }) - - It("marks a child resources as deleted", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - - err = eng.Delete(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0) - }) - }) - - Describe("Restore", func() { - It("also marks child resources as restored", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Delete(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 0) - - err = eng.Restore(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Name+`"`, 1) - assertDocCount(rootResource.ID, `"`+childResource.Name+`"`, 1) - }) - }) - - Describe("Purge", func() { - It("removes a resource from the index", func() { - err := eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - assertDocCount(rootResource.ID, "Name:child.pdf", 1) - - err = eng.Purge(childResource.ID, false) - - Expect(err).ToNot(HaveOccurred()) - assertDocCount(rootResource.ID, "Name:child.pdf", 0) - }) - It("removes a resource and its children from the index", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - - err = eng.Purge(parentResource.ID, false) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 0) - }) - It("removes a resource and ignores its children from the index", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 1) - - err = eng.Delete(parentResource.ID) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - - err = eng.Purge(parentResource.ID, true) - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, `"`+parentResource.Document.Name+`"`, 0) - assertDocCount(rootResource.ID, `"`+childResource.Document.Name+`"`, 1) - }) - }) - - Describe("Move", func() { - It("renames the parent and its child resources", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - parentResource.Path = "newname" - err = eng.Move(parentResource.ID, parentResource.ParentID, "./my/newname") - Expect(err).ToNot(HaveOccurred()) - - assertDocCount(rootResource.ID, parentResource.Name, 0) - - matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1) - Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3")) - Expect(matches[0].Entity.Ref.Path).To(Equal("./my/newname/child.pdf")) - }) - - DescribeTable("keeps the flag in step with the path", - func(from, target string, hidden bool) { - parentResource.Path = from - parentResource.Hidden = search.IsHidden(from) - childResource.Path = from + "/child.pdf" - childResource.Hidden = parentResource.Hidden - - Expect(eng.Upsert(parentResource.ID, parentResource)).To(Succeed()) - Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) - - Expect(eng.Move(parentResource.ID, parentResource.ParentID, target)).To(Succeed()) - - for _, id := range []string{parentResource.ID, childResource.ID} { - Expect(hiddenByID(idx, id)). - To(Equal(hidden), "%s after moving from %s to %s", id, from, target) - } - }, - Entry("into a dot folder", "./parent", "./.trash/parent", true), - Entry("into a plain folder", "./parent", "./archive/parent", false), - Entry("renamed with a leading dot", "./parent", "./.parent", true), - Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false), - Entry("renamed without the leading dot", "./.parent", "./parent", false), - Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true), - ) - - // the trash leaves the path alone, so the flag has to come through untouched - It("carries the flag through the trash and back", func() { - childResource.Path = "./.secret/file.txt" - childResource.Hidden = true - Expect(eng.Upsert(childResource.ID, childResource)).To(Succeed()) - - Expect(eng.Delete(childResource.ID)).To(Succeed()) - Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after trashing") - - Expect(eng.Restore(childResource.ID)).To(Succeed()) - Expect(hiddenByID(idx, childResource.ID)).To(BeTrue(), "after restoring") - }) - - It("moves the parent and its child resources", func() { - err := eng.Upsert(parentResource.ID, parentResource) - Expect(err).ToNot(HaveOccurred()) - - err = eng.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - parentResource.Path = " " - parentResource.ParentID = "1$2!somewhereopaqueid" - - err = eng.Move(parentResource.ID, parentResource.ParentID, "./somewhere/else/newname") - Expect(err).ToNot(HaveOccurred()) - assertDocCount(rootResource.ID, `parent d!r`, 0) - - matches := assertDocCount(rootResource.ID, "Name:child.pdf", 1) - Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("3")) - Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname/child.pdf")) - - matches = assertDocCount(rootResource.ID, `newname`, 1) - Expect(matches[0].Entity.ParentId.OpaqueId).To(Equal("somewhereopaqueid")) - Expect(matches[0].Entity.Ref.Path).To(Equal("./somewhere/else/newname")) - - }) - }) - - Describe("StartBatch", func() { - It("starts a new batch", func() { - b, err := eng.NewBatch(100) - Expect(err).ToNot(HaveOccurred()) - - err = b.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(0))) - - err = b.Push() - Expect(err).ToNot(HaveOccurred()) - - count, err = idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - query := bleveSearch.NewMatchQuery("child.pdf") - res, err := idx.Search(bleveSearch.NewSearchRequest(query)) - Expect(err).ToNot(HaveOccurred()) - Expect(res.Hits.Len()).To(Equal(1)) - }) - - It("doesn't intertwine different batches", func() { - b, err := eng.NewBatch(100) - Expect(err).ToNot(HaveOccurred()) - - err = b.Upsert(childResource.ID, childResource) - Expect(err).ToNot(HaveOccurred()) - - count, err := idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(0))) - - b2, err := eng.NewBatch(100) - Expect(err).ToNot(HaveOccurred()) - - err = b2.Upsert(childResource2.ID, childResource2) - Expect(err).ToNot(HaveOccurred()) - - Expect(b.Push()).To(Succeed()) - count, err = idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - Expect(b2.Push()).To(Succeed()) - count, err = idx.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(2))) - }) - }) - - Describe("File type specific metadata", func() { - - Context("with audio metadata", func() { - BeforeEach(func() { - resource := search.Resource{ - ID: "1$2!7", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./some_song.mp3", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{ - Name: "some_song.mp3", - MimeType: "audio/mpeg", - Audio: &libregraph.Audio{ - Album: libregraph.PtrString("Some Album"), - AlbumArtist: libregraph.PtrString("Some AlbumArtist"), - Artist: libregraph.PtrString("Some Artist"), - Bitrate: libregraph.PtrInt64(192), - Composers: libregraph.PtrString("Some Composers"), - Copyright: libregraph.PtrString(""), - Disc: libregraph.PtrInt32(2), - DiscCount: libregraph.PtrInt32(5), - Duration: libregraph.PtrInt64(225000), - Genre: libregraph.PtrString("Some Genre"), - HasDrm: libregraph.PtrBool(false), - IsVariableBitrate: libregraph.PtrBool(true), - Title: libregraph.PtrString("Some Title"), - Track: libregraph.PtrInt32(34), - TrackCount: libregraph.PtrInt32(99), - Year: libregraph.PtrInt32(2004), - }, - }, - } - err := eng.Upsert(resource.ID, resource) - Expect(err).ToNot(HaveOccurred()) - }) - - It("returns audio metadata for search", func() { - matches := assertDocCount(rootResource.ID, `*song*`, 1) - audio := matches[0].Entity.Audio - - Expect(audio).ToNot(BeNil()) - - Expect(audio.Album).To(Equal(libregraph.PtrString("Some Album"))) - Expect(audio.AlbumArtist).To(Equal(libregraph.PtrString("Some AlbumArtist"))) - Expect(audio.Artist).To(Equal(libregraph.PtrString("Some Artist"))) - Expect(audio.Bitrate).To(Equal(libregraph.PtrInt64(192))) - Expect(audio.Composers).To(Equal(libregraph.PtrString("Some Composers"))) - Expect(audio.Copyright).To(Equal(libregraph.PtrString(""))) - Expect(audio.Disc).To(Equal(libregraph.PtrInt32(2))) - Expect(audio.DiscCount).To(Equal(libregraph.PtrInt32(5))) - Expect(audio.Duration).To(Equal(libregraph.PtrInt64(225000))) - Expect(audio.Genre).To(Equal(libregraph.PtrString("Some Genre"))) - Expect(audio.HasDrm).To(Equal(libregraph.PtrBool(false))) - Expect(audio.IsVariableBitrate).To(Equal(libregraph.PtrBool(true))) - Expect(audio.Title).To(Equal(libregraph.PtrString("Some Title"))) - Expect(audio.Track).To(Equal(libregraph.PtrInt32(34))) - Expect(audio.TrackCount).To(Equal(libregraph.PtrInt32(99))) - Expect(audio.Year).To(Equal(libregraph.PtrInt32(2004))) - }) - }) - - Context("with location metadata", func() { - BeforeEach(func() { - resource := search.Resource{ - ID: "1$2!7", - ParentID: rootResource.ID, - RootID: rootResource.ID, - Path: "./team.jpg", - Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), - Document: content.Document{ - Name: "team.jpg", - MimeType: "image/jpeg", - Location: &libregraph.GeoCoordinates{ - Altitude: libregraph.PtrFloat64(1047.7), - Latitude: libregraph.PtrFloat64(49.48675890884328), - Longitude: libregraph.PtrFloat64(11.103870357204285), - }, - }, - } - err := eng.Upsert(resource.ID, resource) - Expect(err).ToNot(HaveOccurred()) - }) - - It("returns audio metadata for search", func() { - matches := assertDocCount(rootResource.ID, `*team*`, 1) - location := matches[0].Entity.Location - - Expect(location).ToNot(BeNil()) - - Expect(location.Altitude).To(Equal(libregraph.PtrFloat64(1047.7))) - Expect(location.Latitude).To(Equal(libregraph.PtrFloat64(49.48675890884328))) - Expect(location.Longitude).To(Equal(libregraph.PtrFloat64(11.103870357204285))) - }) - }) - }) -}) diff --git a/services/search/pkg/opensearch/backend_test.go b/services/search/pkg/opensearch/backend_test.go index 850b9a7bfa..71ef3860fc 100644 --- a/services/search/pkg/opensearch/backend_test.go +++ b/services/search/pkg/opensearch/backend_test.go @@ -1,9 +1,6 @@ package opensearch_test import ( - "context" - "fmt" - "strings" "testing" . "github.com/onsi/ginkgo/v2" @@ -11,12 +8,7 @@ import ( opensearchgo "github.com/opensearch-project/opensearch-go/v4" opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - - searchService "github.com/opencloud-eu/opencloud/protogen/gen/opencloud/services/search/v0" - opensearchtest "github.com/opencloud-eu/opencloud/services/search/internal/opensearchtest" "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch" - "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) func TestOpenSearchBackend(t *testing.T) { @@ -24,56 +16,8 @@ func TestOpenSearchBackend(t *testing.T) { RunSpecs(t, "OpenSearch Backend Suite") } -func deleteIndexOnCleanup(tc *opensearchtest.TestClient, indexName string) { - DeferCleanup(func() { - Expect(tc.IndicesDelete(context.Background(), []string{indexName})).To(Succeed()) - }) -} - -func resourceByID(tc *opensearchtest.TestClient, index, id string) search.Resource { - GinkgoHelper() - - body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "query": map[string]any{ - "ids": map[string]any{ - "values": []string{id}, - }, - }, - }) - - resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(index, strings.NewReader(body)).Hits) - Expect(resources).To(HaveLen(1)) - return resources[0] -} - -// otherRoot returns a copy of the given resource that lives in a different root (space) -// while keeping the same path, so it can be used to assert that cross-root updates do -// not affect identically-named resources in other roots. -func otherRoot(r search.Resource) search.Resource { - r.ID = "2$2!3" - r.RootID = "2$2!1" - r.ParentID = "2$2!2" - return r -} - -func newBackend(indexName string, resources ...search.Resource) (*opensearch.Backend, *opensearchtest.TestClient) { - GinkgoHelper() - - tc := opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - - backend, err := opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - - for _, r := range resources { - tc.Require.DocumentCreate(indexName, r.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), r))) - } - tc.Require.IndicesCount([]string{indexName}, nil, len(resources)) - - return backend, tc -} - +// what the engine does with its index is covered for both engines by +// services/search/pkg/parity; this is the one thing only OpenSearch can do var _ = Describe("Backend", func() { Describe("NewBackend", func() { It("fails to create if the cluster is not healthy", func() { @@ -89,606 +33,4 @@ var _ = Describe("Backend", func() { Expect(err).To(MatchError(opensearch.ErrUnhealthyCluster)) }) }) - - Describe("Search", func() { - const indexName = "opencloud-test-engine-search" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - document search.Resource - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - - document = opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - }) - - It("performs the most simple search", func() { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ - Query: fmt.Sprintf(`"%s"`, document.Name), - }) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(HaveLen(1)) - Expect(resp.TotalMatches).To(Equal(int32(1))) - Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID)) - }) - - It("ignores files that are marked as deleted", func() { - deletedDocument := opensearchtest.Testdata.Resources.File - deletedDocument.ID = "1$2!4" - deletedDocument.Deleted = true - - tc.Require.DocumentCreate(indexName, deletedDocument.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), deletedDocument))) - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ - Query: fmt.Sprintf(`"%s"`, document.Name), - }) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(HaveLen(1)) - Expect(resp.TotalMatches).To(Equal(int32(1))) - Expect(fmt.Sprintf("%s$%s!%s", resp.Matches[0].Entity.Id.StorageId, resp.Matches[0].Entity.Id.SpaceId, resp.Matches[0].Entity.Id.OpaqueId)).To(Equal(document.ID)) - }) - }) - - Describe("Upsert", func() { - const indexName = "opencloud-test-engine-upsert" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("upserts a full document", func() { - document := opensearchtest.Testdata.Resources.File - Expect(backend.Upsert(document.ID, document)).To(Succeed()) - - tc.Require.IndicesCount([]string{indexName}, nil, 1) - }) - }) - - Describe("Move", func() { - const indexName = "opencloud-test-engine-move" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("moves the document to a new path", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "query": map[string]any{ - "ids": map[string]any{ - "values": []string{document.ID}, - }, - }, - }) - - resources := opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits) - Expect(resources).To(HaveLen(1)) - Expect(resources[0].Path).To(Equal(document.Path)) - - document.Path = "./new/path/to/resource" - Expect(backend.Move(document.ID, document.ParentID, document.Path)).To(Succeed()) - - resources = opensearchtest.SearchHitsMustBeConverted[search.Resource](GinkgoTB(), tc.Require.Search(indexName, strings.NewReader(body)).Hits) - Expect(resources).To(HaveLen(1)) - Expect(resources[0].Path).To(Equal(document.Path)) - }) - }) - - Describe("WriteVisibility", func() { - const indexName = "opencloud-test-engine-write-visibility" - - It("deletes a record that was just written", func() { - document := opensearchtest.Testdata.Resources.File - document.ID = "1$1!95" - document.Name = "textfile.txt" - document.Path = "./textfile.txt" - - backend, tc := newBackend(indexName) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Upsert(document.ID, document)).To(Succeed()) - Expect(backend.Delete(document.ID)).To(Succeed()) - - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{ - Query: fmt.Sprintf(`name:"%s"`, document.Name), - }) - Expect(err).ToNot(HaveOccurred()) - Expect(resp.Matches).To(BeEmpty()) - }) - }) - - Describe("Delete", func() { - const indexName = "opencloud-test-engine-delete" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("marks the document as deleted", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "query": map[string]any{ - "term": map[string]any{ - "Deleted": map[string]any{ - "value": true, - }, - }, - }, - }) - - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0) - - Expect(backend.Delete(document.ID)).To(Succeed()) - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1) - }) - }) - - Describe("Restore", func() { - const indexName = "opencloud-test-engine-restore" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("marks the document as not deleted", func() { - document := opensearchtest.Testdata.Resources.File - document.Deleted = true - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - body := opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "query": map[string]any{ - "term": map[string]any{ - "Deleted": map[string]any{ - "value": true, - }, - }, - }, - }) - - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 1) - - Expect(backend.Restore(document.ID)).To(Succeed()) - tc.Require.IndicesCount([]string{indexName}, strings.NewReader(body), 0) - }) - }) - - Describe("Purge", func() { - const indexName = "opencloud-test-engine-purge" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("purges a full document", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - Expect(backend.Purge(document.ID, false)).To(Succeed()) - - tc.Require.IndicesCount([]string{indexName}, nil, 0) - }) - - It("purges resource trees", func() { - resourceFolder := opensearchtest.Testdata.Resources.Folder - tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder))) - - resourceFile := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile))) - - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - Expect(backend.Purge(resourceFolder.ID, false)).To(Succeed()) - - tc.Require.IndicesCount([]string{indexName}, nil, 0) - }) - - It("purges resource trees and ignores undeleted resources", func() { - resourceFolder := opensearchtest.Testdata.Resources.Folder - tc.Require.DocumentCreate(indexName, resourceFolder.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFolder))) - - resourceFile := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, resourceFile.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), resourceFile))) - - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - Expect(backend.Delete(resourceFile.ID)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - Expect(backend.Purge(resourceFolder.ID, true)).To(Succeed()) - - tc.Require.IndicesCount([]string{indexName}, nil, 1) - }) - }) - - Describe("PurgeSpace", func() { - const indexName = "opencloud-test-engine-purge-space" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("takes every record of that space out of the index", func() { - gone := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, gone.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), gone))) - - stays := opensearchtest.Testdata.Resources.File - stays.ID = "1$2!3" - stays.RootID = "1$2!2" - tc.Require.DocumentCreate(indexName, stays.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), stays))) - - tc.Require.IndicesCount([]string{indexName}, nil, 2) - - Expect(backend.PurgeSpace(gone.RootID)).To(Succeed()) - - tc.Require.IndicesRefresh([]string{indexName}, nil) - left := opensearchtest.SearchHitsMustBeConverted[search.Resource]( - GinkgoTB(), - tc.Require.Search(indexName, strings.NewReader(`{"query":{"match_all":{}}}`)).Hits, - ) - Expect(left).To(HaveLen(1), "only the records of that space are gone") - Expect(left[0].ID).To(Equal(stays.ID)) - }) - }) - - Describe("Hidden", func() { - const indexName = "opencloud-test-engine-hidden" - - DescribeTable("keeps the flag in step with the path", - func(from, target string, hidden bool) { - folder := opensearchtest.Testdata.Resources.Folder - folder.ID = "1$1!30" - folder.Name = "parent" - folder.Path = from - folder.Hidden = search.IsHidden(from) - - child := opensearchtest.Testdata.Resources.File - child.ID = "1$1!31" - child.Name = "child.txt" - child.Path = from + "/child.txt" - child.ParentID = folder.ID - child.Hidden = folder.Hidden - - backend, tc := newBackend(indexName, folder, child) - deleteIndexOnCleanup(tc, indexName) - tc.Require.IndicesRefresh([]string{indexName}, nil) - - Expect(backend.Move(folder.ID, folder.ParentID, target)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - - for _, id := range []string{folder.ID, child.ID} { - Expect(resourceByID(tc, indexName, id).Hidden). - To(Equal(hidden), "%s after moving from %s to %s", id, from, target) - } - }, - Entry("into a dot folder", "./parent", "./.trash/parent", true), - Entry("into a plain folder", "./parent", "./archive/parent", false), - Entry("renamed with a leading dot", "./parent", "./.parent", true), - Entry("out of a dot folder", "./.trash/parent", "./archive/parent", false), - Entry("renamed without the leading dot", "./.parent", "./parent", false), - Entry("within the same dot folder", "./.trash/parent", "./.trash/moved", true), - ) - - It("carries the flag through the trash and back", func() { - hidden := opensearchtest.Testdata.Resources.File - hidden.ID = "1$1!32" - hidden.Path = "./.secret/file.txt" - hidden.Hidden = true - - backend, tc := newBackend(indexName, hidden) - deleteIndexOnCleanup(tc, indexName) - tc.Require.IndicesRefresh([]string{indexName}, nil) - - Expect(backend.Delete(hidden.ID)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - Expect(resourceByID(tc, indexName, hidden.ID).Hidden).To(BeTrue(), "after trashing") - - Expect(backend.Restore(hidden.ID)).To(Succeed()) - tc.Require.IndicesRefresh([]string{indexName}, nil) - Expect(resourceByID(tc, indexName, hidden.ID).Hidden).To(BeTrue(), "after restoring") - }) - }) - - Describe("DocCount", func() { - const indexName = "opencloud-test-engine-doc-count" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tc = opensearchtest.NewDefaultTestClient(GinkgoTB(), defaultConfig.Engine.OpenSearch.Client) - tc.Require.IndicesReset([]string{indexName}) - tc.Require.IndicesCount([]string{indexName}, nil, 0) - deleteIndexOnCleanup(tc, indexName) - - var err error - backend, err = opensearch.NewBackend(indexName, tc.Client()) - Expect(err).ToNot(HaveOccurred()) - }) - - It("ignores deleted documents", func() { - document := opensearchtest.Testdata.Resources.File - tc.Require.DocumentCreate(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), document))) - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - count, err := backend.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(1))) - - tc.Require.Update(indexName, document.ID, strings.NewReader(opensearchtest.JSONMustMarshal(GinkgoTB(), map[string]any{ - "doc": map[string]any{ - "Deleted": true, - }, - }))) - - tc.Require.IndicesCount([]string{indexName}, nil, 1) - - count, err = backend.DocCount() - Expect(err).ToNot(HaveOccurred()) - Expect(count).To(Equal(uint64(0))) - }) - }) - - // The following specs ensure that updates which affect a resource and its descendants - // (Delete, Restore, Move) are scoped to the root (space) of the target resource. Two - // resources living in different roots may share the exact same path, so matching by - // path alone would incorrectly update the wrong resource. - Describe("updateSelfAndDescendants root scope", func() { - It("deletes only the resource in the target root", func() { - const indexName = "opencloud-test-engine-root-scope-delete" - - target := opensearchtest.Testdata.Resources.File - other := otherRoot(target) - - backend, tc := newBackend(indexName, target, other) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Delete(target.ID)).To(Succeed()) - - Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeTrue(), "target resource should be marked as deleted") - Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeFalse(), "resource in a different root must not be affected") - }) - - It("restores only the resource in the target root", func() { - const indexName = "opencloud-test-engine-root-scope-restore" - - target := opensearchtest.Testdata.Resources.File - target.Deleted = true - other := otherRoot(target) - - backend, tc := newBackend(indexName, target, other) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Restore(target.ID)).To(Succeed()) - - Expect(resourceByID(tc, indexName, target.ID).Deleted).To(BeFalse(), "target resource should be restored") - Expect(resourceByID(tc, indexName, other.ID).Deleted).To(BeTrue(), "resource in a different root must not be affected") - }) - - It("moves only the resource in the target root", func() { - const indexName = "opencloud-test-engine-root-scope-move" - - target := opensearchtest.Testdata.Resources.File - other := otherRoot(target) - - backend, tc := newBackend(indexName, target, other) - deleteIndexOnCleanup(tc, indexName) - - Expect(backend.Move(target.ID, target.ParentID, "./new/path/to/resource")).To(Succeed()) - - Expect(resourceByID(tc, indexName, target.ID).Path).To(Equal("./new/path/to/resource"), "target resource should be moved") - Expect(resourceByID(tc, indexName, other.ID).Path).To(Equal(other.Path), "resource in a different root must not be moved") - }) - }) - - Describe("SearchInAnalyzedFields", func() { - const indexName = "opencloud-test-engine-search-analyzed-fields" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - dashed := opensearchtest.Testdata.Resources.Folder - dashed.ID = "1$1!10" - dashed.Name = "new-folder" - dashed.Path = "./new-folder" - dashed.Title = "quarterly report" - - plain := opensearchtest.Testdata.Resources.Folder - plain.ID = "1$1!11" - plain.Name = "documents" - plain.Path = "./documents" - plain.Title = "notes" - - spaced := opensearchtest.Testdata.Resources.Folder - spaced.ID = "1$1!12" - spaced.Name = "foo bar" - spaced.Path = "./foo bar" - spaced.Title = "spaced out" - - backend, tc = newBackend(indexName, dashed, plain, spaced) - deleteIndexOnCleanup(tc, indexName) - tc.Require.IndicesRefresh([]string{indexName}, nil) - }) - - DescribeTable("finds what the analyzer made of the value", - func(query string, want []string) { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query}) - Expect(err).ToNot(HaveOccurred()) - - names := make([]string, 0, len(resp.Matches)) - for _, match := range resp.Matches { - names = append(names, match.Entity.Name) - } - Expect(names).To(ConsistOf(want)) - }, - Entry("the full name with the dash", "new-folder", []string{"new-folder"}), - Entry("one token of it", "new", []string{"new-folder"}), - Entry("a name without a dash", "documents", []string{"documents"}), - Entry("a wildcard", "*folder*", []string{"new-folder"}), - // the shape the web client sends for every name search - Entry("a wildcard around the whole dashed name", `name:"*new-folder*"`, []string{"new-folder"}), - Entry("a wildcard spanning the dash", `name:"*w-fol*"`, []string{"new-folder"}), - Entry("a wildcard in a different case", `name:"*NEW-FOLDER*"`, []string{"new-folder"}), - Entry("a wildcard spanning a space", `name:"*oo ba*"`, []string{"foo bar"}), - Entry("a wildcard around a name with a space", `name:"*foo bar*"`, []string{"foo bar"}), - Entry("a name with a space", `name:"foo bar"`, []string{"foo bar"}), - Entry("a title of two words", `Title:"quarterly report"`, []string{"new-folder"}), - Entry("one token of a title", "Title:quarterly", []string{"new-folder"}), - ) - }) - - Describe("SearchByTag", func() { - const indexName = "opencloud-test-engine-search-by-tag" - - var ( - tc *opensearchtest.TestClient - backend *opensearch.Backend - ) - - BeforeEach(func() { - tagged := opensearchtest.Testdata.Resources.Folder - tagged.ID = "1$1!20" - tagged.Name = "tagged" - tagged.Path = "./tagged" - tagged.Tags = []string{"foo-bar"} - - other := opensearchtest.Testdata.Resources.Folder - other.ID = "1$1!21" - other.Name = "other" - other.Path = "./other" - other.Tags = []string{"foo"} - - backend, tc = newBackend(indexName, tagged, other) - deleteIndexOnCleanup(tc, indexName) - tc.Require.IndicesRefresh([]string{indexName}, nil) - }) - - // a tag is one label, not prose, so it matches as a whole or not at all - DescribeTable("matches a tag as a whole", - func(query string, want []string) { - resp, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: query}) - Expect(err).ToNot(HaveOccurred()) - - names := make([]string, 0, len(resp.Matches)) - for _, match := range resp.Matches { - names = append(names, match.Entity.Name) - } - Expect(names).To(ConsistOf(want)) - }, - Entry("the whole tag", `tag:("foo-bar")`, []string{"tagged"}), - Entry("a token of a tag does not match it", `tag:("foo")`, []string{"other"}), - Entry("a tag in a different case", `tag:("FOO-BAR")`, []string{"tagged"}), - Entry("a wildcard reaches both", `tag:("*foo*")`, []string{"tagged", "other"}), - ) - }) - - Describe("SearchWithAnInvalidQuery", func() { - const indexName = "opencloud-test-engine-search-invalid-query" - - It("answers with a bad request", func() { - backend, tc := newBackend(indexName) - deleteIndexOnCleanup(tc, indexName) - - _, err := backend.Search(context.Background(), &searchService.SearchIndexRequest{Query: "AND mediatype:document"}) - Expect(err).To(HaveOccurred()) - Expect(err).To(BeAssignableToTypeOf(errtypes.BadRequest(""))) - Expect(err.Error()).To(Equal(`error: bad request: the expression can't begin from a binary operator: 'AND'`)) - }) - }) }) diff --git a/services/search/pkg/parity/README.md b/services/search/pkg/parity/README.md index ad880695b4..a8acafba94 100644 --- a/services/search/pkg/parity/README.md +++ b/services/search/pkg/parity/README.md @@ -206,6 +206,7 @@ Fixtures: - `plain.txt`, ID = 1$1!plain.txt - `box`, ID = 1$1!box, folder - `boxed.txt`, ID = 1$1!boxed.txt, Path = ./box/boxed.txt +- `song.mp3`, ID = 1$1!song.mp3, MimeType = audio/mpeg | Case | Query | expected | bleve | OpenSearch | same? | |---|---|---|---|---|---| @@ -221,6 +222,8 @@ Fixtures: | FIELDS-10 | `hidden:TRUE` | hidden.txt | no match | error | ❌ known | | FIELDS-11 | `id:"1$1!AB-23"` | cased.txt | cased.txt | no match | ❌ known | | FIELDS-12 | `id:"1$1!ab-23"` | no match | no match | no match | ✅ | +| FIELDS-13 | `audio.artist:"Some Artist"` | song.mp3 | song.mp3 | no match | ❌ known | +| FIELDS-14 | `audio.artist:"some artist"` | no match | no match | no match | ✅ | ### deleted @@ -582,6 +585,8 @@ Fixtures: | BATCH-03 | takes a resource out the same way a delete does, then `name:"*child*"` | no match | no match | no match | ✅ | | BATCH-04 | moves a resource the same way a move does, then `path:"./my/newname/child.pdf"` | child.pdf | child.pdf | child.pdf | ✅ | | BATCH-04 | moves a resource the same way a move does, then `path:"./parent/child.pdf"` | no match | no match | no match | ✅ | +| BATCH-05 | keeps what another batch holds out of its push, then `name:"*added*"` | added.pdf | added.pdf | added.pdf | ✅ | +| BATCH-05 | keeps what another batch holds out of its push, then `name:"*other*"` | no match | no match | no match | ✅ | ## Response diff --git a/services/search/pkg/parity/lifecycle_batch_test.go b/services/search/pkg/parity/lifecycle_batch_test.go index 5234e7758d..eb718714db 100644 --- a/services/search/pkg/parity/lifecycle_batch_test.go +++ b/services/search/pkg/parity/lifecycle_batch_test.go @@ -9,6 +9,7 @@ func batchLifecycle() lifecycleGroup { parent, child := fixtureTree() added := fixtureDoc("added.pdf", withID("1$1!7"), withParent(parent.ID), withPath("./parent/added.pdf")) + other := fixtureDoc("other.pdf", withID("1$1!8"), withParent(parent.ID), withPath("./parent/other.pdf")) return lifecycleGroup{ name: "batch", @@ -78,6 +79,34 @@ func batchLifecycle() lifecycleGroup { {`path:"./parent/child.pdf"`, nil}, }, }, + { + id: 5, title: "keeps what another batch holds out of its push", + do: func(e search.Engine) error { + first, err := e.NewBatch(100) + if err != nil { + return err + } + + second, err := e.NewBatch(100) + if err != nil { + return err + } + + if err := first.Upsert(added.ID, added); err != nil { + return err + } + + if err := second.Upsert(other.ID, other); err != nil { + return err + } + + return first.Push() + }, + expect: []expectation{ + {`name:"*added*"`, []string{"added.pdf"}}, + {`name:"*other*"`, nil}, + }, + }, }, } } diff --git a/services/search/pkg/parity/query_fields_test.go b/services/search/pkg/parity/query_fields_test.go index 265a7aecd5..a643f0cfae 100644 --- a/services/search/pkg/parity/query_fields_test.go +++ b/services/search/pkg/parity/query_fields_test.go @@ -1,6 +1,8 @@ package parity import ( + libregraph "github.com/opencloud-eu/libre-graph-api-go" + "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) @@ -16,6 +18,7 @@ func fieldsGroup() queryGroup { fixtureDoc("plain.txt"), fixtureFolder("box"), fixtureDoc("boxed.txt", withParent("1$1!box"), withPath("./box/boxed.txt")), + fixtureDoc("song.mp3", withMime("audio/mpeg"), withAudio(&libregraph.Audio{Artist: libregraph.PtrString("Some Artist")})), }, cases: []queryCase{ {id: 1, query: `size:42`, want: []string{"small.txt"}}, @@ -30,6 +33,9 @@ func fieldsGroup() queryGroup { {id: 10, query: `hidden:TRUE`, want: []string{"hidden.txt"}, engineOverrides: map[string]override{"bleve": override{}, "opensearch": override{want: []string{"error"}}}}, {id: 11, query: `id:"1$1!AB-23"`, want: []string{"cased.txt"}, engineOverrides: map[string]override{"opensearch": override{}}}, {id: 12, query: `id:"1$1!ab-23"`}, + // a facet value keeps its case, the field is not marked lowercase + {id: 13, query: `audio.artist:"Some Artist"`, want: []string{"song.mp3"}, engineOverrides: map[string]override{"opensearch": override{}}}, + {id: 14, query: `audio.artist:"some artist"`}, }, } } From 18759b01836345b354a9067e99a3227141a18856 Mon Sep 17 00:00:00 2001 From: Florian Schade Date: Wed, 26 Aug 2026 16:20:02 +0200 Subject: [PATCH 8/9] test(search): cover a deleted space on both engines --- .../features/apiSearch1/search.feature | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/acceptance/features/apiSearch1/search.feature b/tests/acceptance/features/apiSearch1/search.feature index 7f196c0e4f..122d8f2e98 100644 --- a/tests/acceptance/features/apiSearch1/search.feature +++ b/tests/acceptance/features/apiSearch1/search.feature @@ -501,3 +501,19 @@ Feature: Search | old | | new | | spaces | + + + Scenario: a deleted space leaves the search index + Given using spaces DAV path + And the administrator has assigned the role "Space Admin" to user "Alice" using the Graph API + And user "Alice" has created a space "index-space" with the default quota using the Graph API + And user "Alice" has uploaded a file inside space "index-space" with content "some data" to "inSpace.txt" + When user "Alice" searches for "*inSpace*" using the WebDAV API + Then the HTTP status code should be "207" + And the search result of user "Alice" should contain these entries: + | inSpace.txt | + When user "Alice" has disabled a space "index-space" + And user "Alice" has deleted a space "index-space" + And user "Alice" searches for "*inSpace*" using the WebDAV API + Then the HTTP status code should be "207" + And the search result should contain "0" entries From fef2df867186a10b6c6d10488051c87fbad31003 Mon Sep 17 00:00:00 2001 From: Dominik Schmidt Date: Fri, 28 Aug 2026 19:56:57 +0200 Subject: [PATCH 9/9] chore(codacy): exclude the generated search parity matrix Markdown lint has nothing to say about a table the test suite writes. --- .codacy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.codacy.yml b/.codacy.yml index 959f2f3cea..f5eba526d9 100644 --- a/.codacy.yml +++ b/.codacy.yml @@ -18,6 +18,8 @@ exclude_paths: - 'deployments/**' - "release-config.ts" - 'tests/acceptance/expected-failures-*.md' + # written by the search engine parity suite, table rows are as wide as they are + - 'services/search/pkg/parity/README.md' - 'tests/acceptance/bootstrap/**' - 'tests/acceptance/TestHelpers/**' - 'tests/acceptance/scripts/run.sh'