diff --git a/services/search/pkg/bleve/backend.go b/services/search/pkg/bleve/backend.go index 7fc6fe5bfa..d8426e5435 100644 --- a/services/search/pkg/bleve/backend.go +++ b/services/search/pkg/bleve/backend.go @@ -75,14 +75,10 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques }, ) // Scope below the space root: restrict at query level so totals and - // paging respect the path too. Path is a case-preserving keyword - // (paths act as references, /Foo and /foo are distinct), so the exact - // folder or the folder prefix matches all of, and only, the scope. + // paging respect the path too. The folder term matches the folder and + // its descendants (see PathAnalyzer). if requestedPath := utils.MakeRelativePath(sir.Ref.Path); requestedPath != "." { - q.Conjuncts = append(q.Conjuncts, query.NewDisjunctionQuery([]query.Query{ - &query.TermQuery{FieldVal: "Path", Term: requestedPath}, - &query.PrefixQuery{FieldVal: "Path", Prefix: requestedPath + "/"}, - })) + q.Conjuncts = append(q.Conjuncts, &query.TermQuery{FieldVal: "Path", Term: requestedPath}) } } diff --git a/services/search/pkg/bleve/bleve.go b/services/search/pkg/bleve/bleve.go index 1bbf53857d..6c244a542a 100644 --- a/services/search/pkg/bleve/bleve.go +++ b/services/search/pkg/bleve/bleve.go @@ -1,8 +1,6 @@ package bleve import ( - "regexp" - bleveSearch "github.com/blevesearch/bleve/v2/search" storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -11,8 +9,6 @@ import ( "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) -var queryEscape = regexp.MustCompile(`([` + regexp.QuoteMeta(`+=&|> ".", "./a", "./a/b" with +// delimiter "/", one level per byte without. tag_depth prepends "/". +const Name = "hierarchy" + +type Tokenizer struct { + delimiter []byte + tagDepth bool +} + +func (t *Tokenizer) Tokenize(input []byte) analysis.TokenStream { + if len(input) == 0 { + return nil + } + var out analysis.TokenStream + emit := func(depth, end int) { + term := input[:end] + if t.tagDepth { + term = strconv.AppendInt(make([]byte, 0, end+4), int64(depth), 10) + term = append(term, '/') + term = append(term, input[:end]...) + } + out = append(out, &analysis.Token{ + Term: term, + Position: depth, + Start: 0, + End: end, + Type: analysis.AlphaNumeric, + }) + } + + if len(t.delimiter) == 0 { + for i := range input { + emit(i+1, i+1) + } + return out + } + + depth := 0 + for start := 0; start <= len(input); { + i := bytes.Index(input[start:], t.delimiter) + if i < 0 { + if start < len(input) { + depth++ + emit(depth, len(input)) + } + break + } + if i > 0 { + depth++ + emit(depth, start+i) + } + start += i + len(t.delimiter) + } + return out +} + +func Constructor(config map[string]interface{}, _ *registry.Cache) (analysis.Tokenizer, error) { + t := &Tokenizer{} + if d, ok := config["delimiter"].(string); ok { + t.delimiter = []byte(d) + } + if v, ok := config["tag_depth"].(bool); ok { + t.tagDepth = v + } + return t, nil +} + +func init() { + if err := registry.RegisterTokenizer(Name, Constructor); err != nil { + panic(err) + } +} diff --git a/services/search/pkg/bleve/hierarchy/hierarchy_suite_test.go b/services/search/pkg/bleve/hierarchy/hierarchy_suite_test.go new file mode 100644 index 0000000000..a74f37fdd0 --- /dev/null +++ b/services/search/pkg/bleve/hierarchy/hierarchy_suite_test.go @@ -0,0 +1,13 @@ +package hierarchy_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestHierarchy(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "hierarchy tokenizer") +} diff --git a/services/search/pkg/bleve/hierarchy/hierarchy_test.go b/services/search/pkg/bleve/hierarchy/hierarchy_test.go new file mode 100644 index 0000000000..6bc8eaec31 --- /dev/null +++ b/services/search/pkg/bleve/hierarchy/hierarchy_test.go @@ -0,0 +1,56 @@ +package hierarchy_test + +import ( + "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/registry" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/opencloud-eu/opencloud/services/search/pkg/bleve/hierarchy" +) + +func terms(ts analysis.TokenStream) []string { + out := make([]string, 0, len(ts)) + for _, t := range ts { + out = append(out, string(t.Term)) + } + return out +} + +func tokenize(config map[string]any, input string) []string { + tok, err := hierarchy.Constructor(config, registry.NewCache()) + Expect(err).ToNot(HaveOccurred()) + return terms(tok.Tokenize([]byte(input))) +} + +var _ = Describe("hierarchy tokenizer", func() { + path := map[string]any{"delimiter": "/"} + geohash := map[string]any{"tag_depth": true} + + DescribeTable("emits every prefix up to a level boundary", + func(config map[string]any, input string, want []string) { + Expect(tokenize(config, input)).To(Equal(want)) + }, + Entry("relative path", path, "./a/b.txt", []string{".", "./a", "./a/b.txt"}), + Entry("space root", path, ".", []string{"."}), + Entry("trailing delimiter is not a level", path, "./a/", []string{".", "./a"}), + Entry("delimiter only", path, "/", []string{}), + Entry("leading delimiter", path, "/abs/x", []string{"/abs", "/abs/x"}), + Entry("double delimiter", path, "./a//b", []string{".", "./a", "./a//b"}), + Entry("spaces and special characters stay literal", path, "./odd name*[1]/f:x?.txt", + []string{".", "./odd name*[1]", "./odd name*[1]/f:x?.txt"}), + Entry("empty input", path, "", []string{}), + Entry("geohash, one level per byte, depth tagged", geohash, "u4pru", + []string{"1/u", "2/u4", "3/u4p", "4/u4pr", "5/u4pru"}), + ) + + It("keeps byte offsets on the source value", func() { + tok, err := hierarchy.Constructor(path, registry.NewCache()) + Expect(err).ToNot(HaveOccurred()) + ts := tok.Tokenize([]byte("./a/b")) + Expect(ts).To(HaveLen(3)) + Expect(ts[2].Start).To(Equal(0)) + Expect(ts[2].End).To(Equal(5)) + Expect(ts[2].Position).To(Equal(3)) + }) +}) diff --git a/services/search/pkg/bleve/index.go b/services/search/pkg/bleve/index.go index fd4d50e0d6..a11d126130 100644 --- a/services/search/pkg/bleve/index.go +++ b/services/search/pkg/bleve/index.go @@ -19,6 +19,7 @@ import ( storageProvider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/opencloud-eu/opencloud/pkg/log" + "github.com/opencloud-eu/opencloud/services/search/pkg/bleve/hierarchy" searchmapping "github.com/opencloud-eu/opencloud/services/search/pkg/mapping" "github.com/opencloud-eu/opencloud/services/search/pkg/search" ) @@ -208,6 +209,40 @@ func NewMapping() (mapping.IndexMapping, error) { if err != nil { return nil, err } + // path: every ancestor prefix is a term, so one term query matches a folder + // and all of its descendants + err = indexMapping.AddCustomTokenizer("path_hierarchy", map[string]any{ + "type": hierarchy.Name, + "delimiter": "/", + }) + if err != nil { + return nil, err + } + err = indexMapping.AddCustomAnalyzer(searchmapping.PathAnalyzer, map[string]any{ + "type": custom.Name, + "tokenizer": "path_hierarchy", + }) + if err != nil { + return nil, err + } + // geohash: no field uses it yet. It is part of the v5 schema so that #3272 + // can add its geohash field additively: new fields reconcile at startup, + // a changed analysis block does not (classifyStoredMapping), so the names + // and the config below must not change. + err = indexMapping.AddCustomTokenizer("geohash_hierarchy", map[string]any{ + "type": hierarchy.Name, + "tag_depth": true, + }) + if err != nil { + return nil, err + } + err = indexMapping.AddCustomAnalyzer(searchmapping.GeohashAnalyzer, map[string]any{ + "type": custom.Name, + "tokenizer": "geohash_hierarchy", + }) + if err != nil { + return nil, err + } return indexMapping, nil } @@ -226,11 +261,15 @@ func searchResourceByID(id string, index bleve.Index) (*search.Resource, error) return matchToResource(res.Hits[0]), nil } +// searchResourcesByPath returns the descendants of the folder at lookupPath. +// The folder term matches the folder and everything below it in one term +// query (see PathAnalyzer); the folder itself is dropped from the result. func searchResourcesByPath(rootID string, lookupPath string, index bleve.Index) ([]*search.Resource, error) { - q := bleve.NewConjunctionQuery( - bleve.NewQueryStringQuery("RootID:"+rootID), - bleve.NewQueryStringQuery("Path:"+escapeQuery(lookupPath+"/*")), - ) + rootQuery := bleve.NewTermQuery(rootID) + rootQuery.SetField("RootID") + pathQuery := bleve.NewTermQuery(lookupPath) + pathQuery.SetField("Path") + q := bleve.NewConjunctionQuery(rootQuery, pathQuery) bleveReq := bleve.NewSearchRequest(q) bleveReq.Size = math.MaxInt bleveReq.Fields = []string{"*"} @@ -241,7 +280,11 @@ func searchResourcesByPath(rootID string, lookupPath string, index bleve.Index) resources := make([]*search.Resource, 0, res.Hits.Len()) for _, match := range res.Hits { - resources = append(resources, matchToResource(match)) + resource := matchToResource(match) + if resource.Path == lookupPath { + continue + } + resources = append(resources, resource) } return resources, nil diff --git a/services/search/pkg/bleve/testdata/mapping.golden.json b/services/search/pkg/bleve/testdata/mapping.golden.json index 61ea6474f1..5606b61a20 100644 --- a/services/search/pkg/bleve/testdata/mapping.golden.json +++ b/services/search/pkg/bleve/testdata/mapping.golden.json @@ -160,7 +160,7 @@ "fields": [ { "type": "text", - "analyzer": "keyword", + "analyzer": "path", "store": true, "index": true, "include_term_vectors": true, @@ -1259,7 +1259,25 @@ "type": "regexp" } }, + "tokenizers": { + "geohash_hierarchy": { + "tag_depth": true, + "type": "hierarchy" + }, + "path_hierarchy": { + "delimiter": "/", + "type": "hierarchy" + } + }, "analyzers": { + "geohash": { + "tokenizer": "geohash_hierarchy", + "type": "custom" + }, + "path": { + "tokenizer": "path_hierarchy", + "type": "custom" + }, "words": { "char_filters": [ "dot_to_space" diff --git a/services/search/pkg/mapping/bleve.go b/services/search/pkg/mapping/bleve.go index 0a3a55c45b..5c45faa2ef 100644 --- a/services/search/pkg/mapping/bleve.go +++ b/services/search/pkg/mapping/bleve.go @@ -60,7 +60,6 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix } if fieldType == TypeKeyword || fieldType == TypePath { - // bleve has no path tokenizer, so a path is a plain keyword here. base := bleveKeywordMapping(fieldType, opts) doc.AddFieldMappingsAt(fi.Name, base) if opts.caseInsensitive() { @@ -84,8 +83,9 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix return doc, err } -// bleveKeywordMapping is a case-preserving keyword field; path fields stay out -// of _all by default. +// bleveKeywordMapping is a case-preserving keyword field; path fields are +// analyzed into their ancestor prefixes (see PathAnalyzer) and stay out of +// _all by default. func bleveKeywordMapping(fieldType string, opts FieldOpts) *bleveMapping.FieldMapping { fm := bleve.NewKeywordFieldMapping() switch { @@ -94,6 +94,9 @@ func bleveKeywordMapping(fieldType string, opts FieldOpts) *bleveMapping.FieldMa case fieldType == TypePath: fm.IncludeInAll = false } + if fieldType == TypePath { + fm.Analyzer = PathAnalyzer + } return fm } diff --git a/services/search/pkg/mapping/opts.go b/services/search/pkg/mapping/opts.go index d8af5f17bf..fc0fe31c73 100644 --- a/services/search/pkg/mapping/opts.go +++ b/services/search/pkg/mapping/opts.go @@ -26,6 +26,15 @@ const WordsSuffix = "_words" // WordsAnalyzer names the analyzer both engines register for the words sibling. const WordsAnalyzer = "words" +// PathAnalyzer names the bleve analyzer for TypePath fields: every ancestor +// prefix is a term, like path_hierarchy in OpenSearch. +const PathAnalyzer = "path" + +// GeohashAnalyzer names the bleve analyzer for a geohash: every prefix is a +// depth-tagged term (1/u, 2/u4, ...), so a terms facet with TermPrefix +// "/" is a geohash grid at that precision. +const GeohashAnalyzer = "geohash" + // FieldOpts overrides the default type inference for a struct field. Keys in // the override map are json-tag names (e.g. "Name", "location", "audio.artist"), // not Go field names. diff --git a/services/search/pkg/query/bleve/compiler.go b/services/search/pkg/query/bleve/compiler.go index 1ca00dbe7e..a1dba9a65a 100644 --- a/services/search/pkg/query/bleve/compiler.go +++ b/services/search/pkg/query/bleve/compiler.go @@ -123,6 +123,13 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { var q bleveQuery.Query = bleveQuery.NewQueryStringQuery(k + ":" + v) switch { + case searchQuery.FieldIsPath(n.Key) && !isWildcard: + // the folder term matches the folder itself and its descendants + // (see PathAnalyzer); a query string would analyze the value into + // its prefixes and match everything under the root + tq := bleveQuery.NewTermQuery(val) + tq.SetField(k) + q = tq case n.Exact && !isWildcard: // = matches the whole value, on the lowercased sibling for // case-insensitive fields @@ -140,17 +147,6 @@ func walk(offset int, nodes []ast.Node) (bleveQuery.Query, int, error) { bq.SetMinShould(1) q = bq } - if searchQuery.FieldIsPath(n.Key) { - // bleve has no path hierarchy analyzer, unlike OpenSearch: match the - // folder itself and its descendants (`\/*`). A BooleanQuery keeps - // this atomic; a DisjunctionQuery would be redistributed by an - // enclosing AND (mapBinary treats a left disjunction as an OR-chain). - bq := bleve.NewBooleanQuery() - bq.AddShould(q, bleveQuery.NewQueryStringQuery(k+":"+v+`\/*`)) - bq.SetMinShould(1) - q = bq - } - if prev == nil { prev = q } else { diff --git a/services/search/pkg/query/bleve/compiler_test.go b/services/search/pkg/query/bleve/compiler_test.go index c4becff38f..632a29b387 100644 --- a/services/search/pkg/query/bleve/compiler_test.go +++ b/services/search/pkg/query/bleve/compiler_test.go @@ -51,23 +51,17 @@ func Test_compile(t *testing.T) { wantErr: false, }, { - // path fields expand to match the folder itself and its descendants, - // since bleve has no path hierarchy analyzer. + // one term matches the folder itself and its descendants name: `path:/Foo`, args: &ast.Ast{ Nodes: []ast.Node{ &ast.StringNode{Key: "path", Value: "/Foo"}, }, }, - // a BooleanQuery (should: exact OR descendants), not a DisjunctionQuery, - // so an enclosing AND does not redistribute the folder-itself clause. want: func() query.Query { - bq := query.NewBooleanQuery(nil, []query.Query{ - query.NewQueryStringQuery(`Path:\/Foo`), - query.NewQueryStringQuery(`Path:\/Foo\/*`), - }, nil) - bq.SetMinShould(1) - return query.NewConjunctionQuery([]query.Query{bq}) + tq := query.NewTermQuery("/Foo") + tq.SetField("Path") + return query.NewConjunctionQuery([]query.Query{tq}) }(), wantErr: false, }, diff --git a/services/search/pkg/search/search.go b/services/search/pkg/search/search.go index 009c8afa87..fb81135f2f 100644 --- a/services/search/pkg/search/search.go +++ b/services/search/pkg/search/search.go @@ -29,7 +29,7 @@ import ( // on a breaking mapping change: each version gets its own index (OpenSearch name // suffix, bleve path suffix), so the service builds a fresh index instead of // colliding with the old one. No migration; reindex to populate. -const SchemaVersion = 4 +const SchemaVersion = 5 var scopeRegex = regexp.MustCompile(`scope:\s*([^" "\n\r]*)`)