mirror/opencloud
1
0
Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-03-15 14:10:34 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9456 from kobergj/IndexAllSpaces

Browse Source 
Allow reindexing all spaces
This commit is contained in:
kobergj
2024-06-25 10:57:57 +02:00
committed by GitHub
parent 9d7ea44b2e 4e28fc45dc
commit f6de65d3a8
10 changed files with 169 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
changelog/unreleased/index-all-spaces.md Normal file
View File

@@ -0,0 +1,5 @@
Enhancement: Allow reindexing all spaces
Adds a `--all-spaces` flag to the `ocis search index` command to allow reindexing all spaces at once.
https://github.com/owncloud/ocis/pull/9456

47
ocis-pkg/kql/dictionary_gen.go
View File

@@ -3290,7 +3290,7 @@ func (p *parser) parse(g *grammar) (val any, err error) {
}
p.read() // advance to first rune
val, ok = p.parseRule(startRule)
val, ok = p.parseRuleWrap(startRule)
if !ok {
if len(*p.errs) == 0 {
// If parsing fails, but no errors have been recorded, the expected values
@@ -3331,17 +3331,33 @@ func listJoin(list []string, sep string, lastSep string) string {
}
}
func (p *parser) parseRuleWrap(rule *rule) (any, bool) {
var (
val any
ok bool
)
val, ok = p.parseRule(rule)
return val, ok
}
func (p *parser) parseRule(rule *rule) (any, bool) {
p.rstack = append(p.rstack, rule)
p.pushV()
val, ok := p.parseExpr(rule.expr)
val, ok := p.parseExprWrap(rule.expr)
p.popV()
p.rstack = p.rstack[:len(p.rstack)-1]
return val, ok
}
func (p *parser) parseExpr(expr any) (any, bool) {
func (p *parser) parseExprWrap(expr any) (any, bool) {
val, ok := p.parseExpr(expr)
return val, ok
}
func (p *parser) parseExpr(expr any) (any, bool) {
p.ExprCnt++
if p.ExprCnt > p.maxExprCnt {
panic(errMaxExprCnt)
@@ -3392,7 +3408,7 @@ func (p *parser) parseExpr(expr any) (any, bool) {
func (p *parser) parseActionExpr(act *actionExpr) (any, bool) {
start := p.pt
val, ok := p.parseExpr(act.expr)
val, ok := p.parseExprWrap(act.expr)
if ok {
p.cur.pos = start.position
p.cur.text = p.sliceFrom(start)
@@ -3419,7 +3435,7 @@ func (p *parser) parseAndCodeExpr(and *andCodeExpr) (any, bool) {
func (p *parser) parseAndExpr(and *andExpr) (any, bool) {
pt := p.pt
p.pushV()
_, ok := p.parseExpr(and.expr)
_, ok := p.parseExprWrap(and.expr)
p.popV()
p.restore(pt)
@@ -3501,12 +3517,13 @@ func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (any, bool) {
}
func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) {
for altI, alt := range ch.alternatives {
// dummy assignment to prevent compile error if optimized
_ = altI
p.pushV()
val, ok := p.parseExpr(alt)
val, ok := p.parseExprWrap(alt)
p.popV()
if ok {
return val, ok
@@ -3517,7 +3534,7 @@ func (p *parser) parseChoiceExpr(ch *choiceExpr) (any, bool) {
func (p *parser) parseLabeledExpr(lab *labeledExpr) (any, bool) {
p.pushV()
val, ok := p.parseExpr(lab.expr)
val, ok := p.parseExprWrap(lab.expr)
p.popV()
if ok && lab.label != "" {
m := p.vstack[len(p.vstack)-1]
@@ -3557,7 +3574,7 @@ func (p *parser) parseNotExpr(not *notExpr) (any, bool) {
pt := p.pt
p.pushV()
p.maxFailInvertExpected = !p.maxFailInvertExpected
_, ok := p.parseExpr(not.expr)
_, ok := p.parseExprWrap(not.expr)
p.maxFailInvertExpected = !p.maxFailInvertExpected
p.popV()
p.restore(pt)
@@ -3570,7 +3587,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) {
for {
p.pushV()
val, ok := p.parseExpr(expr.expr)
val, ok := p.parseExprWrap(expr.expr)
p.popV()
if !ok {
if len(vals) == 0 {
@@ -3586,7 +3603,7 @@ func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (any, bool) {
func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (any, bool) {
p.pushRecovery(recover.failureLabel, recover.recoverExpr)
val, ok := p.parseExpr(recover.expr)
val, ok := p.parseExprWrap(recover.expr)
p.popRecovery()
return val, ok
@@ -3602,7 +3619,7 @@ func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (any, bool) {
p.addErr(fmt.Errorf("undefined rule: %s", ref.name))
return nil, false
}
return p.parseRule(rule)
return p.parseRuleWrap(rule)
}
func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) {
@@ -3610,7 +3627,7 @@ func (p *parser) parseSeqExpr(seq *seqExpr) (any, bool) {
pt := p.pt
for _, expr := range seq.exprs {
val, ok := p.parseExpr(expr)
val, ok := p.parseExprWrap(expr)
if !ok {
p.restore(pt)
return nil, false
@@ -3624,7 +3641,7 @@ func (p *parser) parseThrowExpr(expr *throwExpr) (any, bool) {
for i := len(p.recoveryStack) - 1; i >= 0; i-- {
if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok {
if val, ok := p.parseExpr(recoverExpr); ok {
if val, ok := p.parseExprWrap(recoverExpr); ok {
return val, ok
}
}
@@ -3638,7 +3655,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) {
for {
p.pushV()
val, ok := p.parseExpr(expr.expr)
val, ok := p.parseExprWrap(expr.expr)
p.popV()
if !ok {
return vals, true
@@ -3649,7 +3666,7 @@ func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (any, bool) {
func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (any, bool) {
p.pushV()
val, _ := p.parseExpr(expr.expr)
val, _ := p.parseExprWrap(expr.expr)
p.popV()
// whether it matched or not, consider it a match
return val, true

23
services/search/pkg/command/index.go
View File

@@ -2,6 +2,7 @@ package command
import (
"context"
"errors"
"fmt"
"time"
@@ -25,22 +26,23 @@ func Index(cfg *config.Config) *cli.Command {
Aliases: []string{"i"},
Flags: []cli.Flag{
&cli.StringFlag{
Name: "space",
Aliases: []string{"s"},
Required: true,
Usage: "the id of the space to travers and index the files of",
Name: "space",
Aliases: []string{"s"},
Usage: "the id of the space to travers and index the files of. This or --all-spaces is required.",
},
&cli.StringFlag{
Name: "user",
Aliases: []string{"u"},
Required: true,
Usage: "the username of the user that shall be used to access the files",
&cli.BoolFlag{
Name: "all-spaces",
Usage: "index all spaces instead. This or --space is required.",
},
},
Before: func(c *cli.Context) error {
Before: func(_ *cli.Context) error {
return configlog.ReturnFatal(parser.ParseConfig(cfg))
},
Action: func(ctx *cli.Context) error {
if ctx.String("space") == "" && !ctx.Bool("all-spaces") {
return errors.New("either --space or --all-spaces is required")
}
traceProvider, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name)
if err != nil {
return err
@@ -57,7 +59,6 @@ func Index(cfg *config.Config) *cli.Command {
c := searchsvc.NewSearchProviderService("com.owncloud.api.search", grpcClient)
_, err = c.IndexSpace(context.Background(), &searchsvc.IndexSpaceRequest{
SpaceId: ctx.String("space"),
UserId: ctx.String("user"),
}, func(opts *client.CallOptions) { opts.RequestTimeout = 10 * time.Minute })
if err != nil {
fmt.Println("failed to index space: " + err.Error())

9
services/search/pkg/search/debouncer.go
View File

@@ -4,14 +4,13 @@ import (
"sync"
"time"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
)
// SpaceDebouncer debounces operations on spaces for a configurable amount of time
type SpaceDebouncer struct {
after time.Duration
f func(id *provider.StorageSpaceId, userID *user.UserId)
f func(id *provider.StorageSpaceId)
pending map[string]*time.Timer
inProgress sync.Map
@@ -19,7 +18,7 @@ type SpaceDebouncer struct {
}
// NewSpaceDebouncer returns a new SpaceDebouncer instance
func NewSpaceDebouncer(d time.Duration, f func(id *provider.StorageSpaceId, userID *user.UserId)) *SpaceDebouncer {
func NewSpaceDebouncer(d time.Duration, f func(id *provider.StorageSpaceId)) *SpaceDebouncer {
return &SpaceDebouncer{
after: d,
f: f,
@@ -29,7 +28,7 @@ func NewSpaceDebouncer(d time.Duration, f func(id *provider.StorageSpaceId, user
}
// Debounce restars the debounce timer for the given space
func (d *SpaceDebouncer) Debounce(id *provider.StorageSpaceId, userID *user.UserId) {
func (d *SpaceDebouncer) Debounce(id *provider.StorageSpaceId) {
d.mutex.Lock()
defer d.mutex.Unlock()
@@ -48,6 +47,6 @@ func (d *SpaceDebouncer) Debounce(id *provider.StorageSpaceId, userID *user.User
d.inProgress.Store(id.OpaqueId, true)
defer d.inProgress.Delete(id.OpaqueId)
d.f(id, userID)
d.f(id)
})
}

28
services/search/pkg/search/debouncer_test.go
View File

@@ -4,7 +4,6 @@ import (
"sync/atomic"
"time"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -17,9 +16,6 @@ var _ = Describe("SpaceDebouncer", func() {
callCount atomic.Int32
userId = &user.UserId{
OpaqueId: "user",
}
spaceid = &sprovider.StorageSpaceId{
OpaqueId: "spaceid",
}
@@ -27,7 +23,7 @@ var _ = Describe("SpaceDebouncer", func() {
BeforeEach(func() {
callCount = atomic.Int32{}
debouncer = search.NewSpaceDebouncer(50*time.Millisecond, func(id *sprovider.StorageSpaceId, _ *user.UserId) {
debouncer = search.NewSpaceDebouncer(50*time.Millisecond, func(id *sprovider.StorageSpaceId) {
if id.OpaqueId == "spaceid" {
callCount.Add(1)
}
@@ -35,22 +31,22 @@ var _ = Describe("SpaceDebouncer", func() {
})
It("debounces", func() {
debouncer.Debounce(spaceid, userId)
debouncer.Debounce(spaceid, userId)
debouncer.Debounce(spaceid, userId)
debouncer.Debounce(spaceid)
debouncer.Debounce(spaceid)
debouncer.Debounce(spaceid)
Eventually(func() int {
return int(callCount.Load())
}, "200ms").Should(Equal(1))
})
It("works multiple times", func() {
debouncer.Debounce(spaceid, userId)
debouncer.Debounce(spaceid, userId)
debouncer.Debounce(spaceid, userId)
debouncer.Debounce(spaceid)
debouncer.Debounce(spaceid)
debouncer.Debounce(spaceid)
time.Sleep(100 * time.Millisecond)
debouncer.Debounce(spaceid, userId)
debouncer.Debounce(spaceid, userId)
debouncer.Debounce(spaceid)
debouncer.Debounce(spaceid)
Eventually(func() int {
return int(callCount.Load())
@@ -58,16 +54,16 @@ var _ = Describe("SpaceDebouncer", func() {
})
It("doesn't trigger twice simultaneously", func() {
debouncer = search.NewSpaceDebouncer(50*time.Millisecond, func(id *sprovider.StorageSpaceId, _ *user.UserId) {
debouncer = search.NewSpaceDebouncer(50*time.Millisecond, func(id *sprovider.StorageSpaceId) {
if id.OpaqueId == "spaceid" {
callCount.Add(1)
}
time.Sleep(300 * time.Millisecond)
})
debouncer.Debounce(spaceid, userId)
debouncer.Debounce(spaceid)
time.Sleep(100 * time.Millisecond) // Let it trigger once
debouncer.Debounce(spaceid, userId)
debouncer.Debounce(spaceid)
time.Sleep(100 * time.Millisecond) // shouldn't trigger as the other run is still in progress
Expect(int(callCount.Load())).To(Equal(1))

54
services/search/pkg/search/events.go
View File

@@ -3,7 +3,6 @@ package search
import (
"time"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/v2/pkg/events"
"github.com/cs3org/reva/v2/pkg/storagespace"
@@ -52,21 +51,9 @@ func HandleEvents(s Searcher, bus events.Consumer, logger log.Logger, cfg *confi
}
}
getUser := func(users ...*user.UserId) *user.UserId {
for _, u := range users {
if u == nil {
continue
}
return u
}
return nil
}
indexSpaceDebouncer := NewSpaceDebouncer(time.Duration(cfg.Events.DebounceDuration)*time.Millisecond, func(id *provider.StorageSpaceId, userID *user.UserId) {
if err := s.IndexSpace(id, userID); err != nil {
logger.Error().Err(err).Interface("spaceID", id).Interface("userID", userID).Msg("error while indexing a space")
indexSpaceDebouncer := NewSpaceDebouncer(time.Duration(cfg.Events.DebounceDuration)*time.Millisecond, func(id *provider.StorageSpaceId) {
if err := s.IndexSpace(id); err != nil {
logger.Error().Err(err).Interface("spaceID", id).Msg("error while indexing a space")
}
})
@@ -77,41 +64,32 @@ func HandleEvents(s Searcher, bus events.Consumer, logger log.Logger, cfg *confi
go func() {
logger.Debug().Interface("event", e).Msg("updating index")
var err error
switch ev := e.Event.(type) {
case events.ItemTrashed:
u := getUser(ev.SpaceOwner, ev.Executant)
s.TrashItem(ev.ID)
indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), u)
indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref))
case events.ItemMoved:
u := getUser(ev.SpaceOwner, ev.Executant)
s.MoveItem(ev.Ref, u)
indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), getUser(ev.SpaceOwner, ev.Executant))
s.MoveItem(ev.Ref)
indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref))
case events.ItemRestored:
u := getUser(ev.SpaceOwner, ev.Executant)
s.RestoreItem(ev.Ref, u)
indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), u)
s.RestoreItem(ev.Ref)
indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref))
case events.ContainerCreated:
indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), getUser(ev.SpaceOwner, ev.Executant))
indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref))
case events.FileTouched:
indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), getUser(ev.SpaceOwner, ev.Executant))
indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref))
case events.FileVersionRestored:
indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), getUser(ev.SpaceOwner, ev.Executant))
indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref))
case events.TagsAdded:
s.UpsertItem(ev.Ref, ev.Executant)
s.UpsertItem(ev.Ref)
case events.TagsRemoved:
s.UpsertItem(ev.Ref, ev.Executant)
s.UpsertItem(ev.Ref)
case events.FileUploaded:
indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref), getUser(ev.SpaceOwner, ev.Executant))
indexSpaceDebouncer.Debounce(getSpaceID(ev.Ref))
case events.UploadReady:
indexSpaceDebouncer.Debounce(getSpaceID(ev.FileRef), getUser(ev.SpaceOwner, ev.ExecutingUser.GetId()))
indexSpaceDebouncer.Debounce(getSpaceID(ev.FileRef))
case events.SpaceRenamed:
indexSpaceDebouncer.Debounce(ev.ID, getUser(ev.Executant))
}
if err != nil {
logger.Error().Err(err).Interface("event", e)
indexSpaceDebouncer.Debounce(ev.ID)
}
}()
}

74
services/search/pkg/search/mocks/searcher.go
View File

@@ -8,8 +8,6 @@ import (
providerv1beta1 "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
mock "github.com/stretchr/testify/mock"
userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
v0 "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0"
)
@@ -26,17 +24,17 @@ func (_m *Searcher) EXPECT() *Searcher_Expecter {
return &Searcher_Expecter{mock: &_m.Mock}
}
// IndexSpace provides a mock function with given fields: rID, uID
func (_m *Searcher) IndexSpace(rID *providerv1beta1.StorageSpaceId, uID *userv1beta1.UserId) error {
ret := _m.Called(rID, uID)
// IndexSpace provides a mock function with given fields: rID
func (_m *Searcher) IndexSpace(rID *providerv1beta1.StorageSpaceId) error {
ret := _m.Called(rID)
if len(ret) == 0 {
panic("no return value specified for IndexSpace")
}
var r0 error
if rf, ok := ret.Get(0).(func(*providerv1beta1.StorageSpaceId, *userv1beta1.UserId) error); ok {
r0 = rf(rID, uID)
if rf, ok := ret.Get(0).(func(*providerv1beta1.StorageSpaceId) error); ok {
r0 = rf(rID)
} else {
r0 = ret.Error(0)
}
@@ -51,14 +49,13 @@ type Searcher_IndexSpace_Call struct {
// IndexSpace is a helper method to define mock.On call
// - rID *providerv1beta1.StorageSpaceId
// - uID *userv1beta1.UserId
func (_e *Searcher_Expecter) IndexSpace(rID interface{}, uID interface{}) *Searcher_IndexSpace_Call {
return &Searcher_IndexSpace_Call{Call: _e.mock.On("IndexSpace", rID, uID)}
func (_e *Searcher_Expecter) IndexSpace(rID interface{}) *Searcher_IndexSpace_Call {
return &Searcher_IndexSpace_Call{Call: _e.mock.On("IndexSpace", rID)}
}
func (_c *Searcher_IndexSpace_Call) Run(run func(rID *providerv1beta1.StorageSpaceId, uID *userv1beta1.UserId)) *Searcher_IndexSpace_Call {
func (_c *Searcher_IndexSpace_Call) Run(run func(rID *providerv1beta1.StorageSpaceId)) *Searcher_IndexSpace_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(*providerv1beta1.StorageSpaceId), args[1].(*userv1beta1.UserId))
run(args[0].(*providerv1beta1.StorageSpaceId))
})
return _c
}
@@ -68,14 +65,14 @@ func (_c *Searcher_IndexSpace_Call) Return(_a0 error) *Searcher_IndexSpace_Call
return _c
}
func (_c *Searcher_IndexSpace_Call) RunAndReturn(run func(*providerv1beta1.StorageSpaceId, *userv1beta1.UserId) error) *Searcher_IndexSpace_Call {
func (_c *Searcher_IndexSpace_Call) RunAndReturn(run func(*providerv1beta1.StorageSpaceId) error) *Searcher_IndexSpace_Call {
_c.Call.Return(run)
return _c
}
// MoveItem provides a mock function with given fields: ref, uID
func (_m *Searcher) MoveItem(ref *providerv1beta1.Reference, uID *userv1beta1.UserId) {
_m.Called(ref, uID)
// MoveItem provides a mock function with given fields: ref
func (_m *Searcher) MoveItem(ref *providerv1beta1.Reference) {
_m.Called(ref)
}
// Searcher_MoveItem_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'MoveItem'
@@ -85,14 +82,13 @@ type Searcher_MoveItem_Call struct {
// MoveItem is a helper method to define mock.On call
// - ref *providerv1beta1.Reference
// - uID *userv1beta1.UserId
func (_e *Searcher_Expecter) MoveItem(ref interface{}, uID interface{}) *Searcher_MoveItem_Call {
return &Searcher_MoveItem_Call{Call: _e.mock.On("MoveItem", ref, uID)}
func (_e *Searcher_Expecter) MoveItem(ref interface{}) *Searcher_MoveItem_Call {
return &Searcher_MoveItem_Call{Call: _e.mock.On("MoveItem", ref)}
}
func (_c *Searcher_MoveItem_Call) Run(run func(ref *providerv1beta1.Reference, uID *userv1beta1.UserId)) *Searcher_MoveItem_Call {
func (_c *Searcher_MoveItem_Call) Run(run func(ref *providerv1beta1.Reference)) *Searcher_MoveItem_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(*providerv1beta1.Reference), args[1].(*userv1beta1.UserId))
run(args[0].(*providerv1beta1.Reference))
})
return _c
}
@@ -102,14 +98,14 @@ func (_c *Searcher_MoveItem_Call) Return() *Searcher_MoveItem_Call {
return _c
}
func (_c *Searcher_MoveItem_Call) RunAndReturn(run func(*providerv1beta1.Reference, *userv1beta1.UserId)) *Searcher_MoveItem_Call {
func (_c *Searcher_MoveItem_Call) RunAndReturn(run func(*providerv1beta1.Reference)) *Searcher_MoveItem_Call {
_c.Call.Return(run)
return _c
}
// RestoreItem provides a mock function with given fields: ref, uID
func (_m *Searcher) RestoreItem(ref *providerv1beta1.Reference, uID *userv1beta1.UserId) {
_m.Called(ref, uID)
// RestoreItem provides a mock function with given fields: ref
func (_m *Searcher) RestoreItem(ref *providerv1beta1.Reference) {
_m.Called(ref)
}
// Searcher_RestoreItem_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RestoreItem'
@@ -119,14 +115,13 @@ type Searcher_RestoreItem_Call struct {
// RestoreItem is a helper method to define mock.On call
// - ref *providerv1beta1.Reference
// - uID *userv1beta1.UserId
func (_e *Searcher_Expecter) RestoreItem(ref interface{}, uID interface{}) *Searcher_RestoreItem_Call {
return &Searcher_RestoreItem_Call{Call: _e.mock.On("RestoreItem", ref, uID)}
func (_e *Searcher_Expecter) RestoreItem(ref interface{}) *Searcher_RestoreItem_Call {
return &Searcher_RestoreItem_Call{Call: _e.mock.On("RestoreItem", ref)}
}
func (_c *Searcher_RestoreItem_Call) Run(run func(ref *providerv1beta1.Reference, uID *userv1beta1.UserId)) *Searcher_RestoreItem_Call {
func (_c *Searcher_RestoreItem_Call) Run(run func(ref *providerv1beta1.Reference)) *Searcher_RestoreItem_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(*providerv1beta1.Reference), args[1].(*userv1beta1.UserId))
run(args[0].(*providerv1beta1.Reference))
})
return _c
}
@@ -136,7 +131,7 @@ func (_c *Searcher_RestoreItem_Call) Return() *Searcher_RestoreItem_Call {
return _c
}
func (_c *Searcher_RestoreItem_Call) RunAndReturn(run func(*providerv1beta1.Reference, *userv1beta1.UserId)) *Searcher_RestoreItem_Call {
func (_c *Searcher_RestoreItem_Call) RunAndReturn(run func(*providerv1beta1.Reference)) *Searcher_RestoreItem_Call {
_c.Call.Return(run)
return _c
}
@@ -233,9 +228,9 @@ func (_c *Searcher_TrashItem_Call) RunAndReturn(run func(*providerv1beta1.Resour
return _c
}
// UpsertItem provides a mock function with given fields: ref, uID
func (_m *Searcher) UpsertItem(ref *providerv1beta1.Reference, uID *userv1beta1.UserId) {
_m.Called(ref, uID)
// UpsertItem provides a mock function with given fields: ref
func (_m *Searcher) UpsertItem(ref *providerv1beta1.Reference) {
_m.Called(ref)
}
// Searcher_UpsertItem_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpsertItem'
@@ -245,14 +240,13 @@ type Searcher_UpsertItem_Call struct {
// UpsertItem is a helper method to define mock.On call
// - ref *providerv1beta1.Reference
// - uID *userv1beta1.UserId
func (_e *Searcher_Expecter) UpsertItem(ref interface{}, uID interface{}) *Searcher_UpsertItem_Call {
return &Searcher_UpsertItem_Call{Call: _e.mock.On("UpsertItem", ref, uID)}
func (_e *Searcher_Expecter) UpsertItem(ref interface{}) *Searcher_UpsertItem_Call {
return &Searcher_UpsertItem_Call{Call: _e.mock.On("UpsertItem", ref)}
}
func (_c *Searcher_UpsertItem_Call) Run(run func(ref *providerv1beta1.Reference, uID *userv1beta1.UserId)) *Searcher_UpsertItem_Call {
func (_c *Searcher_UpsertItem_Call) Run(run func(ref *providerv1beta1.Reference)) *Searcher_UpsertItem_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(*providerv1beta1.Reference), args[1].(*userv1beta1.UserId))
run(args[0].(*providerv1beta1.Reference))
})
return _c
}
@@ -262,7 +256,7 @@ func (_c *Searcher_UpsertItem_Call) Return() *Searcher_UpsertItem_Call {
return _c
}
func (_c *Searcher_UpsertItem_Call) RunAndReturn(run func(*providerv1beta1.Reference, *userv1beta1.UserId)) *Searcher_UpsertItem_Call {
func (_c *Searcher_UpsertItem_Call) RunAndReturn(run func(*providerv1beta1.Reference)) *Searcher_UpsertItem_Call {
_c.Call.Return(run)
return _c
}

27
services/search/pkg/search/service.go
View File

@@ -10,7 +10,6 @@ import (
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
collaborationv1beta1 "github.com/cs3org/go-cs3apis/cs3/sharing/collaboration/v1beta1"
@@ -46,11 +45,11 @@ const (
// Searcher is the interface to the SearchService
type Searcher interface {
Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error)
IndexSpace(rID *provider.StorageSpaceId, uID *user.UserId) error
IndexSpace(rID *provider.StorageSpaceId) error
TrashItem(rID *provider.ResourceId)
UpsertItem(ref *provider.Reference, uID *user.UserId)
RestoreItem(ref *provider.Reference, uID *user.UserId)
MoveItem(ref *provider.Reference, uID *user.UserId)
UpsertItem(ref *provider.Reference)
RestoreItem(ref *provider.Reference)
MoveItem(ref *provider.Reference)
}
// Service is responsible for indexing spaces and pass on a search
@@ -409,7 +408,7 @@ func (s *Service) searchIndex(ctx context.Context, req *searchsvc.SearchRequest,
}
// IndexSpace (re)indexes all resources of a given space.
func (s *Service) IndexSpace(spaceID *provider.StorageSpaceId, uID *user.UserId) error {
func (s *Service) IndexSpace(spaceID *provider.StorageSpaceId) error {
ownerCtx, err := getAuthContext(s.serviceAccountID, s.gatewaySelector, s.serviceAccountSecret, s.logger)
if err != nil {
return err
@@ -456,7 +455,7 @@ func (s *Service) IndexSpace(spaceID *provider.StorageSpaceId, uID *user.UserId)
return nil
}
s.UpsertItem(ref, uID)
s.UpsertItem(ref)
return nil
})
@@ -479,8 +478,8 @@ func (s *Service) TrashItem(rID *provider.ResourceId) {
}
// UpsertItem indexes or stores Resource data fields.
func (s *Service) UpsertItem(ref *provider.Reference, uID *user.UserId) {
ctx, stat, path := s.resInfo(uID, ref)
func (s *Service) UpsertItem(ref *provider.Reference) {
ctx, stat, path := s.resInfo(ref)
if ctx == nil || stat == nil || path == "" {
return
}
@@ -610,8 +609,8 @@ func valueToString(value interface{}) string {
}
// RestoreItem makes the item available again.
func (s *Service) RestoreItem(ref *provider.Reference, uID *user.UserId) {
ctx, stat, path := s.resInfo(uID, ref)
func (s *Service) RestoreItem(ref *provider.Reference) {
ctx, stat, path := s.resInfo(ref)
if ctx == nil || stat == nil || path == "" {
return
}
@@ -622,8 +621,8 @@ func (s *Service) RestoreItem(ref *provider.Reference, uID *user.UserId) {
}
// MoveItem updates the resource location and all of its necessary fields.
func (s *Service) MoveItem(ref *provider.Reference, uID *user.UserId) {
ctx, stat, path := s.resInfo(uID, ref)
func (s *Service) MoveItem(ref *provider.Reference) {
ctx, stat, path := s.resInfo(ref)
if ctx == nil || stat == nil || path == "" {
return
}
@@ -633,7 +632,7 @@ func (s *Service) MoveItem(ref *provider.Reference, uID *user.UserId) {
}
}
func (s *Service) resInfo(uID *user.UserId, ref *provider.Reference) (context.Context, *provider.StatResponse, string) {
func (s *Service) resInfo(ref *provider.Reference) (context.Context, *provider.StatResponse, string) {
ownerCtx, err := getAuthContext(s.serviceAccountID, s.gatewaySelector, s.serviceAccountSecret, s.logger)
if err != nil {
return nil, nil, ""

2
services/search/pkg/search/service_test.go
View File

@@ -128,7 +128,7 @@ var _ = Describe("Searchprovider", func() {
Status: status.NewOK(context.Background()),
Info: ri,
}, nil)
err := s.IndexSpace(&sprovider.StorageSpaceId{OpaqueId: "storageid$spaceid!spaceid"}, user.Id)
err := s.IndexSpace(&sprovider.StorageSpaceId{OpaqueId: "storageid$spaceid!spaceid"})
Expect(err).ShouldNot(HaveOccurred())
})
})

42
services/search/pkg/service/grpc/v0/service.go
View File

@@ -6,7 +6,9 @@ import (
"fmt"
"time"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
revactx "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/errtypes"
@@ -14,6 +16,7 @@ import (
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/v2/pkg/token"
"github.com/cs3org/reva/v2/pkg/token/manager/jwt"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/jellydator/ttlcache/v2"
merrors "go-micro.dev/v4/errors"
"go-micro.dev/v4/metadata"
@@ -23,6 +26,7 @@ import (
"github.com/owncloud/ocis/v2/ocis-pkg/registry"
v0 "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/search/v0"
searchsvc "github.com/owncloud/ocis/v2/protogen/gen/ocis/services/search/v0"
"github.com/owncloud/ocis/v2/services/search/pkg/config"
"github.com/owncloud/ocis/v2/services/search/pkg/content"
"github.com/owncloud/ocis/v2/services/search/pkg/engine"
"github.com/owncloud/ocis/v2/services/search/pkg/query/bleve"
@@ -114,6 +118,8 @@ func NewHandler(opts ...Option) (searchsvc.SearchProviderHandler, func(), error)
searcher: ss,
cache: cache,
tokenManager: tokenManager,
gws: selector,
cfg: cfg,
}, teardown, nil
}
@@ -124,6 +130,8 @@ type Service struct {
searcher search.Searcher
cache *ttlcache.Cache
tokenManager token.Manager
gws *pool.Selector[gateway.GatewayAPIClient]
cfg *config.Config
}
// Search handles the search
@@ -171,8 +179,38 @@ func (s Service) Search(ctx context.Context, in *searchsvc.SearchRequest, out *s
}
// IndexSpace (re)indexes all resources of a given space.
func (s Service) IndexSpace(ctx context.Context, in *searchsvc.IndexSpaceRequest, _ *searchsvc.IndexSpaceResponse) error {
return s.searcher.IndexSpace(&provider.StorageSpaceId{OpaqueId: in.SpaceId}, &user.UserId{OpaqueId: in.UserId})
func (s Service) IndexSpace(_ context.Context, in *searchsvc.IndexSpaceRequest, _ *searchsvc.IndexSpaceResponse) error {
if in.GetSpaceId() != "" {
return s.searcher.IndexSpace(&provider.StorageSpaceId{OpaqueId: in.GetSpaceId()})
}
// index all spaces instead
gwc, err := s.gws.Next()
if err != nil {
return err
}
ctx, err := utils.GetServiceUserContext(s.cfg.ServiceAccount.ServiceAccountID, gwc, s.cfg.ServiceAccount.ServiceAccountSecret)
if err != nil {
return err
}
resp, err := gwc.ListStorageSpaces(ctx, &provider.ListStorageSpacesRequest{})
if err != nil {
return err
}
if resp.GetStatus().GetCode() != rpc.Code_CODE_OK {
return errors.New(resp.GetStatus().GetMessage())
}
for _, space := range resp.GetStorageSpaces() {
if err := s.searcher.IndexSpace(space.GetId()); err != nil {
return err
}
}
return nil
}
// FromCache pulls a search result from cache