From be75d8fe95f0738b578691a68e9bb55d668dd08b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 30 Jul 2026 15:25:13 +0200 Subject: [PATCH 1/3] Reindex spaces concurrently This speeds up reindexing all spaces which happend sequentially space after space until now. The level of concurrency can be configured using the SEARCH_REINDEX_CONCURRENCY env var (3 by default). --- services/search/pkg/config/config.go | 1 + .../pkg/config/defaults/defaultconfig.go | 1 + .../search/pkg/service/grpc/v0/service.go | 26 ++++++++++++++++--- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/services/search/pkg/config/config.go b/services/search/pkg/config/config.go index 128061b76f..c3a653f489 100644 --- a/services/search/pkg/config/config.go +++ b/services/search/pkg/config/config.go @@ -28,6 +28,7 @@ type Config struct { Extractor Extractor `yaml:"extractor"` ContentExtractionSizeLimit uint64 `yaml:"content_extraction_size_limit" env:"SEARCH_CONTENT_EXTRACTION_SIZE_LIMIT" desc:"Maximum file size in bytes that is allowed for content extraction." introductionVersion:"1.0.0"` BatchSize int `yaml:"batch_size" env:"SEARCH_BATCH_SIZE" desc:"The number of documents to process in a single batch. Defaults to 500." introductionVersion:"1.0.0"` + ReindexConcurrency int `yaml:"reindex_concurrency" env:"SEARCH_REINDEX_CONCURRENCY" desc:"The maximum number of spaces that are reindexed concurrently when reindexing all spaces." introductionVersion:"%%NEXT%%"` ServiceAccount ServiceAccount `yaml:"service_account"` diff --git a/services/search/pkg/config/defaults/defaultconfig.go b/services/search/pkg/config/defaults/defaultconfig.go index 3e504350fa..35e3331359 100644 --- a/services/search/pkg/config/defaults/defaultconfig.go +++ b/services/search/pkg/config/defaults/defaultconfig.go @@ -65,6 +65,7 @@ func DefaultConfig() *config.Config { }, ContentExtractionSizeLimit: 20 * 1024 * 1024, // Limit content extraction to <20MB files by default BatchSize: 50, + ReindexConcurrency: 3, } } diff --git a/services/search/pkg/service/grpc/v0/service.go b/services/search/pkg/service/grpc/v0/service.go index dddb64d4b6..f2c38983a2 100644 --- a/services/search/pkg/service/grpc/v0/service.go +++ b/services/search/pkg/service/grpc/v0/service.go @@ -19,6 +19,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/utils" merrors "go-micro.dev/v4/errors" "go-micro.dev/v4/metadata" + "golang.org/x/sync/errgroup" grpcmetadata "google.golang.org/grpc/metadata" "github.com/opencloud-eu/opencloud/pkg/log" @@ -144,13 +145,32 @@ func (s Service) IndexSpace(_ context.Context, in *searchsvc.IndexSpaceRequest, return errors.New(resp.GetStatus().GetMessage()) } + // Index all spaces concurrently, limited to a configurable number of spaces + // being reindexed at the same time. + concurrency := max(s.cfg.ReindexConcurrency, 1) + + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) + for _, space := range resp.GetStorageSpaces() { - if err := s.searcher.IndexSpace(space.GetId(), in.GetForceReindex()); err != nil { - return err + // stop scheduling new work once one of the goroutines failed + if gctx.Err() != nil { + break } + + g.Go(func() error { + s.log.Info().Str("space_id", space.GetId().GetOpaqueId()).Msg("indexing space") + err := s.searcher.IndexSpace(space.GetId(), in.GetForceReindex()) + if err != nil { + s.log.Error().Err(err).Str("space_id", space.GetId().GetOpaqueId()).Msg("failed to index space") + } else { + s.log.Info().Str("space_id", space.GetId().GetOpaqueId()).Msg("finished indexing space") + } + return err + }) } - return nil + return g.Wait() } // FromCache pulls a search result from cache From 243514ec17ab25806f38c5632e0990ab266849a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 30 Jul 2026 15:34:57 +0200 Subject: [PATCH 2/3] Do not abort reindexing all spaces if one of the spaces fails --- services/search/pkg/service/grpc/v0/service.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/services/search/pkg/service/grpc/v0/service.go b/services/search/pkg/service/grpc/v0/service.go index f2c38983a2..e65b082a71 100644 --- a/services/search/pkg/service/grpc/v0/service.go +++ b/services/search/pkg/service/grpc/v0/service.go @@ -148,25 +148,21 @@ func (s Service) IndexSpace(_ context.Context, in *searchsvc.IndexSpaceRequest, // Index all spaces concurrently, limited to a configurable number of spaces // being reindexed at the same time. concurrency := max(s.cfg.ReindexConcurrency, 1) - - g, gctx := errgroup.WithContext(ctx) + var g errgroup.Group g.SetLimit(concurrency) for _, space := range resp.GetStorageSpaces() { - // stop scheduling new work once one of the goroutines failed - if gctx.Err() != nil { - break - } - g.Go(func() error { s.log.Info().Str("space_id", space.GetId().GetOpaqueId()).Msg("indexing space") + err := s.searcher.IndexSpace(space.GetId(), in.GetForceReindex()) if err != nil { s.log.Error().Err(err).Str("space_id", space.GetId().GetOpaqueId()).Msg("failed to index space") } else { s.log.Info().Str("space_id", space.GetId().GetOpaqueId()).Msg("finished indexing space") } - return err + + return nil }) } From d50d8d890042c24c8a56fae7ac2e02e1d30b3699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Fri, 31 Jul 2026 12:04:44 +0200 Subject: [PATCH 3/3] Rename max concurrency option --- services/search/pkg/config/config.go | 2 +- services/search/pkg/config/defaults/defaultconfig.go | 2 +- services/search/pkg/service/grpc/v0/service.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/search/pkg/config/config.go b/services/search/pkg/config/config.go index c3a653f489..a5ce9abdc4 100644 --- a/services/search/pkg/config/config.go +++ b/services/search/pkg/config/config.go @@ -28,7 +28,7 @@ type Config struct { Extractor Extractor `yaml:"extractor"` ContentExtractionSizeLimit uint64 `yaml:"content_extraction_size_limit" env:"SEARCH_CONTENT_EXTRACTION_SIZE_LIMIT" desc:"Maximum file size in bytes that is allowed for content extraction." introductionVersion:"1.0.0"` BatchSize int `yaml:"batch_size" env:"SEARCH_BATCH_SIZE" desc:"The number of documents to process in a single batch. Defaults to 500." introductionVersion:"1.0.0"` - ReindexConcurrency int `yaml:"reindex_concurrency" env:"SEARCH_REINDEX_CONCURRENCY" desc:"The maximum number of spaces that are reindexed concurrently when reindexing all spaces." introductionVersion:"%%NEXT%%"` + ReindexMaxConcurrency int `yaml:"reindex_concurrency" env:"SEARCH_REINDEX_MAX_CONCURRENCY" desc:"The maximum number of spaces that are reindexed concurrently when reindexing all spaces." introductionVersion:"%%NEXT%%"` ServiceAccount ServiceAccount `yaml:"service_account"` diff --git a/services/search/pkg/config/defaults/defaultconfig.go b/services/search/pkg/config/defaults/defaultconfig.go index 35e3331359..7145dfdc70 100644 --- a/services/search/pkg/config/defaults/defaultconfig.go +++ b/services/search/pkg/config/defaults/defaultconfig.go @@ -65,7 +65,7 @@ func DefaultConfig() *config.Config { }, ContentExtractionSizeLimit: 20 * 1024 * 1024, // Limit content extraction to <20MB files by default BatchSize: 50, - ReindexConcurrency: 3, + ReindexMaxConcurrency: 3, } } diff --git a/services/search/pkg/service/grpc/v0/service.go b/services/search/pkg/service/grpc/v0/service.go index e65b082a71..f382159c31 100644 --- a/services/search/pkg/service/grpc/v0/service.go +++ b/services/search/pkg/service/grpc/v0/service.go @@ -147,7 +147,7 @@ func (s Service) IndexSpace(_ context.Context, in *searchsvc.IndexSpaceRequest, // Index all spaces concurrently, limited to a configurable number of spaces // being reindexed at the same time. - concurrency := max(s.cfg.ReindexConcurrency, 1) + concurrency := max(s.cfg.ReindexMaxConcurrency, 1) var g errgroup.Group g.SetLimit(concurrency)