From c618c7436a6a47337ab6de8f0e3e94965d2826e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 14 May 2025 09:02:57 +0200 Subject: [PATCH] Make the write buffer duration configurable --- .woodpecker.star | 1 + services/activitylog/pkg/config/config.go | 2 ++ .../pkg/config/defaults/defaultconfig.go | 3 ++ .../activitylog/pkg/server/http/server.go | 1 + services/activitylog/pkg/service/options.go | 30 ++++++++++++------- services/activitylog/pkg/service/service.go | 4 +-- 6 files changed, 29 insertions(+), 12 deletions(-) diff --git a/.woodpecker.star b/.woodpecker.star index eb59f5bb8d..82b43fa2e3 100644 --- a/.woodpecker.star +++ b/.woodpecker.star @@ -1886,6 +1886,7 @@ def opencloudServer(storage = "decomposed", accounts_hash_difficulty = 4, depend "OC_JWT_SECRET": "some-opencloud-jwt-secret", "EVENTHISTORY_STORE": "memory", "OC_TRANSLATION_PATH": "%s/tests/config/translations" % dirs["base"], + "ACTIVITYLOG_WRITE_BUFFER_DURATION": "0", # Disable write buffer so that test expectations are met in time # debug addresses required for running services health tests "ACTIVITYLOG_DEBUG_ADDR": "0.0.0.0:9197", "APP_PROVIDER_DEBUG_ADDR": "0.0.0.0:9165", diff --git a/services/activitylog/pkg/config/config.go b/services/activitylog/pkg/config/config.go index cc3987a982..84a42e49cd 100644 --- a/services/activitylog/pkg/config/config.go +++ b/services/activitylog/pkg/config/config.go @@ -32,6 +32,8 @@ type Config struct { ServiceAccount ServiceAccount `yaml:"service_account"` Context context.Context `yaml:"-"` + + WriteBufferDuration time.Duration `yaml:"write_buffer_duration" env:"ACTIVITYLOG_WRITE_BUFFER_DURATION" desc:"The duration to wait before flushing the write buffer. This is used to reduce the number of writes to the store." introductionVersion:"%%NEXT%%"` } // Events combines the configuration options for the event bus. diff --git a/services/activitylog/pkg/config/defaults/defaultconfig.go b/services/activitylog/pkg/config/defaults/defaultconfig.go index 9500c04c12..70b445398a 100644 --- a/services/activitylog/pkg/config/defaults/defaultconfig.go +++ b/services/activitylog/pkg/config/defaults/defaultconfig.go @@ -1,6 +1,8 @@ package defaults import ( + "time" + "github.com/opencloud-eu/opencloud/pkg/shared" "github.com/opencloud-eu/opencloud/pkg/structs" "github.com/opencloud-eu/opencloud/services/activitylog/pkg/config" @@ -50,6 +52,7 @@ func DefaultConfig() *config.Config { AllowCredentials: true, }, }, + WriteBufferDuration: 10 * time.Second, } } diff --git a/services/activitylog/pkg/server/http/server.go b/services/activitylog/pkg/server/http/server.go index 1b0b914050..5a6c4ccdbc 100644 --- a/services/activitylog/pkg/server/http/server.go +++ b/services/activitylog/pkg/server/http/server.go @@ -88,6 +88,7 @@ func Server(opts ...Option) (http.Service, error) { svc.HistoryClient(options.HistoryClient), svc.ValueClient(options.ValueClient), svc.RegisteredEvents(options.RegisteredEvents), + svc.WriteBufferDuration(options.Config.WriteBufferDuration), ) if err != nil { return http.Service{}, err diff --git a/services/activitylog/pkg/service/options.go b/services/activitylog/pkg/service/options.go index 4b5f227fee..3427f326f9 100644 --- a/services/activitylog/pkg/service/options.go +++ b/services/activitylog/pkg/service/options.go @@ -1,6 +1,8 @@ package service import ( + "time" + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" "github.com/go-chi/chi/v5" "github.com/opencloud-eu/opencloud/pkg/log" @@ -18,16 +20,17 @@ type Option func(*Options) // Options for the activitylog service type Options struct { - Logger log.Logger - Config *config.Config - TraceProvider trace.TracerProvider - Stream events.Stream - RegisteredEvents []events.Unmarshaller - Store microstore.Store - GatewaySelector pool.Selectable[gateway.GatewayAPIClient] - Mux *chi.Mux - HistoryClient ehsvc.EventHistoryService - ValueClient settingssvc.ValueService + Logger log.Logger + Config *config.Config + TraceProvider trace.TracerProvider + Stream events.Stream + RegisteredEvents []events.Unmarshaller + Store microstore.Store + GatewaySelector pool.Selectable[gateway.GatewayAPIClient] + Mux *chi.Mux + HistoryClient ehsvc.EventHistoryService + ValueClient settingssvc.ValueService + WriteBufferDuration time.Duration } // Logger configures a logger for the activitylog service @@ -99,3 +102,10 @@ func ValueClient(vs settingssvc.ValueService) Option { o.ValueClient = vs } } + +// WriteBufferDuration sets the write buffer duration +func WriteBufferDuration(d time.Duration) Option { + return func(o *Options) { + o.WriteBufferDuration = d + } +} diff --git a/services/activitylog/pkg/service/service.go b/services/activitylog/pkg/service/service.go index 2c95fefa1d..2be7ba8db1 100644 --- a/services/activitylog/pkg/service/service.go +++ b/services/activitylog/pkg/service/service.go @@ -143,7 +143,7 @@ func New(opts ...Option) (*ActivitylogService, error) { } cache := ttlcache.NewCache() - err = cache.SetTTL(10 * time.Second) + err = cache.SetTTL(30 * time.Second) if err != nil { return nil, err } @@ -163,7 +163,7 @@ func New(opts ...Option) (*ActivitylogService, error) { tracer: o.TraceProvider.Tracer("github.com/opencloud-eu/opencloud/services/activitylog/pkg/service"), parentIdCache: cache, } - s.debouncer = NewDebouncer(10*time.Second, s.storeActivity) + s.debouncer = NewDebouncer(o.WriteBufferDuration, s.storeActivity) s.mux.Get("/graph/v1beta1/extensions/org.libregraph/activities", s.HandleGetItemActivities)