From b480fc09b0afc2ab19be4cd28178174897e2169c Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Tue, 18 Jul 2023 12:29:08 +0200 Subject: [PATCH 1/7] Bump reva, for 'graceful_shutdown_timeout' setting --- go.mod | 2 +- go.sum | 4 +- .../reva/v2/cmd/revad/internal/grace/grace.go | 117 +++++++++++------- .../reva/v2/cmd/revad/runtime/runtime.go | 18 +-- .../storage/utils/decomposedfs/node/xattrs.go | 10 ++ .../pkg/storage/utils/decomposedfs/spaces.go | 3 + vendor/modules.txt | 2 +- 7 files changed, 98 insertions(+), 58 deletions(-) diff --git a/go.mod b/go.mod index da9d71f61d..425b38eb71 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/coreos/go-oidc/v3 v3.6.0 github.com/cs3org/go-cs3apis v0.0.0-20230516150832-730ac860c71d - github.com/cs3org/reva/v2 v2.15.0 + github.com/cs3org/reva/v2 v2.15.1-0.20230718140539-0af2a07c7fd9 github.com/disintegration/imaging v1.6.2 github.com/dutchcoders/go-clamd v0.0.0-20170520113014-b970184f4d9e github.com/egirna/icap-client v0.1.1 diff --git a/go.sum b/go.sum index a023b23656..525a672a52 100644 --- a/go.sum +++ b/go.sum @@ -625,8 +625,8 @@ github.com/crewjam/httperr v0.2.0 h1:b2BfXR8U3AlIHwNeFFvZ+BV1LFvKLlzMjzaTnZMybNo github.com/crewjam/httperr v0.2.0/go.mod h1:Jlz+Sg/XqBQhyMjdDiC+GNNRzZTD7x39Gu3pglZ5oH4= github.com/crewjam/saml v0.4.13 h1:TYHggH/hwP7eArqiXSJUvtOPNzQDyQ7vwmwEqlFWhMc= github.com/crewjam/saml v0.4.13/go.mod h1:igEejV+fihTIlHXYP8zOec3V5A8y3lws5bQBFsTm4gA= -github.com/cs3org/reva/v2 v2.15.0 h1:saU2Heig/HswkNdDHh2Jttmhsn0nfTkvaYbXUspcNOM= -github.com/cs3org/reva/v2 v2.15.0/go.mod h1:4z5EQghS2LhSWZWocH51Dw9VAs16No1zSFvFgQtgS7w= +github.com/cs3org/reva/v2 v2.15.1-0.20230718140539-0af2a07c7fd9 h1:ycV7H1siLmMiRmc9kaS0WonysHYUT7irvH4FDDAghqQ= +github.com/cs3org/reva/v2 v2.15.1-0.20230718140539-0af2a07c7fd9/go.mod h1:4z5EQghS2LhSWZWocH51Dw9VAs16No1zSFvFgQtgS7w= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8/go.mod h1:4abs/jPXcmJzYoYGF91JF9Uq9s/KL5n1jvFDix8KcqY= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= diff --git a/vendor/github.com/cs3org/reva/v2/cmd/revad/internal/grace/grace.go b/vendor/github.com/cs3org/reva/v2/cmd/revad/internal/grace/grace.go index 339a8c5c1e..8aaf18aaf5 100644 --- a/vendor/github.com/cs3org/reva/v2/cmd/revad/internal/grace/grace.go +++ b/vendor/github.com/cs3org/reva/v2/cmd/revad/internal/grace/grace.go @@ -36,13 +36,14 @@ import ( // Watcher watches a process for a graceful restart // preserving open network sockets to avoid packet loss. type Watcher struct { - log zerolog.Logger - graceful bool - ppid int - lns map[string]net.Listener - ss map[string]Server - pidFile string - childPIDs []int + log zerolog.Logger + graceful bool + ppid int + lns map[string]net.Listener + ss map[string]Server + pidFile string + childPIDs []int + gracefulShutdownTimeout int } // Option represent an option. @@ -62,6 +63,12 @@ func WithPIDFile(fn string) Option { } } +func WithGracefuleShutdownTimeout(seconds int) Option { + return func(w *Watcher) { + w.gracefulShutdownTimeout = seconds + } +} + // NewWatcher creates a Watcher. func NewWatcher(opts ...Option) *Watcher { w := &Watcher{ @@ -279,51 +286,69 @@ func (w *Watcher) TrapSignals() { } case syscall.SIGQUIT: - w.log.Info().Msg("preparing for a graceful shutdown with deadline of 10 seconds") - go func() { - count := 10 - ticker := time.NewTicker(time.Second) - for ; true; <-ticker.C { - w.log.Info().Msgf("shutting down in %d seconds", count-1) - count-- - if count <= 0 { - w.log.Info().Msg("deadline reached before draining active conns, hard stopping ...") - for _, s := range w.ss { - err := s.Stop() - if err != nil { - w.log.Error().Err(err).Msg("error stopping server") - } - w.log.Info().Msgf("fd to %s:%s abruptly closed", s.Network(), s.Address()) - } - w.Exit(1) - } - } - }() - for _, s := range w.ss { - w.log.Info().Msgf("fd to %s:%s gracefully closed ", s.Network(), s.Address()) - err := s.GracefulStop() - if err != nil { - w.log.Error().Err(err).Msg("error stopping server") - w.log.Info().Msg("exit with error code 1") - w.Exit(1) - } - } - w.log.Info().Msg("exit with error code 0") - w.Exit(0) + gracefulShutdown(w) case syscall.SIGINT, syscall.SIGTERM: - w.log.Info().Msg("preparing for hard shutdown, aborting all conns") - for _, s := range w.ss { - w.log.Info().Msgf("fd to %s:%s abruptly closed", s.Network(), s.Address()) - err := s.Stop() - if err != nil { - w.log.Error().Err(err).Msg("error stopping server") - } + if w.gracefulShutdownTimeout == 0 { + hardShutdown(w) } - w.Exit(0) + gracefulShutdown(w) } } } +// TODO: Ideally this would call exit() but properly return an error. The +// exit() is problematic (i.e. racey) especiaily when orchestrating multiple +// reva services from some external runtime (like in the "ocis server" case +func gracefulShutdown(w *Watcher) { + w.log.Info().Int("Timeout", w.gracefulShutdownTimeout).Msg("preparing for a graceful shutdown with deadline") + go func() { + count := w.gracefulShutdownTimeout + ticker := time.NewTicker(time.Second) + for ; true; <-ticker.C { + w.log.Info().Msgf("shutting down in %d seconds", count-1) + count-- + if count <= 0 { + w.log.Info().Msg("deadline reached before draining active conns, hard stopping ...") + for _, s := range w.ss { + err := s.Stop() + if err != nil { + w.log.Error().Err(err).Msg("error stopping server") + } + w.log.Info().Msgf("fd to %s:%s abruptly closed", s.Network(), s.Address()) + } + w.Exit(1) + } + } + }() + for _, s := range w.ss { + w.log.Info().Msgf("fd to %s:%s gracefully closed ", s.Network(), s.Address()) + err := s.GracefulStop() + if err != nil { + w.log.Error().Err(err).Msg("error stopping server") + w.log.Info().Msg("exit with error code 1") + + w.Exit(1) + } + } + w.log.Info().Msg("exit with error code 0") + w.Exit(0) +} + +// TODO: Ideally this would call exit() but properly return an error. The +// exit() is problematic (i.e. racey) especiaily when orchestrating multiple +// reva services from some external runtime (like in the "ocis server" case +func hardShutdown(w *Watcher) { + w.log.Info().Msg("preparing for hard shutdown, aborting all conns") + for _, s := range w.ss { + w.log.Info().Msgf("fd to %s:%s abruptly closed", s.Network(), s.Address()) + err := s.Stop() + if err != nil { + w.log.Error().Err(err).Msg("error stopping server") + } + } + w.Exit(0) +} + func getListenerFile(ln net.Listener) (*os.File, error) { switch t := ln.(type) { case *net.TCPListener: diff --git a/vendor/github.com/cs3org/reva/v2/cmd/revad/runtime/runtime.go b/vendor/github.com/cs3org/reva/v2/cmd/revad/runtime/runtime.go index 948397b7fe..cc908d3e22 100644 --- a/vendor/github.com/cs3org/reva/v2/cmd/revad/runtime/runtime.go +++ b/vendor/github.com/cs3org/reva/v2/cmd/revad/runtime/runtime.go @@ -72,6 +72,8 @@ type coreConf struct { // TracingService specifies the service. i.e OpenCensus, OpenTelemetry, OpenTracing... TracingService string `mapstructure:"tracing_service"` + + GracefulShutdownTimeout int `mapstructure:"graceful_shutdown_timeout"` } func run( @@ -92,7 +94,7 @@ func run( initCPUCount(coreConf, logger) servers := initServers(mainConf, logger, tp) - watcher, err := initWatcher(logger, filename) + watcher, err := initWatcher(logger, filename, coreConf.GracefulShutdownTimeout) if err != nil { log.Panic(err) } @@ -110,8 +112,8 @@ func initListeners(watcher *grace.Watcher, servers map[string]grace.Server, log return listeners } -func initWatcher(log *zerolog.Logger, filename string) (*grace.Watcher, error) { - watcher, err := handlePIDFlag(log, filename) +func initWatcher(log *zerolog.Logger, filename string, gracefulShutdownTimeout int) (*grace.Watcher, error) { + watcher, err := handlePIDFlag(log, filename, gracefulShutdownTimeout) // TODO(labkode): maybe pidfile can be created later on? like once a server is going to be created? if err != nil { log.Error().Err(err).Msg("error creating grace watcher") @@ -187,11 +189,11 @@ func initLogger(conf *logConf) *zerolog.Logger { return log } -func handlePIDFlag(l *zerolog.Logger, pidFile string) (*grace.Watcher, error) { - var opts []grace.Option - opts = append(opts, grace.WithPIDFile(pidFile)) - opts = append(opts, grace.WithLogger(l.With().Str("pkg", "grace").Logger())) - w := grace.NewWatcher(opts...) +func handlePIDFlag(l *zerolog.Logger, pidFile string, gracefulShutdownTimeout int) (*grace.Watcher, error) { + w := grace.NewWatcher(grace.WithPIDFile(pidFile), + grace.WithLogger(l.With().Str("pkg", "grace").Logger()), + grace.WithGracefuleShutdownTimeout(gracefulShutdownTimeout), + ) err := w.WritePID() if err != nil { return nil, err diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/xattrs.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/xattrs.go index 5545f29271..39e1725476 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/xattrs.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node/xattrs.go @@ -49,6 +49,16 @@ func (md Attributes) SetInt64(key string, val int64) { md[key] = []byte(strconv.FormatInt(val, 10)) } +// UInt64 reads an uint64 value +func (md Attributes) UInt64(key string) (uint64, error) { + return strconv.ParseUint(string(md[key]), 10, 64) +} + +// SetInt64 sets an uint64 value +func (md Attributes) SetUInt64(key string, val uint64) { + md[key] = []byte(strconv.FormatUint(val, 10)) +} + // SetXattrs sets multiple extended attributes on the write-through cache/node func (n *Node) SetXattrsWithContext(ctx context.Context, attribs map[string][]byte, acquireLock bool) (err error) { if n.xattrsCache != nil { diff --git a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaces.go b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaces.go index e04927e2a2..cc72cced20 100644 --- a/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaces.go +++ b/vendor/github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/spaces.go @@ -122,6 +122,9 @@ func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.Cr metadata.SetString(prefixes.NameAttr, req.Name) metadata.SetString(prefixes.SpaceNameAttr, req.Name) + // This space is empty so set initial treesize to 0 + metadata.SetUInt64(prefixes.TreesizeAttr, 0) + if req.Type != "" { metadata.SetString(prefixes.SpaceTypeAttr, req.Type) } diff --git a/vendor/modules.txt b/vendor/modules.txt index 3bdf891d84..2c0054fa3c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -352,7 +352,7 @@ github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1 github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1 github.com/cs3org/go-cs3apis/cs3/tx/v1beta1 github.com/cs3org/go-cs3apis/cs3/types/v1beta1 -# github.com/cs3org/reva/v2 v2.15.0 +# github.com/cs3org/reva/v2 v2.15.1-0.20230718140539-0af2a07c7fd9 ## explicit; go 1.20 github.com/cs3org/reva/v2/cmd/revad/internal/grace github.com/cs3org/reva/v2/cmd/revad/runtime From e32aef17cd820d41872d664ce23ac958f49a71cf Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Tue, 18 Jul 2023 12:29:47 +0200 Subject: [PATCH 2/7] storage-users: Add config option for graceful shutdown timeout Add STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT setting to allow a graceful shutdown of the storage-users service. This currently only applicable when running storage-user as a sepearate service. Setting STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT to a non-zero value gives the storage-users service a chance to cleanly shutdown and finish any in progess tasks (e.g. metadata propagation) before exiting. Partial-Fix: #6602 --- services/storage-users/pkg/config/config.go | 3 ++- .../pkg/config/defaults/defaultconfig.go | 13 +++++++------ services/storage-users/pkg/revaconfig/config.go | 12 +++++++----- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index 6c5d12792d..f250b6e1a6 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -21,7 +21,8 @@ type Config struct { TokenManager *TokenManager `yaml:"token_manager"` Reva *shared.Reva `yaml:"reva"` - SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_USERS_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token."` + SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_USERS_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token."` + GracefulShutdownTimeout int `yaml:"graceful_shutdown_timeout" env:"STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT" desc:"The number of seconds to wait for the service to shutdown cleanly before exiting with an error. Note: This setting is only applicable when running the 'storage-users' service as a standalone service."` Driver string `yaml:"driver" env:"STORAGE_USERS_DRIVER" desc:"The storage driver which should be used by the service. Defaults to 'ocis', Supported values are: 'ocis', 's3ng' and 'owncloudsql'. The 'ocis' driver stores all data (blob and meta data) in an POSIX compliant volume. The 's3ng' driver stores metadata in a POSIX compliant volume and uploads blobs to the s3 bucket."` Drivers Drivers `yaml:"drivers"` diff --git a/services/storage-users/pkg/config/defaults/defaultconfig.go b/services/storage-users/pkg/config/defaults/defaultconfig.go index f8673c23e1..4674a3b1e8 100644 --- a/services/storage-users/pkg/config/defaults/defaultconfig.go +++ b/services/storage-users/pkg/config/defaults/defaultconfig.go @@ -41,12 +41,13 @@ func DefaultConfig() *config.Config { Service: config.Service{ Name: "storage-users", }, - Reva: shared.DefaultRevaConfig(), - DataServerURL: "http://localhost:9158/data", - DataGatewayURL: "https://localhost:9200/data", - TransferExpires: 86400, - UploadExpiration: 24 * 60 * 60, - Driver: "ocis", + Reva: shared.DefaultRevaConfig(), + DataServerURL: "http://localhost:9158/data", + DataGatewayURL: "https://localhost:9200/data", + TransferExpires: 86400, + UploadExpiration: 24 * 60 * 60, + GracefulShutdownTimeout: 30, + Driver: "ocis", Drivers: config.Drivers{ OwnCloudSQL: config.OwnCloudSQLDriver{ Root: filepath.Join(defaults.BaseDataPath(), "storage", "owncloud"), diff --git a/services/storage-users/pkg/revaconfig/config.go b/services/storage-users/pkg/revaconfig/config.go index 1a486ebf2a..851a38569a 100644 --- a/services/storage-users/pkg/revaconfig/config.go +++ b/services/storage-users/pkg/revaconfig/config.go @@ -1,3 +1,4 @@ +// Package revaconfig contains the config for the reva service package revaconfig import ( @@ -10,11 +11,12 @@ import ( func StorageUsersConfigFromStruct(cfg *config.Config) map[string]interface{} { rcfg := map[string]interface{}{ "core": map[string]interface{}{ - "tracing_enabled": cfg.Tracing.Enabled, - "tracing_exporter": cfg.Tracing.Type, - "tracing_endpoint": cfg.Tracing.Endpoint, - "tracing_collector": cfg.Tracing.Collector, - "tracing_service_name": cfg.Service.Name, + "tracing_enabled": cfg.Tracing.Enabled, + "tracing_exporter": cfg.Tracing.Type, + "tracing_endpoint": cfg.Tracing.Endpoint, + "tracing_collector": cfg.Tracing.Collector, + "tracing_service_name": cfg.Service.Name, + "graceful_shutdown_timeout": cfg.GracefulShutdownTimeout, }, "shared": map[string]interface{}{ "jwt_secret": cfg.TokenManager.JWTSecret, From 5f72c610890c6a5494cd7457a299dc4773820793 Mon Sep 17 00:00:00 2001 From: mmattel Date: Wed, 19 Jul 2023 11:06:29 +0200 Subject: [PATCH 3/7] envvar text fix and readme addon --- services/storage-users/README.md | 12 ++++++++++++ services/storage-users/pkg/config/config.go | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/services/storage-users/README.md b/services/storage-users/README.md index 1622fa11f2..e61c29c0aa 100644 --- a/services/storage-users/README.md +++ b/services/storage-users/README.md @@ -6,6 +6,18 @@ Purpose and description to be added Starting with ocis version 3.0.0, the default backend for metadata switched to messagepack. If the setting `STORAGE_USERS_OCIS_METADATA_BACKEND` has not been defined manually, the backend will be migrated to `messagepack` automatically. Though still possible to manually configure `xattrs`, this setting should not be used anymore as it will be removed in a later version. +## Graceful Shutdown + +Starting with Infinite Scale version 3.1, you can define a graceful shutdown period for the `storage-users` service. + +IMPORTANT: The graceful shtutdown period is only applicapable if the `storage-users` service runs as stand-alone service. It does not apply if the `storage-users` service runs as part of the single binary or as single Docker environment. To build an environment where the `storage-users` service runs as a stand alone service, you must start two instances, one _without_ the `storage-users` service and one _only with_ the the `storage-users` service. Note that the instances configured must be able to communicate on the same network. + +When hard stopping Infinite Scale, it is possible that not all data from the decomposedfs (metadata) has been written down to the storage. Such a hard stop may result in an inconsistent decomposedfs. To mitigate that situation, the following things have been implemented: + +* With the value of the environment variable `STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT`, the `storage-users` service will delay its shutdown giving it time to finalize writing down necessary data. This delay can be necessary if there is a lot of data to be written and/or if storage access/thruput is slow. Though it should not occur but in case you would recieve an error log entry because of not being able to write down data in time, you must increase the default value. + +* If a shutdown error has been logged, the command line maintenance tool [Inspect and Manipulate Node Metadata](https://doc.owncloud.com/ocis/next/maintenance/commands/commands.html#inspect-and-manipulate-node-metadata) can help to fix the issue. Please contact support for details. + ## CLI Commands ### Manage Unfinished Uploads diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index f250b6e1a6..cec4dcdf34 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -22,7 +22,7 @@ type Config struct { Reva *shared.Reva `yaml:"reva"` SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_USERS_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token."` - GracefulShutdownTimeout int `yaml:"graceful_shutdown_timeout" env:"STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT" desc:"The number of seconds to wait for the service to shutdown cleanly before exiting with an error. Note: This setting is only applicable when running the 'storage-users' service as a standalone service."` + GracefulShutdownTimeout int `yaml:"graceful_shutdown_timeout" env:"STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT" desc:"The number of seconds to wait for the 'storage-users' service to shutdown cleanly before exiting with an error that gets logged. Note: This setting is only applicable when running the 'storage-users' service as a standalone service. See the text description for more details."` Driver string `yaml:"driver" env:"STORAGE_USERS_DRIVER" desc:"The storage driver which should be used by the service. Defaults to 'ocis', Supported values are: 'ocis', 's3ng' and 'owncloudsql'. The 'ocis' driver stores all data (blob and meta data) in an POSIX compliant volume. The 's3ng' driver stores metadata in a POSIX compliant volume and uploads blobs to the s3 bucket."` Drivers Drivers `yaml:"drivers"` From 1ec451d3d512905c22cd49716c947bffa5050608 Mon Sep 17 00:00:00 2001 From: Martin Date: Wed, 19 Jul 2023 13:02:25 +0200 Subject: [PATCH 4/7] Apply suggestions from code review Co-authored-by: Edith Parzefall --- services/storage-users/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/storage-users/README.md b/services/storage-users/README.md index e61c29c0aa..5ecae4902c 100644 --- a/services/storage-users/README.md +++ b/services/storage-users/README.md @@ -12,11 +12,11 @@ Starting with Infinite Scale version 3.1, you can define a graceful shutdown per IMPORTANT: The graceful shtutdown period is only applicapable if the `storage-users` service runs as stand-alone service. It does not apply if the `storage-users` service runs as part of the single binary or as single Docker environment. To build an environment where the `storage-users` service runs as a stand alone service, you must start two instances, one _without_ the `storage-users` service and one _only with_ the the `storage-users` service. Note that the instances configured must be able to communicate on the same network. -When hard stopping Infinite Scale, it is possible that not all data from the decomposedfs (metadata) has been written down to the storage. Such a hard stop may result in an inconsistent decomposedfs. To mitigate that situation, the following things have been implemented: +When hard-stopping Infinite Scale, it is possible that not all data from the decomposedfs (metadata) has been written to the storage. Such a hard stop may result in an inconsistent decomposedfs. To mitigate that situation, the following things have been implemented: -* With the value of the environment variable `STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT`, the `storage-users` service will delay its shutdown giving it time to finalize writing down necessary data. This delay can be necessary if there is a lot of data to be written and/or if storage access/thruput is slow. Though it should not occur but in case you would recieve an error log entry because of not being able to write down data in time, you must increase the default value. +* With the value of the environment variable `STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT`, the `storage-users` service will delay its shutdown giving it time to finalize writing necessary data. This delay can be necessary if there is a lot of data to be saved and/or if storage access/thruput is slow. In such a case you would receive an error log entry informing you that not all data could be saved in time. To prevent such occurrences you must increase the default value. -* If a shutdown error has been logged, the command line maintenance tool [Inspect and Manipulate Node Metadata](https://doc.owncloud.com/ocis/next/maintenance/commands/commands.html#inspect-and-manipulate-node-metadata) can help to fix the issue. Please contact support for details. +* If a shutdown error has been logged, the command-line maintenance tool [Inspect and Manipulate Node Metadata](https://doc.owncloud.com/ocis/next/maintenance/commands/commands.html#inspect-and-manipulate-node-metadata) can help to fix the issue. Please contact support for details. ## CLI Commands From 35dd1fa173a7c7b9580326100cd52fbe81385324 Mon Sep 17 00:00:00 2001 From: Martin Date: Wed, 19 Jul 2023 13:02:48 +0200 Subject: [PATCH 5/7] Update services/storage-users/README.md Co-authored-by: Edith Parzefall --- services/storage-users/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage-users/README.md b/services/storage-users/README.md index 5ecae4902c..b26ce49a30 100644 --- a/services/storage-users/README.md +++ b/services/storage-users/README.md @@ -10,7 +10,7 @@ Starting with ocis version 3.0.0, the default backend for metadata switched to m Starting with Infinite Scale version 3.1, you can define a graceful shutdown period for the `storage-users` service. -IMPORTANT: The graceful shtutdown period is only applicapable if the `storage-users` service runs as stand-alone service. It does not apply if the `storage-users` service runs as part of the single binary or as single Docker environment. To build an environment where the `storage-users` service runs as a stand alone service, you must start two instances, one _without_ the `storage-users` service and one _only with_ the the `storage-users` service. Note that the instances configured must be able to communicate on the same network. +IMPORTANT: The graceful shutdown period is only applicable if the `storage-users` service runs as standalone service. It does not apply if the `storage-users` service runs as part of the single binary or as single Docker environment. To build an environment where the `storage-users` service runs as a standalone service, you must start two instances, one _without_ the `storage-users` service and one _only with_ the the `storage-users` service. Note that both instances must be able to communicate on the same network. When hard-stopping Infinite Scale, it is possible that not all data from the decomposedfs (metadata) has been written to the storage. Such a hard stop may result in an inconsistent decomposedfs. To mitigate that situation, the following things have been implemented: From 6c33d945f350de9546350bb3071d172cf34db779 Mon Sep 17 00:00:00 2001 From: mmattel Date: Wed, 19 Jul 2023 13:27:30 +0200 Subject: [PATCH 6/7] update readme --- services/storage-users/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/storage-users/README.md b/services/storage-users/README.md index b26ce49a30..14a0718f8e 100644 --- a/services/storage-users/README.md +++ b/services/storage-users/README.md @@ -12,9 +12,9 @@ Starting with Infinite Scale version 3.1, you can define a graceful shutdown per IMPORTANT: The graceful shutdown period is only applicable if the `storage-users` service runs as standalone service. It does not apply if the `storage-users` service runs as part of the single binary or as single Docker environment. To build an environment where the `storage-users` service runs as a standalone service, you must start two instances, one _without_ the `storage-users` service and one _only with_ the the `storage-users` service. Note that both instances must be able to communicate on the same network. -When hard-stopping Infinite Scale, it is possible that not all data from the decomposedfs (metadata) has been written to the storage. Such a hard stop may result in an inconsistent decomposedfs. To mitigate that situation, the following things have been implemented: +When hard-stopping Infinite Scale like when using the `kill ` command (SIGKILL), it is possible and likely that not all data from the decomposedfs (metadata) has been written to the storage which may result in an inconsistent decomposedfs. When gracefully shutdown Infinite Scale like using a command like SIGTERM, the process will not longer accept any write requests from _other_ services and will try to write down the internal open requests which can have an undefined duration based on many factors. To mitigate that situation, the following things have been implemented: -* With the value of the environment variable `STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT`, the `storage-users` service will delay its shutdown giving it time to finalize writing necessary data. This delay can be necessary if there is a lot of data to be saved and/or if storage access/thruput is slow. In such a case you would receive an error log entry informing you that not all data could be saved in time. To prevent such occurrences you must increase the default value. +* With the value of the environment variable `STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT`, the `storage-users` service will delay its shutdown giving it time to finalize writing necessary data. This delay can be necessary if there is a lot of data to be saved and/or if storage access/thruput is slow. In such a case you would receive an error log entry informing you that not all data could be saved in time. To prevent such occurrences, you must increase the default value. * If a shutdown error has been logged, the command-line maintenance tool [Inspect and Manipulate Node Metadata](https://doc.owncloud.com/ocis/next/maintenance/commands/commands.html#inspect-and-manipulate-node-metadata) can help to fix the issue. Please contact support for details. From 6be90847b7c17ed736931654759ceded3534ab31 Mon Sep 17 00:00:00 2001 From: Martin Date: Wed, 19 Jul 2023 14:35:52 +0200 Subject: [PATCH 7/7] Update services/storage-users/README.md Co-authored-by: Edith Parzefall --- services/storage-users/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage-users/README.md b/services/storage-users/README.md index 14a0718f8e..1ceb861c58 100644 --- a/services/storage-users/README.md +++ b/services/storage-users/README.md @@ -12,7 +12,7 @@ Starting with Infinite Scale version 3.1, you can define a graceful shutdown per IMPORTANT: The graceful shutdown period is only applicable if the `storage-users` service runs as standalone service. It does not apply if the `storage-users` service runs as part of the single binary or as single Docker environment. To build an environment where the `storage-users` service runs as a standalone service, you must start two instances, one _without_ the `storage-users` service and one _only with_ the the `storage-users` service. Note that both instances must be able to communicate on the same network. -When hard-stopping Infinite Scale like when using the `kill ` command (SIGKILL), it is possible and likely that not all data from the decomposedfs (metadata) has been written to the storage which may result in an inconsistent decomposedfs. When gracefully shutdown Infinite Scale like using a command like SIGTERM, the process will not longer accept any write requests from _other_ services and will try to write down the internal open requests which can have an undefined duration based on many factors. To mitigate that situation, the following things have been implemented: +When hard-stopping Infinite Scale, for example with the `kill ` command (SIGKILL), it is possible and likely that not all data from the decomposedfs (metadata) has been written to the storage which may result in an inconsistent decomposedfs. When gracefully shutting down Infinite Scale, using a command like SIGTERM, the process will no longer accept any write requests from _other_ services and will try to write the internal open requests which can take an undefined duration based on many factors. To mitigate that situation, the following things have been implemented: * With the value of the environment variable `STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT`, the `storage-users` service will delay its shutdown giving it time to finalize writing necessary data. This delay can be necessary if there is a lot of data to be saved and/or if storage access/thruput is slow. In such a case you would receive an error log entry informing you that not all data could be saved in time. To prevent such occurrences, you must increase the default value.