From 89785d097631f6583fdf7406c993559761cdb289 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 19 Jul 2023 12:16:03 +0200 Subject: [PATCH 1/2] configurable max message size Signed-off-by: jkoberg --- .../configurable_max_message_size.md | 5 +++++ .../general-info/additional-information.md | 20 +++++++++++++++++++ ocis-pkg/shared/shared_types.go | 4 ++++ 3 files changed, 29 insertions(+) create mode 100644 changelog/unreleased/configurable_max_message_size.md create mode 100644 docs/services/general-info/additional-information.md diff --git a/changelog/unreleased/configurable_max_message_size.md b/changelog/unreleased/configurable_max_message_size.md new file mode 100644 index 0000000000..b210e76012 --- /dev/null +++ b/changelog/unreleased/configurable_max_message_size.md @@ -0,0 +1,5 @@ +Enhancement: configure max message size + +Add a configuration option for the grpc max message size + +https://github.com/owncloud/ocis/pull/6849 diff --git a/docs/services/general-info/additional-information.md b/docs/services/general-info/additional-information.md new file mode 100644 index 0000000000..7b24700fd2 --- /dev/null +++ b/docs/services/general-info/additional-information.md @@ -0,0 +1,20 @@ +--- +title: Additinal Information +date: 2018-05-02T00:00:00+00:00 +weight: 20 +geekdocRepo: https://github.com/owncloud/ocis +geekdocEditPath: edit/master/docs/services/general-info +geekdocFilePath: additional-information.md +geekdocCollapseSection: true +--- + +This section contains information on general topics + +## GRPC Maximum message size + +Ocis is using grpc for inter service communication. When having a folder with a lot of files (25000+, size doesn't matter) and doing a `PROPFIND` on the folder, the server will run into errors because the +grpc message body gets to big. We introduced the envvar `OCIS_GRPC_MAX_RECEIVED_MESSAGE_SIZE` to raise the max size for the grpc body. + +NOTE: With a certain amount of files even raising the grpc message size will not suffice as the requests will run into network timeouts. Also generally the more files are in a folder, the longer it will take to load. + +It is recommended to use `OCIS_GRPC_MAX_RECEIVED_MESSAGE_SIZE` only temporary to copy files out of the folder (e.g. via web ui) and use the default value in general. diff --git a/ocis-pkg/shared/shared_types.go b/ocis-pkg/shared/shared_types.go index a20d0dee61..efc9062c44 100644 --- a/ocis-pkg/shared/shared_types.go +++ b/ocis-pkg/shared/shared_types.go @@ -81,4 +81,8 @@ type Commons struct { SystemUserID string `yaml:"system_user_id" env:"OCIS_SYSTEM_USER_ID" desc:"ID of the oCIS storage-system system user. Admins need to set the ID for the storage-system system user in this config option which is then used to reference the user. Any reasonable long string is possible, preferably this would be an UUIDv4 format."` SystemUserAPIKey string `mask:"password" yaml:"system_user_api_key" env:"SYSTEM_USER_API_KEY"` AdminUserID string `yaml:"admin_user_id" env:"OCIS_ADMIN_USER_ID" desc:"ID of a user, that should receive admin privileges. Consider that the UUID can be encoded in some LDAP deployment configurations like in .ldif files. These need to be decoded beforehand."` + + // NOTE: you will not fing GRPCMaxReceivedMessageSize size being used in the code. The envvar is actually extracted in revas `pool` package: https://github.com/cs3org/reva/blob/edge/pkg/rgrpc/todo/pool/connection.go + // It is mentioned here again so it is documented + GRPCMaxReceivedMessageSize int `env:"OCIS_GRPC_MAX_RECEIVED_MESSAGE_SIZE" desc:"The maximum body size for grpc requests. Defaults to '10240000'. Note that large values can potentially hide errors and can cause network timeouts. Can be used temporarily to repair unaccessable large folders (25000+ files)"` } From 31226e2134b03c24afa2432684ea81ce9861a3d7 Mon Sep 17 00:00:00 2001 From: kobergj Date: Thu, 20 Jul 2023 15:59:31 +0200 Subject: [PATCH 2/2] improve documentation Co-authored-by: Martin --- changelog/unreleased/configurable_max_message_size.md | 2 +- docs/services/general-info/additional-information.md | 9 ++++----- ocis-pkg/shared/shared_types.go | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/changelog/unreleased/configurable_max_message_size.md b/changelog/unreleased/configurable_max_message_size.md index b210e76012..5ea428b7a6 100644 --- a/changelog/unreleased/configurable_max_message_size.md +++ b/changelog/unreleased/configurable_max_message_size.md @@ -1,4 +1,4 @@ -Enhancement: configure max message size +Enhancement: configure max grpc message size Add a configuration option for the grpc max message size diff --git a/docs/services/general-info/additional-information.md b/docs/services/general-info/additional-information.md index 7b24700fd2..ba4e081454 100644 --- a/docs/services/general-info/additional-information.md +++ b/docs/services/general-info/additional-information.md @@ -10,11 +10,10 @@ geekdocCollapseSection: true This section contains information on general topics -## GRPC Maximum message size +## GRPC Maximum Message Size -Ocis is using grpc for inter service communication. When having a folder with a lot of files (25000+, size doesn't matter) and doing a `PROPFIND` on the folder, the server will run into errors because the -grpc message body gets to big. We introduced the envvar `OCIS_GRPC_MAX_RECEIVED_MESSAGE_SIZE` to raise the max size for the grpc body. +ocis is using grpc for inter-service communication. When having a folder with a lot of files (25.000+, the size does not matter) and doing a `PROPFIND` on that folder, the server will run into errors. This is because the grpc message body becomes to big. With introduction of the envvar `OCIS_GRPC_MAX_RECEIVED_MESSAGE_SIZE`, the max size for the grpc body can be raised. -NOTE: With a certain amount of files even raising the grpc message size will not suffice as the requests will run into network timeouts. Also generally the more files are in a folder, the longer it will take to load. +NOTE: With a certain amount of files even raising the grpc message size will not suffice as the requests will run into network timeouts. Also generally the more files are in a folder, the longer it will take time to load. -It is recommended to use `OCIS_GRPC_MAX_RECEIVED_MESSAGE_SIZE` only temporary to copy files out of the folder (e.g. via web ui) and use the default value in general. +It is recommended to use `OCIS_GRPC_MAX_RECEIVED_MESSAGE_SIZE` only *temporary* to copy files out of the folder (like via the web ui) and use the default value in general. diff --git a/ocis-pkg/shared/shared_types.go b/ocis-pkg/shared/shared_types.go index efc9062c44..ddfab30cf1 100644 --- a/ocis-pkg/shared/shared_types.go +++ b/ocis-pkg/shared/shared_types.go @@ -84,5 +84,5 @@ type Commons struct { // NOTE: you will not fing GRPCMaxReceivedMessageSize size being used in the code. The envvar is actually extracted in revas `pool` package: https://github.com/cs3org/reva/blob/edge/pkg/rgrpc/todo/pool/connection.go // It is mentioned here again so it is documented - GRPCMaxReceivedMessageSize int `env:"OCIS_GRPC_MAX_RECEIVED_MESSAGE_SIZE" desc:"The maximum body size for grpc requests. Defaults to '10240000'. Note that large values can potentially hide errors and can cause network timeouts. Can be used temporarily to repair unaccessable large folders (25000+ files)"` + GRPCMaxReceivedMessageSize int `env:"OCIS_GRPC_MAX_RECEIVED_MESSAGE_SIZE" desc:"The maximum body size for grpc requests. Defaults to '10240000' bytes (10MB). Note that large values can potentially hide errors but may lead to network timeouts. Should only be changed temporarily to regain access for large folders with 25.000+ files to copy out data."` }