From f88cf46c75788f0f7010feeadc4a13201f57fdee Mon Sep 17 00:00:00 2001 From: jkoberg Date: Tue, 27 Dec 2022 16:02:01 +0100 Subject: [PATCH 1/4] add readme for postprocessing Signed-off-by: jkoberg --- services/postprocessing/README.md | 29 ++++++++++++++++++++ services/postprocessing/pkg/config/config.go | 4 +-- 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 services/postprocessing/README.md diff --git a/services/postprocessing/README.md b/services/postprocessing/README.md new file mode 100644 index 000000000..bb127cd66 --- /dev/null +++ b/services/postprocessing/README.md @@ -0,0 +1,29 @@ +# Postprocessing service + +The `postprocessing` service handles coordination of asynchronous postprocessing. + +## Postprocessing functionality + +The storageprovider service (`storage-users`) can be configured to do asynchronous postprocessing by setting the `STORAGE_USERS_OCIS_ASYNC_UPLOADS` envvar to true. +If this is the case, the storageprovider will initiate an asynchronous postprocessing after he has reveived all bytes of an upload. The `postprocessing` service will then +coordinate various postprocessing steps (like e.g. scan the file for viruses). During postprocessing the file will be in a `processing` state during which only limited actions are available. + +## Prerequisites for using `postprocessing` service + +In the storageprovider (`storage-users`) set `STORAGE_USERS_OCIS_ASYNC_UPLOADS` envvar to `true`. Configuring any postprocessing step will require an additional service to be enabled and configured. +For example to use `virusscan` step one needs to have an enabled and configured `antivirus` service. + +All of this functionality will need an event system to be configured for all services: `ocis` ships with +`nats` enabled by default. + +## Postprocessing steps + +As of now ocis allows two different postprocessing steps to be enabled via envvar + +### Virus scanning + +Can be set via envvar `POSTPROCESSING_VIRUSSCAN`. This means that each upload is virus scanned during postprocessing. `antivirus` service is needed for this to work. + +### Delay + +Can be set via envvar `POSTPROCESSING_DELAY`. This step will just sleep for the configured amount of time. Intended for testing postprocessing functionality. NOT RECOMMENDED on productive systems. diff --git a/services/postprocessing/pkg/config/config.go b/services/postprocessing/pkg/config/config.go index 72a2ee38c..c7bb9d69d 100644 --- a/services/postprocessing/pkg/config/config.go +++ b/services/postprocessing/pkg/config/config.go @@ -32,7 +32,7 @@ type Events struct { Endpoint string `yaml:"endpoint" env:"POSTPROCESSING_EVENTS_ENDPOINT" desc:"The address of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture."` Cluster string `yaml:"cluster" env:"POSTPROCESSING_EVENTS_CLUSTER" desc:"The clusterID of the event system. The event system is the message queuing service. It is used as message broker for the microservice architecture. Mandatory when using NATS as event system."` - TLSInsecure bool `yaml:"tls_insecure" env:"OCIS_INSECURE;SEARCH_EVENTS_TLS_INSECURE" desc:"Whether the ocis server should skip the client certificate verification during the TLS handshake."` - TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"SEARCH_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided SEARCH_EVENTS_TLS_INSECURE will be seen as false."` + TLSInsecure bool `yaml:"tls_insecure" env:"OCIS_INSECURE;POSTPROCESSING_EVENTS_TLS_INSECURE" desc:"Whether the ocis server should skip the client certificate verification during the TLS handshake."` + TLSRootCACertificate string `yaml:"tls_root_ca_certificate" env:"POSTPROCESSING_EVENTS_TLS_ROOT_CA_CERTIFICATE" desc:"The root CA certificate used to validate the server's TLS certificate. If provided POSTPROCESSING_EVENTS_TLS_INSECURE will be seen as false."` EnableTLS bool `yaml:"enable_tls" env:"OCIS_EVENTS_ENABLE_TLS;POSTPROCESSING_EVENTS_ENABLE_TLS" desc:"Enable TLS for the connection to the events broker. The events broker is the ocis service which receives and delivers events between the services."` } From cb9e1eaf9b02359d4f43aba08e5a62ec70722143 Mon Sep 17 00:00:00 2001 From: mmattel Date: Tue, 27 Dec 2022 17:17:27 +0100 Subject: [PATCH 2/4] text improvements --- services/postprocessing/README.md | 34 ++++++++++++++++--------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/services/postprocessing/README.md b/services/postprocessing/README.md index bb127cd66..4e0c3ea7a 100644 --- a/services/postprocessing/README.md +++ b/services/postprocessing/README.md @@ -1,29 +1,31 @@ -# Postprocessing service +# Postprocessing Service -The `postprocessing` service handles coordination of asynchronous postprocessing. +The `postprocessing` service handles the coordination of asynchronous postprocessing steps. -## Postprocessing functionality +## General Prerequisites -The storageprovider service (`storage-users`) can be configured to do asynchronous postprocessing by setting the `STORAGE_USERS_OCIS_ASYNC_UPLOADS` envvar to true. -If this is the case, the storageprovider will initiate an asynchronous postprocessing after he has reveived all bytes of an upload. The `postprocessing` service will then -coordinate various postprocessing steps (like e.g. scan the file for viruses). During postprocessing the file will be in a `processing` state during which only limited actions are available. +To use the postprocessing service, an event system needs to be configured for all services. By default, `ocis` ships with a preconfigured `nats` service. -## Prerequisites for using `postprocessing` service +## Postprocessing Functionality -In the storageprovider (`storage-users`) set `STORAGE_USERS_OCIS_ASYNC_UPLOADS` envvar to `true`. Configuring any postprocessing step will require an additional service to be enabled and configured. -For example to use `virusscan` step one needs to have an enabled and configured `antivirus` service. +The storageprovider service (`storage-users`) can be configured to initiate asynchronous postprocessing by setting the `STORAGE_USERS_OCIS_ASYNC_UPLOADS` environment variable to `true`. If this is the case, postprocessing will get initiated *after* uploading a file and all bytes have been recieved. -All of this functionality will need an event system to be configured for all services: `ocis` ships with -`nats` enabled by default. +The `postprocessing` service will then coordinate configured postprocessing steps like scanning the file for viruses. During postprocessing, the file will be in a `processing state` where only a limited set of actions are available. Note that this processing state excludes file accessability by users. -## Postprocessing steps +When all postprocessing steps have completed successfully, the file will be made accessible for users. -As of now ocis allows two different postprocessing steps to be enabled via envvar +## Additional Prerequisites for the `postprocessing` Service -### Virus scanning +When postprocessing has been enabled, configuring any postprocessing step will require the requested services to be enabled and pre-configured. For example, to use the `virusscan` step, one needs to have an enabled and configured `antivirus` service. -Can be set via envvar `POSTPROCESSING_VIRUSSCAN`. This means that each upload is virus scanned during postprocessing. `antivirus` service is needed for this to work. +## Postprocessing Steps + +As of now, `ocis` allows two different postprocessing steps to be enabled via an environment variable. + +### Virus Scanning + +To enable virus scanning as postprocessing step after uploading a file, the environment variable `POSTPROCESSING_VIRUSSCAN` needs to be set to ` true`. As a result, each uploaded file gets virus scanned as part of the postprocessing steps. Note that the `antivirus` service is required to be enabled and configured for this to work. ### Delay -Can be set via envvar `POSTPROCESSING_DELAY`. This step will just sleep for the configured amount of time. Intended for testing postprocessing functionality. NOT RECOMMENDED on productive systems. +Though this is for development purposes only and NOT RECOMMENDED on productive systems, setting the environment variable `POSTPROCESSING_DELAY` to a duration not equal to zero will add a delay step with the configured amount of time. ocis will continue postprocessing the file after the configured delay. From 7fcb31ab421837de7c8a9e5f4ce9eff657381185 Mon Sep 17 00:00:00 2001 From: mmattel Date: Tue, 27 Dec 2022 17:18:48 +0100 Subject: [PATCH 3/4] fix --- services/postprocessing/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/postprocessing/README.md b/services/postprocessing/README.md index 4e0c3ea7a..f4d14aaae 100644 --- a/services/postprocessing/README.md +++ b/services/postprocessing/README.md @@ -8,7 +8,7 @@ To use the postprocessing service, an event system needs to be configured for al ## Postprocessing Functionality -The storageprovider service (`storage-users`) can be configured to initiate asynchronous postprocessing by setting the `STORAGE_USERS_OCIS_ASYNC_UPLOADS` environment variable to `true`. If this is the case, postprocessing will get initiated *after* uploading a file and all bytes have been recieved. +The storageprovider service (`storage-users`) can be configured to initiate asynchronous postprocessing by setting the `STORAGE_USERS_OCIS_ASYNC_UPLOADS` environment variable to `true`. If this is the case, postprocessing will get initiated *after* uploading a file and all bytes have been received. The `postprocessing` service will then coordinate configured postprocessing steps like scanning the file for viruses. During postprocessing, the file will be in a `processing state` where only a limited set of actions are available. Note that this processing state excludes file accessability by users. From abba39e471518a4984a2c02df84c09110e8357c6 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Wed, 28 Dec 2022 06:48:11 +0545 Subject: [PATCH 4/4] Apply suggestions from code review --- services/postprocessing/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/postprocessing/README.md b/services/postprocessing/README.md index f4d14aaae..102dee52b 100644 --- a/services/postprocessing/README.md +++ b/services/postprocessing/README.md @@ -24,8 +24,8 @@ As of now, `ocis` allows two different postprocessing steps to be enabled via an ### Virus Scanning -To enable virus scanning as postprocessing step after uploading a file, the environment variable `POSTPROCESSING_VIRUSSCAN` needs to be set to ` true`. As a result, each uploaded file gets virus scanned as part of the postprocessing steps. Note that the `antivirus` service is required to be enabled and configured for this to work. +To enable virus scanning as a postprocessing step after uploading a file, the environment variable `POSTPROCESSING_VIRUSSCAN` needs to be set to ` true`. As a result, each uploaded file gets virus scanned as part of the postprocessing steps. Note that the `antivirus` service is required to be enabled and configured for this to work. ### Delay -Though this is for development purposes only and NOT RECOMMENDED on productive systems, setting the environment variable `POSTPROCESSING_DELAY` to a duration not equal to zero will add a delay step with the configured amount of time. ocis will continue postprocessing the file after the configured delay. +Though this is for development purposes only and NOT RECOMMENDED on production systems, setting the environment variable `POSTPROCESSING_DELAY` to a duration not equal to zero will add a delay step with the configured amount of time. ocis will continue postprocessing the file after the configured delay.