From 5073e1040564f0be19a7dc7c3e195628b415a4f5 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Wed, 29 Sep 2021 08:43:33 +0200 Subject: [PATCH 1/3] make mimetype allow list configurable and add example config --- .gitignore | 1 + .../make-allow-list-mimetypes-configurable.md | 7 ++ go.mod | 1 + storage/config/mimetypes.json | 72 +++++++++++ storage/pkg/command/gateway.go | 118 ++++++++++++++++++ storage/pkg/config/config.go | 7 ++ storage/pkg/flagset/gateway.go | 17 +++ 7 files changed, 223 insertions(+) create mode 100644 changelog/unreleased/make-allow-list-mimetypes-configurable.md create mode 100644 storage/config/mimetypes.json diff --git a/.gitignore b/.gitignore index 9a2e33a360..0f6d809d95 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ node_modules/ ocis/ocis ocis/cmd/ocis/__debug_bin ocis/cmd/ocis/config/ +config/ .idea diff --git a/changelog/unreleased/make-allow-list-mimetypes-configurable.md b/changelog/unreleased/make-allow-list-mimetypes-configurable.md new file mode 100644 index 0000000000..95d1f8b186 --- /dev/null +++ b/changelog/unreleased/make-allow-list-mimetypes-configurable.md @@ -0,0 +1,7 @@ +Enhancement: Make mimetype allow list configurable for app provider + +We've added a configuration option to configure the mimetype allow list introduced in cs3org/reva#2095. This also makes it possible to set one application per mime type as a default. + +https://github.com/owncloud/ocis/pull/2553 +https://github.com/cs3org/reva/pull/2095 +https://github.com/owncloud/ocis/issues/2563 diff --git a/go.mod b/go.mod index 5aaa90ec2b..5a84fdc51a 100644 --- a/go.mod +++ b/go.mod @@ -38,6 +38,7 @@ require ( github.com/justinas/alice v1.2.0 github.com/libregraph/lico v0.34.1-0.20210803054646-b584e0372224 github.com/mennanov/fieldmask-utils v0.4.0 + github.com/mitchellh/mapstructure v1.4.2 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/nmcclain/asn1-ber v0.0.0-20170104154839-2661553a0484 github.com/nmcclain/ldap v0.0.0-20210720162743-7f8d1e44eeba diff --git a/storage/config/mimetypes.json b/storage/config/mimetypes.json new file mode 100644 index 0000000000..935b3d26ef --- /dev/null +++ b/storage/config/mimetypes.json @@ -0,0 +1,72 @@ +{ + "application/pdf": { + "extension": "pdf", + "name": "PDF", + "description": "PDF document", + "icon": "", + "default_app": "" + }, + "application/vnd.oasis.opendocument.text": { + "extension": "odt", + "name": "OpenDocument", + "description": "OpenDocument text document", + "icon": "", + "default_app": "" + }, + "application/vnd.oasis.opendocument.spreadsheet": { + "extension": "ods", + "name": "OpenSpreadsheet", + "description": "OpenDocument spreadsheet document", + "icon": "", + "default_app": "" + }, + "application/vnd.oasis.opendocument.presentation": { + "extension": "odp", + "name": "OpenPresentation", + "description": "OpenDocument presentation document", + "icon": "", + "default_app": "" + }, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": { + "extension": "docx", + "name": "Microsoft Word", + "description": "Microsoft Word document", + "icon": "", + "default_app": "" + }, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": { + "extension": "xlsx", + "name": "Microsoft Excel", + "description": "Microsoft Excel document", + "icon": "", + "default_app": "" + }, + "application/vnd.openxmlformats-officedocument.presentationml.presentation": { + "extension": "pptx", + "name": "Microsoft PowerPoint", + "description": "Microsoft PowerPoint document", + "icon": "", + "default_app": "" + }, + "application/vnd.jupyter": { + "extension": "ipynb", + "name": "Jupyter Notebook", + "description": "Jupyter Notebook", + "icon": "", + "default_app": "" + }, + "text/markdown": { + "extension": "md", + "name": "Markdown file", + "description": "Markdown file", + "icon": "", + "default_app": "" + }, + "application/compressed-markdown": { + "extension": "zmd", + "name": "Compressed markdown file", + "description": "Compressed markdown file", + "icon": "", + "default_app": "" + } +} diff --git a/storage/pkg/command/gateway.go b/storage/pkg/command/gateway.go index fe17764363..f5865be5db 100644 --- a/storage/pkg/command/gateway.go +++ b/storage/pkg/command/gateway.go @@ -9,6 +9,7 @@ import ( "path" "strings" + "github.com/mitchellh/mapstructure" "github.com/owncloud/ocis/storage/pkg/tracing" "github.com/owncloud/ocis/ocis-pkg/sync" @@ -167,6 +168,11 @@ func gatewayConfigFromStruct(c *cli.Context, cfg *config.Config, logger log.Logg }, "appregistry": map[string]interface{}{ "driver": "static", + "drivers": map[string]interface{}{ + "static": map[string]interface{}{ + "mime_types": mimetypes(cfg, logger), + }, + }, }, "storageregistry": map[string]interface{}{ "driver": cfg.Reva.StorageRegistry.Driver, @@ -222,6 +228,118 @@ func rules(cfg *config.Config, logger log.Logger) map[string]map[string]interfac } } +func mimetypes(cfg *config.Config, logger log.Logger) (m map[string]map[string]string) { + + type mimeTypeConfig struct { + Extension string `json:"extension" mapstructure:"extension"` + Name string `json:"name" mapstructure:"name"` + Description string `json:"description" mapstructure:"description"` + Icon string `json:"icon" mapstructure:"icon"` + DefaultApp string `json:"default_app" mapstructure:"default_app"` + } + type mimetypesConfig map[string]mimeTypeConfig + + var mimetypes mimetypesConfig + + // load default app mimetypes from a json file + if cfg.Reva.AppRegistry.MimetypesJSON != "" { + data, err := ioutil.ReadFile(cfg.Reva.AppRegistry.MimetypesJSON) + if err != nil { + logger.Error().Err(err).Msg("Failed to read app registry mimetypes from JSON file: " + cfg.Reva.AppRegistry.MimetypesJSON) + return nil + } + if err = json.Unmarshal(data, &mimetypes); err != nil { + logger.Error().Err(err).Msg("Failed to unmarshal storage registry rules") + return nil + } + if err := mapstructure.Decode(mimetypes, &m); err != nil { + return nil + } + return m + } + + logger.Info().Msg("No app registry mimetypes JSON file provided, loading default configuration") + + mimetypes = map[string]mimeTypeConfig{ + "application/pdf": { + Extension: "pdf", + Name: "PDF", + Description: "PDF document", + Icon: "", + DefaultApp: "", + }, + "application/vnd.oasis.opendocument.text": { + Extension: "odt", + Name: "OpenDocument", + Description: "OpenDocument text document", + Icon: "", + DefaultApp: "", + }, + "application/vnd.oasis.opendocument.spreadsheet": { + Extension: "ods", + Name: "OpenSpreadsheet", + Description: "OpenDocument spreadsheet document", + Icon: "", + DefaultApp: "", + }, + "application/vnd.oasis.opendocument.presentation": { + Extension: "odp", + Name: "OpenPresentation", + Description: "OpenDocument presentation document", + Icon: "", + DefaultApp: "", + }, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document": { + Extension: "docx", + Name: "Microsoft Word", + Description: "Microsoft Word document", + Icon: "", + DefaultApp: "", + }, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": { + Extension: "xlsx", + Name: "Microsoft Excel", + Description: "Microsoft Excel document", + Icon: "", + DefaultApp: "", + }, + "application/vnd.openxmlformats-officedocument.presentationml.presentation": { + Extension: "pptx", + Name: "Microsoft PowerPoint", + Description: "Microsoft PowerPoint document", + Icon: "", + DefaultApp: "", + }, + "application/vnd.jupyter": { + Extension: "ipynb", + Name: "Jupyter Notebook", + Description: "Jupyter Notebook", + Icon: "", + DefaultApp: "", + }, + "text/markdown": { + Extension: "md", + Name: "Markdown file", + Description: "Markdown file", + Icon: "", + DefaultApp: "", + }, + "application/compressed-markdown": { + Extension: "zmd", + Name: "Compressed markdown file", + Description: "Compressed markdown file", + Icon: "", + DefaultApp: "", + }, + } + + if err := mapstructure.Decode(mimetypes, &m); err != nil { + return nil + } + return m + +} + // GatewaySutureService allows for the storage-gateway command to be embedded and supervised by a suture supervisor tree. type GatewaySutureService struct { cfg *config.Config diff --git a/storage/pkg/config/config.go b/storage/pkg/config/config.go index 5d377820fa..7d34e71871 100644 --- a/storage/pkg/config/config.go +++ b/storage/pkg/config/config.go @@ -39,6 +39,12 @@ type StorageRegistry struct { JSON string } +// AppRegistry defines the available app registry configuration +type AppRegistry struct { + Driver string + MimetypesJSON string +} + // AppProvider defines the available app provider configuration type AppProvider struct { Port @@ -438,6 +444,7 @@ type Reva struct { DataGateway DataGatewayPort Gateway Gateway StorageRegistry StorageRegistry + AppRegistry AppRegistry Users Users Groups Groups AuthProvider Users diff --git a/storage/pkg/flagset/gateway.go b/storage/pkg/flagset/gateway.go index 9b5ef80cf5..59795d467d 100644 --- a/storage/pkg/flagset/gateway.go +++ b/storage/pkg/flagset/gateway.go @@ -155,6 +155,23 @@ func GatewayWithConfig(cfg *config.Config) []cli.Flag { Destination: &cfg.Reva.StorageRegistry.JSON, }, + // app registry + + &cli.StringFlag{ + Name: "app-registry-driver", + Value: flags.OverrideDefaultString(cfg.Reva.AppRegistry.Driver, "static"), + Usage: "driver of the app registry", + EnvVars: []string{"STORAGE_APP_REGISTRY_DRIVER"}, + Destination: &cfg.Reva.AppRegistry.Driver, + }, + &cli.StringFlag{ + Name: "app-registry-mimetypes-json", + Value: flags.OverrideDefaultString(cfg.Reva.AppRegistry.MimetypesJSON, ""), + Usage: "JSON file containing the storage registry rules", + EnvVars: []string{"STORAGE_APP_REGISTRY_MIMETYPES_JSON"}, + Destination: &cfg.Reva.AppRegistry.MimetypesJSON, + }, + // please note that STORAGE_FRONTEND_PUBLIC_URL is also defined in // storage/pkg/flagset/frontend.go because this setting may be consumed // by both the gateway and frontend service From 423e0d6450f73b1ae51017c1a4bcb8382a8ff202 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Thu, 30 Sep 2021 15:31:00 +0200 Subject: [PATCH 2/3] remove named return and add loggin --- storage/pkg/command/gateway.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/storage/pkg/command/gateway.go b/storage/pkg/command/gateway.go index f5865be5db..8c550a19eb 100644 --- a/storage/pkg/command/gateway.go +++ b/storage/pkg/command/gateway.go @@ -228,7 +228,7 @@ func rules(cfg *config.Config, logger log.Logger) map[string]map[string]interfac } } -func mimetypes(cfg *config.Config, logger log.Logger) (m map[string]map[string]string) { +func mimetypes(cfg *config.Config, logger log.Logger) map[string]map[string]string { type mimeTypeConfig struct { Extension string `json:"extension" mapstructure:"extension"` @@ -252,7 +252,9 @@ func mimetypes(cfg *config.Config, logger log.Logger) (m map[string]map[string]s logger.Error().Err(err).Msg("Failed to unmarshal storage registry rules") return nil } + var m map[string]map[string]string if err := mapstructure.Decode(mimetypes, &m); err != nil { + logger.Error().Err(err).Msg("Failed to decode defaultapp registry mimetypes to mapstructure") return nil } return m @@ -333,7 +335,9 @@ func mimetypes(cfg *config.Config, logger log.Logger) (m map[string]map[string]s }, } + var m map[string]map[string]string if err := mapstructure.Decode(mimetypes, &m); err != nil { + logger.Error().Err(err).Msg("Failed to decode defaultapp registry mimetypes to mapstructure") return nil } return m From 66f430bff1b259e16608025e62fb6c3937ac5d70 Mon Sep 17 00:00:00 2001 From: Willy Kloucek Date: Thu, 30 Sep 2021 16:00:43 +0200 Subject: [PATCH 3/3] remove empty fields --- storage/pkg/command/gateway.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/storage/pkg/command/gateway.go b/storage/pkg/command/gateway.go index 8c550a19eb..f7bfa6c55b 100644 --- a/storage/pkg/command/gateway.go +++ b/storage/pkg/command/gateway.go @@ -267,71 +267,51 @@ func mimetypes(cfg *config.Config, logger log.Logger) map[string]map[string]stri Extension: "pdf", Name: "PDF", Description: "PDF document", - Icon: "", - DefaultApp: "", }, "application/vnd.oasis.opendocument.text": { Extension: "odt", Name: "OpenDocument", Description: "OpenDocument text document", - Icon: "", - DefaultApp: "", }, "application/vnd.oasis.opendocument.spreadsheet": { Extension: "ods", Name: "OpenSpreadsheet", Description: "OpenDocument spreadsheet document", - Icon: "", - DefaultApp: "", }, "application/vnd.oasis.opendocument.presentation": { Extension: "odp", Name: "OpenPresentation", Description: "OpenDocument presentation document", - Icon: "", - DefaultApp: "", }, "application/vnd.openxmlformats-officedocument.wordprocessingml.document": { Extension: "docx", Name: "Microsoft Word", Description: "Microsoft Word document", - Icon: "", - DefaultApp: "", }, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": { Extension: "xlsx", Name: "Microsoft Excel", Description: "Microsoft Excel document", - Icon: "", - DefaultApp: "", }, "application/vnd.openxmlformats-officedocument.presentationml.presentation": { Extension: "pptx", Name: "Microsoft PowerPoint", Description: "Microsoft PowerPoint document", - Icon: "", - DefaultApp: "", }, "application/vnd.jupyter": { Extension: "ipynb", Name: "Jupyter Notebook", Description: "Jupyter Notebook", - Icon: "", - DefaultApp: "", }, "text/markdown": { Extension: "md", Name: "Markdown file", Description: "Markdown file", - Icon: "", - DefaultApp: "", }, "application/compressed-markdown": { Extension: "zmd", Name: "Compressed markdown file", Description: "Compressed markdown file", - Icon: "", - DefaultApp: "", }, }