From 416c966dbc408e1e93f76b938db69fe5ec5e4bac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Tue, 26 Apr 2022 08:24:27 +0000 Subject: [PATCH 01/13] introduce metadata gateway MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- .../pkg/config/defaults/defaultconfig.go | 4 +- .../storage-metadata/pkg/command/command.go | 75 ++++++++++++++++++- .../storage-metadata/pkg/config/config.go | 5 +- .../pkg/config/defaults/defaultconfig.go | 24 ++++-- .../pkg/config/defaults/defaultconfig.go | 4 +- 5 files changed, 95 insertions(+), 17 deletions(-) diff --git a/extensions/settings/pkg/config/defaults/defaultconfig.go b/extensions/settings/pkg/config/defaults/defaultconfig.go index 4a3a4cd31..bfb6d4844 100644 --- a/extensions/settings/pkg/config/defaults/defaultconfig.go +++ b/extensions/settings/pkg/config/defaults/defaultconfig.go @@ -55,10 +55,10 @@ func DefaultConfig() *config.Config { }, Metadata: config.Metadata{ - GatewayAddress: "127.0.0.1:9142", + GatewayAddress: "127.0.0.1:9215", StorageAddress: "127.0.0.1:9215", ServiceUserID: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", - ServiceUserIDP: "https://localhost:9200", + ServiceUserIDP: "internal", MachineAuthAPIKey: "change-me-please", }, } diff --git a/extensions/storage-metadata/pkg/command/command.go b/extensions/storage-metadata/pkg/command/command.go index 06e5c2245..bd0f0b565 100644 --- a/extensions/storage-metadata/pkg/command/command.go +++ b/extensions/storage-metadata/pkg/command/command.go @@ -125,16 +125,83 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in }, "shared": map[string]interface{}{ "jwt_secret": cfg.JWTSecret, - "gatewaysvc": cfg.GatewayEndpoint, + "gatewaysvc": cfg.GRPC.Addr, "skip_user_groups_in_token": cfg.SkipUserGroupsInToken, }, "grpc": map[string]interface{}{ "network": cfg.GRPC.Protocol, "address": cfg.GRPC.Addr, - "interceptors": map[string]interface{}{ - "log": map[string]interface{}{}, - }, + //"interceptors": map[string]interface{}{ + // "log": map[string]interface{}{}, + //}, "services": map[string]interface{}{ + "gateway": map[string]interface{}{ + // registries are located on the gateway + "authregistrysvc": cfg.GRPC.Addr, + "storageregistrysvc": cfg.GRPC.Addr, + // user metadata is located on the users services + "userprovidersvc": cfg.GRPC.Addr, + "groupprovidersvc": cfg.GRPC.Addr, + "permissionssvc": cfg.GRPC.Addr, + // other + "disable_home_creation_on_login": true, + //"datagateway": cfg.Reva.StorageMetadata.HTTPAddr, // needs to start with a protocol + "transfer_shared_secret": cfg.TransferSecret, + "transfer_expires": cfg.TransferExpires, + //"home_mapping": cfg.Reva.Gateway.HomeMapping, + //"etag_cache_ttl": cfg.Reva.Gateway.EtagCacheTTL, + }, + "userprovider": map[string]interface{}{ + "driver": "memory", + "drivers": map[string]interface{}{ + "memory": map[string]interface{}{ + "users": map[string]interface{}{ + "serviceuser": map[string]interface{}{ + "id": map[string]interface{}{ + "opaqueId": "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", + "idp": "internal", + "type": 1, // user.UserType_USER_TYPE_PRIMARY + }, + "username": "serviceuser", + // "secret": // TODO should not have a secret + "mail": "admin@example.org", + "display_name": "System User", + }, + }, + }, + }, + }, + "authregistry": map[string]interface{}{ + "driver": "static", + "drivers": map[string]interface{}{ + "static": map[string]interface{}{ + "rules": map[string]interface{}{ + "machine": cfg.GRPC.Addr, + }, + }, + }, + }, + "authprovider": map[string]interface{}{ + "auth_manager": "machine", + "auth_managers": map[string]interface{}{ + "machine": map[string]interface{}{ + "api_key": cfg.MachineAuthAPIKey, + "gateway_addr": cfg.GRPC.Addr, + }, + }, + }, + "storageregistry": map[string]interface{}{ + "driver": "static", + "drivers": map[string]interface{}{ + "static": map[string]interface{}{ + "rules": map[string]interface{}{ + "/": map[string]interface{}{ + "address": cfg.GRPC.Addr, + }, + }, + }, + }, + }, "storageprovider": map[string]interface{}{ "driver": cfg.Driver, "drivers": config.MetadataDrivers(cfg), diff --git a/extensions/storage-metadata/pkg/config/config.go b/extensions/storage-metadata/pkg/config/config.go index 526a4eabc..30608f421 100644 --- a/extensions/storage-metadata/pkg/config/config.go +++ b/extensions/storage-metadata/pkg/config/config.go @@ -25,7 +25,10 @@ type Config struct { Drivers Drivers `yaml:"drivers"` DataServerURL string TempFolder string - DataProviderInsecure bool `env:"OCIS_INSECURE;STORAGE_METADATA_DATAPROVIDER_INSECURE"` + TransferSecret string `yaml:"transfer_secret" env:"STORAGE_METADATA_TRANSFER_SECRET"` + TransferExpires int `yaml:"transfer_expires" env:"STORAGE_METADATA_TRANSFER_EXPIRES"` + DataProviderInsecure bool `env:"OCIS_INSECURE;STORAGE_METADATA_DATAPROVIDER_INSECURE"` + MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;STORAGE_METADATA_MACHINE_AUTH_API_KEY"` } type Tracing struct { Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_METADATA_TRACING_ENABLED" desc:"Activates tracing."` diff --git a/extensions/storage-metadata/pkg/config/defaults/defaultconfig.go b/extensions/storage-metadata/pkg/config/defaults/defaultconfig.go index 298d31eb5..a50d301c7 100644 --- a/extensions/storage-metadata/pkg/config/defaults/defaultconfig.go +++ b/extensions/storage-metadata/pkg/config/defaults/defaultconfig.go @@ -24,6 +24,11 @@ func DefaultConfig() *config.Config { Pprof: false, Zpages: false, }, + Logging: &config.Logging{ + Level: "debug", + Pretty: true, + Color: true, + }, GRPC: config.GRPCConfig{ Addr: "127.0.0.1:9215", Protocol: "tcp", @@ -35,11 +40,14 @@ func DefaultConfig() *config.Config { Service: config.Service{ Name: "storage-metadata", }, - GatewayEndpoint: "127.0.0.1:9142", - JWTSecret: "Pive-Fumkiu4", - TempFolder: filepath.Join(defaults.BaseDataPath(), "tmp", "metadata"), - DataServerURL: "http://localhost:9216/data", - Driver: "ocis", + GatewayEndpoint: "127.0.0.1:9215", // metadata is a self contained reva instance + JWTSecret: "Pive-Fumkiu4", + TempFolder: filepath.Join(defaults.BaseDataPath(), "tmp", "metadata"), + DataServerURL: "http://localhost:9216/data", + TransferSecret: "replace-me-with-a-transfer-secret-for-metadata", + TransferExpires: 24 * 60 * 60, + MachineAuthAPIKey: "change-me-please", + Driver: "ocis", Drivers: config.Drivers{ EOS: config.EOSDriver{ Root: "/eos/dockertest/reva", @@ -59,7 +67,7 @@ func DefaultConfig() *config.Config { SecProtocol: "", Keytab: "", SingleUsername: "", - GatewaySVC: "127.0.0.1:9142", + GatewaySVC: "127.0.0.1:9215", }, Local: config.LocalDriver{ Root: filepath.Join(defaults.BaseDataPath(), "storage", "local", "metadata"), @@ -71,12 +79,12 @@ func DefaultConfig() *config.Config { Root: filepath.Join(defaults.BaseDataPath(), "storage", "metadata"), UserLayout: "{{.Id.OpaqueId}}", Region: "default", - PermissionsEndpoint: "127.0.0.1:9191", + PermissionsEndpoint: "127.0.0.1:9191", // fixme }, OCIS: config.OCISDriver{ Root: filepath.Join(defaults.BaseDataPath(), "storage", "metadata"), UserLayout: "{{.Id.OpaqueId}}", - PermissionsEndpoint: "127.0.0.1:9191", + PermissionsEndpoint: "127.0.0.1:9191", // fixme }, }, } diff --git a/extensions/storage/pkg/config/defaults/defaultconfig.go b/extensions/storage/pkg/config/defaults/defaultconfig.go index c14ac52f0..499e08873 100644 --- a/extensions/storage/pkg/config/defaults/defaultconfig.go +++ b/extensions/storage/pkg/config/defaults/defaultconfig.go @@ -339,9 +339,9 @@ func DefaultConfig() *config.Config { GRPCAddr: "127.0.0.1:9150", Services: []string{"usershareprovider", "publicshareprovider"}, }, - CS3ProviderAddr: "127.0.0.1:9215", + CS3ProviderAddr: "127.0.0.1:9215", // metadata storage CS3ServiceUser: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", - CS3ServiceUserIdp: "https://localhost:9200", + CS3ServiceUserIdp: "internal", UserDriver: "json", UserJSONFile: path.Join(defaults.BaseDataPath(), "storage", "shares.json"), UserSQLUsername: "", From ec86cd82f71896d54b34565a7ebf2cdc51b555a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 27 Apr 2022 13:33:56 +0000 Subject: [PATCH 02/13] use demo permissions service for metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- extensions/storage-metadata/pkg/command/command.go | 6 ++++++ extensions/storage-metadata/pkg/config/metadata.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/extensions/storage-metadata/pkg/command/command.go b/extensions/storage-metadata/pkg/command/command.go index bd0f0b565..813f6955b 100644 --- a/extensions/storage-metadata/pkg/command/command.go +++ b/extensions/storage-metadata/pkg/command/command.go @@ -190,6 +190,12 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in }, }, }, + "permissions": map[string]interface{}{ + "driver": "demo", + "drivers": map[string]interface{}{ + "demo": map[string]interface{}{}, + }, + }, "storageregistry": map[string]interface{}{ "driver": "static", "drivers": map[string]interface{}{ diff --git a/extensions/storage-metadata/pkg/config/metadata.go b/extensions/storage-metadata/pkg/config/metadata.go index e58088237..849883626 100644 --- a/extensions/storage-metadata/pkg/config/metadata.go +++ b/extensions/storage-metadata/pkg/config/metadata.go @@ -49,7 +49,7 @@ func MetadataDrivers(cfg *Config) map[string]interface{} { "user_layout": cfg.Drivers.OCIS.UserLayout, "treetime_accounting": false, "treesize_accounting": false, - "permissionssvc": cfg.Drivers.OCIS.PermissionsEndpoint, + "permissionssvc": cfg.GRPC.Addr, }, "s3": map[string]interface{}{ "region": cfg.Drivers.S3.Region, From ce16c4f92c0ce51638cea2077aa76613f28ce620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 27 Apr 2022 14:12:33 +0000 Subject: [PATCH 03/13] minor config cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- .../storage-metadata/pkg/command/command.go | 17 ++++------------- .../storage-metadata/pkg/config/config.go | 3 --- .../pkg/config/defaults/defaultconfig.go | 12 ++---------- .../storage-metadata/pkg/config/metadata.go | 2 +- 4 files changed, 7 insertions(+), 27 deletions(-) diff --git a/extensions/storage-metadata/pkg/command/command.go b/extensions/storage-metadata/pkg/command/command.go index 813f6955b..ad77c557c 100644 --- a/extensions/storage-metadata/pkg/command/command.go +++ b/extensions/storage-metadata/pkg/command/command.go @@ -131,9 +131,6 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in "grpc": map[string]interface{}{ "network": cfg.GRPC.Protocol, "address": cfg.GRPC.Addr, - //"interceptors": map[string]interface{}{ - // "log": map[string]interface{}{}, - //}, "services": map[string]interface{}{ "gateway": map[string]interface{}{ // registries are located on the gateway @@ -144,12 +141,8 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in "groupprovidersvc": cfg.GRPC.Addr, "permissionssvc": cfg.GRPC.Addr, // other - "disable_home_creation_on_login": true, - //"datagateway": cfg.Reva.StorageMetadata.HTTPAddr, // needs to start with a protocol - "transfer_shared_secret": cfg.TransferSecret, - "transfer_expires": cfg.TransferExpires, - //"home_mapping": cfg.Reva.Gateway.HomeMapping, - //"etag_cache_ttl": cfg.Reva.Gateway.EtagCacheTTL, + "disable_home_creation_on_login": true, // metadata manually creates a space + // metadata always uses the simple upload, so no transfer secret or datagateway needed }, "userprovider": map[string]interface{}{ "driver": "memory", @@ -162,9 +155,7 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in "idp": "internal", "type": 1, // user.UserType_USER_TYPE_PRIMARY }, - "username": "serviceuser", - // "secret": // TODO should not have a secret - "mail": "admin@example.org", + "username": "serviceuser", "display_name": "System User", }, }, @@ -219,7 +210,7 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in "http": map[string]interface{}{ "network": cfg.HTTP.Protocol, "address": cfg.HTTP.Addr, - // TODO build services dynamically + // no datagateway needed as the metadata clients directly talk to the dataprovider with the simple protocol "services": map[string]interface{}{ "dataprovider": map[string]interface{}{ "prefix": "data", diff --git a/extensions/storage-metadata/pkg/config/config.go b/extensions/storage-metadata/pkg/config/config.go index 30608f421..41beb6e1c 100644 --- a/extensions/storage-metadata/pkg/config/config.go +++ b/extensions/storage-metadata/pkg/config/config.go @@ -19,14 +19,11 @@ type Config struct { Context context.Context JWTSecret string - GatewayEndpoint string SkipUserGroupsInToken bool Driver string `yaml:"driver" env:"STORAGE_METADATA_DRIVER" desc:"The driver which should be used by the service"` Drivers Drivers `yaml:"drivers"` DataServerURL string TempFolder string - TransferSecret string `yaml:"transfer_secret" env:"STORAGE_METADATA_TRANSFER_SECRET"` - TransferExpires int `yaml:"transfer_expires" env:"STORAGE_METADATA_TRANSFER_EXPIRES"` DataProviderInsecure bool `env:"OCIS_INSECURE;STORAGE_METADATA_DATAPROVIDER_INSECURE"` MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;STORAGE_METADATA_MACHINE_AUTH_API_KEY"` } diff --git a/extensions/storage-metadata/pkg/config/defaults/defaultconfig.go b/extensions/storage-metadata/pkg/config/defaults/defaultconfig.go index a50d301c7..3bbf5004c 100644 --- a/extensions/storage-metadata/pkg/config/defaults/defaultconfig.go +++ b/extensions/storage-metadata/pkg/config/defaults/defaultconfig.go @@ -24,11 +24,6 @@ func DefaultConfig() *config.Config { Pprof: false, Zpages: false, }, - Logging: &config.Logging{ - Level: "debug", - Pretty: true, - Color: true, - }, GRPC: config.GRPCConfig{ Addr: "127.0.0.1:9215", Protocol: "tcp", @@ -40,12 +35,9 @@ func DefaultConfig() *config.Config { Service: config.Service{ Name: "storage-metadata", }, - GatewayEndpoint: "127.0.0.1:9215", // metadata is a self contained reva instance JWTSecret: "Pive-Fumkiu4", TempFolder: filepath.Join(defaults.BaseDataPath(), "tmp", "metadata"), DataServerURL: "http://localhost:9216/data", - TransferSecret: "replace-me-with-a-transfer-secret-for-metadata", - TransferExpires: 24 * 60 * 60, MachineAuthAPIKey: "change-me-please", Driver: "ocis", Drivers: config.Drivers{ @@ -79,12 +71,12 @@ func DefaultConfig() *config.Config { Root: filepath.Join(defaults.BaseDataPath(), "storage", "metadata"), UserLayout: "{{.Id.OpaqueId}}", Region: "default", - PermissionsEndpoint: "127.0.0.1:9191", // fixme + PermissionsEndpoint: "127.0.0.1:9215", }, OCIS: config.OCISDriver{ Root: filepath.Join(defaults.BaseDataPath(), "storage", "metadata"), UserLayout: "{{.Id.OpaqueId}}", - PermissionsEndpoint: "127.0.0.1:9191", // fixme + PermissionsEndpoint: "127.0.0.1:9215", }, }, } diff --git a/extensions/storage-metadata/pkg/config/metadata.go b/extensions/storage-metadata/pkg/config/metadata.go index 849883626..e58088237 100644 --- a/extensions/storage-metadata/pkg/config/metadata.go +++ b/extensions/storage-metadata/pkg/config/metadata.go @@ -49,7 +49,7 @@ func MetadataDrivers(cfg *Config) map[string]interface{} { "user_layout": cfg.Drivers.OCIS.UserLayout, "treetime_accounting": false, "treesize_accounting": false, - "permissionssvc": cfg.GRPC.Addr, + "permissionssvc": cfg.Drivers.OCIS.PermissionsEndpoint, }, "s3": map[string]interface{}{ "region": cfg.Drivers.S3.Region, From b553f7a5f3001ae85bff1ce45610df93c22b1649 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 27 Apr 2022 14:15:13 +0000 Subject: [PATCH 04/13] add changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- changelog/unreleased/metadata-gateway.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/metadata-gateway.md diff --git a/changelog/unreleased/metadata-gateway.md b/changelog/unreleased/metadata-gateway.md new file mode 100644 index 000000000..3205409fb --- /dev/null +++ b/changelog/unreleased/metadata-gateway.md @@ -0,0 +1,5 @@ +Enhancement: wrap metadata storage with dedicated reva gateway + +We wrapped the metadata storage in a minimal reva instance with a dedicated gateway, including static storage registry, static auth registry, in memory userprovider, machine authprovider and demo permissions service. This allows us to preconfigure the service user for the ocis settings service, share and public share providers. + +https://github.com/owncloud/ocis/pull/3602 From 5f7c40373ade7b2e4a1ddc748e35b29790090ccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 27 Apr 2022 15:49:04 +0000 Subject: [PATCH 05/13] update reva MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- changelog/unreleased/update-reva.md | 1 + go.sum | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/changelog/unreleased/update-reva.md b/changelog/unreleased/update-reva.md index 0cce56ff6..fb8e40e23 100644 --- a/changelog/unreleased/update-reva.md +++ b/changelog/unreleased/update-reva.md @@ -7,5 +7,6 @@ Updated reva to version 2.x.x. This update includes: https://github.com/owncloud/ocis/pull/3552 https://github.com/owncloud/ocis/pull/3570 https://github.com/owncloud/ocis/pull/3601 +https://github.com/owncloud/ocis/pull/3602 https://github.com/owncloud/ocis/pull/3605 https://github.com/owncloud/ocis/pull/3611 diff --git a/go.sum b/go.sum index 97e90625a..1843c2b33 100644 --- a/go.sum +++ b/go.sum @@ -318,8 +318,6 @@ github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde h1:WrD9O8ZaWvsm0 github.com/cs3org/go-cs3apis v0.0.0-20220412090512-93c5918b4bde/go.mod h1:UXha4TguuB52H14EMoSsCqDj7k8a/t7g4gVP+bgY5LY= github.com/cs3org/reva v1.18.0 h1:MbPS5ZAa8RzKcTxAVeSDdISB3XXqLIxqB03BTN5ReBY= github.com/cs3org/reva v1.18.0/go.mod h1:e5VDUDu4vVWIeVkZcW//n6UZzhGGMa+Tz/whCiX3N6o= -github.com/cs3org/reva/v2 v2.0.0-20220427133111-618964eed515 h1:8pPCLxNXVz/q7PMM6Zq1lff3P8SFAu8/CXwB2eA21xc= -github.com/cs3org/reva/v2 v2.0.0-20220427133111-618964eed515/go.mod h1:2e/4HcIy54Mic3V7Ow0bz4n5dkZU0dHIZSWomFe5vng= github.com/cs3org/reva/v2 v2.0.0-20220427203355-0164880ac7d3 h1:6sKjGI0AUW5tBXWBduaBoc+9sNYZWQR894G0oFCbus0= github.com/cs3org/reva/v2 v2.0.0-20220427203355-0164880ac7d3/go.mod h1:2e/4HcIy54Mic3V7Ow0bz4n5dkZU0dHIZSWomFe5vng= github.com/cubewise-code/go-mime v0.0.0-20200519001935-8c5762b177d8 h1:Z9lwXumT5ACSmJ7WGnFl+OMLLjpz5uR2fyz7dC255FI= From c655e6bfe957362d82884b615176e252f106a42f Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Fri, 29 Apr 2022 18:55:00 +0200 Subject: [PATCH 06/13] Increase log level for ci debugging --- .drone.star | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.star b/.drone.star index 16542d0f7..f8c8c5c08 100644 --- a/.drone.star +++ b/.drone.star @@ -1710,7 +1710,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "PROXY_ENABLE_BASIC_AUTH": True, "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json", "IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml", - "OCIS_LOG_LEVEL": "error", + "OCIS_LOG_LEVEL": "debug", "SETTINGS_DATA_PATH": "/srv/app/tmp/ocis/settings", "OCIS_INSECURE": "true", "IDM_CREATE_DEMO_USERS": True, From 34554f319a0a5f93ccffcba0fb70e597fe57d50e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Fri, 29 Apr 2022 18:37:10 +0000 Subject: [PATCH 07/13] fix some storage env vars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- .../storage-metadata/pkg/config/config.go | 4 +-- .../storage-publiclink/pkg/config/config.go | 28 ++++++++-------- .../storage-shares/pkg/config/config.go | 32 +++++++++---------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/extensions/storage-metadata/pkg/config/config.go b/extensions/storage-metadata/pkg/config/config.go index 41beb6e1c..07698478f 100644 --- a/extensions/storage-metadata/pkg/config/config.go +++ b/extensions/storage-metadata/pkg/config/config.go @@ -58,8 +58,8 @@ type GRPCConfig struct { } type HTTPConfig struct { - Addr string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."` - Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."` + Addr string `yaml:"addr" env:"STORAGE_METADATA_HTTP_ADDR" desc:"The address of the http service."` + Protocol string `yaml:"protocol" env:"STORAGE_METADATA_HTTP_PROTOCOL" desc:"The transport protocol of the http service."` } type Drivers struct { diff --git a/extensions/storage-publiclink/pkg/config/config.go b/extensions/storage-publiclink/pkg/config/config.go index 3766e35ea..bc0bee265 100644 --- a/extensions/storage-publiclink/pkg/config/config.go +++ b/extensions/storage-publiclink/pkg/config/config.go @@ -24,17 +24,17 @@ type Config struct { StorageProvider StorageProvider } type Tracing struct { - Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_METADATA_TRACING_ENABLED" desc:"Activates tracing."` - Type string `yaml:"type" env:"OCIS_TRACING_TYPE;STORAGE_METADATA_TRACING_TYPE"` - Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;STORAGE_METADATA_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."` - Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORAGE_METADATA_TRACING_COLLECTOR"` + Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_PUBLICLINK_TRACING_ENABLED" desc:"Activates tracing."` + Type string `yaml:"type" env:"OCIS_TRACING_TYPE;STORAGE_PUBLICLINK_TRACING_TYPE"` + Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;STORAGE_PUBLICLINK_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."` + Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORAGE_PUBLICLINK_TRACING_COLLECTOR"` } type Logging struct { - Level string `yaml:"level" env:"OCIS_LOG_LEVEL;STORAGE_METADATA_LOG_LEVEL" desc:"The log level."` - Pretty bool `yaml:"pretty" env:"OCIS_LOG_PRETTY;STORAGE_METADATA_LOG_PRETTY" desc:"Activates pretty log output."` - Color bool `yaml:"color" env:"OCIS_LOG_COLOR;STORAGE_METADATA_LOG_COLOR" desc:"Activates colorized log output."` - File string `yaml:"file" env:"OCIS_LOG_FILE;STORAGE_METADATA_LOG_FILE" desc:"The target log file."` + Level string `yaml:"level" env:"OCIS_LOG_LEVEL;STORAGE_PUBLICLINK_LOG_LEVEL" desc:"The log level."` + Pretty bool `yaml:"pretty" env:"OCIS_LOG_PRETTY;STORAGE_PUBLICLINK_LOG_PRETTY" desc:"Activates pretty log output."` + Color bool `yaml:"color" env:"OCIS_LOG_COLOR;STORAGE_PUBLICLINK_LOG_COLOR" desc:"Activates colorized log output."` + File string `yaml:"file" env:"OCIS_LOG_FILE;STORAGE_PUBLICLINK_LOG_FILE" desc:"The target log file."` } type Service struct { @@ -42,15 +42,15 @@ type Service struct { } type Debug struct { - Addr string `yaml:"addr" env:"STORAGE_METADATA_DEBUG_ADDR"` - Token string `yaml:"token" env:"STORAGE_METADATA_DEBUG_TOKEN"` - Pprof bool `yaml:"pprof" env:"STORAGE_METADATA_DEBUG_PPROF"` - Zpages bool `yaml:"zpages" env:"STORAGE_METADATA_DEBUG_ZPAGES"` + Addr string `yaml:"addr" env:"STORAGE_PUBLICLINK_DEBUG_ADDR"` + Token string `yaml:"token" env:"STORAGE_PUBLICLINK_DEBUG_TOKEN"` + Pprof bool `yaml:"pprof" env:"STORAGE_PUBLICLINK_DEBUG_PPROF"` + Zpages bool `yaml:"zpages" env:"STORAGE_PUBLICLINK_DEBUG_ZPAGES"` } type GRPCConfig struct { - Addr string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."` - Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."` + Addr string `yaml:"addr" env:"STORAGE_PUBLICLINK_GRPC_ADDR" desc:"The address of the grpc service."` + Protocol string `yaml:"protocol" env:"STORAGE_PUBLICLINK_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."` } type AuthProvider struct { diff --git a/extensions/storage-shares/pkg/config/config.go b/extensions/storage-shares/pkg/config/config.go index 8c1345601..8769be874 100644 --- a/extensions/storage-shares/pkg/config/config.go +++ b/extensions/storage-shares/pkg/config/config.go @@ -25,17 +25,17 @@ type Config struct { SharesProviderEndpoint string } type Tracing struct { - Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_METADATA_TRACING_ENABLED" desc:"Activates tracing."` - Type string `yaml:"type" env:"OCIS_TRACING_TYPE;STORAGE_METADATA_TRACING_TYPE"` - Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;STORAGE_METADATA_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."` - Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORAGE_METADATA_TRACING_COLLECTOR"` + Enabled bool `yaml:"enabled" env:"OCIS_TRACING_ENABLED;STORAGE_SHARES_TRACING_ENABLED" desc:"Activates tracing."` + Type string `yaml:"type" env:"OCIS_TRACING_TYPE;STORAGE_SHARES_TRACING_TYPE"` + Endpoint string `yaml:"endpoint" env:"OCIS_TRACING_ENDPOINT;STORAGE_SHARES_TRACING_ENDPOINT" desc:"The endpoint to the tracing collector."` + Collector string `yaml:"collector" env:"OCIS_TRACING_COLLECTOR;STORAGE_SHARES_TRACING_COLLECTOR"` } type Logging struct { - Level string `yaml:"level" env:"OCIS_LOG_LEVEL;STORAGE_METADATA_LOG_LEVEL" desc:"The log level."` - Pretty bool `yaml:"pretty" env:"OCIS_LOG_PRETTY;STORAGE_METADATA_LOG_PRETTY" desc:"Activates pretty log output."` - Color bool `yaml:"color" env:"OCIS_LOG_COLOR;STORAGE_METADATA_LOG_COLOR" desc:"Activates colorized log output."` - File string `yaml:"file" env:"OCIS_LOG_FILE;STORAGE_METADATA_LOG_FILE" desc:"The target log file."` + Level string `yaml:"level" env:"OCIS_LOG_LEVEL;STORAGE_SHARES_LOG_LEVEL" desc:"The log level."` + Pretty bool `yaml:"pretty" env:"OCIS_LOG_PRETTY;STORAGE_SHARES_LOG_PRETTY" desc:"Activates pretty log output."` + Color bool `yaml:"color" env:"OCIS_LOG_COLOR;STORAGE_SHARES_LOG_COLOR" desc:"Activates colorized log output."` + File string `yaml:"file" env:"OCIS_LOG_FILE;STORAGE_SHARES_LOG_FILE" desc:"The target log file."` } type Service struct { @@ -43,18 +43,18 @@ type Service struct { } type Debug struct { - Addr string `yaml:"addr" env:"STORAGE_METADATA_DEBUG_ADDR"` - Token string `yaml:"token" env:"STORAGE_METADATA_DEBUG_TOKEN"` - Pprof bool `yaml:"pprof" env:"STORAGE_METADATA_DEBUG_PPROF"` - Zpages bool `yaml:"zpages" env:"STORAGE_METADATA_DEBUG_ZPAGES"` + Addr string `yaml:"addr" env:"STORAGE_SHARES_DEBUG_ADDR"` + Token string `yaml:"token" env:"STORAGE_SHARES_DEBUG_TOKEN"` + Pprof bool `yaml:"pprof" env:"STORAGE_SHARES_DEBUG_PPROF"` + Zpages bool `yaml:"zpages" env:"STORAGE_SHARES_DEBUG_ZPAGES"` } type GRPCConfig struct { - Addr string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."` - Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."` + Addr string `yaml:"addr" env:"STORAGE_SHARES_GRPC_ADDR" desc:"The address of the grpc service."` + Protocol string `yaml:"protocol" env:"STORAGE_SHARES_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."` } type HTTPConfig struct { - Addr string `yaml:"addr" env:"STORAGE_METADATA_GRPC_ADDR" desc:"The address of the grpc service."` - Protocol string `yaml:"protocol" env:"STORAGE_METADATA_GRPC_PROTOCOL" desc:"The transport protocol of the grpc service."` + Addr string `yaml:"addr" env:"STORAGE_SHARES_HTTP_ADDR" desc:"The address of the grpc service."` + Protocol string `yaml:"protocol" env:"STORAGE_SHARES_HTTP_PROTOCOL" desc:"The transport protocol of the grpc service."` } From fd292563d88ac1786b25b9d50f01e6f633c0bf13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 2 May 2022 10:30:51 +0000 Subject: [PATCH 08/13] do not overwrite metadata IDP with oCIS IDP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- extensions/settings/pkg/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/settings/pkg/config/config.go b/extensions/settings/pkg/config/config.go index 24de34c3a..8182911bf 100644 --- a/extensions/settings/pkg/config/config.go +++ b/extensions/settings/pkg/config/config.go @@ -40,6 +40,6 @@ type Metadata struct { StorageAddress string `yaml:"storage_addr" env:"STORAGE_GRPC_ADDR"` ServiceUserID string `yaml:"service_user_id" env:"METADATA_SERVICE_USER_UUID"` - ServiceUserIDP string `yaml:"service_user_idp" env:"OCIS_URL;METADATA_SERVICE_USER_IDP"` + ServiceUserIDP string `yaml:"service_user_idp" env:"METADATA_SERVICE_USER_IDP"` MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY"` } From 7dd486ba1719a01baf65c0e63b23589b508ebff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 2 May 2022 11:17:08 +0000 Subject: [PATCH 09/13] use cs3 user type constant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- extensions/storage-metadata/pkg/command/command.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extensions/storage-metadata/pkg/command/command.go b/extensions/storage-metadata/pkg/command/command.go index 674abfa1c..c9e5e0932 100644 --- a/extensions/storage-metadata/pkg/command/command.go +++ b/extensions/storage-metadata/pkg/command/command.go @@ -7,18 +7,18 @@ import ( "os" "path" - "github.com/owncloud/ocis/extensions/storage-metadata/pkg/config/parser" - "github.com/owncloud/ocis/ocis-pkg/log" - "github.com/owncloud/ocis/ocis-pkg/sync" - "github.com/owncloud/ocis/ocis-pkg/tracing" - + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" "github.com/cs3org/reva/v2/cmd/revad/runtime" "github.com/gofrs/uuid" "github.com/oklog/run" "github.com/owncloud/ocis/extensions/storage-metadata/pkg/config" + "github.com/owncloud/ocis/extensions/storage-metadata/pkg/config/parser" "github.com/owncloud/ocis/extensions/storage/pkg/server/debug" "github.com/owncloud/ocis/extensions/storage/pkg/service/external" ociscfg "github.com/owncloud/ocis/ocis-pkg/config" + "github.com/owncloud/ocis/ocis-pkg/log" + "github.com/owncloud/ocis/ocis-pkg/sync" + "github.com/owncloud/ocis/ocis-pkg/tracing" "github.com/owncloud/ocis/ocis-pkg/version" "github.com/thejerf/suture/v4" "github.com/urfave/cli/v2" @@ -162,7 +162,7 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in "id": map[string]interface{}{ "opaqueId": "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", // FIXME generate service user id "idp": "internal", - "type": 1, // user.UserType_USER_TYPE_PRIMARY + "type": userpb.UserType_USER_TYPE_PRIMARY, }, "username": "serviceuser", "display_name": "System User", From 161c23976f88f3e4c1b058507c132f6ca1a04177 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 2 May 2022 11:30:36 +0000 Subject: [PATCH 10/13] remove unused system user initialization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- extensions/idm/ldif/base.ldif.tmpl | 13 ------- .../pkg/config/defaults/defaultconfig.go | 2 +- .../settings/pkg/service/v0/settings.go | 26 ------------- .../settings/pkg/store/defaults/defaults.go | 39 ------------------- .../pkg/store/metadata/assignments.go | 2 +- .../pkg/config/defaults/defaultconfig.go | 8 ++-- 6 files changed, 6 insertions(+), 84 deletions(-) diff --git a/extensions/idm/ldif/base.ldif.tmpl b/extensions/idm/ldif/base.ldif.tmpl index 1cbaaec4c..e29221c62 100644 --- a/extensions/idm/ldif/base.ldif.tmpl +++ b/extensions/idm/ldif/base.ldif.tmpl @@ -40,17 +40,4 @@ userPassword:: {{ .Password }} {{ end -}} -## Service user for the settings service -dn: uid=95cb8724-03b2-11eb-a0a6-c33ef8ef53ad,ou=users,o=libregraph-idm -objectClass: inetOrgPerson -objectClass: organizationalPerson -objectClass: ownCloud -objectClass: person -objectClass: top -uid: 95cb8724-03b2-11eb-a0a6-c33ef8ef53ad -givenName: 95cb8724-03b2-11eb-a0a6-c33ef8ef53ad -sn: 95cb8724-03b2-11eb-a0a6-c33ef8ef53ad -cn: 95cb8724-03b2-11eb-a0a6-c33ef8ef53ad -displayName: 95cb8724-03b2-11eb-a0a6-c33ef8ef53ad -ownCloudUUID: 95cb8724-03b2-11eb-a0a6-c33ef8ef53ad diff --git a/extensions/settings/pkg/config/defaults/defaultconfig.go b/extensions/settings/pkg/config/defaults/defaultconfig.go index 1f5b97c3c..29cc21d03 100644 --- a/extensions/settings/pkg/config/defaults/defaultconfig.go +++ b/extensions/settings/pkg/config/defaults/defaultconfig.go @@ -50,7 +50,7 @@ func DefaultConfig() *config.Config { }, Metadata: config.Metadata{ - GatewayAddress: "127.0.0.1:9215", + GatewayAddress: "127.0.0.1:9215", // metadata storage StorageAddress: "127.0.0.1:9215", ServiceUserID: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", ServiceUserIDP: "internal", diff --git a/extensions/settings/pkg/service/v0/settings.go b/extensions/settings/pkg/service/v0/settings.go index abf541a4a..404845e51 100644 --- a/extensions/settings/pkg/service/v0/settings.go +++ b/extensions/settings/pkg/service/v0/settings.go @@ -6,9 +6,6 @@ import ( ) const ( - // BundleUUIDRoleMetadata represents the metadata user role - BundleUUIDRoleMetadata = "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad" - // BundleUUIDRoleAdmin represents the admin role BundleUUIDRoleAdmin = "71881883-1768-46bd-a24d-a356a2afdf7f" @@ -532,34 +529,11 @@ func generatePermissionRequests() []*settingssvc.AddSettingToBundleRequest { }, }, }, - { - BundleId: BundleUUIDRoleMetadata, - Setting: &settingsmsg.Setting{ - Id: CreateSpacePermissionID, - Name: CreateSpacePermissionName, - DisplayName: "Create own Space", - Description: "This permission allows to create a space owned by the current user.", - Resource: &settingsmsg.Resource{ - Type: settingsmsg.Resource_TYPE_SYSTEM, // TODO resource type space? self? me? own? - }, - Value: &settingsmsg.Setting_PermissionValue{ - PermissionValue: &settingsmsg.Permission{ - Operation: settingsmsg.Permission_OPERATION_CREATE, - Constraint: settingsmsg.Permission_CONSTRAINT_OWN, - }, - }, - }, - }, } } func defaultRoleAssignments() []*settingsmsg.UserRoleAssignment { return []*settingsmsg.UserRoleAssignment{ - // accounts service user for the metadata user is allowed to create spaces - { - AccountUuid: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", - RoleId: BundleUUIDRoleAdmin, - }, // default admin users { AccountUuid: "058bff95-6708-4fe5-91e4-9ea3d377588b", diff --git a/extensions/settings/pkg/store/defaults/defaults.go b/extensions/settings/pkg/store/defaults/defaults.go index b20357257..27f3d43f9 100644 --- a/extensions/settings/pkg/store/defaults/defaults.go +++ b/extensions/settings/pkg/store/defaults/defaults.go @@ -17,9 +17,6 @@ const ( // BundleUUIDRoleGuest represents the guest role. BundleUUIDRoleGuest = "38071a68-456a-4553-846a-fa67bf5596cc" - // BundleUUIDRoleMetadata represents the metadata user role - BundleUUIDRoleMetadata = "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad" - // RoleManagementPermissionID is the hardcoded setting UUID for the role management permission RoleManagementPermissionID string = "a53e601e-571f-4f86-8fec-d4576ef49c62" // RoleManagementPermissionName is the hardcoded setting name for the role management permission @@ -68,7 +65,6 @@ func GenerateBundlesDefaultRoles() []*settingsmsg.Bundle { generateBundleUserRole(), generateBundleGuestRole(), generateBundleProfileRequest(), - generateBundleMetadataRole(), generateBundleSpaceAdminRole(), } } @@ -434,36 +430,6 @@ func generateBundleProfileRequest() *settingsmsg.Bundle { } } -func generateBundleMetadataRole() *settingsmsg.Bundle { - return &settingsmsg.Bundle{ - Id: BundleUUIDRoleMetadata, - Name: "metadata", - Type: settingsmsg.Bundle_TYPE_ROLE, - Extension: "ocis-roles", - DisplayName: "Metadata", - Resource: &settingsmsg.Resource{ - Type: settingsmsg.Resource_TYPE_SYSTEM, - }, - Settings: []*settingsmsg.Setting{ - { - Id: CreateSpacePermissionID, - Name: CreateSpacePermissionName, - DisplayName: "Create own Space", - Description: "This permission allows to create a space owned by the current user.", - Resource: &settingsmsg.Resource{ - Type: settingsmsg.Resource_TYPE_SYSTEM, // TODO resource type space? self? me? own? - }, - Value: &settingsmsg.Setting_PermissionValue{ - PermissionValue: &settingsmsg.Permission{ - Operation: settingsmsg.Permission_OPERATION_CREATE, - Constraint: settingsmsg.Permission_CONSTRAINT_OWN, - }, - }, - }, - }, - } -} - // TODO: languageSetting needed? var languageSetting = settingsmsg.Setting_SingleChoiceValue{ SingleChoiceValue: &settingsmsg.SingleChoiceList{ @@ -532,11 +498,6 @@ var languageSetting = settingsmsg.Setting_SingleChoiceValue{ // DefaultRoleAssignments returns (as one might guess) the default role assignments func DefaultRoleAssignments() []*settingsmsg.UserRoleAssignment { return []*settingsmsg.UserRoleAssignment{ - // accounts service user for the metadata user is allowed to create spaces - { - AccountUuid: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", - RoleId: BundleUUIDRoleAdmin, - }, // default admin users { AccountUuid: "058bff95-6708-4fe5-91e4-9ea3d377588b", diff --git a/extensions/settings/pkg/store/metadata/assignments.go b/extensions/settings/pkg/store/metadata/assignments.go index 11fafbccb..88ce7e1c1 100644 --- a/extensions/settings/pkg/store/metadata/assignments.go +++ b/extensions/settings/pkg/store/metadata/assignments.go @@ -13,7 +13,7 @@ import ( // ListRoleAssignments loads and returns all role assignments matching the given assignment identifier. func (s *Store) ListRoleAssignments(accountUUID string) ([]*settingsmsg.UserRoleAssignment, error) { - if s.mdc == nil || accountUUID == "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad" { + if s.mdc == nil { return defaultRoleAssignments(accountUUID), nil } s.Init() diff --git a/extensions/sharing/pkg/config/defaults/defaultconfig.go b/extensions/sharing/pkg/config/defaults/defaultconfig.go index 2c00c4267..04868d9b6 100644 --- a/extensions/sharing/pkg/config/defaults/defaultconfig.go +++ b/extensions/sharing/pkg/config/defaults/defaultconfig.go @@ -48,9 +48,9 @@ func DefaultConfig() *config.Config { JanitorRunInterval: 60, }, CS3: config.UserSharingCS3Driver{ - ProviderAddr: "127.0.0.1:9215", + ProviderAddr: "127.0.0.1:9215", // metadata storage ServiceUserID: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", - ServiceUserIDP: "https://localhost:9200", + ServiceUserIDP: "internal", }, }, PublicSharingDriver: "json", @@ -69,9 +69,9 @@ func DefaultConfig() *config.Config { JanitorRunInterval: 60, }, CS3: config.PublicSharingCS3Driver{ - ProviderAddr: "127.0.0.1:9215", + ProviderAddr: "127.0.0.1:9215", // metadata storage ServiceUserID: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", - ServiceUserIDP: "https://localhost:9200", + ServiceUserIDP: "internal", }, }, Events: config.Events{ From df8fd7626ddf4ad054bceb0cf2f562f57ef1c250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 2 May 2022 11:31:22 +0000 Subject: [PATCH 11/13] revert a launch.json change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- .vscode/launch.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 200611e97..ddaf13bf7 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,7 @@ "mode": "debug", "program": "${workspaceFolder}/ocis/cmd/ocis", "args": [ - "storage-metadata" + "server" ], "env": { // log settings for human developers From f88c000bacbd3d31e95c86a577928033b473449c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 2 May 2022 12:36:30 +0000 Subject: [PATCH 12/13] generate metadata user id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- .../settings/pkg/config/defaults/defaultconfig.go | 5 ++++- .../sharing/pkg/config/defaults/defaultconfig.go | 10 ++++++++-- extensions/sharing/pkg/config/parser/parse.go | 8 ++++++++ extensions/storage-metadata/pkg/command/command.go | 2 +- extensions/storage-metadata/pkg/config/config.go | 3 ++- .../pkg/config/defaults/defaultconfig.go | 5 +++++ extensions/storage-metadata/pkg/config/parser/parse.go | 4 ++++ ocis-pkg/config/config.go | 1 + ocis-pkg/config/parser/parse.go | 8 ++++++++ ocis-pkg/shared/errors.go | 8 ++++++++ ocis-pkg/shared/shared_types.go | 1 + ocis/pkg/init/init.go | 5 +++++ 12 files changed, 55 insertions(+), 5 deletions(-) diff --git a/extensions/settings/pkg/config/defaults/defaultconfig.go b/extensions/settings/pkg/config/defaults/defaultconfig.go index 29cc21d03..5e87d0702 100644 --- a/extensions/settings/pkg/config/defaults/defaultconfig.go +++ b/extensions/settings/pkg/config/defaults/defaultconfig.go @@ -52,7 +52,6 @@ func DefaultConfig() *config.Config { Metadata: config.Metadata{ GatewayAddress: "127.0.0.1:9215", // metadata storage StorageAddress: "127.0.0.1:9215", - ServiceUserID: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", ServiceUserIDP: "internal", }, } @@ -93,6 +92,10 @@ func EnsureDefaults(cfg *config.Config) { if cfg.Metadata.MachineAuthAPIKey == "" && cfg.Commons != nil && cfg.Commons.MachineAuthAPIKey != "" { cfg.Metadata.MachineAuthAPIKey = cfg.Commons.MachineAuthAPIKey } + + if cfg.Metadata.ServiceUserID == "" && cfg.Commons != nil && cfg.Commons.MetadataUserID != "" { + cfg.Metadata.ServiceUserID = cfg.Commons.MetadataUserID + } } func Sanitize(cfg *config.Config) { diff --git a/extensions/sharing/pkg/config/defaults/defaultconfig.go b/extensions/sharing/pkg/config/defaults/defaultconfig.go index 04868d9b6..924e43228 100644 --- a/extensions/sharing/pkg/config/defaults/defaultconfig.go +++ b/extensions/sharing/pkg/config/defaults/defaultconfig.go @@ -49,7 +49,6 @@ func DefaultConfig() *config.Config { }, CS3: config.UserSharingCS3Driver{ ProviderAddr: "127.0.0.1:9215", // metadata storage - ServiceUserID: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", ServiceUserIDP: "internal", }, }, @@ -70,7 +69,6 @@ func DefaultConfig() *config.Config { }, CS3: config.PublicSharingCS3Driver{ ProviderAddr: "127.0.0.1:9215", // metadata storage - ServiceUserID: "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", ServiceUserIDP: "internal", }, }, @@ -125,9 +123,17 @@ func EnsureDefaults(cfg *config.Config) { cfg.UserSharingDrivers.CS3.MachineAuthAPIKey = cfg.Commons.MachineAuthAPIKey } + if cfg.UserSharingDrivers.CS3.ServiceUserID == "" && cfg.Commons != nil && cfg.Commons.MetadataUserID != "" { + cfg.UserSharingDrivers.CS3.ServiceUserID = cfg.Commons.MetadataUserID + } + if cfg.PublicSharingDrivers.CS3.MachineAuthAPIKey == "" && cfg.Commons != nil && cfg.Commons.MachineAuthAPIKey != "" { cfg.PublicSharingDrivers.CS3.MachineAuthAPIKey = cfg.Commons.MachineAuthAPIKey } + + if cfg.PublicSharingDrivers.CS3.ServiceUserID == "" && cfg.Commons != nil && cfg.Commons.MetadataUserID != "" { + cfg.PublicSharingDrivers.CS3.ServiceUserID = cfg.Commons.MetadataUserID + } } func Sanitize(cfg *config.Config) { diff --git a/extensions/sharing/pkg/config/parser/parse.go b/extensions/sharing/pkg/config/parser/parse.go index a8a7b00e2..afc4d88b8 100644 --- a/extensions/sharing/pkg/config/parser/parse.go +++ b/extensions/sharing/pkg/config/parser/parse.go @@ -42,9 +42,17 @@ func Validate(cfg *config.Config) error { return shared.MissingMachineAuthApiKeyError(cfg.Service.Name) } + if cfg.PublicSharingDriver == "cs3" && cfg.PublicSharingDrivers.CS3.ServiceUserID == "" { + return shared.MissingMetadataUserID(cfg.Service.Name) + } + if cfg.UserSharingDriver == "cs3" && cfg.UserSharingDrivers.CS3.MachineAuthAPIKey == "" { return shared.MissingMachineAuthApiKeyError(cfg.Service.Name) } + if cfg.UserSharingDriver == "cs3" && cfg.UserSharingDrivers.CS3.ServiceUserID == "" { + return shared.MissingMetadataUserID(cfg.Service.Name) + } + return nil } diff --git a/extensions/storage-metadata/pkg/command/command.go b/extensions/storage-metadata/pkg/command/command.go index c9e5e0932..54eff79d4 100644 --- a/extensions/storage-metadata/pkg/command/command.go +++ b/extensions/storage-metadata/pkg/command/command.go @@ -160,7 +160,7 @@ func storageMetadataFromStruct(c *cli.Context, cfg *config.Config) map[string]in "users": map[string]interface{}{ "serviceuser": map[string]interface{}{ "id": map[string]interface{}{ - "opaqueId": "95cb8724-03b2-11eb-a0a6-c33ef8ef53ad", // FIXME generate service user id + "opaqueId": cfg.MetadataUserID, "idp": "internal", "type": userpb.UserType_USER_TYPE_PRIMARY, }, diff --git a/extensions/storage-metadata/pkg/config/config.go b/extensions/storage-metadata/pkg/config/config.go index 2d8869eac..8c4475600 100644 --- a/extensions/storage-metadata/pkg/config/config.go +++ b/extensions/storage-metadata/pkg/config/config.go @@ -21,7 +21,8 @@ type Config struct { TokenManager *TokenManager `yaml:"token_manager"` Reva *Reva `yaml:"reva"` - MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY;STORAGE_METADATA_MACHINE_AUTH_API_KEY"` + MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"STORAGE_METADATA_MACHINE_AUTH_API_KEY"` + MetadataUserID string `yaml:"metadata_user_id"` SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token"` Driver string `yaml:"driver" env:"STORAGE_METADATA_DRIVER" desc:"The driver which should be used by the service"` diff --git a/extensions/storage-metadata/pkg/config/defaults/defaultconfig.go b/extensions/storage-metadata/pkg/config/defaults/defaultconfig.go index 2b3d84f42..4f274aa0c 100644 --- a/extensions/storage-metadata/pkg/config/defaults/defaultconfig.go +++ b/extensions/storage-metadata/pkg/config/defaults/defaultconfig.go @@ -125,6 +125,11 @@ func EnsureDefaults(cfg *config.Config) { if cfg.MachineAuthAPIKey == "" && cfg.Commons != nil && cfg.Commons.MachineAuthAPIKey != "" { cfg.MachineAuthAPIKey = cfg.Commons.MachineAuthAPIKey } + + if cfg.MetadataUserID == "" && cfg.Commons != nil && cfg.Commons.MetadataUserID != "" { + cfg.MetadataUserID = cfg.Commons.MetadataUserID + } + } func Sanitize(cfg *config.Config) { diff --git a/extensions/storage-metadata/pkg/config/parser/parse.go b/extensions/storage-metadata/pkg/config/parser/parse.go index 019438ab2..413bbd52c 100644 --- a/extensions/storage-metadata/pkg/config/parser/parse.go +++ b/extensions/storage-metadata/pkg/config/parser/parse.go @@ -41,5 +41,9 @@ func Validate(cfg *config.Config) error { if cfg.MachineAuthAPIKey == "" { return shared.MissingMachineAuthApiKeyError(cfg.Service.Name) } + + if cfg.MetadataUserID == "" { + return shared.MissingMetadataUserID(cfg.Service.Name) + } return nil } diff --git a/ocis-pkg/config/config.go b/ocis-pkg/config/config.go index 33b9645d2..edd2d4977 100644 --- a/ocis-pkg/config/config.go +++ b/ocis-pkg/config/config.go @@ -67,6 +67,7 @@ type Config struct { TokenManager *shared.TokenManager `yaml:"token_manager"` MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY"` TransferSecret string `yaml:"transfer_secret" env:"STORAGE_TRANSFER_SECRET"` + MetadataUserID string `yaml:"metadata_user_id"` Runtime Runtime `yaml:"runtime"` Audit *audit.Config `yaml:"audit"` diff --git a/ocis-pkg/config/parser/parse.go b/ocis-pkg/config/parser/parse.go index 3c4939a23..cd5f8ab32 100644 --- a/ocis-pkg/config/parser/parse.go +++ b/ocis-pkg/config/parser/parse.go @@ -94,6 +94,10 @@ func EnsureCommons(cfg *config.Config) { cfg.Commons.TransferSecret = cfg.TransferSecret } + // copy metadata user id to the commons part if set + if cfg.MetadataUserID != "" { + cfg.Commons.MetadataUserID = cfg.MetadataUserID + } } func Validate(cfg *config.Config) error { @@ -109,5 +113,9 @@ func Validate(cfg *config.Config) error { return shared.MissingMachineAuthApiKeyError("ocis") } + if cfg.MetadataUserID == "" { + return shared.MissingMetadataUserID("ocis") + } + return nil } diff --git a/ocis-pkg/shared/errors.go b/ocis-pkg/shared/errors.go index bb4b5f4ec..de1ed5a82 100644 --- a/ocis-pkg/shared/errors.go +++ b/ocis-pkg/shared/errors.go @@ -45,3 +45,11 @@ func MissingServiceUserPassword(service, serviceUser string) error { "the config/corresponding environment variable).", serviceUser, service, defaults.BaseConfigPath()) } + +func MissingMetadataUserID(service string) error { + return fmt.Errorf("The metadata user ID has not been configured for %s. "+ + "Make sure your %s config contains the proper values "+ + "(e.g. by running ocis init or setting it manually in "+ + "the config/corresponding environment variable).", + service, defaults.BaseConfigPath()) +} diff --git a/ocis-pkg/shared/shared_types.go b/ocis-pkg/shared/shared_types.go index f4cf19fc0..3497bed61 100644 --- a/ocis-pkg/shared/shared_types.go +++ b/ocis-pkg/shared/shared_types.go @@ -44,4 +44,5 @@ type Commons struct { Reva *Reva `yaml:"reva"` MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY"` TransferSecret string `yaml:"transfer_secret,omitempty" env:"REVA_TRANSFER_SECRET"` + MetadataUserID string `yaml:"metadata_user_id" env:"METADATA_USER_ID"` } diff --git a/ocis/pkg/init/init.go b/ocis/pkg/init/init.go index 8b2ca85bf..5cce91746 100644 --- a/ocis/pkg/init/init.go +++ b/ocis/pkg/init/init.go @@ -9,6 +9,7 @@ import ( "path" "time" + "github.com/gofrs/uuid" "github.com/owncloud/ocis/ocis-pkg/generators" "gopkg.in/yaml.v2" ) @@ -99,6 +100,7 @@ type OcisConfig struct { TokenManager TokenManager `yaml:"token_manager"` MachineAuthApiKey string `yaml:"machine_auth_api_key"` TransferSecret string `yaml:"transfer_secret"` + MetadataUserID string `yaml:"metadata_user_id"` Graph GraphExtension Idp LdapBasedExtension Idm IdmExtension @@ -160,6 +162,8 @@ func CreateConfig(insecure, forceOverwrite bool, configPath, adminPassword strin return err } + metadataUserID := uuid.Must(uuid.NewV4()).String() + idmServicePassword, err := generators.GenerateRandomPassword(passwordLength) if err != nil { return fmt.Errorf("could not generate random password for idm: %s", err) @@ -199,6 +203,7 @@ func CreateConfig(insecure, forceOverwrite bool, configPath, adminPassword strin }, MachineAuthApiKey: machineAuthApiKey, TransferSecret: revaTransferSecret, + MetadataUserID: metadataUserID, Idm: IdmExtension{ ServiceUserPasswords: ServiceUserPasswordsSettings{ AdminPassword: ocisAdminServicePassword, From 3074d70cae1329dfa0648f0b0b2844e7c3b18f4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 2 May 2022 12:38:58 +0000 Subject: [PATCH 13/13] revert some debug changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- .drone.star | 2 +- .vscode/launch.json | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.drone.star b/.drone.star index 84e755fe0..98b27ef53 100644 --- a/.drone.star +++ b/.drone.star @@ -1710,7 +1710,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "PROXY_ENABLE_BASIC_AUTH": True, "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json", "IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml", - "OCIS_LOG_LEVEL": "debug", + "OCIS_LOG_LEVEL": "error", "SETTINGS_DATA_PATH": "/srv/app/tmp/ocis/settings", "IDM_CREATE_DEMO_USERS": True, "IDM_ADMIN_PASSWORD": "admin", # override the random admin password from `ocis init` diff --git a/.vscode/launch.json b/.vscode/launch.json index ddaf13bf7..aec90a875 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -17,6 +17,23 @@ "OCIS_LOG_COLOR": "true", // enable basic auth for dev setup so that we can use curl for testing "PROXY_ENABLE_BASIC_AUTH": "true", + // set insecure options because we don't have valid certificates in dev environments + "OCIS_INSECURE": "true", + // set some hardcoded secrets + "OCIS_JWT_SECRET": "some-ocis-jwt-secret", + "STORAGE_TRANSFER_SECRET": "some-ocis-transfer-secret", + "OCIS_MACHINE_AUTH_API_KEY": "some-ocis-machine-auth-api-key", + // idm ldap + "IDM_SVC_PASSWORD": "some-ldap-idm-password", + "GRAPH_LDAP_BIND_PASSWORD": "some-ldap-idm-password", + // reva ldap + "IDM_REVASVC_PASSWORD": "some-ldap-reva-password", + "GROUPS_LDAP_BIND_PASSWORD": "some-ldap-reva-password", + "USERS_LDAP_BIND_PASSWORD": "some-ldap-reva-password", + "AUTH_BASIC_LDAP_BIND_PASSWORD": "some-ldap-reva-password", + // idp ldap + "IDM_IDPSVC_PASSWORD": "some-ldap-idp-password", + "IDP_LDAP_BIND_PASSWORD": "some-ldap-idp-password", // admin user default password "IDM_ADMIN_PASSWORD": "admin", // demo users