From 1a38f3623c69a9986eeed417652556c620b88ec8 Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Wed, 9 Mar 2022 11:42:51 +0100 Subject: [PATCH 1/7] switch default config to idm - The accounts and glauth service are turned off by default - proxy is switch from "accounts" to "cs3" for the account backend - The LDAP configuration (graph, idp, storage) of all services now points to idm instead of glauth --- extensions/auth-basic/pkg/command/command.go | 2 ++ extensions/auth-basic/pkg/config/config.go | 2 ++ .../pkg/config/defaults/defaultconfig.go | 24 ++++++++++--------- .../pkg/config/defaults/defaultconfig.go | 16 ++++++------- extensions/group/pkg/command/command.go | 2 ++ extensions/group/pkg/config/config.go | 2 ++ .../pkg/config/defaults/defaultconfig.go | 24 ++++++++++--------- .../idp/pkg/config/defaults/defaultconfig.go | 12 +++++----- .../ocs/pkg/config/defaults/defaultconfig.go | 2 +- .../pkg/config/defaults/defaultconfig.go | 2 +- .../pkg/config/defaults/defaultconfig.go | 22 ++++++++--------- extensions/user/pkg/command/command.go | 2 ++ extensions/user/pkg/config/config.go | 2 ++ .../user/pkg/config/defaults/defaultconfig.go | 24 ++++++++++--------- ocis/pkg/runtime/service/service.go | 8 +++++++ 15 files changed, 86 insertions(+), 60 deletions(-) diff --git a/extensions/auth-basic/pkg/command/command.go b/extensions/auth-basic/pkg/command/command.go index 44745e4825..7835e9f09f 100644 --- a/extensions/auth-basic/pkg/command/command.go +++ b/extensions/auth-basic/pkg/command/command.go @@ -193,6 +193,8 @@ func ldapConfigFromString(cfg config.LDAPProvider) map[string]interface{} { "group_base_dn": cfg.GroupBaseDN, "user_filter": cfg.UserFilter, "group_filter": cfg.GroupFilter, + "user_scope": cfg.UserScope, + "group_scope": cfg.GroupScope, "user_objectclass": cfg.UserObjectClass, "group_objectclass": cfg.GroupObjectClass, "login_attributes": cfg.LoginAttributes, diff --git a/extensions/auth-basic/pkg/config/config.go b/extensions/auth-basic/pkg/config/config.go index 8557e3e7f1..e3706473cc 100644 --- a/extensions/auth-basic/pkg/config/config.go +++ b/extensions/auth-basic/pkg/config/config.go @@ -66,6 +66,8 @@ type LDAPProvider struct { BindPassword string `env:"LDAP_BIND_PASSWORD;AUTH_BASIC_LDAP_BIND_PASSWORD"` UserBaseDN string `env:"LDAP_USER_BASE_DN;AUTH_BASIC_LDAP_USER_BASE_DN"` GroupBaseDN string `env:"LDAP_GROUP_BASE_DN;AUTH_BASIC_LDAP_GROUP_BASE_DN"` + UserScope string `env:"LDAP_USER_SCOPE;AUTH_BASIC_LDAP_USER_SCOPE"` + GroupScope string `env:"LDAP_GROUP_SCOPE;AUTH_BASIC_LDAP_GROUP_SCOPE"` UserFilter string `env:"LDAP_USERFILTER;AUTH_BASIC_LDAP_USERFILTER"` GroupFilter string `env:"LDAP_GROUPFILTER;AUTH_BASIC_LDAP_USERFILTER"` UserObjectClass string `env:"LDAP_USER_OBJECTCLASS;AUTH_BASIC_LDAP_USER_OBJECTCLASS"` diff --git a/extensions/auth-basic/pkg/config/defaults/defaultconfig.go b/extensions/auth-basic/pkg/config/defaults/defaultconfig.go index 4d23247193..309b132446 100644 --- a/extensions/auth-basic/pkg/config/defaults/defaultconfig.go +++ b/extensions/auth-basic/pkg/config/defaults/defaultconfig.go @@ -35,31 +35,33 @@ func DefaultConfig() *config.Config { AuthProvider: "ldap", AuthProviders: config.AuthProviders{ LDAP: config.LDAPProvider{ - URI: "ldaps://localhost:9126", - CACert: filepath.Join(defaults.BaseDataPath(), "ldap", "ldap.crt"), + URI: "ldaps://localhost:9235", + CACert: filepath.Join(defaults.BaseDataPath(), "idm", "ldap.crt"), Insecure: false, - UserBaseDN: "dc=ocis,dc=test", - GroupBaseDN: "dc=ocis,dc=test", - LoginAttributes: []string{"cn", "mail"}, + UserBaseDN: "ou=users,o=libregraph-idm", + GroupBaseDN: "ou=groups,o=libregraph-idm", + UserScope: "sub", + GroupScope: "sub", + LoginAttributes: []string{"uid", "mail"}, UserFilter: "", GroupFilter: "", - UserObjectClass: "posixAccount", - GroupObjectClass: "posixGroup", - BindDN: "cn=reva,ou=sysusers,dc=ocis,dc=test", + UserObjectClass: "inetOrgPerson", + GroupObjectClass: "groupOfNames", + BindDN: "uid=reva,ou=sysusers,o=libregraph-idm", BindPassword: "reva", IDP: "https://localhost:9200", UserSchema: config.LDAPUserSchema{ ID: "ownclouduuid", Mail: "mail", DisplayName: "displayname", - Username: "cn", + Username: "uid", }, GroupSchema: config.LDAPGroupSchema{ - ID: "cn", + ID: "ownclouduuid", Mail: "mail", DisplayName: "cn", Groupname: "cn", - Member: "cn", + Member: "member", }, }, JSON: config.JSONProvider{}, diff --git a/extensions/graph/pkg/config/defaults/defaultconfig.go b/extensions/graph/pkg/config/defaults/defaultconfig.go index 49cd9916b5..512fa68d10 100644 --- a/extensions/graph/pkg/config/defaults/defaultconfig.go +++ b/extensions/graph/pkg/config/defaults/defaultconfig.go @@ -33,15 +33,15 @@ func DefaultConfig() *config.Config { Insecure: false, }, Identity: config.Identity{ - Backend: "cs3", + Backend: "ldap", LDAP: config.LDAP{ - URI: "ldap://localhost:9125", - Insecure: false, - BindDN: "", - BindPassword: "", + URI: "ldaps://localhost:9235", + Insecure: true, + BindDN: "uid=libregraph,ou=sysusers,o=libregraph-idm", + BindPassword: "idm", UseServerUUID: false, - WriteEnabled: false, - UserBaseDN: "ou=users,dc=ocis,dc=test", + WriteEnabled: true, + UserBaseDN: "ou=users,o=libregraph-idm", UserSearchScope: "sub", UserFilter: "", UserObjectClass: "inetOrgPerson", @@ -51,7 +51,7 @@ func DefaultConfig() *config.Config { // FIXME: switch this to some more widely available attribute by default // ideally this needs to be constant for the lifetime of a users UserIDAttribute: "owncloudUUID", - GroupBaseDN: "ou=groups,dc=ocis,dc=test", + GroupBaseDN: "ou=groups,o=libregraph-idm", GroupSearchScope: "sub", GroupFilter: "", GroupObjectClass: "groupOfNames", diff --git a/extensions/group/pkg/command/command.go b/extensions/group/pkg/command/command.go index ab71caef11..0f2162e0a3 100644 --- a/extensions/group/pkg/command/command.go +++ b/extensions/group/pkg/command/command.go @@ -190,6 +190,8 @@ func ldapConfigFromString(cfg config.LDAPDriver) map[string]interface{} { "bind_password": cfg.BindPassword, "user_base_dn": cfg.UserBaseDN, "group_base_dn": cfg.GroupBaseDN, + "user_scope": cfg.UserScope, + "group_scope": cfg.GroupScope, "user_filter": cfg.UserFilter, "group_filter": cfg.GroupFilter, "user_objectclass": cfg.UserObjectClass, diff --git a/extensions/group/pkg/config/config.go b/extensions/group/pkg/config/config.go index 9940bd7f26..c0eb2a4a71 100644 --- a/extensions/group/pkg/config/config.go +++ b/extensions/group/pkg/config/config.go @@ -67,6 +67,8 @@ type LDAPDriver struct { BindPassword string `env:"LDAP_BIND_PASSWORD;GROUPS_LDAP_BIND_PASSWORD"` UserBaseDN string `env:"LDAP_USER_BASE_DN;GROUPS_LDAP_USER_BASE_DN"` GroupBaseDN string `env:"LDAP_GROUP_BASE_DN;GROUPS_LDAP_GROUP_BASE_DN"` + UserScope string `env:"LDAP_USER_SCOPE;GROUPS_LDAP_USER_SCOPE"` + GroupScope string `env:"LDAP_GROUP_SCOPE;GROUPS_LDAP_GROUP_SCOPE"` UserFilter string `env:"LDAP_USERFILTER;GROUPS_LDAP_USERFILTER"` GroupFilter string `env:"LDAP_GROUPFILTER;GROUPS_LDAP_USERFILTER"` UserObjectClass string `env:"LDAP_USER_OBJECTCLASS;GROUPS_LDAP_USER_OBJECTCLASS"` diff --git a/extensions/group/pkg/config/defaults/defaultconfig.go b/extensions/group/pkg/config/defaults/defaultconfig.go index d7b0d988a8..372bcc6952 100644 --- a/extensions/group/pkg/config/defaults/defaultconfig.go +++ b/extensions/group/pkg/config/defaults/defaultconfig.go @@ -36,31 +36,33 @@ func DefaultConfig() *config.Config { Driver: "ldap", Drivers: config.Drivers{ LDAP: config.LDAPDriver{ - URI: "ldaps://localhost:9126", - CACert: filepath.Join(defaults.BaseDataPath(), "ldap", "ldap.crt"), + URI: "ldaps://localhost:9235", + CACert: filepath.Join(defaults.BaseDataPath(), "idm", "ldap.crt"), Insecure: false, - UserBaseDN: "dc=ocis,dc=test", - GroupBaseDN: "dc=ocis,dc=test", - LoginAttributes: []string{"cn", "mail"}, + UserBaseDN: "ou=users,o=libregraph-idm", + GroupBaseDN: "ou=groups,o=libregraph-idm", + UserScope: "sub", + GroupScope: "sub", + LoginAttributes: []string{"uid", "mail"}, UserFilter: "", GroupFilter: "", - UserObjectClass: "posixAccount", - GroupObjectClass: "posixGroup", - BindDN: "cn=reva,ou=sysusers,dc=ocis,dc=test", + UserObjectClass: "inetOrgPerson", + GroupObjectClass: "groupOfNames", + BindDN: "uid=reva,ou=sysusers,o=libregraph-idm", BindPassword: "reva", IDP: "https://localhost:9200", UserSchema: config.LDAPUserSchema{ ID: "ownclouduuid", Mail: "mail", DisplayName: "displayname", - Username: "cn", + Username: "uid", }, GroupSchema: config.LDAPGroupSchema{ - ID: "cn", + ID: "ownclouduuid", Mail: "mail", DisplayName: "cn", Groupname: "cn", - Member: "cn", + Member: "member", }, }, JSON: config.JSONDriver{}, diff --git a/extensions/idp/pkg/config/defaults/defaultconfig.go b/extensions/idp/pkg/config/defaults/defaultconfig.go index 2be18b92a4..fe328b2bb3 100644 --- a/extensions/idp/pkg/config/defaults/defaultconfig.go +++ b/extensions/idp/pkg/config/defaults/defaultconfig.go @@ -42,7 +42,7 @@ func DefaultConfig() *config.Config { SignedOutURI: "", AuthorizationEndpointURI: "", EndsessionEndpointURI: "", - Insecure: false, + Insecure: true, TrustedProxy: nil, AllowScope: nil, AllowClientGuests: false, @@ -68,18 +68,18 @@ func DefaultConfig() *config.Config { DyamicClientSecretDurationSeconds: 0, }, Ldap: config.Ldap{ - URI: "ldap://localhost:9125", - BindDN: "cn=idp,ou=sysusers,dc=ocis,dc=test", + URI: "ldaps://localhost:9235", + BindDN: "uid=idp,ou=sysusers,o=libregraph-idm", BindPassword: "idp", - BaseDN: "ou=users,dc=ocis,dc=test", + BaseDN: "ou=users,o=libregraph-idm", Scope: "sub", - LoginAttribute: "cn", + LoginAttribute: "uid", EmailAttribute: "mail", NameAttribute: "displayName", UUIDAttribute: "uid", UUIDAttributeType: "text", Filter: "", - ObjectClass: "posixAccount", + ObjectClass: "inetOrgPerson", }, } } diff --git a/extensions/ocs/pkg/config/defaults/defaultconfig.go b/extensions/ocs/pkg/config/defaults/defaultconfig.go index 90edea71eb..6038e0c8d5 100644 --- a/extensions/ocs/pkg/config/defaults/defaultconfig.go +++ b/extensions/ocs/pkg/config/defaults/defaultconfig.go @@ -41,7 +41,7 @@ func DefaultConfig() *config.Config { TokenManager: config.TokenManager{ JWTSecret: "Pive-Fumkiu4", }, - AccountBackend: "accounts", + AccountBackend: "cs3", Reva: config.Reva{ Address: "127.0.0.1:9142", }, diff --git a/extensions/proxy/pkg/config/defaults/defaultconfig.go b/extensions/proxy/pkg/config/defaults/defaultconfig.go index 487f9f09ab..d646436d81 100644 --- a/extensions/proxy/pkg/config/defaults/defaultconfig.go +++ b/extensions/proxy/pkg/config/defaults/defaultconfig.go @@ -45,7 +45,7 @@ func DefaultConfig() *config.Config { AllowedHTTPMethods: []string{"GET"}, Enabled: true, }, - AccountBackend: "accounts", + AccountBackend: "cs3", UserOIDCClaim: "email", UserCS3Claim: "mail", MachineAuthAPIKey: "change-me-please", diff --git a/extensions/storage/pkg/config/defaults/defaultconfig.go b/extensions/storage/pkg/config/defaults/defaultconfig.go index 95cc5c6cd2..c14ac52f0d 100644 --- a/extensions/storage/pkg/config/defaults/defaultconfig.go +++ b/extensions/storage/pkg/config/defaults/defaultconfig.go @@ -44,35 +44,35 @@ func DefaultConfig() *config.Config { IDClaim: "preferred_username", }, LDAP: config.LDAP{ - URI: "ldaps://localhost:9126", - CACert: path.Join(defaults.BaseDataPath(), "ldap", "ldap.crt"), + URI: "ldaps://localhost:9235", + CACert: path.Join(defaults.BaseDataPath(), "idm", "ldap.crt"), Insecure: false, - UserBaseDN: "dc=ocis,dc=test", - GroupBaseDN: "dc=ocis,dc=test", + UserBaseDN: "ou=users,o=libregraph-idm", + GroupBaseDN: "ou=groups,o=libregraph-idm", UserScope: "sub", GroupScope: "sub", - LoginAttributes: []string{"cn", "mail"}, + LoginAttributes: []string{"uid", "mail"}, UserFilter: "", GroupFilter: "", - UserObjectClass: "posixAccount", - GroupObjectClass: "posixGroup", - BindDN: "cn=reva,ou=sysusers,dc=ocis,dc=test", + UserObjectClass: "inetOrgPerson", + GroupObjectClass: "groupOfNames", + BindDN: "uid=reva,ou=sysusers,o=libregraph-idm", BindPassword: "reva", IDP: defaultPublicURL, UserSchema: config.LDAPUserSchema{ ID: "ownclouduuid", Mail: "mail", DisplayName: "displayname", - Username: "cn", + Username: "uid", UIDNumber: "uidnumber", GIDNumber: "gidnumber", }, GroupSchema: config.LDAPGroupSchema{ - ID: "cn", + ID: "ownclouduuid", Mail: "mail", DisplayName: "cn", Groupname: "cn", - Member: "cn", + Member: "member", GIDNumber: "gidnumber", }, }, diff --git a/extensions/user/pkg/command/command.go b/extensions/user/pkg/command/command.go index 31035acda1..a77f23f4c1 100644 --- a/extensions/user/pkg/command/command.go +++ b/extensions/user/pkg/command/command.go @@ -209,6 +209,8 @@ func ldapConfigFromString(cfg config.LDAPDriver) map[string]interface{} { "bind_password": cfg.BindPassword, "user_base_dn": cfg.UserBaseDN, "group_base_dn": cfg.GroupBaseDN, + "user_scope": cfg.UserScope, + "group_scope": cfg.GroupScope, "user_filter": cfg.UserFilter, "group_filter": cfg.GroupFilter, "user_objectclass": cfg.UserObjectClass, diff --git a/extensions/user/pkg/config/config.go b/extensions/user/pkg/config/config.go index efdcd54430..fdb08f931d 100644 --- a/extensions/user/pkg/config/config.go +++ b/extensions/user/pkg/config/config.go @@ -67,6 +67,8 @@ type LDAPDriver struct { BindPassword string `env:"LDAP_BIND_PASSWORD;USERS_LDAP_BIND_PASSWORD"` UserBaseDN string `env:"LDAP_USER_BASE_DN;USERS_LDAP_USER_BASE_DN"` GroupBaseDN string `env:"LDAP_GROUP_BASE_DN;USERS_LDAP_GROUP_BASE_DN"` + UserScope string `env:"LDAP_USER_SCOPE;USERS_LDAP_USER_SCOPE"` + GroupScope string `env:"LDAP_GROUP_SCOPE;USERS_LDAP_GROUP_SCOPE"` UserFilter string `env:"LDAP_USERFILTER;USERS_LDAP_USERFILTER"` GroupFilter string `env:"LDAP_GROUPFILTER;USERS_LDAP_USERFILTER"` UserObjectClass string `env:"LDAP_USER_OBJECTCLASS;USERS_LDAP_USER_OBJECTCLASS"` diff --git a/extensions/user/pkg/config/defaults/defaultconfig.go b/extensions/user/pkg/config/defaults/defaultconfig.go index 09f4abe003..20a486f47a 100644 --- a/extensions/user/pkg/config/defaults/defaultconfig.go +++ b/extensions/user/pkg/config/defaults/defaultconfig.go @@ -36,31 +36,33 @@ func DefaultConfig() *config.Config { Driver: "ldap", Drivers: config.Drivers{ LDAP: config.LDAPDriver{ - URI: "ldaps://localhost:9126", - CACert: filepath.Join(defaults.BaseDataPath(), "ldap", "ldap.crt"), + URI: "ldaps://localhost:9235", + CACert: filepath.Join(defaults.BaseDataPath(), "idm", "ldap.crt"), Insecure: false, - UserBaseDN: "dc=ocis,dc=test", - GroupBaseDN: "dc=ocis,dc=test", - LoginAttributes: []string{"cn", "mail"}, + UserBaseDN: "ou=users,o=libregraph-idm", + GroupBaseDN: "ou=groups,o=libregraph-idm", + UserScope: "sub", + GroupScope: "sub", + LoginAttributes: []string{"uid", "mail"}, UserFilter: "", GroupFilter: "", - UserObjectClass: "posixAccount", - GroupObjectClass: "posixGroup", - BindDN: "cn=reva,ou=sysusers,dc=ocis,dc=test", + UserObjectClass: "inetOrgPerson", + GroupObjectClass: "groupOfNames", + BindDN: "uid=reva,ou=sysusers,o=libregraph-idm", BindPassword: "reva", IDP: "https://localhost:9200", UserSchema: config.LDAPUserSchema{ ID: "ownclouduuid", Mail: "mail", DisplayName: "displayname", - Username: "cn", + Username: "uid", }, GroupSchema: config.LDAPGroupSchema{ - ID: "cn", + ID: "ownclouduuid", Mail: "mail", DisplayName: "cn", Groupname: "cn", - Member: "cn", + Member: "member", }, }, JSON: config.JSONDriver{}, diff --git a/ocis/pkg/runtime/service/service.go b/ocis/pkg/runtime/service/service.go index e7ca1fc71e..7215d8098d 100644 --- a/ocis/pkg/runtime/service/service.go +++ b/ocis/pkg/runtime/service/service.go @@ -254,10 +254,18 @@ func (s *Service) generateRunSet(cfg *ociscfg.Config) { } for name := range s.ServicesRegistry { + // don't run glauth by default but keep the possiblity to start it via cfg.Runtime.Extensions for now + if name == "glauth" { + continue + } runset = append(runset, name) } for name := range s.Delayed { + // don't run accounts by default but keep the possiblity to start it via cfg.Runtime.Extensions for now + if name == "accounts" { + continue + } runset = append(runset, name) } } From 45f0940071bd0208f6afdbf35a8449e9cd8bb658 Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Wed, 16 Mar 2022 12:40:57 +0100 Subject: [PATCH 2/7] Turn of "insecure" of built-in IDP Setup idp to verify the LDAP server certificate. As this certificate might be generated on startup, this also moved the IDP to the "delayed" set of services. So it starts after "idm". --- extensions/idp/pkg/config/config.go | 3 ++- extensions/idp/pkg/config/defaults/defaultconfig.go | 3 ++- extensions/idp/pkg/service/v0/service.go | 4 ++++ ocis/pkg/runtime/service/service.go | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/extensions/idp/pkg/config/config.go b/extensions/idp/pkg/config/config.go index 83bd84554d..4979fb0f38 100644 --- a/extensions/idp/pkg/config/config.go +++ b/extensions/idp/pkg/config/config.go @@ -27,7 +27,8 @@ type Config struct { // Ldap defines the available LDAP configuration. type Ldap struct { - URI string `yaml:"uri" env:"LDAP_URI;IDP_LDAP_URI"` + URI string `yaml:"uri" env:"LDAP_URI;IDP_LDAP_URI"` + TLSCACert string `yaml:"cacert" env:"LDAP_CACERT;IDP_LDAP_TLS_CACERT"` BindDN string `yaml:"bind_dn" env:"LDAP_BIND_DN;IDP_LDAP_BIND_DN"` BindPassword string `yaml:"bind_password" env:"LDAP_BIND_PASSWORD;IDP_LDAP_BIND_PASSWORD"` diff --git a/extensions/idp/pkg/config/defaults/defaultconfig.go b/extensions/idp/pkg/config/defaults/defaultconfig.go index fe328b2bb3..d9b68fb506 100644 --- a/extensions/idp/pkg/config/defaults/defaultconfig.go +++ b/extensions/idp/pkg/config/defaults/defaultconfig.go @@ -42,7 +42,7 @@ func DefaultConfig() *config.Config { SignedOutURI: "", AuthorizationEndpointURI: "", EndsessionEndpointURI: "", - Insecure: true, + Insecure: false, TrustedProxy: nil, AllowScope: nil, AllowClientGuests: false, @@ -69,6 +69,7 @@ func DefaultConfig() *config.Config { }, Ldap: config.Ldap{ URI: "ldaps://localhost:9235", + TLSCACert: path.Join(defaults.BaseDataPath(), "idm", "ldap.crt"), BindDN: "uid=idp,ou=sysusers,o=libregraph-idm", BindPassword: "idp", BaseDN: "ou=users,o=libregraph-idm", diff --git a/extensions/idp/pkg/service/v0/service.go b/extensions/idp/pkg/service/v0/service.go index 71270f0c44..1b1a8bf1d0 100644 --- a/extensions/idp/pkg/service/v0/service.go +++ b/extensions/idp/pkg/service/v0/service.go @@ -142,6 +142,10 @@ func initLicoInternalEnvVars(ldap *config.Ldap) error { "LDAP_FILTER": filter, } + if ldap.TLSCACert != "" { + defaults["LDAP_TLS_CACERT"] = ldap.TLSCACert + } + for k, v := range defaults { if err := os.Setenv(k, v); err != nil { return fmt.Errorf("could not set env var %s=%s", k, v) diff --git a/ocis/pkg/runtime/service/service.go b/ocis/pkg/runtime/service/service.go index 7215d8098d..f791543a45 100644 --- a/ocis/pkg/runtime/service/service.go +++ b/ocis/pkg/runtime/service/service.go @@ -112,7 +112,6 @@ func NewService(options ...Option) (*Service, error) { s.ServicesRegistry["glauth"] = glauth.NewSutureService s.ServicesRegistry["graph"] = graph.NewSutureService s.ServicesRegistry["graph-explorer"] = graphExplorer.NewSutureService - s.ServicesRegistry["idp"] = idp.NewSutureService s.ServicesRegistry["idm"] = idm.NewSutureService s.ServicesRegistry["ocs"] = ocs.NewSutureService s.ServicesRegistry["store"] = store.NewSutureService @@ -137,6 +136,7 @@ func NewService(options ...Option) (*Service, error) { s.Delayed["accounts"] = accounts.NewSutureService s.Delayed["proxy"] = proxy.NewSutureService s.Delayed["ocdav"] = ocdav.NewOCDav + s.Delayed["idp"] = idp.NewSutureService return s, nil } From 72cb96079052609eda88225685ab03453bd372db Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Thu, 21 Apr 2022 14:34:14 +0200 Subject: [PATCH 3/7] Adjust drone config for defaulting to libreidm/graph --- .drone.star | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/.drone.star b/.drone.star index 20aa53b7b5..bb288b03a4 100644 --- a/.drone.star +++ b/.drone.star @@ -76,16 +76,16 @@ config = { }, "uiTests": { "filterTags": "@ocisSmokeTest", - "skip": False, + "skip": True, "skipExceptParts": [], "earlyFail": True, }, "accountsUITests": { - "skip": False, + "skip": True, "earlyFail": True, }, "settingsUITests": { - "skip": False, + "skip": True, "earlyFail": True, }, "parallelApiTests": { @@ -111,7 +111,7 @@ config = { }, }, "graphApiTests": { - "skip": False, + "skip": True, "earlyFali": False, "numberOfParts": 10, "skipExceptParts": [], @@ -505,6 +505,9 @@ def localApiTests(ctx, storage, suite, accounts_hash_difficulty = 4): "name": "localApiTests-%s-%s" % (suite, storage), "image": OC_CI_PHP % DEFAULT_PHP_VERSION, "environment": { + "TEST_WITH_GRAPH_API": "true", + "PATH_TO_OCIS": "/drone/src", + "PATH_TO_CORE": "/srv/app/testrunner", "TEST_SERVER_URL": "https://ocis-server:9200", "OCIS_REVA_DATA_ROOT": "%s" % ("/srv/app/tmp/ocis/owncloud/data/" if storage == "owncloud" else ""), "SKELETON_DIR": "/srv/app/tmp/testing/data/apiSkeleton", @@ -513,8 +516,7 @@ def localApiTests(ctx, storage, suite, accounts_hash_difficulty = 4): "SEND_SCENARIO_LINE_REFERENCES": "true", "STORAGE_DRIVER": storage, "BEHAT_SUITE": suite, - "BEHAT_FILTER_TAGS": "~@skip&&~@skipOnOcis-%s-Storage" % ("OC" if storage == "owncloud" else "OCIS"), - "PATH_TO_CORE": "/srv/app/testrunner", + "BEHAT_FILTER_TAGS": "~@skip&&~@skipOnGraph&&~@skipOnOcis-%s-Storage" % ("OC" if storage == "owncloud" else "OCIS"), "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-localAPI-on-%s-storage.md" % (storage.upper()), "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0, }, @@ -570,6 +572,8 @@ def cs3ApiTests(ctx, storage, accounts_hash_difficulty = 4): def coreApiTests(ctx, part_number = 1, number_of_parts = 1, storage = "ocis", accounts_hash_difficulty = 4): early_fail = config["apiTests"]["earlyFail"] if "earlyFail" in config["apiTests"] else False + filterTags = "~@skipOnGraph&&~@skipOnOcis&&~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@local_storage&&~@skipOnOcis-%s-Storage&&~@issue-ocis-3023" % ("OC" if storage == "owncloud" else "OCIS") + expectedFailuresFile = "/drone/src/tests/acceptance/expected-failures-graphAPI-on-%s-storage.md" % (storage.upper()) return { "kind": "pipeline", @@ -586,6 +590,9 @@ def coreApiTests(ctx, part_number = 1, number_of_parts = 1, storage = "ocis", ac "name": "oC10ApiTests-%s-storage-%s" % (storage, part_number), "image": OC_CI_PHP % DEFAULT_PHP_VERSION, "environment": { + "TEST_WITH_GRAPH_API": "true", + "PATH_TO_OCIS": "/drone/src", + "PATH_TO_CORE": "/srv/app/testrunner", "TEST_SERVER_URL": "https://ocis-server:9200", "OCIS_REVA_DATA_ROOT": "%s" % ("/srv/app/tmp/ocis/owncloud/data/" if storage == "owncloud" else ""), "SKELETON_DIR": "/srv/app/tmp/testing/data/apiSkeleton", @@ -593,10 +600,10 @@ def coreApiTests(ctx, part_number = 1, number_of_parts = 1, storage = "ocis", ac "TEST_OCIS": "true", "SEND_SCENARIO_LINE_REFERENCES": "true", "STORAGE_DRIVER": storage, - "BEHAT_FILTER_TAGS": "~@skipOnOcis&&~@notToImplementOnOCIS&&~@toImplementOnOCIS&&~comments-app-required&&~@federation-app-required&&~@notifications-app-required&&~systemtags-app-required&&~@local_storage&&~@skipOnOcis-%s-Storage&&~@issue-ocis-3023" % ("OC" if storage == "owncloud" else "OCIS"), + "BEHAT_FILTER_TAGS": filterTags, "DIVIDE_INTO_NUM_PARTS": number_of_parts, "RUN_PART": part_number, - "EXPECTED_FAILURES_FILE": "/drone/src/tests/acceptance/expected-failures-API-on-%s-storage.md" % (storage.upper()), + "EXPECTED_FAILURES_FILE": expectedFailuresFile, "UPLOAD_DELETE_WAIT_TIME": "1" if storage == "owncloud" else 0, }, "commands": [ @@ -1696,7 +1703,6 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "OCIS_LOG_LEVEL": "error", "SETTINGS_DATA_PATH": "/srv/app/tmp/ocis/settings", "OCIS_INSECURE": "true", - "ACCOUNTS_DEMO_USERS_AND_GROUPS": True, # deprecated, remove after switching to LibreIDM "IDM_CREATE_DEMO_USERS": True, } wait_for_ocis = { @@ -1710,6 +1716,8 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = else: user = "33:33" environment = { + "GRAPH_IDENTITY_BACKEND": "cs3", + "GRAPH_LDAP_SERVER_WRITE_ENABLED": "false", # Keycloak IDP specific configuration "PROXY_OIDC_ISSUER": "https://keycloak/auth/realms/owncloud", "LDAP_IDP": "https://keycloak/auth/realms/owncloud", @@ -1781,6 +1789,7 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = "OCIS_MACHINE_AUTH_API_KEY": "change-me-please", "OCIS_INSECURE": "true", "PROXY_ENABLE_BASIC_AUTH": "true", + "IDM_CREATE_DEMO_USERS": True, } wait_for_ocis = { "name": "wait-for-ocis-server", @@ -2508,7 +2517,7 @@ def graphApiTests(ctx, part_number = 1, number_of_parts = 1): }, "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") + - ocisServerWithIdp() + + ocisServer() + cloneCoreRepos() + [ { "name": "Graph-oC10ApiTests-%s-storage-%s" % (storage, part_number), From 5d4800b570cdfc111de558f78d5c43bc03642fb2 Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Mon, 25 Apr 2022 11:35:50 +0200 Subject: [PATCH 4/7] move test away from accounts api, use graph to get userid --- .../features/bootstrap/SpacesContext.php | 52 ++++++++----------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/tests/acceptance/features/bootstrap/SpacesContext.php b/tests/acceptance/features/bootstrap/SpacesContext.php index bbe9eda598..8200217e2f 100644 --- a/tests/acceptance/features/bootstrap/SpacesContext.php +++ b/tests/acceptance/features/bootstrap/SpacesContext.php @@ -30,6 +30,7 @@ use GuzzleHttp\Exception\GuzzleException; use Psr\Http\Message\ResponseInterface; use TestHelpers\HttpRequestHelper; use TestHelpers\SetupHelper; +use TestHelpers\GraphHelper; use PHPUnit\Framework\Assert; require_once 'bootstrap.php'; @@ -285,31 +286,23 @@ class SpacesContext implements Context { * @return string */ public function getUserIdByUserName(string $userName): string { - $fullUrl = $this->baseUrl . "/api/v0/accounts/accounts-list"; - $this->featureContext->setResponse( - HttpRequestHelper::post( - $fullUrl, - "", - $this->featureContext->getAdminUsername(), - $this->featureContext->getAdminPassword(), - [], - "{}" - ) - ); + $this->featureContext->setResponse(GraphHelper::getUser( + $this->featureContext->getBaseUrl(), + $this->featureContext->getStepLineRef(), + $this->featureContext->getAdminUsername(), + $this->featureContext->getAdminPassword(), + $userName + )); if ($this->featureContext->getResponse()) { $rawBody = $this->featureContext->getResponse()->getBody()->getContents(); $response = \json_decode($rawBody, true, 512, JSON_THROW_ON_ERROR); - if (isset($response["accounts"])) { - $accounts = $response["accounts"]; + if (isset($response["id"])) { + $user = $response; } else { throw new Exception(__METHOD__ . " accounts-list is empty"); } } - foreach ($accounts as $account) { - if ($account["preferredName"] === $userName) { - return $account["id"]; - } - } + return $user["id"]; throw new Exception(__METHOD__ . " user with name $userName not found"); } @@ -607,7 +600,6 @@ class SpacesContext implements Context { $password = $this->featureContext->getAdminPassword(); $headers = []; $bundles = []; - $accounts = []; $assignment = []; // get the roles list first @@ -628,22 +620,20 @@ class SpacesContext implements Context { } Assert::assertNotEmpty($roleToAssign, "The selected role $role could not be found"); - // get the accounts list first - $fullUrl = $this->baseUrl . "/api/v0/accounts/accounts-list"; - $this->featureContext->setResponse(HttpRequestHelper::post($fullUrl, "", $admin, $password, $headers, "{}")); + $this->featureContext->setResponse(GraphHelper::getUser( + $this->featureContext->getBaseUrl(), + $this->featureContext->getStepLineRef(), + $this->featureContext->getAdminUsername(), + $this->featureContext->getAdminPassword(), + $user + )); if ($this->featureContext->getResponse()) { $rawBody = $this->featureContext->getResponse()->getBody()->getContents(); - if (isset(\json_decode($rawBody, true, 512, JSON_THROW_ON_ERROR)["accounts"])) { - $accounts = \json_decode($rawBody, true, 512, JSON_THROW_ON_ERROR)["accounts"]; - } - } - $accountToChange = ""; - foreach ($accounts as $account) { - // find the selected user - if ($account["preferredName"] === $user) { - $accountToChange = $account; + if (isset(\json_decode($rawBody, true, 512, JSON_THROW_ON_ERROR)["id"])) { + $accountToChange = \json_decode($rawBody, true, 512, JSON_THROW_ON_ERROR); } } + Assert::assertNotEmpty($accountToChange, "The selected account $user does not exist"); // set the new role From 095e3c3f779700ed913753f20bd30fba96ef11c0 Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Thu, 21 Apr 2022 17:37:07 +0200 Subject: [PATCH 5/7] skip test that requires ocs provisioning API When using the GraphAPI skip ocs specific test --- .../features/apiAccountsHashDifficulty/addUser.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature b/tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature index eb271388b9..869a9eada5 100644 --- a/tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature +++ b/tests/acceptance/features/apiAccountsHashDifficulty/addUser.feature @@ -1,4 +1,4 @@ -@api @provisioning_api-app-required @skipOnLDAP +@api @provisioning_api-app-required @skipOnLDAP @skipOnGraph Feature: add user As an admin I want to be able to add users and store their password with the full hash difficulty From 33cb8a4e63171a939f580ee9b77d531fc6bfee2b Mon Sep 17 00:00:00 2001 From: Viktor Scharf Date: Mon, 25 Apr 2022 13:19:46 +0200 Subject: [PATCH 6/7] addSteps: clean project space after test --- .../features/bootstrap/SpacesContext.php | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/acceptance/features/bootstrap/SpacesContext.php b/tests/acceptance/features/bootstrap/SpacesContext.php index 8200217e2f..61d8f20b0d 100644 --- a/tests/acceptance/features/bootstrap/SpacesContext.php +++ b/tests/acceptance/features/bootstrap/SpacesContext.php @@ -24,6 +24,7 @@ declare(strict_types=1); use Behat\Behat\Context\Context; use Behat\Behat\Hook\Scope\BeforeScenarioScope; +use Behat\Behat\Hook\Call\AfterScenario; use Behat\Gherkin\Node\TableNode; use Behat\Testwork\Environment\Environment; use GuzzleHttp\Exception\GuzzleException; @@ -333,6 +334,55 @@ class SpacesContext implements Context { ); } + /** + * @AfterScenario + * + * @return void + * + * @throws Exception + */ + public function cleanDataAfterTests(): void + { + $this->deleteAllSpacesOfTheType('project'); + } + + /** + * The method first disables and then deletes spaces + * @param string $driveType + * + * @return void + * + * @throws Exception + */ + public function deleteAllSpacesOfTheType(string $driveType): void + { + $query = "\$filter=driveType eq $driveType"; + $userAdmin = $this->featureContext->getAdminUsername(); + + for ($i = 0; $i < 2; ++$i) { + $this->theUserListsAllHisAvailableSpacesUsingTheGraphApiWithFilter( + $userAdmin, + $query + ); + + $rawBody = $this->featureContext->getResponse()->getBody()->getContents(); + $drives = json_decode($rawBody, true, 512, JSON_THROW_ON_ERROR); + if (isset($drives["value"])) { + $drives = $drives["value"]; + } + + if (!empty($drives)) { + foreach ($drives as $value) { + if (!array_key_exists("deleted", $value["root"])) { + $this->sendDisableSpaceRequest($userAdmin, $value["name"]); + } else { + $this->sendDeleteSpaceRequest($userAdmin, $value["name"]); + } + } + } + } + } + /** * Send Graph List My Spaces Request * From aa7d75c84583e51fcf93815b6ae992087c94d1a5 Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Tue, 26 Apr 2022 08:29:10 +0200 Subject: [PATCH 7/7] Run UI test with accounts service enabled until the ui tests are able to switch to the Graph API for user provisioning --- .drone.star | 72 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/.drone.star b/.drone.star index bb288b03a4..e5794bea7a 100644 --- a/.drone.star +++ b/.drone.star @@ -76,7 +76,7 @@ config = { }, "uiTests": { "filterTags": "@ocisSmokeTest", - "skip": True, + "skip": False, "skipExceptParts": [], "earlyFail": True, }, @@ -698,7 +698,7 @@ def uiTestPipeline(ctx, filterTags, early_fail, runPart = 1, numberOfParts = 1, "arch": "amd64", }, "steps": skipIfUnchanged(ctx, "acceptance-tests") + restoreBuildArtifactCache(ctx, "ocis-binary-amd64", "ocis/bin/ocis") + - ocisServer(storage, accounts_hash_difficulty, [stepVolumeOC10Tests]) + waitForSeleniumService() + waitForMiddlewareService() + [ + ocisServerWithAccounts(storage, accounts_hash_difficulty, [stepVolumeOC10Tests]) + waitForSeleniumService() + waitForMiddlewareService() + [ { "name": "webUITests", "image": OC_CI_NODEJS % DEFAULT_NODEJS_VERSION, @@ -1629,40 +1629,50 @@ def notify(ctx): }, } -def ocisServerWithIdp(): +def ocisServerWithAccounts(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = []): environment = { - "GRAPH_IDENTITY_BACKEND": "ldap", - "GRAPH_LDAP_SERVER_WRITE_ENABLED": "true", - "LDAP_URI": "ldaps://0.0.0.0:9235", + "GRAPH_IDENTITY_BACKEND": "cs3", + "GRAPH_LDAP_SERVER_WRITE_ENABLED": "false", + "LDAP_URI": "ldaps://0.0.0.0:9126", "LDAP_INSECURE": "true", - "GRAPH_LDAP_BIND_DN": "uid=libregraph,ou=sysusers,o=libregraph-idm", - "GRAPH_LDAP_BIND_PASSWORD": "idm", - "LDAP_USER_BASE_DN": "ou=users,o=libregraph-idm", + "LDAP_BIND_DN": "cn=admin,dc=ocis,dc=test", + "LDAP_BIND_PASSWORD": "admin", + "LDAP_USER_BASE_DN": "dc=ocis,dc=test", "LDAP_USER_SCHEMA_ID": "ownclouduuid", "LDAP_USER_SCHEMA_MAIL": "mail", - "LDAP_USER_SCHEMA_USERNAME": "uid", - "LDAP_USER_OBJECTCLASS": "inetOrgPerson", - "LDAP_GROUP_BASE_DN": "ou=groups,o=libregraph-idm", - "LDAP_GROUP_SCHEMA_ID": "ownclouduuid", + "LDAP_USER_SCHEMA_USERNAME": "cn", + "LDAP_USER_OBJECTCLASS": "posixAccount", + "LDAP_GROUP_BASE_DN": "dc=ocis,dc=test", + "LDAP_GROUP_SCHEMA_ID": "cn", "LDAP_GROUP_SCHEMA_MAIL": "mail", "LDAP_GROUP_SCHEMA_GROUPNAME": "cn", - "LDAP_GROUP_SCHEMA_MEMBER": "member", - "LDAP_GROUP_OBJECTCLASS": "groupOfNames", - "IDP_INSECURE": "true", - "IDP_LDAP_BIND_DN": "uid=idp,ou=sysusers,o=libregraph-idm", - "IDP_LDAP_BIND_PASSWORD": "idp", - "IDP_LDAP_BASE_DN": "ou=users,o=libregraph-idm", + "LDAP_GROUP_SCHEMA_MEMBER": "cn", + "LDAP_GROUP_OBJECTCLASS": "posixGroup", + "IDP_LDAP_BIND_DN": "cn=admin,dc=ocis,dc=test", + "LDAP_CACERT": "/root/.ocis/ldap/ldap.crt", + "IDP_LDAP_BIND_PASSWORD": "admin", "IDP_LDAP_LOGIN_ATTRIBUTE": "uid", - "PROXY_ACCOUNT_BACKEND_TYPE": "cs3", - "PROXY_ENABLE_BASIC_AUTH": "true", - "LDAP_BIND_DN": "uid=reva,ou=sysusers,o=libregraph-idm", - "LDAP_BIND_PASSWORD": "reva", - "OCS_ACCOUNT_BACKEND_TYPE": "cs3", - "OCIS_RUN_EXTENSIONS": "settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,storage-appprovider,storage-sharing,proxy,idp,nats,idm,ocdav", - "OCIS_LOG_LEVEL": "error", + "PROXY_ACCOUNT_BACKEND_TYPE": "accounts", + "OCS_ACCOUNT_BACKEND_TYPE": "accounts", + "OCIS_RUN_EXTENSIONS": "settings,storage-metadata,graph,graph-explorer,ocs,store,thumbnails,web,webdav,storage-frontend,storage-gateway,storage-userprovider,storage-groupprovider,storage-authbasic,storage-authbearer,storage-authmachine,storage-users,storage-shares,storage-public-link,storage-appprovider,storage-sharing,proxy,idp,nats,accounts,glauth,ocdav", "OCIS_INSECURE": "true", + "PROXY_ENABLE_BASIC_AUTH": "true", + "IDP_INSECURE": "true", + "OCIS_LOG_LEVEL": "error", "OCIS_URL": "https://ocis-server:9200", + "ACCOUNTS_DEMO_USERS_AND_GROUPS": True, + "STORAGE_HOME_DRIVER": "%s" % (storage), + "STORAGE_USERS_DRIVER": "%s" % (storage), + "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json", } + + # Pass in "default" accounts_hash_difficulty to not set this environment variable. + # That will allow OCIS to use whatever its built-in default is. + # Otherwise pass in a value from 4 to about 11 or 12 (default 4, for making regular tests fast) + # The high values cause lots of CPU to be used when hashing passwords, and really slow down the tests. + if (accounts_hash_difficulty != "default"): + environment["ACCOUNTS_HASH_DIFFICULTY"] = accounts_hash_difficulty + return [ { "name": "ocis-server", @@ -1672,16 +1682,16 @@ def ocisServerWithIdp(): "commands": [ "ocis/bin/ocis server", ], - "volumes": [stepVolumeOC10Tests], - "depends_on": [], + "volumes": volumes, + "depends_on": depends_on, }, { "name": "wait-for-ocis-server", - "image": OC_CI_WAIT_FOR, + "image": OC_CI_ALPINE, "commands": [ - "wait-for -it ocis-server:9200 -t 300", + "curl -k -u admin:admin --fail --retry-connrefused --retry 10 --retry-all-errors 'https://ocis-server:9200/graph/v1.0/users/ddc2004c-0977-11eb-9d3f-a793888cd0f8'", ], - "depends_on": [], + "depends_on": depends_on, }, ]