From 9147c96ce89c818d7ac5108d03514018760fe217 Mon Sep 17 00:00:00 2001 From: saw-jan Date: Wed, 12 Jan 2022 14:23:40 +0545 Subject: [PATCH] build ocis server for parallel deploy tests --- .drone.star | 164 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 121 insertions(+), 43 deletions(-) diff --git a/.drone.star b/.drone.star index fbe76ef342..8de24ed03f 100644 --- a/.drone.star +++ b/.drone.star @@ -156,18 +156,17 @@ def main(ctx): test_pipelines = \ cancelPreviousBuilds() + \ [buildOcisBinaryForTesting(ctx)] + \ - testOcisModules(ctx) + \ testPipelines(ctx) - build_release_pipelines = \ - dockerReleases(ctx) + \ - binaryReleases(ctx) + \ - [releaseSubmodule(ctx)] + # build_release_pipelines = \ + # dockerReleases(ctx) + \ + # binaryReleases(ctx) + \ + # [releaseSubmodule(ctx)] - build_release_helpers = [ - changelog(ctx), - docs(ctx), - ] + # build_release_helpers = [ + # changelog(ctx), + # docs(ctx), + # ] test_pipelines.append( pipelineDependsOn( @@ -176,7 +175,7 @@ def main(ctx): ), ) - pipelines = test_pipelines + build_release_pipelines + build_release_helpers + pipelines = test_pipelines # + build_release_pipelines + build_release_helpers pipelines = \ pipelines + \ @@ -236,21 +235,21 @@ def testPipelines(ctx): if "skip" not in config["localApiTests"] or not config["localApiTests"]["skip"]: pipelines = [ localApiTests(ctx, "ocis", "apiAccountsHashDifficulty", "default"), - localApiTests(ctx, "ocis", "apiSpaces", "default"), - localApiTests(ctx, "ocis", "apiArchiver", "default"), + # localApiTests(ctx, "ocis", "apiSpaces", "default"), + # localApiTests(ctx, "ocis", "apiArchiver", "default"), ] - if "skip" not in config["apiTests"] or not config["apiTests"]["skip"]: - pipelines += apiTests(ctx) + # if "skip" not in config["apiTests"] or not config["apiTests"]["skip"]: + # pipelines += apiTests(ctx) - if "skip" not in config["uiTests"] or not config["uiTests"]["skip"]: - pipelines += uiTests(ctx) + # if "skip" not in config["uiTests"] or not config["uiTests"]["skip"]: + # pipelines += uiTests(ctx) - if "skip" not in config["accountsUITests"] or not config["accountsUITests"]["skip"]: - pipelines.append(accountsUITests(ctx)) + # if "skip" not in config["accountsUITests"] or not config["accountsUITests"]["skip"]: + # pipelines.append(accountsUITests(ctx)) - if "skip" not in config["settingsUITests"] or not config["settingsUITests"]["skip"]: - pipelines.append(settingsUITests(ctx)) + # if "skip" not in config["settingsUITests"] or not config["settingsUITests"]["skip"]: + # pipelines.append(settingsUITests(ctx)) if "skip" not in config["parallelApiTests"] or not config["parallelApiTests"]["skip"]: pipelines += parallelDeployAcceptancePipeline(ctx) @@ -1405,22 +1404,93 @@ def notify(ctx): }, } -def ocisServer(storage, accounts_hash_difficulty = 4, volumes = []): - environment = { - "OCIS_URL": "https://ocis-server:9200", - "STORAGE_USERS_DRIVER": "%s" % (storage), - "STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root", - "STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR": "/srv/app/tmp/ocis/owncloud/data", - "STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users", - "STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata", - "STORAGE_SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json", - "PROXY_ENABLE_BASIC_AUTH": True, - "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json", - "IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml", - "OCIS_LOG_LEVEL": "error", - "SETTINGS_DATA_PATH": "/srv/app/tmp/ocis/settings", - "OCIS_INSECURE": "true", - } +def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = [], testing_parallel_deploy = False): + if not testing_parallel_deploy: + environment = { + "OCIS_URL": "https://ocis-server:9200", + "STORAGE_HOME_DRIVER": "%s" % (storage), + "STORAGE_USERS_DRIVER": "%s" % (storage), + "STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root", + "STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR": "/srv/app/tmp/ocis/owncloud/data", + "STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users", + "STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata", + "STORAGE_SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json", + "PROXY_ENABLE_BASIC_AUTH": True, + "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json", + "IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml", + "OCIS_LOG_LEVEL": "error", + "SETTINGS_DATA_PATH": "/srv/app/tmp/ocis/settings", + "OCIS_INSECURE": "true", + } + else: + environment = { + # Keycloak IDP specific configuration + "PROXY_OIDC_ISSUER": "https://keycloak/auth/realmsowncloud", + "WEB_OIDC_AUTHORITY": "https://keycloak/auth/realms/owncloud", + "WEB_OIDC_CLIENT_ID": "ocis-web", + "WEB_OIDC_METADATA_URL": "https://keycloak/auth/realms/owncloud/.well-known/openid-configuration", + "STORAGE_OIDC_ISSUER": "https://keycloak", + "STORAGE_LDAP_IDP": "https://keycloak/auth/realms/owncloud", + "WEB_OIDC_SCOPE": "openid profile email owncloud", + # LDAP bind + "STORAGE_LDAP_HOSTNAME": "openldap", + "STORAGE_LDAP_PORT": 636, + "STORAGE_LDAP_INSECURE": "true", + "STORAGE_LDAP_BIND_DN": "cn=admin,dc=owncloud,dc=com", + "STORAGE_LDAP_BIND_PASSWORD": "admin", + # LDAP user settings + "PROXY_AUTOPROVISION_ACCOUNTS": "true", # automatically create users when they login + "PROXY_ACCOUNT_BACKEND_TYPE": "cs3", # proxy should get users from CS3APIS (which gets it from LDAP) + "PROXY_USER_OIDC_CLAIM": "ocis.user.uuid", # claim was added in Keycloak + "PROXY_USER_CS3_CLAIM": "userid", # equals STORAGE_LDAP_USER_SCHEMA_UID + "STORAGE_LDAP_BASE_DN": "dc=owncloud,dc=com", + "STORAGE_LDAP_GROUP_SCHEMA_DISPLAYNAME": "cn", + "STORAGE_LDAP_GROUP_SCHEMA_GID_NUMBER": "gidnumber", + "STORAGE_LDAP_GROUP_SCHEMA_GID": "cn", + "STORAGE_LDAP_GROUP_SCHEMA_MAIL": "mail", + "STORAGE_LDAP_GROUPATTRIBUTEFILTER": "(&(objectclass=posixGroup)(objectclass=owncloud)({{attr}}={{value}}))", + "STORAGE_LDAP_GROUPFILTER": "(&(objectclass=groupOfUniqueNames)(objectclass=owncloud)(ownclouduuid={{.OpaqueId}}*))", + "STORAGE_LDAP_GROUPMEMBERFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)(ownclouduuid={{.OpaqueId}}*))", + "STORAGE_LDAP_USERGROUPFILTER": "(&(objectclass=posixGroup)(objectclass=owncloud)(ownclouduuid={{.OpaqueId}}*))", + "STORAGE_LDAP_USER_SCHEMA_CN": "cn", + "STORAGE_LDAP_USER_SCHEMA_DISPLAYNAME": "displayname", + "STORAGE_LDAP_USER_SCHEMA_GID_NUMBER": "gidnumber", + "STORAGE_LDAP_USER_SCHEMA_MAIL": "mail", + "STORAGE_LDAP_USER_SCHEMA_UID_NUMBER": "uidnumber", + "STORAGE_LDAP_USER_SCHEMA_UID": "ownclouduuid", + "STORAGE_LDAP_LOGINFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)(|(uid={{login}})(mail={{login}})))", + "STORAGE_LDAP_USERATTRIBUTEFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)({{attr}}={{value}}))", + "STORAGE_LDAP_USERFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)(|(ownclouduuid={{.OpaqueId}})(uid={{.OpaqueId}})))", + "STORAGE_LDAP_USERFINDFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)(|(cn={{query}}*)(displayname={{query}}*)(mail={{query}}*)))", + # ownCloud storage driver + "STORAGE_HOME_DRIVER": "owncloudsql", + "STORAGE_USERS_DRIVER": "owncloudsql", + "STORAGE_METADATA_DRIVER": "ocis", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR": "/mnt/data/files", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR": "/tmp", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_SHARE_FOLDER": "/Shares", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_LAYOUT": "{{.Username}}", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBUSERNAME": "owncloud", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPASSWORD": "owncloud", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBHOST": "oc10-db", + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPORT": 3306, + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBNAME": "owncloud", + # TODO: redis is not yet supported + "STORAGE_USERS_DRIVER_OWNCLOUDSQL_REDIS_ADDR": "redis:6379", + # ownCloud storage readonly + # TODO: conflict with OWNCLOUDSQL -> https://github.com/owncloud/ocis/issues/2303 + "OCIS_STORAGE_READ_ONLY": "false", + # General oCIS config + "OCIS_LOG_LEVEL": "error", + "OCIS_URL": OCIS_URL, + "PROXY_TLS": "true", + # change default secrets + "OCIS_JWT_SECRET": "Pive-Fumkiu4", + "STORAGE_TRANSFER_SECRET": "replace-me-with-a-transfer-secret", + "OCIS_MACHINE_AUTH_API_KEY": "change-me-please", + "OCIS_INSECURE": "true", + "PROXY_ENABLE_BASIC_AUTH": "true", + } # Pass in "default" accounts_hash_difficulty to not set this environment variable. # That will allow OCIS to use whatever its built-in default is. @@ -1439,7 +1509,9 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = []): "apk add mailcap", # install /etc/mime.types "ocis/bin/ocis server", ], + "user": "33:33", "volumes": volumes, + "depends_on": depends_on, }, { "name": "wait-for-ocis-server", @@ -1879,8 +1951,8 @@ QUAY_IO_KEYCLOAK = "quay.io/keycloak/keycloak:latest" POSTGRES = "postgres:alpine" # configs -OCIS_URL = "https://ocis:9200" -OCIS_DOMAIN = "ocis:9200" +OCIS_URL = "https://ocis-server:9200" +OCIS_DOMAIN = "ocis-server:9200" OC10_URL = "http://oc10:8080" PARALLEL_DEPLOY_CONFIG_PATH = "/drone/src/tests/parallelDeployAcceptance/drone" @@ -1981,7 +2053,13 @@ def parallelDeployAcceptancePipeline(ctx): oC10Server() + owncloudLog() + fixSharedDataPermissions() + - latestOcisServer() + + ocisServer( + "ocis", + "default", + [stepVolumeOC10OCISData, stepVolumeOCISConfig], + ["fix-shared-data-permissions"], + True, + ) + parallelAcceptance(environment) + failEarly(ctx, early_fail), "services": oc10DbService() + @@ -2118,7 +2196,7 @@ def latestOcisServer(): return [ { - "name": "ocis", + "name": "ocis-server", "image": OC_OCIS, "environment": environment, "detach": True, @@ -2130,13 +2208,13 @@ def latestOcisServer(): stepVolumeOCISConfig, ], "user": "33:33", - "depends_on": ["fix-permissions"], + "depends_on": ["fix-shared-data-permissions"], }, { "name": "wait-for-ocis", "image": OC_CI_WAIT_FOR, "commands": [ - "wait-for -it ocis:9200 -t 300", + "wait-for -it ocis-server:9200 -t 300", ], "depends_on": ["wait-for-oc10"], }, @@ -2293,7 +2371,7 @@ def owncloudLog(): def fixSharedDataPermissions(): return [{ - "name": "fix-permissions", + "name": "fix-shared-data-permissions", "image": OC_CI_PHP, "pull": "always", "commands": [