build ocis server for parallel deploy tests

2026-03-05 15:57:01 -05:00 · 2022-01-12 14:23:40 +05:45
parent 0de87a940f
commit 9147c96ce8
1 changed files with 121 additions and 43 deletions
--- a/.drone.star
+++ b/.drone.star
@@ -156,18 +156,17 @@ def main(ctx):
    test_pipelines = \
        cancelPreviousBuilds() + \
        [buildOcisBinaryForTesting(ctx)] + \
-        testOcisModules(ctx) + \
        testPipelines(ctx)

-    build_release_pipelines = \
-        dockerReleases(ctx) + \
-        binaryReleases(ctx) + \
-        [releaseSubmodule(ctx)]
+    # build_release_pipelines = \
+    #     dockerReleases(ctx) + \
+    #     binaryReleases(ctx) + \
+    #     [releaseSubmodule(ctx)]

-    build_release_helpers = [
-        changelog(ctx),
-        docs(ctx),
-    ]
+    # build_release_helpers = [
+    #     changelog(ctx),
+    #     docs(ctx),
+    # ]

    test_pipelines.append(
        pipelineDependsOn(
@@ -176,7 +175,7 @@ def main(ctx):
        ),
    )

-    pipelines = test_pipelines + build_release_pipelines + build_release_helpers
+    pipelines = test_pipelines  # + build_release_pipelines + build_release_helpers

    pipelines = \
        pipelines + \
@@ -236,21 +235,21 @@ def testPipelines(ctx):
    if "skip" not in config["localApiTests"] or not config["localApiTests"]["skip"]:
        pipelines = [
            localApiTests(ctx, "ocis", "apiAccountsHashDifficulty", "default"),
-            localApiTests(ctx, "ocis", "apiSpaces", "default"),
-            localApiTests(ctx, "ocis", "apiArchiver", "default"),
+            # localApiTests(ctx, "ocis", "apiSpaces", "default"),
+            # localApiTests(ctx, "ocis", "apiArchiver", "default"),
        ]

-    if "skip" not in config["apiTests"] or not config["apiTests"]["skip"]:
-        pipelines += apiTests(ctx)
+    # if "skip" not in config["apiTests"] or not config["apiTests"]["skip"]:
+    #     pipelines += apiTests(ctx)

-    if "skip" not in config["uiTests"] or not config["uiTests"]["skip"]:
-        pipelines += uiTests(ctx)
+    # if "skip" not in config["uiTests"] or not config["uiTests"]["skip"]:
+    #     pipelines += uiTests(ctx)

-    if "skip" not in config["accountsUITests"] or not config["accountsUITests"]["skip"]:
-        pipelines.append(accountsUITests(ctx))
+    # if "skip" not in config["accountsUITests"] or not config["accountsUITests"]["skip"]:
+    #     pipelines.append(accountsUITests(ctx))

-    if "skip" not in config["settingsUITests"] or not config["settingsUITests"]["skip"]:
-        pipelines.append(settingsUITests(ctx))
+    # if "skip" not in config["settingsUITests"] or not config["settingsUITests"]["skip"]:
+    #     pipelines.append(settingsUITests(ctx))

    if "skip" not in config["parallelApiTests"] or not config["parallelApiTests"]["skip"]:
        pipelines += parallelDeployAcceptancePipeline(ctx)
@@ -1405,22 +1404,93 @@ def notify(ctx):
        },
    }

-def ocisServer(storage, accounts_hash_difficulty = 4, volumes = []):
-    environment = {
-        "OCIS_URL": "https://ocis-server:9200",
-        "STORAGE_USERS_DRIVER": "%s" % (storage),
-        "STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root",
-        "STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR": "/srv/app/tmp/ocis/owncloud/data",
-        "STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users",
-        "STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata",
-        "STORAGE_SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json",
-        "PROXY_ENABLE_BASIC_AUTH": True,
-        "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json",
-        "IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml",
-        "OCIS_LOG_LEVEL": "error",
-        "SETTINGS_DATA_PATH": "/srv/app/tmp/ocis/settings",
-        "OCIS_INSECURE": "true",
-    }
+def ocisServer(storage, accounts_hash_difficulty = 4, volumes = [], depends_on = [], testing_parallel_deploy = False):
+    if not testing_parallel_deploy:
+        environment = {
+            "OCIS_URL": "https://ocis-server:9200",
+            "STORAGE_HOME_DRIVER": "%s" % (storage),
+            "STORAGE_USERS_DRIVER": "%s" % (storage),
+            "STORAGE_USERS_DRIVER_LOCAL_ROOT": "/srv/app/tmp/ocis/local/root",
+            "STORAGE_USERS_DRIVER_OWNCLOUD_DATADIR": "/srv/app/tmp/ocis/owncloud/data",
+            "STORAGE_USERS_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/users",
+            "STORAGE_METADATA_DRIVER_OCIS_ROOT": "/srv/app/tmp/ocis/storage/metadata",
+            "STORAGE_SHARING_USER_JSON_FILE": "/srv/app/tmp/ocis/shares.json",
+            "PROXY_ENABLE_BASIC_AUTH": True,
+            "WEB_UI_CONFIG": "/drone/src/tests/config/drone/ocis-config.json",
+            "IDP_IDENTIFIER_REGISTRATION_CONF": "/drone/src/tests/config/drone/identifier-registration.yml",
+            "OCIS_LOG_LEVEL": "error",
+            "SETTINGS_DATA_PATH": "/srv/app/tmp/ocis/settings",
+            "OCIS_INSECURE": "true",
+        }
+    else:
+        environment = {
+            # Keycloak IDP specific configuration
+            "PROXY_OIDC_ISSUER": "https://keycloak/auth/realmsowncloud",
+            "WEB_OIDC_AUTHORITY": "https://keycloak/auth/realms/owncloud",
+            "WEB_OIDC_CLIENT_ID": "ocis-web",
+            "WEB_OIDC_METADATA_URL": "https://keycloak/auth/realms/owncloud/.well-known/openid-configuration",
+            "STORAGE_OIDC_ISSUER": "https://keycloak",
+            "STORAGE_LDAP_IDP": "https://keycloak/auth/realms/owncloud",
+            "WEB_OIDC_SCOPE": "openid profile email owncloud",
+            # LDAP bind
+            "STORAGE_LDAP_HOSTNAME": "openldap",
+            "STORAGE_LDAP_PORT": 636,
+            "STORAGE_LDAP_INSECURE": "true",
+            "STORAGE_LDAP_BIND_DN": "cn=admin,dc=owncloud,dc=com",
+            "STORAGE_LDAP_BIND_PASSWORD": "admin",
+            # LDAP user settings
+            "PROXY_AUTOPROVISION_ACCOUNTS": "true",  # automatically create users when they login
+            "PROXY_ACCOUNT_BACKEND_TYPE": "cs3",  # proxy should get users from CS3APIS (which gets it from LDAP)
+            "PROXY_USER_OIDC_CLAIM": "ocis.user.uuid",  # claim was added in Keycloak
+            "PROXY_USER_CS3_CLAIM": "userid",  # equals STORAGE_LDAP_USER_SCHEMA_UID
+            "STORAGE_LDAP_BASE_DN": "dc=owncloud,dc=com",
+            "STORAGE_LDAP_GROUP_SCHEMA_DISPLAYNAME": "cn",
+            "STORAGE_LDAP_GROUP_SCHEMA_GID_NUMBER": "gidnumber",
+            "STORAGE_LDAP_GROUP_SCHEMA_GID": "cn",
+            "STORAGE_LDAP_GROUP_SCHEMA_MAIL": "mail",
+            "STORAGE_LDAP_GROUPATTRIBUTEFILTER": "(&(objectclass=posixGroup)(objectclass=owncloud)({{attr}}={{value}}))",
+            "STORAGE_LDAP_GROUPFILTER": "(&(objectclass=groupOfUniqueNames)(objectclass=owncloud)(ownclouduuid={{.OpaqueId}}*))",
+            "STORAGE_LDAP_GROUPMEMBERFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)(ownclouduuid={{.OpaqueId}}*))",
+            "STORAGE_LDAP_USERGROUPFILTER": "(&(objectclass=posixGroup)(objectclass=owncloud)(ownclouduuid={{.OpaqueId}}*))",
+            "STORAGE_LDAP_USER_SCHEMA_CN": "cn",
+            "STORAGE_LDAP_USER_SCHEMA_DISPLAYNAME": "displayname",
+            "STORAGE_LDAP_USER_SCHEMA_GID_NUMBER": "gidnumber",
+            "STORAGE_LDAP_USER_SCHEMA_MAIL": "mail",
+            "STORAGE_LDAP_USER_SCHEMA_UID_NUMBER": "uidnumber",
+            "STORAGE_LDAP_USER_SCHEMA_UID": "ownclouduuid",
+            "STORAGE_LDAP_LOGINFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)(|(uid={{login}})(mail={{login}})))",
+            "STORAGE_LDAP_USERATTRIBUTEFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)({{attr}}={{value}}))",
+            "STORAGE_LDAP_USERFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)(|(ownclouduuid={{.OpaqueId}})(uid={{.OpaqueId}})))",
+            "STORAGE_LDAP_USERFINDFILTER": "(&(objectclass=posixAccount)(objectclass=owncloud)(|(cn={{query}}*)(displayname={{query}}*)(mail={{query}}*)))",
+            # ownCloud storage driver
+            "STORAGE_HOME_DRIVER": "owncloudsql",
+            "STORAGE_USERS_DRIVER": "owncloudsql",
+            "STORAGE_METADATA_DRIVER": "ocis",
+            "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DATADIR": "/mnt/data/files",
+            "STORAGE_USERS_DRIVER_OWNCLOUDSQL_UPLOADINFO_DIR": "/tmp",
+            "STORAGE_USERS_DRIVER_OWNCLOUDSQL_SHARE_FOLDER": "/Shares",
+            "STORAGE_USERS_DRIVER_OWNCLOUDSQL_LAYOUT": "{{.Username}}",
+            "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBUSERNAME": "owncloud",
+            "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPASSWORD": "owncloud",
+            "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBHOST": "oc10-db",
+            "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBPORT": 3306,
+            "STORAGE_USERS_DRIVER_OWNCLOUDSQL_DBNAME": "owncloud",
+            # TODO: redis is not yet supported
+            "STORAGE_USERS_DRIVER_OWNCLOUDSQL_REDIS_ADDR": "redis:6379",
+            # ownCloud storage readonly
+            # TODO: conflict with OWNCLOUDSQL -> https://github.com/owncloud/ocis/issues/2303
+            "OCIS_STORAGE_READ_ONLY": "false",
+            # General oCIS config
+            "OCIS_LOG_LEVEL": "error",
+            "OCIS_URL": OCIS_URL,
+            "PROXY_TLS": "true",
+            # change default secrets
+            "OCIS_JWT_SECRET": "Pive-Fumkiu4",
+            "STORAGE_TRANSFER_SECRET": "replace-me-with-a-transfer-secret",
+            "OCIS_MACHINE_AUTH_API_KEY": "change-me-please",
+            "OCIS_INSECURE": "true",
+            "PROXY_ENABLE_BASIC_AUTH": "true",
+        }

    # Pass in "default" accounts_hash_difficulty to not set this environment variable.
    # That will allow OCIS to use whatever its built-in default is.
@@ -1439,7 +1509,9 @@ def ocisServer(storage, accounts_hash_difficulty = 4, volumes = []):
                "apk add mailcap",  # install /etc/mime.types
                "ocis/bin/ocis server",
            ],
+            "user": "33:33",
            "volumes": volumes,
+            "depends_on": depends_on,
        },
        {
            "name": "wait-for-ocis-server",
@@ -1879,8 +1951,8 @@ QUAY_IO_KEYCLOAK = "quay.io/keycloak/keycloak:latest"
 POSTGRES = "postgres:alpine"

 # configs
-OCIS_URL = "https://ocis:9200"
-OCIS_DOMAIN = "ocis:9200"
+OCIS_URL = "https://ocis-server:9200"
+OCIS_DOMAIN = "ocis-server:9200"
 OC10_URL = "http://oc10:8080"
 PARALLEL_DEPLOY_CONFIG_PATH = "/drone/src/tests/parallelDeployAcceptance/drone"

@@ -1981,7 +2053,13 @@ def parallelDeployAcceptancePipeline(ctx):
                         oC10Server() +
                         owncloudLog() +
                         fixSharedDataPermissions() +
-                         latestOcisServer() +
+                         ocisServer(
+                             "ocis",
+                             "default",
+                             [stepVolumeOC10OCISData, stepVolumeOCISConfig],
+                             ["fix-shared-data-permissions"],
+                             True,
+                         ) +
                         parallelAcceptance(environment) +
                         failEarly(ctx, early_fail),
                "services": oc10DbService() +
@@ -2118,7 +2196,7 @@ def latestOcisServer():

    return [
        {
-            "name": "ocis",
+            "name": "ocis-server",
            "image": OC_OCIS,
            "environment": environment,
            "detach": True,
@@ -2130,13 +2208,13 @@ def latestOcisServer():
                stepVolumeOCISConfig,
            ],
            "user": "33:33",
-            "depends_on": ["fix-permissions"],
+            "depends_on": ["fix-shared-data-permissions"],
        },
        {
            "name": "wait-for-ocis",
            "image": OC_CI_WAIT_FOR,
            "commands": [
-                "wait-for -it ocis:9200 -t 300",
+                "wait-for -it ocis-server:9200 -t 300",
            ],
            "depends_on": ["wait-for-oc10"],
        },
@@ -2293,7 +2371,7 @@ def owncloudLog():

 def fixSharedDataPermissions():
    return [{
-        "name": "fix-permissions",
+        "name": "fix-shared-data-permissions",
        "image": OC_CI_PHP,
        "pull": "always",
        "commands": [