Merge pull request #2241 from owncloud/remove_eos

remove EOS stuff
This commit is contained in:
Willy Kloucek
2021-12-07 13:17:33 +01:00
committed by GitHub
12 changed files with 0 additions and 643 deletions

View File

@@ -12,8 +12,6 @@ exclude_paths:
- 'idp/scripts/**'
- 'settings/rollup.config.js'
- 'accounts/rollup.config.js'
- 'ocis/docker/eos-ocis/Dockerfile'
- 'ocis/docker/eos-ocis-dev/Dockerfile'
- 'deployments/**'
- 'tests/acceptance/expected-failures-*.md'
- 'tests/acceptance/features/bootstrap/**'

View File

@@ -140,7 +140,6 @@ def main(ctx):
build_release_pipelines = \
dockerReleases(ctx) + \
[dockerEos(ctx)] + \
binaryReleases(ctx) + \
[releaseSubmodule(ctx)]
@@ -921,78 +920,6 @@ def dockerRelease(ctx, arch):
"volumes": [pipelineVolumeGo],
}
def dockerEos(ctx):
return {
"kind": "pipeline",
"type": "docker",
"name": "docker-eos-ocis",
"platform": {
"os": "linux",
"arch": "amd64",
},
"steps": skipIfUnchanged(ctx, "build-docker") +
makeNodeGenerate("") +
makeGoGenerate("") + [
{
"name": "build",
"image": OC_CI_GOLANG,
"commands": [
"make -C ocis release-linux-docker-amd64",
],
},
{
"name": "dryrun-eos-ocis",
"image": "plugins/docker:latest",
"settings": {
"dry_run": True,
"context": "ocis",
"tags": "linux-eos-ocis",
"dockerfile": "ocis/docker/eos-ocis/Dockerfile",
"repo": "owncloud/eos-ocis",
},
"when": {
"ref": {
"include": [
"refs/pull/**",
],
},
},
},
{
"name": "docker-eos-ocis",
"image": "plugins/docker:latest",
"settings": {
"username": {
"from_secret": "docker_username",
},
"password": {
"from_secret": "docker_password",
},
"auto_tag": True,
"context": "ocis",
"dockerfile": "ocis/docker/eos-ocis/Dockerfile",
"repo": "owncloud/eos-ocis",
},
"when": {
"ref": {
"exclude": [
"refs/pull/**",
],
},
},
},
],
"depends_on": getPipelineNames(testOcisModules(ctx) + testPipelines(ctx)),
"trigger": {
"ref": [
"refs/heads/master",
"refs/tags/v*",
"refs/pull/**",
],
},
"volumes": [pipelineVolumeGo],
}
def binaryReleases(ctx):
pipelines = []
for os in config["binaryReleases"]["os"]:

View File

@@ -1,179 +0,0 @@
---
version: '3.5'
networks:
testnet:
name: testnet
services:
ocis:
container_name: ocis
## Comment out either the 'build:' section or the 'image:' line. They are mutually exclusive.
## Using an image is prefered, as this is the published way of building and faster.
image: owncloud/eos-ocis:${EOS_OCIS_TAG:-1.0.0-rc5}
#build:
# context: ./docker/eos-ocis
# args:
# BRANCH: ${BRANCH:-v1.0.0-rc5}
tty: true
privileged: true
stdin_open: true
ports:
- 9200:9200
env_file:
- ./config/eos-docker.env
hostname: ocis
networks:
- testnet
environment:
# ocis log level will be used for all services
OCIS_LOG_LEVEL: debug
# domain setup
# TODO currently the below lines hardcode the port to 9200, use an OCIS_URL that includes protocol and port
OCIS_URL: https://${OCIS_DOMAIN:-localhost:9200}
# make home and users storages use eos
STORAGE_HOME_DRIVER: eoshome
STORAGE_USERS_DRIVER: eos
# make accounts use ocis storage driver
# TODO provision metadata storage in eos and switch to cs3 backend for accounts
ACCOUNTS_LOG_LEVEL: debug
ACCOUNTS_STORAGE_DISK_PATH: /var/tmp/ocis/accounts
# TODO make id the default in ocis-storage
STORAGE_DRIVER_EOS_LAYOUT: "{{substr 0 1 .Id.OpaqueId}}/{{.Id.OpaqueId}}"
# common eos settings used for both drivers: eos and eoshome
STORAGE_DRIVER_EOS_MASTER_URL: ${EOS_MGM_URL:-root://mgm-master.testnet:1094}
STORAGE_DRIVER_EOS_SLAVE_URL: ${EOS_MGM_URL:-root://mgm-master.testnet:1094}
STORAGE_DRIVER_EOS_NAMESPACE: "/eos/dockertest/reva/users"
# the eos end xrdcopy binaries use this env var to find the eos mgm
EOS_MGM_URL: ${EOS_MGM_URL:-root://mgm-master.testnet:1094}
# TODO without this the /etc/nclcd.conf file is empty
LDAP_BINDDN: "cn=reva,ou=sysusers,dc=ocis,dc=test"
LDAP_BINDPW: "reva"
mgm-master:
container_name: mgm-master
image: owncloud/eos-mgm:4.8.26
tty: true
privileged: true
stdin_open: true
env_file:
- ./config/eos-docker.env
hostname: mgm-master.testnet
networks:
- testnet
volumes:
- eos-mgm-master-log:/var/log/eos
- eos-mgm-master-config:/var/eos/config
- eos-mgm-master-ns-queue:/var/eos/ns-queue
# this volume kills mgm-master during startup
# - ./e/master/var/eos/md:/var/eos/md
environment:
EOS_SET_MASTER: 1
mq-master:
container_name: mq-master
image: owncloud/eos-mq:4.8.26
tty: true
privileged: true
stdin_open: true
env_file:
- ./config/eos-docker.env
hostname: mq-master.testnet
networks:
- testnet
volumes:
- eos-mq-master-log:/var/log/eos
- eos-mq-master-config:/var/eos/config
- eos-mq-master-ns-queue:/var/eos/ns-queue
environment:
EOS_SET_MASTER: 1
fst:
container_name: fst
image: owncloud/eos-fst:4.8.26
tty: true
privileged: true
stdin_open: true
env_file:
- ./config/eos-docker.env
hostname: fst.testnet
networks:
- testnet
volumes:
- eos-fst-log:/var/log/eos
- eos-fst-disks:/disks
environment:
EOS_MGM_URL: "root://mgm-master.testnet"
quark-1:
container_name: quark-1
image: owncloud/eos-qdb:4.8.26
tty: true
privileged: true
stdin_open: true
env_file:
- ./config/eos-docker.env
hostname: quark-1.testnet
networks:
- testnet
volumes:
- eos-quarkdb1:/var/lib/quarkdb
environment:
EOS_QDB_DIR: "/var/lib/quarkdb/eosns"
EOS_QDB_PORT: "7777"
EOS_QDB_MODE: "raft"
EOS_QDB_CLUSTER_ID: "3d659c1a-e70f-43f0-bed4-941a2ca0765b"
EOS_QDB_NODES: "quark-1.testnet:7777,quark-2.testnet:7777,quark-3.testnet:7777"
quark-2:
container_name: quark-2
image: owncloud/eos-qdb:4.8.26
tty: true
privileged: true
stdin_open: true
env_file:
- ./config/eos-docker.env
hostname: quark-2.testnet
networks:
- testnet
volumes:
- eos-quarkdb2:/var/lib/quarkdb
environment:
EOS_QDB_DIR: "/var/lib/quarkdb/eosns"
EOS_QDB_PORT: "7777"
EOS_QDB_MODE: "raft"
EOS_QDB_CLUSTER_ID: "3d659c1a-e70f-43f0-bed4-941a2ca0765b"
EOS_QDB_NODES: "quark-1.testnet:7777,quark-2.testnet:7777,quark-3.testnet:7777"
quark-3:
container_name: quark-3
image: owncloud/eos-qdb:4.8.26
tty: true
privileged: true
stdin_open: true
env_file:
- ./config/eos-docker.env
hostname: quark-3.testnet
networks:
- testnet
volumes:
- eos-quarkdb3:/var/lib/quarkdb
environment:
EOS_QDB_DIR: "/var/lib/quarkdb/eosns"
EOS_QDB_PORT: "7777"
EOS_QDB_MODE: "raft"
EOS_QDB_CLUSTER_ID: "3d659c1a-e70f-43f0-bed4-941a2ca0765b"
EOS_QDB_NODES: "quark-1.testnet:7777,quark-2.testnet:7777,quark-3.testnet:7777"
volumes:
eos-mgm-master-log:
eos-mgm-master-config:
eos-mgm-master-ns-queue:
eos-mq-master-log:
eos-mq-master-config:
eos-mq-master-ns-queue:
eos-fst-log:
eos-fst-disks:
eos-quarkdb1:
eos-quarkdb2:
eos-quarkdb3:

View File

@@ -1,177 +0,0 @@
---
version: '3.5'
networks:
testnet:
name: testnet
services:
ocis:
container_name: ocis
image: owncloud/eos-ocis-dev:latest
tty: true
privileged: true
stdin_open: true
ports:
- 9200:9200
env_file:
- ./config/eos-docker.env
hostname: ocis
networks:
- testnet
volumes:
- .:/ocis
- ../accounts:/accounts
- ../glauth:/glauth
- ../idp:/idp
- ../web:/web
- ../ocis-pkg:/ocis-pkg
- ../ocs:/ocs
- ../proxy:/proxy
- ../settings:/settings
- ../store:/store
- ../thumbnails:/thumbnails
- ../webdav:/webdav
- ../storage:/storage
environment:
# ocis log level will be used for all services
OCIS_LOG_LEVEL: debug
# domain setup
# TODO currently the below lines hardcode the port to 9200, use an OCIS_URL that includes protocol and port
OCIS_URL: https://${OCIS_DOMAIN:-localhost:9200}
# TODO make id the default in ocis-storage
STORAGE_DRIVER_EOS_LAYOUT: "{{substr 0 1 .Id.OpaqueId}}/{{.Id.OpaqueId}}"
# common eos settings used for both drivers: eos and eoshome
STORAGE_DRIVER_EOS_MASTER_URL: ${EOS_MGM_URL:-root://mgm-master.testnet:1094}
STORAGE_DRIVER_EOS_SLAVE_URL: ${EOS_MGM_URL:-root://mgm-master.testnet:1094}
STORAGE_DRIVER_EOS_NAMESPACE: "/eos/dockertest/reva/users"
# the eos end xrdcopy binaries use this env var to find the eos mgm
EOS_MGM_URL: ${EOS_MGM_URL:-root://mgm-master.testnet:1094}
mgm-master:
container_name: mgm-master
image: owncloud/eos-mgm:4.8.26
tty: true
privileged: true
stdin_open: true
env_file:
- ./config/eos-docker.env
hostname: mgm-master.testnet
networks:
- testnet
volumes:
- eos-mgm-master-log:/var/log/eos
- eos-mgm-master-config:/var/eos/config
- eos-mgm-master-ns-queue:/var/eos/ns-queue
# this volume kills mgm-master during startup
# - ./e/master/var/eos/md:/var/eos/md
environment:
EOS_SET_MASTER: 1
mq-master:
container_name: mq-master
image: owncloud/eos-mq:4.8.26
tty: true
privileged: true
stdin_open: true
env_file:
- ./config/eos-docker.env
hostname: mq-master.testnet
networks:
- testnet
volumes:
- eos-mq-master-log:/var/log/eos
- eos-mq-master-config:/var/eos/config
- eos-mq-master-ns-queue:/var/eos/ns-queue
environment:
EOS_SET_MASTER: 1
fst:
container_name: fst
image: owncloud/eos-fst:4.8.26
tty: true
privileged: true
stdin_open: true
env_file:
- ./config/eos-docker.env
hostname: fst.testnet
networks:
- testnet
volumes:
- eos-fst-log:/var/log/eos
- eos-fst-disks:/disks
environment:
EOS_MGM_URL: "root://mgm-master.testnet"
quark-1:
container_name: quark-1
image: owncloud/eos-qdb:4.8.26
tty: true
privileged: true
stdin_open: true
env_file:
- ./config/eos-docker.env
hostname: quark-1.testnet
networks:
- testnet
volumes:
- eos-quarkdb1:/var/lib/quarkdb
environment:
EOS_QDB_DIR: "/var/lib/quarkdb/eosns"
EOS_QDB_PORT: "7777"
EOS_QDB_MODE: "raft"
EOS_QDB_CLUSTER_ID: "3d659c1a-e70f-43f0-bed4-941a2ca0765b"
EOS_QDB_NODES: "quark-1.testnet:7777,quark-2.testnet:7777,quark-3.testnet:7777"
quark-2:
container_name: quark-2
image: owncloud/eos-qdb:4.8.26
tty: true
privileged: true
stdin_open: true
env_file:
- ./config/eos-docker.env
hostname: quark-2.testnet
networks:
- testnet
volumes:
- eos-quarkdb2:/var/lib/quarkdb
environment:
EOS_QDB_DIR: "/var/lib/quarkdb/eosns"
EOS_QDB_PORT: "7777"
EOS_QDB_MODE: "raft"
EOS_QDB_CLUSTER_ID: "3d659c1a-e70f-43f0-bed4-941a2ca0765b"
EOS_QDB_NODES: "quark-1.testnet:7777,quark-2.testnet:7777,quark-3.testnet:7777"
quark-3:
container_name: quark-3
image: owncloud/eos-qdb:4.8.26
tty: true
privileged: true
stdin_open: true
env_file:
- ./config/eos-docker.env
hostname: quark-3.testnet
networks:
- testnet
volumes:
- eos-quarkdb3:/var/lib/quarkdb
environment:
EOS_QDB_DIR: "/var/lib/quarkdb/eosns"
EOS_QDB_PORT: "7777"
EOS_QDB_MODE: "raft"
EOS_QDB_CLUSTER_ID: "3d659c1a-e70f-43f0-bed4-941a2ca0765b"
EOS_QDB_NODES: "quark-1.testnet:7777,quark-2.testnet:7777,quark-3.testnet:7777"
volumes:
eos-mgm-master-log:
eos-mgm-master-config:
eos-mgm-master-ns-queue:
eos-mq-master-log:
eos-mq-master-config:
eos-mq-master-ns-queue:
eos-fst-log:
eos-fst-disks:
eos-quarkdb1:
eos-quarkdb2:
eos-quarkdb3:

View File

@@ -1,24 +0,0 @@
FROM owncloud/eos-base:4.8.26
ENV CGO_ENABLED=1
ENV GOOS=linux
RUN rpm --rebuilddb && yum -y install \
wget \
time \
make \
gcc \
git
RUN wget -q https://dl.google.com/go/go1.15.7.linux-amd64.tar.gz
RUN mkdir -p /usr/local/bin
RUN tar xf go1.15.7.linux-amd64.tar.gz -C /usr/local
RUN ln -s /usr/local/go/bin/* /usr/local/bin
COPY entrypoint /entrypoint
COPY start-ldap /start-ldap
VOLUME [ "/ocis" ]
WORKDIR /ocis

View File

@@ -1,88 +0,0 @@
# Docker image for oCIS development with eos
Image is based on [owncloud/eos-base](https://hub.docker.com/r/owncloud/eos-base) from [eos-stack](https://github.com/owncloud-docker/eos-stack)
{{< hint info >}}
On MacOS do not mount a local folder to the `/var/tmp/reva/` path. The fuse driver used by docker [does not support extended attributes](https://docs.docker.com/v18.09/docker-for-mac/osxfs/). See [#182](https://github.com/owncloud/ocis/issues/182) for more details.
{{< /hint >}}
## Build
```shell
docker build -t owncloud/eos-ocis-dev:latest .
```
## Publish
```shell
docker push owncloud/eos-ocis-dev:latest
```
## Maintainer
* [Felix Böhm](https://github.com/felixboehm)
## Disclaimer
Use only for development or testing. Setup is not secured nor tested.
## oCIS development on eos
### Setup oCIS
To build and run your local ocis code with default storage driver
```shell
docker run --rm -ti --name ocis -v $PWD:/ocis -p 9200:9200 owncloud/eos-ocis-dev
```
ocis will use the owncloud storage driver and store files in the container at /var/tmp/reva/data/<username>/files
Data is here: `docker exec -it ocis ll /var/tmp/reva/`
Alternative: With the [docker-compose.yml file in ocis repo](https://github.com/owncloud/ocis/blob/master/docker-compose.yml) you can also start ocis via compose:
```shell
docker-compose up -d ocis
```
Now try to list the running services
```shell
docker-compose exec ocis ./bin/ocis list
```
## Setup eos storage
1. Start the eos cluster and ocis via the compose stack
```shell
docker-compose up -d
```
2. Configure to use eos storage driver instead of default storage driver
* kill the home storage and data providers. we need to switch them to the eoshome driver:
```shell
docker-compose exec ocis ./bin/ocis kill reva-storage-home
docker-compose exec ocis ./bin/ocis kill reva-storage-home-data
```
* restart them with the eoshome driver and a new layout:
```shell
docker-compose exec -e REVA_STORAGE_EOS_LAYOUT="{{substr 0 1 .Username}}/{{.Username}}" -e REVA_STORAGE_HOME_DRIVER=eoshome -d ocis ./bin/ocis run reva-storage-home
docker-compose exec -e REVA_STORAGE_EOS_LAYOUT="{{substr 0 1 .Username}}/{{.Username}}" -e REVA_STORAGE_HOME_DATA_DRIVER=eoshome -d ocis ./bin/ocis run reva-storage-home-data
```
* restart the reva frontend with a new namespace (pointing to the eos storage provider) for the dav files endpoint
```shell
docker-compose exec ocis ./bin/ocis kill reva-frontend
docker-compose exec -e DAV_FILES_NAMESPACE="/eos/" -d ocis ./bin/ocis run reva-frontend
```
* login with `einstein / relativity`, upload a file to einsteins home and verify the file is there using
```shell
docker-compose exec ocis eos ls -l /eos/dockertest/reva/users/e/einstein/
-rw-r--r-- 1 einstein users 10 Jul 1 15:24 newfile.txt
```

View File

@@ -1,6 +0,0 @@
#!/bin/sh
make build
/start-ldap &
exec bin/ocis server

View File

@@ -1,17 +0,0 @@
#!/bin/sh
LDAP_BINDDN=${LDAP_BINDDN:-cn=reva,ou=sysusers,dc=ocis,dc=test}
LDAP_BINDPW=${LDAP_BINDPW:-reva}
echo "Waiting for EOS MGM"
echo "until nc -z -w 3 $EOS_MGM_ALIAS 1094; do sleep 2; done;" > /wait-for-mgm
chmod +x /wait-for-mgm
/wait-for-mgm;
echo "----- [ocis] LDAP setup -----";
authconfig --enableldap --enableldapauth --ldapserver=${EOS_LDAP_HOST} --ldapbasedn="dc=ocis,dc=test" --update;
sed -i "s/#binddn cn=.*/binddn ${LDAP_BINDDN}/" /etc/nslcd.conf
sed -i "s/#bindpw .*/bindpw ${LDAP_BINDPW}/" /etc/nslcd.conf
# start in debug mode;
nslcd -d

View File

@@ -1,7 +0,0 @@
FROM owncloud/eos-base:4.8.26
RUN mkdir -p /usr/local/bin
COPY dist/binaries/ocis-linux-amd64 /usr/local/bin/ocis
RUN chmod +x /usr/local/bin/ocis
COPY docker/eos-ocis/setup /setup
COPY docker/eos-ocis/entrypoint /entrypoint

View File

@@ -1,30 +0,0 @@
# Docker image for oCIS running on eos
Image is based on [owncloud/eos-base](https://hub.docker.com/r/owncloud/eos-base) from [eos-stack](https://github.com/owncloud-docker/eos-stack)
## Build
Build owncloud/ocis master branch
```shell
docker build -t owncloud/eos-ocis:latest .
```
Or build a certain branch / tag
```shell
docker build -t owncloud/eos-ocis:1.0.0 --build-arg BRANCH=v1.0.0./eos-ocis
```
## Publish
```shell
docker push owncloud/eos-ocis:latest
```
## Maintainer
* [Felix Böhm](https://github.com/felixboehm)
## Disclaimer
Use only for development or testing. Setup is not secured nor tested.
## Example Usage
See <https://github.com/owncloud-docker/compose-playground/tree/master/examples/eos-compose>

View File

@@ -1,17 +0,0 @@
#!/bin/sh
# todo: log all ocis services to stdout
echo "Waiting for EOS MGM"
echo "until nc -z -w 3 $EOS_MGM_ALIAS 1094; do sleep 2; done;" > /wait-for-mgm
chmod +x /wait-for-mgm
mkdir -p /var/log/ocis
(/wait-for-mgm; /setup > /var/log/ocis/setup.log; cat /var/log/ocis/setup.log ) &
# TODO do we still need to precreate this folder?
mkdir -p /var/tmp/reva
echo "----- [ocis] Starting oCIS -----"
# todo start ocis as daemon not as root
exec ocis server

View File

@@ -1,23 +0,0 @@
#!/bin/bash
set -x
echo "----- [ocis] LDAP setup -----"
authconfig --enableldap --enableldapauth --ldapserver=${EOS_LDAP_HOST} --ldapbasedn="dc=ocis,dc=test" --update
sed -i "s/#binddn cn=.*/binddn ${LDAP_BINDDN}/" /etc/nslcd.conf
sed -i "s/#bindpw .*/bindpw ${LDAP_BINDPW}/" /etc/nslcd.conf
# start in debug mode
nslcd -d &
# echo "----- [ocis] eos setup -----"
# eos -r 0 0 -b vid set membership daemon -uids adm
# eos -r 0 0 -b vid set membership daemon -gids adm
# eos -r 0 0 -b vid set membership daemon +sudo
# eos -r 0 0 -b vid set map -unix "<pwd>" vuid:0 vgid:0
# # eos -r 0 0 -b vid add gateway ocis.testnet
# eos -r 0 0 -b vid add gateway ocis
# todo start ocis as daemon not as root
# eos -r 0 0 -b vid set membership root -uids adm
# eos -r 0 0 -b vid set membership root -gids adm
# eos -r 0 0 -b vid set membership root +sudo