From 8a33993d766fe0bf4c520ab5b546dcee9d44e0ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Sat, 12 Dec 2020 20:24:51 +0000 Subject: [PATCH] Make all paths configurable and default to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- accounts/pkg/flagset/flagset.go | 2 +- changelog/unreleased/make-all-paths-configurable.md | 5 +++++ docs/extensions/storage/storages.md | 2 +- docs/ocis/getting-started.md | 2 +- konnectd/pkg/flagset/flagset.go | 2 +- ocis-pkg/indexer/index/cs3/non_unique.go | 2 +- ocis/docker-compose-eos-test.yml | 2 +- settings/pkg/flagset/flagset.go | 2 +- settings/ui/tests/run-acceptance-test.sh | 2 +- storage/pkg/command/frontend.go | 2 +- storage/pkg/command/sharing.go | 5 +++++ storage/pkg/command/storagehome.go | 1 + storage/pkg/command/storagemetadata.go | 1 + storage/pkg/command/storageusers.go | 1 + storage/pkg/config/config.go | 10 ++++++---- storage/pkg/flagset/driverlocal.go | 2 +- storage/pkg/flagset/driverowncloud.go | 4 ++-- storage/pkg/flagset/frontend.go | 11 +++++++++-- storage/pkg/flagset/sharing.go | 9 ++++++++- storage/pkg/flagset/storagehome.go | 7 +++++++ storage/pkg/flagset/storagemetadata.go | 9 ++++++++- storage/pkg/flagset/storageusers.go | 7 +++++++ store/pkg/flagset/flagset.go | 2 +- store/pkg/service/v0/service.go | 2 +- 24 files changed, 72 insertions(+), 22 deletions(-) create mode 100644 changelog/unreleased/make-all-paths-configurable.md diff --git a/accounts/pkg/flagset/flagset.go b/accounts/pkg/flagset/flagset.go index 8ff3687194..b58bcce547 100644 --- a/accounts/pkg/flagset/flagset.go +++ b/accounts/pkg/flagset/flagset.go @@ -143,7 +143,7 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag { &cli.StringFlag{ Name: "storage-disk-path", Value: "", - Usage: "Path on the local disk, e.g. /var/tmp/ocis-accounts", + Usage: "Path on the local disk, e.g. /var/tmp/ocis/accounts", EnvVars: []string{"ACCOUNTS_STORAGE_DISK_PATH"}, Destination: &cfg.Repo.Disk.Path, }, diff --git a/changelog/unreleased/make-all-paths-configurable.md b/changelog/unreleased/make-all-paths-configurable.md new file mode 100644 index 0000000000..ef450114da --- /dev/null +++ b/changelog/unreleased/make-all-paths-configurable.md @@ -0,0 +1,5 @@ +Change: Make all paths configurable and default to `/var/tmp/ocis//...` + +Aligned all services to use a subdir of `/var/tmp/ocis/` by default. Also made some missing temp paths configurable via env vars and config flags. + +https://github.com/owncloud/ocis/pulls/1080 diff --git a/docs/extensions/storage/storages.md b/docs/extensions/storage/storages.md index 2992dfda0e..a05917a32f 100644 --- a/docs/extensions/storage/storages.md +++ b/docs/extensions/storage/storages.md @@ -120,7 +120,7 @@ To provide the other storage aspects we plan to implement a FUSE overlay filesys This is the current default storage driver. While it implements the file tree (using redis, including id based lookup), ETag propagation, trash, versions and sharing (including expiry) using the data directory layout of ownCloud 10 it has [known limitations](https://github.com/owncloud/core/issues/28095) that cannot be fixed without changing the actual layout on disk. To setup it up properly in a distributed fashion, the storage-home and the storage-oc need to share the same underlying FS. Their "data" counterparts also need access to the same shared FS. -For a simple docker-compose setup, you can create a volume which will be used by the "storage-storage-home", "storage-storage-home-data", "storage-storage-oc" and "storage-storage-oc-data" containers. Using the `owncloud/ocis` docker image, the volume would need to be hooked in the `/var/tmp/ocis` folder insde the containers. +For a simple docker-compose setup, you can create a volume which will be used by the "storage-storage-home", "storage-storage-home-data", "storage-storage-oc" and "storage-storage-oc-data" containers. Using the `owncloud/ocis` docker image, the volume would need to be hooked in the `/var/tmp/ocis` folder inside the containers. - tree provided by a POSIX filesystem - file layout is mapped to the old ownCloud 10 layout diff --git a/docs/ocis/getting-started.md b/docs/ocis/getting-started.md index a61c1d8f11..2c68f85e8d 100644 --- a/docs/ocis/getting-started.md +++ b/docs/ocis/getting-started.md @@ -29,7 +29,7 @@ chmod +x ocis ./ocis server ``` -The default primary storage location is `/var/tmp/`. You can change that value by configuration. +The default primary storage location is `/var/tmp/ocis`. You can change that value by configuration. ### Docker diff --git a/konnectd/pkg/flagset/flagset.go b/konnectd/pkg/flagset/flagset.go index 243d3e3166..0dce352911 100644 --- a/konnectd/pkg/flagset/flagset.go +++ b/konnectd/pkg/flagset/flagset.go @@ -251,7 +251,7 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag { Name: "identifier-client-path", Usage: "Path to the identifier web client base folder", EnvVars: []string{"KONNECTD_IDENTIFIER_CLIENT_PATH"}, - Value: "/var/tmp/konnectd", + Value: "/var/tmp/ocis/konnectd", Destination: &cfg.Konnectd.IdentifierClientPath, }, &cli.StringFlag{ diff --git a/ocis-pkg/indexer/index/cs3/non_unique.go b/ocis-pkg/indexer/index/cs3/non_unique.go index c93376891c..fd0e5e6306 100644 --- a/ocis-pkg/indexer/index/cs3/non_unique.go +++ b/ocis-pkg/indexer/index/cs3/non_unique.go @@ -45,7 +45,7 @@ type NonUnique struct { } // NewNonUniqueIndexWithOptions instantiates a new NonUniqueIndex instance. -// /var/tmp/ocis-accounts/index.cs3/Pets/Bro* +// /var/tmp/ocis/accounts/index.cs3/Pets/Bro* // ├── Brown/ // │ └── rebef-123 -> /var/tmp/testfiles-395764020/pets/rebef-123 // ├── Green/ diff --git a/ocis/docker-compose-eos-test.yml b/ocis/docker-compose-eos-test.yml index a8a341be0d..f796d8a4ca 100644 --- a/ocis/docker-compose-eos-test.yml +++ b/ocis/docker-compose-eos-test.yml @@ -44,7 +44,7 @@ services: # make accounts use ocis storage driver # TODO provision metadata storage in eos and switch to cs3 backend for accounts ACCOUNTS_LOG_LEVEL: debug - ACCOUNTS_STORAGE_DISK_PATH: /var/tmp/ocis-accounts + ACCOUNTS_STORAGE_DISK_PATH: /var/tmp/ocis/accounts # TODO make id the default in ocis-storage STORAGE_DRIVER_EOS_LAYOUT: "{{substr 0 1 .Id.OpaqueId}}/{{.Id.OpaqueId}}" STORAGE_FRONTEND_PUBLIC_URL: https://${OCIS_DOMAIN:-localhost}:9200 diff --git a/settings/pkg/flagset/flagset.go b/settings/pkg/flagset/flagset.go index ab3adfdc13..f4d04aba48 100644 --- a/settings/pkg/flagset/flagset.go +++ b/settings/pkg/flagset/flagset.go @@ -173,7 +173,7 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag { }, &cli.StringFlag{ Name: "data-path", - Value: "/var/tmp/ocis-settings", + Value: "/var/tmp/ocis/settings", Usage: "Mount path for the storage", EnvVars: []string{"SETTINGS_DATA_PATH"}, Destination: &cfg.Service.DataPath, diff --git a/settings/ui/tests/run-acceptance-test.sh b/settings/ui/tests/run-acceptance-test.sh index 939af07707..53f732533b 100755 --- a/settings/ui/tests/run-acceptance-test.sh +++ b/settings/ui/tests/run-acceptance-test.sh @@ -51,7 +51,7 @@ cp -r "$WEB_PATH/tests" "./$testFolder" export NODE_TLS_REJECT_UNAUTHORIZED='0' export SERVER_HOST=${SERVER_HOST:-https://localhost:9200} export BACKEND_HOST=${BACKEND_HOST:-https://localhost:9200} -export OCIS_SETTINGS_STORE=${OCIS_SETTINGS_STORE:-"/var/tmp/ocis-settings"} +export OCIS_SETTINGS_STORE=${OCIS_SETTINGS_STORE:-"/var/tmp/ocis/settings"} export RUN_ON_OCIS=true export TEST_TAGS=${TEST_TAGS:-"not @skip"} diff --git a/storage/pkg/command/frontend.go b/storage/pkg/command/frontend.go index 60455238f2..7b122057b6 100644 --- a/storage/pkg/command/frontend.go +++ b/storage/pkg/command/frontend.go @@ -130,7 +130,7 @@ func Frontend(cfg *config.Config) *cli.Command { }, "ocdav": map[string]interface{}{ "prefix": cfg.Reva.Frontend.OCDavPrefix, - "chunk_folder": "/var/tmp/ocis/chunks", + "chunk_folder": cfg.Reva.OCDav.ChunkFolder, "files_namespace": cfg.Reva.OCDav.DavFilesNamespace, "webdav_namespace": cfg.Reva.OCDav.WebdavNamespace, "timeout": 86400, diff --git a/storage/pkg/command/sharing.go b/storage/pkg/command/sharing.go index 4d48fb8e3a..b8daaf1070 100644 --- a/storage/pkg/command/sharing.go +++ b/storage/pkg/command/sharing.go @@ -97,6 +97,11 @@ func Sharing(cfg *config.Config) *cli.Command { }, "publicshareprovider": map[string]interface{}{ "driver": cfg.Reva.Sharing.PublicDriver, + "drivers": map[string]interface{}{ + "json": map[string]interface{}{ + "file": cfg.Reva.Sharing.PublicJSONFile, + }, + }, }, }, }, diff --git a/storage/pkg/command/storagehome.go b/storage/pkg/command/storagehome.go index 3e7cee1922..2815c6d598 100644 --- a/storage/pkg/command/storagehome.go +++ b/storage/pkg/command/storagehome.go @@ -102,6 +102,7 @@ func StorageHome(cfg *config.Config) *cli.Command { "mount_id": cfg.Reva.StorageHome.MountID, "expose_data_server": cfg.Reva.StorageHome.ExposeDataServer, "data_server_url": cfg.Reva.StorageHome.DataServerURL, + "tmp_folder": cfg.Reva.StorageHome.TempFolder, }, }, }, diff --git a/storage/pkg/command/storagemetadata.go b/storage/pkg/command/storagemetadata.go index c21277e75b..51c5c790f9 100644 --- a/storage/pkg/command/storagemetadata.go +++ b/storage/pkg/command/storagemetadata.go @@ -111,6 +111,7 @@ func StorageMetadata(cfg *config.Config) *cli.Command { "driver": cfg.Reva.StorageMetadata.Driver, "drivers": drivers(cfg), "data_server_url": cfg.Reva.StorageMetadata.DataServerURL, + "tmp_folder": cfg.Reva.StorageMetadata.TempFolder, }, }, }, diff --git a/storage/pkg/command/storageusers.go b/storage/pkg/command/storageusers.go index 6e752adc0f..17c9a93fb8 100644 --- a/storage/pkg/command/storageusers.go +++ b/storage/pkg/command/storageusers.go @@ -102,6 +102,7 @@ func StorageUsers(cfg *config.Config) *cli.Command { "mount_id": cfg.Reva.StorageUsers.MountID, "expose_data_server": cfg.Reva.StorageUsers.ExposeDataServer, "data_server_url": cfg.Reva.StorageUsers.DataServerURL, + "tmp_folder": cfg.Reva.StorageUsers.TempFolder, }, }, }, diff --git a/storage/pkg/config/config.go b/storage/pkg/config/config.go index d60a166fa2..0a38f0f8fd 100644 --- a/storage/pkg/config/config.go +++ b/storage/pkg/config/config.go @@ -36,9 +36,10 @@ type StorageRegistry struct { // Sharing defines the available sharing configuration. type Sharing struct { Port - UserDriver string - UserJSONFile string - PublicDriver string + UserDriver string + UserJSONFile string + PublicDriver string + PublicJSONFile string } // Port defines the available port configuration. @@ -82,7 +83,7 @@ type FrontendPort struct { DatagatewayPrefix string OCDavPrefix string OCSPrefix string - OCSSharePrefix string + OCSSharePrefix string PublicURL string Middleware Middleware } @@ -282,6 +283,7 @@ type LDAPSchema struct { // OCDav defines the available ocdav configuration. type OCDav struct { + ChunkFolder string WebdavNamespace string DavFilesNamespace string } diff --git a/storage/pkg/flagset/driverlocal.go b/storage/pkg/flagset/driverlocal.go index 6acd21030d..6184fa2b38 100644 --- a/storage/pkg/flagset/driverlocal.go +++ b/storage/pkg/flagset/driverlocal.go @@ -10,7 +10,7 @@ func DriverLocalWithConfig(cfg *config.Config) []cli.Flag { return []cli.Flag{ &cli.StringFlag{ Name: "storage-local-root", - Value: "/var/tmp/ocis/local", + Value: "/var/tmp/ocis/storage/local", Usage: "the path to the local storage root", EnvVars: []string{"STORAGE_DRIVER_LOCAL_ROOT"}, Destination: &cfg.Reva.Storages.Local.Root, diff --git a/storage/pkg/flagset/driverowncloud.go b/storage/pkg/flagset/driverowncloud.go index c747291948..633c0b4df6 100644 --- a/storage/pkg/flagset/driverowncloud.go +++ b/storage/pkg/flagset/driverowncloud.go @@ -10,14 +10,14 @@ func DriverOwnCloudWithConfig(cfg *config.Config) []cli.Flag { return []cli.Flag{ &cli.StringFlag{ Name: "storage-owncloud-datadir", - Value: "/var/tmp/ocis/owncloud", + Value: "/var/tmp/ocis/storage/owncloud", Usage: "the path to the owncloud data directory", EnvVars: []string{"STORAGE_DRIVER_OWNCLOUD_DATADIR"}, Destination: &cfg.Reva.Storages.OwnCloud.Root, }, &cli.StringFlag{ Name: "storage-owncloud-uploadinfo-dir", - Value: "/var/tmp/ocis/uploadinfo", + Value: "/var/tmp/ocis/storage/uploadinfo", Usage: "the path to the tus upload info directory", EnvVars: []string{"STORAGE_DRIVER_OWNCLOUD_UPLOADINFO_DIR"}, Destination: &cfg.Reva.Storages.OwnCloud.UploadInfoDir, diff --git a/storage/pkg/flagset/frontend.go b/storage/pkg/flagset/frontend.go index cddbc5e5dc..6cfaf820af 100644 --- a/storage/pkg/flagset/frontend.go +++ b/storage/pkg/flagset/frontend.go @@ -30,11 +30,18 @@ func FrontendWithConfig(cfg *config.Config) []cli.Flag { // OCDav + &cli.StringFlag{ + Name: "chunk-folder", + Value: "/var/tmp/ocis/tmp/chunks", + Usage: "temp directory for chunked uploads", + EnvVars: []string{"STORAGE_CHUNK_FOLDER"}, + Destination: &cfg.Reva.OCDav.WebdavNamespace, + }, &cli.StringFlag{ Name: "webdav-namespace", Value: "/home/", Usage: "Namespace prefix for the /webdav endpoint", - EnvVars: []string{"WEBDAV_NAMESPACE"}, + EnvVars: []string{"STORAGE_WEBDAV_NAMESPACE"}, Destination: &cfg.Reva.OCDav.WebdavNamespace, }, @@ -44,7 +51,7 @@ func FrontendWithConfig(cfg *config.Config) []cli.Flag { Name: "dav-files-namespace", Value: "/users/", Usage: "Namespace prefix for the webdav /dav/files endpoint", - EnvVars: []string{"DAV_FILES_NAMESPACE"}, + EnvVars: []string{"STORAGE_DAV_FILES_NAMESPACE"}, Destination: &cfg.Reva.OCDav.DavFilesNamespace, }, diff --git a/storage/pkg/flagset/sharing.go b/storage/pkg/flagset/sharing.go index 3b6129f478..fac4fc0cc3 100644 --- a/storage/pkg/flagset/sharing.go +++ b/storage/pkg/flagset/sharing.go @@ -51,7 +51,7 @@ func SharingWithConfig(cfg *config.Config) []cli.Flag { }, &cli.StringFlag{ Name: "user-json-file", - Value: "/var/tmp/ocis/shares.json", + Value: "/var/tmp/ocis/storage/shares.json", Usage: "file used to persist shares for the UserShareProvider", EnvVars: []string{"STORAGE_SHARING_USER_JSON_FILE"}, Destination: &cfg.Reva.Sharing.UserJSONFile, @@ -63,6 +63,13 @@ func SharingWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_SHARING_PUBLIC_DRIVER"}, Destination: &cfg.Reva.Sharing.PublicDriver, }, + &cli.StringFlag{ + Name: "public-json-file", + Value: "/var/tmp/ocis/storage/publicshares.json", + Usage: "file used to persist shares for the PublicShareProvider", + EnvVars: []string{"STORAGE_SHARING_PUBLIC_JSON_FILE"}, + Destination: &cfg.Reva.Sharing.PublicJSONFile, + }, } flags = append(flags, TracingWithConfig(cfg)...) diff --git a/storage/pkg/flagset/storagehome.go b/storage/pkg/flagset/storagehome.go index 72d0d0c547..b6c0fe9fc4 100644 --- a/storage/pkg/flagset/storagehome.go +++ b/storage/pkg/flagset/storagehome.go @@ -111,6 +111,13 @@ func StorageHomeWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_HOME_HTTP_PREFIX"}, Destination: &cfg.Reva.StorageHome.HTTPPrefix, }, + &cli.StringFlag{ + Name: "tmp-folder", + Value: "/var/tmp/ocis/tmp/home", + Usage: "path to tmp folder", + EnvVars: []string{"STORAGE_HOME_TMP_FOLDER"}, + Destination: &cfg.Reva.StorageHome.TempFolder, + }, &cli.BoolFlag{ Name: "enable-home", Value: true, diff --git a/storage/pkg/flagset/storagemetadata.go b/storage/pkg/flagset/storagemetadata.go index f0585c023c..8119834474 100644 --- a/storage/pkg/flagset/storagemetadata.go +++ b/storage/pkg/flagset/storagemetadata.go @@ -50,6 +50,13 @@ func StorageMetadata(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_METADATA_HTTP_ADDR"}, Destination: &cfg.Reva.StorageMetadata.HTTPAddr, }, + &cli.StringFlag{ + Name: "tmp-folder", + Value: "/var/tmp/ocis/tmp/metadata", + Usage: "path to tmp folder", + EnvVars: []string{"STORAGE_METADATA_TMP_FOLDER"}, + Destination: &cfg.Reva.StorageMetadata.TempFolder, + }, &cli.StringFlag{ Name: "driver", Value: "ocis", @@ -91,7 +98,7 @@ func StorageMetadata(cfg *config.Config) []cli.Flag { flags = append(flags, &cli.StringFlag{ Name: "storage-root", - Value: "/var/tmp/ocis/metadata", + Value: "/var/tmp/ocis/storage/metadata", Usage: "the path to the metadata storage root", EnvVars: []string{"STORAGE_METADATA_ROOT"}, Destination: &cfg.Reva.Storages.Common.Root, diff --git a/storage/pkg/flagset/storageusers.go b/storage/pkg/flagset/storageusers.go index c5a53d198b..4e6c3be755 100644 --- a/storage/pkg/flagset/storageusers.go +++ b/storage/pkg/flagset/storageusers.go @@ -108,6 +108,13 @@ func StorageUsersWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"STORAGE_USERS_HTTP_PREFIX"}, Destination: &cfg.Reva.StorageUsers.HTTPPrefix, }, + &cli.StringFlag{ + Name: "tmp-folder", + Value: "/var/tmp/ocis/tmp/users", + Usage: "path to tmp folder", + EnvVars: []string{"STORAGE_USERS_TMP_FOLDER"}, + Destination: &cfg.Reva.StorageUsers.TempFolder, + }, // some drivers need to look up users at the gateway diff --git a/store/pkg/flagset/flagset.go b/store/pkg/flagset/flagset.go index b731319da0..cfb8c7bd4e 100644 --- a/store/pkg/flagset/flagset.go +++ b/store/pkg/flagset/flagset.go @@ -131,7 +131,7 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag { }, &cli.StringFlag{ Name: "data-path", - Value: "/var/tmp/ocis-store", + Value: "/var/tmp/ocis/store", Usage: "location of the store data path", EnvVars: []string{"STORE_DATA_PATH"}, Destination: &cfg.Datapath, diff --git a/store/pkg/service/v0/service.go b/store/pkg/service/v0/service.go index c773cc66c8..f8e8576a2a 100644 --- a/store/pkg/service/v0/service.go +++ b/store/pkg/service/v0/service.go @@ -242,7 +242,7 @@ func (s *Service) Tables(ctx context.Context, in *proto.TablesRequest, out *prot } // TODO sanitize key. As it may contain invalid characters, such as slashes. -// file: /var/tmp/ocis-store/databases/{database}/{table}/{record.key}. +// file: /var/tmp/ocis/store/databases/{database}/{table}/{record.key}. func getID(database string, table string, key string) string { // TODO sanitize input. return filepath.Join(database, table, key)