mirror/opencloud
1
0
Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-03-23 09:52:12 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #118 from opencloud-eu/switch-to-decomposed-decomposed-s3

Browse Source 
Switch from ocis/s3ng to decomposed/decomposed_s3 storage drivers
This commit is contained in:
Jörn Friedrich Dreyer
2025-01-27 16:43:40 +01:00
committed by GitHub
parent 1bd1c38cf7 f00537dfc6
commit 3900a08049
26 changed files with 334 additions and 319 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
opencloud/README.md
View File

@@ -55,7 +55,7 @@ A node that doesn't have any (or malformed) metadata.
This command provides additional options:
* `-b` / `--blobstore`\
Allows specifying the blobstore to use. Defaults to `ocis`. Empty blobs will not be checked. Can also be switched to `s3ng`, but needs addtional envvar configuration (see the `storage-users` service for more details).
Allows specifying the blobstore to use. Defaults to `decomposed`. Empty blobs will not be checked. Can also be switched to `decomposed_s3`, but needs addtional envvar configuration (see the `storage-users` service for more details).
* `--fail`\
Exits with non-zero exit code if inconsistencies are found. Useful for automation.
@@ -86,7 +86,7 @@ This command provides additional options:
* `--dry-run` (default: `true`)\
Do not remove any revisions but print the revisions that would be removed.
* `-b` / `--blobstore`\
Allows specifying the blobstore to use. Defaults to `ocis`. Can be switched to `s3ng` but needs addtional envvar configuration (see the `storage-users` service for more details).
Allows specifying the blobstore to use. Defaults to `decomposed`. Can be switched to `decomposed_s3` but needs addtional envvar configuration (see the `storage-users` service for more details).
* `-v` / `--verbose`\
Prints additional information about the revisions that are removed.
* `--glob-mechanism` (default: `glob`\

4
opencloud/pkg/backup/provider.go
View File

@@ -9,7 +9,7 @@ import (
"strings"
"sync"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/shamaton/msgpack/v2"
)
@@ -132,7 +132,7 @@ func (dp *DataProvider) getBlobPath(path string) (string, Inconsistency) {
// FIXME: how to check if metadata is complete?
if bid := m["user.ocis.blobid"]; string(bid) != "" {
if bid := m["user.oc.blobid"]; string(bid) != "" {
spaceID, _ := getIDsFromPath(filepath.Join(dp.discpath, path))
return dp.lbs.Path(&node.Node{BlobID: string(bid), SpaceID: spaceID}), ""
}

22
opencloud/pkg/command/backup.go
View File

@@ -9,8 +9,8 @@ import (
"github.com/opencloud-eu/opencloud/pkg/config"
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
"github.com/opencloud-eu/opencloud/pkg/config/parser"
ocbs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis/blobstore"
s3bs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/blobstore"
ocbs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/blobstore"
s3bs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed_s3/blobstore"
"github.com/urfave/cli/v2"
)
@@ -47,8 +47,8 @@ func ConsistencyCommand(cfg *config.Config) *cli.Command {
&cli.StringFlag{
Name: "blobstore",
Aliases: []string{"b"},
Usage: "the blobstore type. Can be (none, ocis, s3ng). Default ocis",
Value: "ocis",
Usage: "the blobstore type. Can be (none, decomposed, decomposed_s3). Default decomposed",
Value: "decomposed",
},
&cli.BoolFlag{
Name: "fail",
@@ -67,16 +67,16 @@ func ConsistencyCommand(cfg *config.Config) *cli.Command {
err error
)
switch c.String("blobstore") {
case "s3ng":
case "decomposed_s3":
bs, err = s3bs.New(
cfg.StorageUsers.Drivers.S3NG.Endpoint,
cfg.StorageUsers.Drivers.S3NG.Region,
cfg.StorageUsers.Drivers.S3NG.Bucket,
cfg.StorageUsers.Drivers.S3NG.AccessKey,
cfg.StorageUsers.Drivers.S3NG.SecretKey,
cfg.StorageUsers.Drivers.DecomposedS3.Endpoint,
cfg.StorageUsers.Drivers.DecomposedS3.Region,
cfg.StorageUsers.Drivers.DecomposedS3.Bucket,
cfg.StorageUsers.Drivers.DecomposedS3.AccessKey,
cfg.StorageUsers.Drivers.DecomposedS3.SecretKey,
s3bs.Options{},
)
case "ocis":
case "decomposed":
bs, err = ocbs.New(basePath)
case "none":
bs = nil

12
opencloud/pkg/command/decomposedfs.go
View File

@@ -15,13 +15,13 @@ import (
"github.com/opencloud-eu/opencloud/pkg/config"
revactx "github.com/opencloud-eu/reva/v2/pkg/ctx"
"github.com/opencloud-eu/reva/v2/pkg/storage/cache"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis/blobstore"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/blobstore"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/opencloud-eu/reva/v2/pkg/store"
"github.com/rs/zerolog"

6
opencloud/pkg/command/migrate.go
View File

@@ -23,9 +23,9 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/shareid"
"github.com/opencloud-eu/reva/v2/pkg/share/manager/registry"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/migrator"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
"github.com/pkg/errors"
"github.com/rs/zerolog"

22
opencloud/pkg/command/revisions.go
View File

@@ -11,9 +11,9 @@ import (
"github.com/opencloud-eu/opencloud/pkg/config"
"github.com/opencloud-eu/opencloud/pkg/config/configlog"
"github.com/opencloud-eu/opencloud/pkg/config/parser"
ocbs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis/blobstore"
ocbs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed/blobstore"
s3bs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/decomposed_s3/blobstore"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup"
s3bs "github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/blobstore"
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
"github.com/urfave/cli/v2"
)
@@ -56,8 +56,8 @@ func PurgeRevisionsCommand(cfg *config.Config) *cli.Command {
&cli.StringFlag{
Name: "blobstore",
Aliases: []string{"b"},
Usage: "the blobstore type. Can be (none, ocis, s3ng). Default ocis. Note: When using s3ng this needs same configuration as the storage-users service",
Value: "ocis",
Usage: "the blobstore type. Can be (none, decomposed, decomposed_s3). Default decomposed. Note: When using s3ng this needs same configuration as the storage-users service",
Value: "decomposed",
},
&cli.BoolFlag{
Name: "dry-run",
@@ -93,16 +93,16 @@ func PurgeRevisionsCommand(cfg *config.Config) *cli.Command {
err error
)
switch c.String("blobstore") {
case "s3ng":
case "decomposed_s3":
bs, err = s3bs.New(
cfg.StorageUsers.Drivers.S3NG.Endpoint,
cfg.StorageUsers.Drivers.S3NG.Region,
cfg.StorageUsers.Drivers.S3NG.Bucket,
cfg.StorageUsers.Drivers.S3NG.AccessKey,
cfg.StorageUsers.Drivers.S3NG.SecretKey,
cfg.StorageUsers.Drivers.DecomposedS3.Endpoint,
cfg.StorageUsers.Drivers.DecomposedS3.Region,
cfg.StorageUsers.Drivers.DecomposedS3.Bucket,
cfg.StorageUsers.Drivers.DecomposedS3.AccessKey,
cfg.StorageUsers.Drivers.DecomposedS3.SecretKey,
s3bs.Options{},
)
case "ocis":
case "decomposed":
bs, err = ocbs.New(basePath)
case "none":
bs = nil

4
opencloud/pkg/revisions/revisions.go
View File

@@ -9,7 +9,7 @@ import (
"strings"
"sync"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
"github.com/shamaton/msgpack/v2"
)
@@ -247,7 +247,7 @@ func getBlobID(path string) (string, error) {
return "", err
}
if bid := m["user.ocis.blobid"]; string(bid) != "" {
if bid := m["user.oc.blobid"]; string(bid) != "" {
return string(bid), nil
}

2
opencloud/pkg/revisions/revisions_test.go
View File

@@ -9,7 +9,7 @@ import (
"testing"
"github.com/google/uuid"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup"
"github.com/test-go/testify/require"
)