mirror/opencloud
1
0
Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-07-19 12:04:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add config for repo implementations

Browse Source
This commit is contained in:
Benedikt Kulmann
2020-09-29 14:40:50 +02:00
parent 4428b771bc
commit 5f37acbb00
8 changed files with 172 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
accounts/go.mod
View File

@@ -5,6 +5,8 @@ go 1.13
require (
github.com/CiscoM31/godata v0.0.0-20191007193734-c2c4ebb1b415
github.com/blevesearch/bleve v1.0.9
github.com/cs3org/go-cs3apis v0.0.0-20200730121022-c4f3d4f7ddfd
github.com/cs3org/reva v1.1.0
github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect
@@ -32,6 +34,7 @@ require (
github.com/tredoe/osutil v1.0.5
golang.org/x/net v0.0.0-20200822124328-c89045814202
google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad
google.golang.org/grpc v1.31.0
google.golang.org/protobuf v1.25.0
)

16
accounts/pkg/config/config.go
View File

@@ -61,6 +61,21 @@ type Log struct {
Color bool
}
type Repo struct {
Disk Disk
CS3 CS3
}
type Disk struct {
Path string
}
type CS3 struct {
ProviderAddr string
DriverURL string
DataPrefix string
}
// Config merges all Account config parameters.
type Config struct {
LDAP LDAP
@@ -70,6 +85,7 @@ type Config struct {
Asset Asset
Log Log
TokenManager TokenManager
Repo Repo
}
// New returns a new config.

28
accounts/pkg/flagset/flagset.go
View File

@@ -99,6 +99,34 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag {
EnvVars: []string{"ACCOUNTS_JWT_SECRET"},
Destination: &cfg.TokenManager.JWTSecret,
},
&cli.StringFlag{
Name: "storage-disk-path",
Value: "",
Usage: "Path on the local disk, e.g. /var/tmp/ocis-accounts",
EnvVars: []string{"ACCOUNTS_STORAGE_CS3_PROVIDER_ADDR"},
Destination: &cfg.Repo.CS3.ProviderAddr,
},
&cli.StringFlag{
Name: "storage-cs3-provider-addr",
Value: "0.0.0.0:9215",
Usage: "bind address for the metadata storage provider",
EnvVars: []string{"ACCOUNTS_STORAGE_CS3_PROVIDER_ADDR"},
Destination: &cfg.Repo.CS3.ProviderAddr,
},
&cli.StringFlag{
Name: "storage-cs3-driver-url",
Value: "http://localhost:9216",
Usage: "http endpoint of the metadata storage, without trailing slash",
EnvVars: []string{"ACCOUNTS_STORAGE_CS3_DRIVER_URL"},
Destination: &cfg.Repo.CS3.DriverURL,
},
&cli.StringFlag{
Name: "storage-cs3-data-prefix",
Value: "data",
Usage: "path prefix for the http endpoint of the metadata storage, without leading slash",
EnvVars: []string{"ACCOUNTS_STORAGE_CS3_DATA_PREFIX"},
Destination: &cfg.Repo.CS3.DataPrefix,
},
}
}

19
accounts/pkg/service/v0/service.go
View File

@@ -51,12 +51,14 @@ func New(opts ...Option) (s *Service, err error) {
roleManager = &m
}
serviceID := cfg.GRPC.Namespace + "." + cfg.Server.Name
s = &Service{
id: cfg.GRPC.Namespace + "." + cfg.Server.Name,
id: serviceID,
log: logger,
Config: cfg,
RoleService: roleService,
RoleManager: roleManager,
repo: createMetadataStorage(serviceID, cfg, logger),
}
// build an index
@@ -398,6 +400,19 @@ func assignRoleToUser(accountID, roleID string, rs settings.RoleService, logger
return true
}
func createMetadataStorage(serviceID string, cfg *config.Config, logger log.Logger) storage.Repo {
// for now we detect the used storage implementation based on which storage is configured
// the config with defaults needs to be checked last
if cfg.Repo.Disk.Path != "" {
return storage.NewDiskRepo(serviceID, cfg, logger)
}
repo, err := storage.NewCS3Repo(serviceID, cfg)
if err != nil {
logger.Fatal().Err(err).Msg("cs3 storage was configured but failed to start")
}
return repo
}
// Service implements the AccountsServiceHandler interface
type Service struct {
id string
@@ -406,7 +421,7 @@ type Service struct {
index bleve.Index
RoleService settings.RoleService
RoleManager *roles.Manager
repo storage.DiskRepo
repo storage.Repo
}
func cleanupID(id string) (string, error) {

100
accounts/pkg/storage/cs3.go
View File

@@ -12,37 +12,41 @@ import (
"github.com/cs3org/reva/pkg/token"
"github.com/cs3org/reva/pkg/token/manager/jwt"
merrors "github.com/micro/go-micro/v2/errors"
"github.com/owncloud/ocis/accounts/pkg/config"
"github.com/owncloud/ocis/accounts/pkg/proto/v0"
"google.golang.org/grpc/metadata"
"io/ioutil"
"net/http"
"path"
)
type CS3Repo struct {
serviceID string
dataPath string
rootPath string
serviceID string
cfg *config.Config
tm token.Manager
storageClient provider.ProviderAPIClient
}
func NewCS3Repo(secret string) (Repo, error) {
func NewCS3Repo(serviceID string, cfg *config.Config) (Repo, error) {
tokenManager, err := jwt.New(map[string]interface{}{
"secret": "Pive-Fumkiu4",
"secret": cfg.TokenManager.JWTSecret,
})
if err != nil {
return nil, err
}
client, err := pool.GetStorageProviderServiceClient("localhost:9185")
client, err := pool.GetStorageProviderServiceClient(cfg.Repo.CS3.ProviderAddr)
if err != nil {
return nil, err
}
return CS3Repo{tm: tokenManager, storageClient: client}, nil
return CS3Repo{
serviceID: serviceID,
cfg: cfg,
tm: tokenManager,
storageClient: client,
}, nil
}
func (r CS3Repo) WriteAccount(ctx context.Context, a *proto.Account) (err error) {
@@ -52,7 +56,7 @@ func (r CS3Repo) WriteAccount(ctx context.Context, a *proto.Account) (err error)
}
ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t)
if err := r.makeRootDirIfNotExist(ctx); err != nil {
if err := r.makeRootDirIfNotExist(ctx, accountsFolder); err != nil {
return err
}
@@ -61,7 +65,7 @@ func (r CS3Repo) WriteAccount(ctx context.Context, a *proto.Account) (err error)
return merrors.InternalServerError(r.serviceID, "could not marshal account: %v", err.Error())
}
ureq, err := http.NewRequest("PUT", fmt.Sprintf("http://localhost:9187/data/accounts/%s", a.Id), bytes.NewReader(by))
ureq, err := http.NewRequest("PUT", r.accountUrl(a.Id), bytes.NewReader(by))
if err != nil {
return err
}
@@ -86,7 +90,7 @@ func (r CS3Repo) LoadAccount(ctx context.Context, id string, a *proto.Account) (
ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t)
ureq, err := http.NewRequest("GET", fmt.Sprintf("http://localhost:9187/data/accounts/%s", id), nil)
ureq, err := http.NewRequest("GET", r.accountUrl(id), nil)
if err != nil {
return err
}
@@ -135,33 +139,6 @@ func (r CS3Repo) DeleteAccount(ctx context.Context, id string) (err error) {
return nil
}
func (r CS3Repo) DeleteGroup(ctx context.Context, id string) (err error) {
t, err := r.authenticate(ctx)
if err != nil {
return err
}
ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t)
ureq, err := http.NewRequest("DELETE", fmt.Sprintf("http://localhost:9187/data/groups/%s", id), nil)
if err != nil {
return err
}
ureq.Header.Add("x-access-token", t)
cl := http.Client{
Transport: http.DefaultTransport,
}
resp, err := cl.Do(ureq)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
func (r CS3Repo) WriteGroup(ctx context.Context, g *proto.Group) (err error) {
t, err := r.authenticate(ctx)
if err != nil {
@@ -169,7 +146,7 @@ func (r CS3Repo) WriteGroup(ctx context.Context, g *proto.Group) (err error) {
}
ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t)
if err := r.makeRootDirIfNotExist(ctx); err != nil {
if err := r.makeRootDirIfNotExist(ctx, groupsFolder); err != nil {
return err
}
@@ -178,7 +155,7 @@ func (r CS3Repo) WriteGroup(ctx context.Context, g *proto.Group) (err error) {
return merrors.InternalServerError(r.serviceID, "could not marshal account: %v", err.Error())
}
ureq, err := http.NewRequest("PUT", fmt.Sprintf("http://localhost:9187/data/groups/%s", g.Id), bytes.NewReader(by))
ureq, err := http.NewRequest("PUT", r.groupUrl(g.Id), bytes.NewReader(by))
if err != nil {
return err
}
@@ -203,7 +180,7 @@ func (r CS3Repo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err
ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t)
ureq, err := http.NewRequest("GET", fmt.Sprintf("http://localhost:9187/data/groups/%s", id), nil)
ureq, err := http.NewRequest("GET", r.groupUrl(id), nil)
if err != nil {
return err
}
@@ -231,6 +208,33 @@ func (r CS3Repo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err
return nil
}
func (r CS3Repo) DeleteGroup(ctx context.Context, id string) (err error) {
t, err := r.authenticate(ctx)
if err != nil {
return err
}
ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t)
ureq, err := http.NewRequest("DELETE", r.groupUrl(id), nil)
if err != nil {
return err
}
ureq.Header.Add("x-access-token", t)
cl := http.Client{
Transport: http.DefaultTransport,
}
resp, err := cl.Do(ureq)
if err != nil {
return err
}
defer resp.Body.Close()
return nil
}
func (r CS3Repo) authenticate(ctx context.Context) (token string, err error) {
return r.tm.MintToken(ctx, &user.User{
Id: &user.UserId{},
@@ -238,9 +242,17 @@ func (r CS3Repo) authenticate(ctx context.Context) (token string, err error) {
})
}
func (r CS3Repo) makeRootDirIfNotExist(ctx context.Context) error {
func (r CS3Repo) accountUrl(id string) string {
return path.Join(r.cfg.Repo.CS3.DriverURL, r.cfg.Repo.CS3.DataPrefix, accountsFolder, id)
}
func (r CS3Repo) groupUrl(id string) string {
return path.Join(r.cfg.Repo.CS3.DriverURL, r.cfg.Repo.CS3.DataPrefix, groupsFolder, id)
}
func (r CS3Repo) makeRootDirIfNotExist(ctx context.Context, folder string) error {
var rootPathRef = &provider.Reference{
Spec: &provider.Reference_Path{Path: "/meta/accounts"},
Spec: &provider.Reference_Path{Path: fmt.Sprintf("/meta/%v", folder)},
}
resp, err := r.storageClient.Stat(ctx, &provider.StatRequest{

13
accounts/pkg/storage/cs3_test.go
View File

@@ -2,13 +2,20 @@ package storage
import (
"context"
"github.com/owncloud/ocis/accounts/pkg/config"
"github.com/owncloud/ocis/accounts/pkg/proto/v0"
"github.com/stretchr/testify/assert"
"testing"
)
var cfg = &config.Config{
TokenManager: config.TokenManager{
JWTSecret: "Pive-Fumkiu4",
},
}
func TestCS3Repo_WriteAccount(t *testing.T) {
r, err := NewCS3Repo("fooo")
r, err := NewCS3Repo(cfg)
assert.NoError(t, err)
err = r.WriteAccount(context.Background(), &proto.Account{
@@ -22,7 +29,7 @@ func TestCS3Repo_WriteAccount(t *testing.T) {
}
func TestCS3Repo_LoadAccount(t *testing.T) {
r, err := NewCS3Repo("fooo")
r, err := NewCS3Repo(cfg)
assert.NoError(t, err)
err = r.WriteAccount(context.Background(), &proto.Account{
@@ -42,7 +49,7 @@ func TestCS3Repo_LoadAccount(t *testing.T) {
}
func TestCS3Repo_DeleteAccount(t *testing.T) {
r, err := NewCS3Repo("fooo")
r, err := NewCS3Repo(cfg)
assert.NoError(t, err)
err = r.WriteAccount(context.Background(), &proto.Account{

71
accounts/pkg/storage/disk.go
View File

@@ -3,28 +3,30 @@ package storage
import (
"context"
"encoding/json"
merrors "github.com/micro/go-micro/v2/errors"
"github.com/owncloud/ocis/accounts/pkg/proto/v0"
"github.com/rs/zerolog"
"github.com/owncloud/ocis/accounts/pkg/config"
"io/ioutil"
"os"
"path/filepath"
"sync"
merrors "github.com/micro/go-micro/v2/errors"
"github.com/owncloud/ocis/accounts/pkg/proto/v0"
olog "github.com/owncloud/ocis/ocis-pkg/log"
)
var groupLock sync.Mutex
type DiskRepo struct {
serviceID string
dataPath string
log zerolog.Logger
cfg *config.Config
log olog.Logger
}
func New(serviceID string, dataPath string, log zerolog.Logger) DiskRepo {
func NewDiskRepo(serviceID string, cfg *config.Config, log olog.Logger) DiskRepo {
return DiskRepo{
serviceID: serviceID,
dataPath: dataPath,
log: log.With().Str("id", serviceID).Logger(),
cfg: cfg,
log: log,
}
}
@@ -38,7 +40,7 @@ func (r DiskRepo) WriteAccount(ctx context.Context, a *proto.Account) (err error
return merrors.InternalServerError(r.serviceID, "could not marshal account: %v", err.Error())
}
path := filepath.Join(r.dataPath, "accounts", a.Id)
path := filepath.Join(r.cfg.Repo.Disk.Path, accountsFolder, a.Id)
if err = ioutil.WriteFile(path, bytes, 0600); err != nil {
return merrors.InternalServerError(r.serviceID, "could not write account: %v", err.Error())
@@ -48,7 +50,7 @@ func (r DiskRepo) WriteAccount(ctx context.Context, a *proto.Account) (err error
// LoadAccount from the storage
func (r DiskRepo) LoadAccount(ctx context.Context, id string, a *proto.Account) (err error) {
path := filepath.Join(r.dataPath, "accounts", id)
path := filepath.Join(r.cfg.Repo.Disk.Path, accountsFolder, id)
var data []byte
if data, err = ioutil.ReadFile(path); err != nil {
@@ -61,6 +63,17 @@ func (r DiskRepo) LoadAccount(ctx context.Context, id string, a *proto.Account)
return
}
// DeleteAccount from the storage
func (r DiskRepo) DeleteAccount(ctx context.Context, id string) (err error) {
path := filepath.Join(r.cfg.Repo.Disk.Path, accountsFolder, id)
if err = os.Remove(path); err != nil {
r.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove account")
return merrors.InternalServerError(r.serviceID, "could not remove account: %v", err.Error())
}
return nil
}
// WriteGroup persists a given group to the storage
func (r DiskRepo) WriteGroup(ctx context.Context, g *proto.Group) (err error) {
// leave only the member id
@@ -71,7 +84,7 @@ func (r DiskRepo) WriteGroup(ctx context.Context, g *proto.Group) (err error) {
return merrors.InternalServerError(r.serviceID, "could not marshal group: %v", err.Error())
}
path := filepath.Join(r.dataPath, "groups", g.Id)
path := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder, g.Id)
groupLock.Lock()
defer groupLock.Unlock()
@@ -83,7 +96,7 @@ func (r DiskRepo) WriteGroup(ctx context.Context, g *proto.Group) (err error) {
// LoadGroup from the storage
func (r DiskRepo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err error) {
path := filepath.Join(r.dataPath, "groups", id)
path := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder, id)
groupLock.Lock()
defer groupLock.Unlock()
@@ -99,12 +112,22 @@ func (r DiskRepo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err
return
}
func (r DiskRepo) DeleteGroup(ctx context.Context, id string) (err error) {
path := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder, id)
if err = os.Remove(path); err != nil {
r.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove group")
return merrors.InternalServerError(r.serviceID, "could not remove group: %v", err.Error())
}
return nil
}
// deflateMemberOf replaces the groups of a user with an instance that only contains the id
func (r DiskRepo) deflateMemberOf(a *proto.Account) {
if a == nil {
return
}
deflated := []*proto.Group{}
var deflated []*proto.Group
for i := range a.MemberOf {
if a.MemberOf[i].Id != "" {
deflated = append(deflated, &proto.Group{Id: a.MemberOf[i].Id})
@@ -116,32 +139,12 @@ func (r DiskRepo) deflateMemberOf(a *proto.Account) {
a.MemberOf = deflated
}
func (r DiskRepo) DeleteAccount(ctx context.Context, id string) (err error) {
path := filepath.Join(r.dataPath, "accounts", id)
if err = os.Remove(path); err != nil {
r.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove account")
return merrors.InternalServerError(r.serviceID, "could not remove account: %v", err.Error())
}
return nil
}
func (r DiskRepo) DeleteGroup(ctx context.Context, id string) (err error) {
path := filepath.Join(r.dataPath, "groups", id)
if err = os.Remove(path); err != nil {
r.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove group")
return merrors.InternalServerError(r.serviceID, "could not remove group: %v", err.Error())
}
return nil
}
// deflateMembers replaces the users of a group with an instance that only contains the id
func (r DiskRepo) deflateMembers(g *proto.Group) {
if g == nil {
return
}
deflated := []*proto.Account{}
var deflated []*proto.Account
for i := range g.Members {
if g.Members[i].Id != "" {
deflated = append(deflated, &proto.Account{Id: g.Members[i].Id})

5
accounts/pkg/storage/repo.go
View File

@@ -5,6 +5,11 @@ import (
"github.com/owncloud/ocis/accounts/pkg/proto/v0"
)
const (
accountsFolder = "accounts"
groupsFolder = "groups"
)
type Repo interface {
WriteAccount(ctx context.Context, a *proto.Account) (err error)
LoadAccount(ctx context.Context, id string, a *proto.Account) (err error)