diff --git a/accounts/go.mod b/accounts/go.mod index 81a05293ca..dc13f404b0 100644 --- a/accounts/go.mod +++ b/accounts/go.mod @@ -5,6 +5,8 @@ go 1.13 require ( github.com/CiscoM31/godata v0.0.0-20191007193734-c2c4ebb1b415 github.com/blevesearch/bleve v1.0.9 + github.com/cs3org/go-cs3apis v0.0.0-20200730121022-c4f3d4f7ddfd + github.com/cs3org/reva v1.1.0 github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect @@ -32,6 +34,7 @@ require ( github.com/tredoe/osutil v1.0.5 golang.org/x/net v0.0.0-20200822124328-c89045814202 google.golang.org/genproto v0.0.0-20200624020401-64a14ca9d1ad + google.golang.org/grpc v1.31.0 google.golang.org/protobuf v1.25.0 ) diff --git a/accounts/pkg/config/config.go b/accounts/pkg/config/config.go index fea59089e9..7237030450 100644 --- a/accounts/pkg/config/config.go +++ b/accounts/pkg/config/config.go @@ -61,6 +61,21 @@ type Log struct { Color bool } +type Repo struct { + Disk Disk + CS3 CS3 +} + +type Disk struct { + Path string +} + +type CS3 struct { + ProviderAddr string + DriverURL string + DataPrefix string +} + // Config merges all Account config parameters. type Config struct { LDAP LDAP @@ -70,6 +85,7 @@ type Config struct { Asset Asset Log Log TokenManager TokenManager + Repo Repo } // New returns a new config. diff --git a/accounts/pkg/flagset/flagset.go b/accounts/pkg/flagset/flagset.go index 4e0b41212b..98da16cc67 100644 --- a/accounts/pkg/flagset/flagset.go +++ b/accounts/pkg/flagset/flagset.go @@ -99,6 +99,34 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag { EnvVars: []string{"ACCOUNTS_JWT_SECRET"}, Destination: &cfg.TokenManager.JWTSecret, }, + &cli.StringFlag{ + Name: "storage-disk-path", + Value: "", + Usage: "Path on the local disk, e.g. /var/tmp/ocis-accounts", + EnvVars: []string{"ACCOUNTS_STORAGE_CS3_PROVIDER_ADDR"}, + Destination: &cfg.Repo.CS3.ProviderAddr, + }, + &cli.StringFlag{ + Name: "storage-cs3-provider-addr", + Value: "0.0.0.0:9215", + Usage: "bind address for the metadata storage provider", + EnvVars: []string{"ACCOUNTS_STORAGE_CS3_PROVIDER_ADDR"}, + Destination: &cfg.Repo.CS3.ProviderAddr, + }, + &cli.StringFlag{ + Name: "storage-cs3-driver-url", + Value: "http://localhost:9216", + Usage: "http endpoint of the metadata storage, without trailing slash", + EnvVars: []string{"ACCOUNTS_STORAGE_CS3_DRIVER_URL"}, + Destination: &cfg.Repo.CS3.DriverURL, + }, + &cli.StringFlag{ + Name: "storage-cs3-data-prefix", + Value: "data", + Usage: "path prefix for the http endpoint of the metadata storage, without leading slash", + EnvVars: []string{"ACCOUNTS_STORAGE_CS3_DATA_PREFIX"}, + Destination: &cfg.Repo.CS3.DataPrefix, + }, } } diff --git a/accounts/pkg/service/v0/service.go b/accounts/pkg/service/v0/service.go index 6b50bf3266..eb2aa566e4 100644 --- a/accounts/pkg/service/v0/service.go +++ b/accounts/pkg/service/v0/service.go @@ -51,12 +51,14 @@ func New(opts ...Option) (s *Service, err error) { roleManager = &m } + serviceID := cfg.GRPC.Namespace + "." + cfg.Server.Name s = &Service{ - id: cfg.GRPC.Namespace + "." + cfg.Server.Name, + id: serviceID, log: logger, Config: cfg, RoleService: roleService, RoleManager: roleManager, + repo: createMetadataStorage(serviceID, cfg, logger), } // build an index @@ -398,6 +400,19 @@ func assignRoleToUser(accountID, roleID string, rs settings.RoleService, logger return true } +func createMetadataStorage(serviceID string, cfg *config.Config, logger log.Logger) storage.Repo { + // for now we detect the used storage implementation based on which storage is configured + // the config with defaults needs to be checked last + if cfg.Repo.Disk.Path != "" { + return storage.NewDiskRepo(serviceID, cfg, logger) + } + repo, err := storage.NewCS3Repo(serviceID, cfg) + if err != nil { + logger.Fatal().Err(err).Msg("cs3 storage was configured but failed to start") + } + return repo +} + // Service implements the AccountsServiceHandler interface type Service struct { id string @@ -406,7 +421,7 @@ type Service struct { index bleve.Index RoleService settings.RoleService RoleManager *roles.Manager - repo storage.DiskRepo + repo storage.Repo } func cleanupID(id string) (string, error) { diff --git a/accounts/pkg/storage/cs3.go b/accounts/pkg/storage/cs3.go index 47466ac4cd..39865a39a8 100644 --- a/accounts/pkg/storage/cs3.go +++ b/accounts/pkg/storage/cs3.go @@ -12,37 +12,41 @@ import ( "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/token/manager/jwt" merrors "github.com/micro/go-micro/v2/errors" + "github.com/owncloud/ocis/accounts/pkg/config" "github.com/owncloud/ocis/accounts/pkg/proto/v0" "google.golang.org/grpc/metadata" "io/ioutil" "net/http" + "path" ) type CS3Repo struct { - serviceID string - dataPath string - rootPath string - + serviceID string + cfg *config.Config tm token.Manager storageClient provider.ProviderAPIClient } -func NewCS3Repo(secret string) (Repo, error) { +func NewCS3Repo(serviceID string, cfg *config.Config) (Repo, error) { tokenManager, err := jwt.New(map[string]interface{}{ - "secret": "Pive-Fumkiu4", + "secret": cfg.TokenManager.JWTSecret, }) if err != nil { return nil, err } - client, err := pool.GetStorageProviderServiceClient("localhost:9185") + client, err := pool.GetStorageProviderServiceClient(cfg.Repo.CS3.ProviderAddr) if err != nil { return nil, err } - return CS3Repo{tm: tokenManager, storageClient: client}, nil - + return CS3Repo{ + serviceID: serviceID, + cfg: cfg, + tm: tokenManager, + storageClient: client, + }, nil } func (r CS3Repo) WriteAccount(ctx context.Context, a *proto.Account) (err error) { @@ -52,7 +56,7 @@ func (r CS3Repo) WriteAccount(ctx context.Context, a *proto.Account) (err error) } ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) - if err := r.makeRootDirIfNotExist(ctx); err != nil { + if err := r.makeRootDirIfNotExist(ctx, accountsFolder); err != nil { return err } @@ -61,7 +65,7 @@ func (r CS3Repo) WriteAccount(ctx context.Context, a *proto.Account) (err error) return merrors.InternalServerError(r.serviceID, "could not marshal account: %v", err.Error()) } - ureq, err := http.NewRequest("PUT", fmt.Sprintf("http://localhost:9187/data/accounts/%s", a.Id), bytes.NewReader(by)) + ureq, err := http.NewRequest("PUT", r.accountUrl(a.Id), bytes.NewReader(by)) if err != nil { return err } @@ -86,7 +90,7 @@ func (r CS3Repo) LoadAccount(ctx context.Context, id string, a *proto.Account) ( ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) - ureq, err := http.NewRequest("GET", fmt.Sprintf("http://localhost:9187/data/accounts/%s", id), nil) + ureq, err := http.NewRequest("GET", r.accountUrl(id), nil) if err != nil { return err } @@ -135,33 +139,6 @@ func (r CS3Repo) DeleteAccount(ctx context.Context, id string) (err error) { return nil } -func (r CS3Repo) DeleteGroup(ctx context.Context, id string) (err error) { - t, err := r.authenticate(ctx) - if err != nil { - return err - } - - ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) - ureq, err := http.NewRequest("DELETE", fmt.Sprintf("http://localhost:9187/data/groups/%s", id), nil) - if err != nil { - return err - } - - ureq.Header.Add("x-access-token", t) - cl := http.Client{ - Transport: http.DefaultTransport, - } - - resp, err := cl.Do(ureq) - if err != nil { - return err - } - - defer resp.Body.Close() - - return nil -} - func (r CS3Repo) WriteGroup(ctx context.Context, g *proto.Group) (err error) { t, err := r.authenticate(ctx) if err != nil { @@ -169,7 +146,7 @@ func (r CS3Repo) WriteGroup(ctx context.Context, g *proto.Group) (err error) { } ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) - if err := r.makeRootDirIfNotExist(ctx); err != nil { + if err := r.makeRootDirIfNotExist(ctx, groupsFolder); err != nil { return err } @@ -178,7 +155,7 @@ func (r CS3Repo) WriteGroup(ctx context.Context, g *proto.Group) (err error) { return merrors.InternalServerError(r.serviceID, "could not marshal account: %v", err.Error()) } - ureq, err := http.NewRequest("PUT", fmt.Sprintf("http://localhost:9187/data/groups/%s", g.Id), bytes.NewReader(by)) + ureq, err := http.NewRequest("PUT", r.groupUrl(g.Id), bytes.NewReader(by)) if err != nil { return err } @@ -203,7 +180,7 @@ func (r CS3Repo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) - ureq, err := http.NewRequest("GET", fmt.Sprintf("http://localhost:9187/data/groups/%s", id), nil) + ureq, err := http.NewRequest("GET", r.groupUrl(id), nil) if err != nil { return err } @@ -231,6 +208,33 @@ func (r CS3Repo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err return nil } +func (r CS3Repo) DeleteGroup(ctx context.Context, id string) (err error) { + t, err := r.authenticate(ctx) + if err != nil { + return err + } + + ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) + ureq, err := http.NewRequest("DELETE", r.groupUrl(id), nil) + if err != nil { + return err + } + + ureq.Header.Add("x-access-token", t) + cl := http.Client{ + Transport: http.DefaultTransport, + } + + resp, err := cl.Do(ureq) + if err != nil { + return err + } + + defer resp.Body.Close() + + return nil +} + func (r CS3Repo) authenticate(ctx context.Context) (token string, err error) { return r.tm.MintToken(ctx, &user.User{ Id: &user.UserId{}, @@ -238,9 +242,17 @@ func (r CS3Repo) authenticate(ctx context.Context) (token string, err error) { }) } -func (r CS3Repo) makeRootDirIfNotExist(ctx context.Context) error { +func (r CS3Repo) accountUrl(id string) string { + return path.Join(r.cfg.Repo.CS3.DriverURL, r.cfg.Repo.CS3.DataPrefix, accountsFolder, id) +} + +func (r CS3Repo) groupUrl(id string) string { + return path.Join(r.cfg.Repo.CS3.DriverURL, r.cfg.Repo.CS3.DataPrefix, groupsFolder, id) +} + +func (r CS3Repo) makeRootDirIfNotExist(ctx context.Context, folder string) error { var rootPathRef = &provider.Reference{ - Spec: &provider.Reference_Path{Path: "/meta/accounts"}, + Spec: &provider.Reference_Path{Path: fmt.Sprintf("/meta/%v", folder)}, } resp, err := r.storageClient.Stat(ctx, &provider.StatRequest{ diff --git a/accounts/pkg/storage/cs3_test.go b/accounts/pkg/storage/cs3_test.go index a99efecbd4..67ee881759 100644 --- a/accounts/pkg/storage/cs3_test.go +++ b/accounts/pkg/storage/cs3_test.go @@ -2,13 +2,20 @@ package storage import ( "context" + "github.com/owncloud/ocis/accounts/pkg/config" "github.com/owncloud/ocis/accounts/pkg/proto/v0" "github.com/stretchr/testify/assert" "testing" ) +var cfg = &config.Config{ + TokenManager: config.TokenManager{ + JWTSecret: "Pive-Fumkiu4", + }, +} + func TestCS3Repo_WriteAccount(t *testing.T) { - r, err := NewCS3Repo("fooo") + r, err := NewCS3Repo(cfg) assert.NoError(t, err) err = r.WriteAccount(context.Background(), &proto.Account{ @@ -22,7 +29,7 @@ func TestCS3Repo_WriteAccount(t *testing.T) { } func TestCS3Repo_LoadAccount(t *testing.T) { - r, err := NewCS3Repo("fooo") + r, err := NewCS3Repo(cfg) assert.NoError(t, err) err = r.WriteAccount(context.Background(), &proto.Account{ @@ -42,7 +49,7 @@ func TestCS3Repo_LoadAccount(t *testing.T) { } func TestCS3Repo_DeleteAccount(t *testing.T) { - r, err := NewCS3Repo("fooo") + r, err := NewCS3Repo(cfg) assert.NoError(t, err) err = r.WriteAccount(context.Background(), &proto.Account{ diff --git a/accounts/pkg/storage/disk.go b/accounts/pkg/storage/disk.go index a9ca4b15b5..dfb0a56275 100644 --- a/accounts/pkg/storage/disk.go +++ b/accounts/pkg/storage/disk.go @@ -3,28 +3,30 @@ package storage import ( "context" "encoding/json" - merrors "github.com/micro/go-micro/v2/errors" - "github.com/owncloud/ocis/accounts/pkg/proto/v0" - "github.com/rs/zerolog" + "github.com/owncloud/ocis/accounts/pkg/config" "io/ioutil" "os" "path/filepath" "sync" + + merrors "github.com/micro/go-micro/v2/errors" + "github.com/owncloud/ocis/accounts/pkg/proto/v0" + olog "github.com/owncloud/ocis/ocis-pkg/log" ) var groupLock sync.Mutex type DiskRepo struct { serviceID string - dataPath string - log zerolog.Logger + cfg *config.Config + log olog.Logger } -func New(serviceID string, dataPath string, log zerolog.Logger) DiskRepo { +func NewDiskRepo(serviceID string, cfg *config.Config, log olog.Logger) DiskRepo { return DiskRepo{ serviceID: serviceID, - dataPath: dataPath, - log: log.With().Str("id", serviceID).Logger(), + cfg: cfg, + log: log, } } @@ -38,7 +40,7 @@ func (r DiskRepo) WriteAccount(ctx context.Context, a *proto.Account) (err error return merrors.InternalServerError(r.serviceID, "could not marshal account: %v", err.Error()) } - path := filepath.Join(r.dataPath, "accounts", a.Id) + path := filepath.Join(r.cfg.Repo.Disk.Path, accountsFolder, a.Id) if err = ioutil.WriteFile(path, bytes, 0600); err != nil { return merrors.InternalServerError(r.serviceID, "could not write account: %v", err.Error()) @@ -48,7 +50,7 @@ func (r DiskRepo) WriteAccount(ctx context.Context, a *proto.Account) (err error // LoadAccount from the storage func (r DiskRepo) LoadAccount(ctx context.Context, id string, a *proto.Account) (err error) { - path := filepath.Join(r.dataPath, "accounts", id) + path := filepath.Join(r.cfg.Repo.Disk.Path, accountsFolder, id) var data []byte if data, err = ioutil.ReadFile(path); err != nil { @@ -61,6 +63,17 @@ func (r DiskRepo) LoadAccount(ctx context.Context, id string, a *proto.Account) return } +// DeleteAccount from the storage +func (r DiskRepo) DeleteAccount(ctx context.Context, id string) (err error) { + path := filepath.Join(r.cfg.Repo.Disk.Path, accountsFolder, id) + if err = os.Remove(path); err != nil { + r.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove account") + return merrors.InternalServerError(r.serviceID, "could not remove account: %v", err.Error()) + } + + return nil +} + // WriteGroup persists a given group to the storage func (r DiskRepo) WriteGroup(ctx context.Context, g *proto.Group) (err error) { // leave only the member id @@ -71,7 +84,7 @@ func (r DiskRepo) WriteGroup(ctx context.Context, g *proto.Group) (err error) { return merrors.InternalServerError(r.serviceID, "could not marshal group: %v", err.Error()) } - path := filepath.Join(r.dataPath, "groups", g.Id) + path := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder, g.Id) groupLock.Lock() defer groupLock.Unlock() @@ -83,7 +96,7 @@ func (r DiskRepo) WriteGroup(ctx context.Context, g *proto.Group) (err error) { // LoadGroup from the storage func (r DiskRepo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err error) { - path := filepath.Join(r.dataPath, "groups", id) + path := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder, id) groupLock.Lock() defer groupLock.Unlock() @@ -99,12 +112,22 @@ func (r DiskRepo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err return } +func (r DiskRepo) DeleteGroup(ctx context.Context, id string) (err error) { + path := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder, id) + if err = os.Remove(path); err != nil { + r.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove group") + return merrors.InternalServerError(r.serviceID, "could not remove group: %v", err.Error()) + } + + return nil +} + // deflateMemberOf replaces the groups of a user with an instance that only contains the id func (r DiskRepo) deflateMemberOf(a *proto.Account) { if a == nil { return } - deflated := []*proto.Group{} + var deflated []*proto.Group for i := range a.MemberOf { if a.MemberOf[i].Id != "" { deflated = append(deflated, &proto.Group{Id: a.MemberOf[i].Id}) @@ -116,32 +139,12 @@ func (r DiskRepo) deflateMemberOf(a *proto.Account) { a.MemberOf = deflated } -func (r DiskRepo) DeleteAccount(ctx context.Context, id string) (err error) { - path := filepath.Join(r.dataPath, "accounts", id) - if err = os.Remove(path); err != nil { - r.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove account") - return merrors.InternalServerError(r.serviceID, "could not remove account: %v", err.Error()) - } - - return nil -} - -func (r DiskRepo) DeleteGroup(ctx context.Context, id string) (err error) { - path := filepath.Join(r.dataPath, "groups", id) - if err = os.Remove(path); err != nil { - r.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove group") - return merrors.InternalServerError(r.serviceID, "could not remove group: %v", err.Error()) - } - - return nil -} - // deflateMembers replaces the users of a group with an instance that only contains the id func (r DiskRepo) deflateMembers(g *proto.Group) { if g == nil { return } - deflated := []*proto.Account{} + var deflated []*proto.Account for i := range g.Members { if g.Members[i].Id != "" { deflated = append(deflated, &proto.Account{Id: g.Members[i].Id}) diff --git a/accounts/pkg/storage/repo.go b/accounts/pkg/storage/repo.go index bac46dc639..12f9327ee5 100644 --- a/accounts/pkg/storage/repo.go +++ b/accounts/pkg/storage/repo.go @@ -5,6 +5,11 @@ import ( "github.com/owncloud/ocis/accounts/pkg/proto/v0" ) +const ( + accountsFolder = "accounts" + groupsFolder = "groups" +) + type Repo interface { WriteAccount(ctx context.Context, a *proto.Account) (err error) LoadAccount(ctx context.Context, id string, a *proto.Account) (err error)