diff --git a/ocis-pkg/indexer/config/config.go b/ocis-pkg/indexer/config/config.go deleted file mode 100644 index 010d2f5de..000000000 --- a/ocis-pkg/indexer/config/config.go +++ /dev/null @@ -1,48 +0,0 @@ -// Package config should be moved to internal -package config - -import ( - "github.com/owncloud/ocis/v2/extensions/accounts/pkg/config" -) - -// Repo defines which storage implementation is to be used. -type Repo struct { - Backend string - Disk Disk - CS3 CS3 -} - -// Disk is the local disk implementation of the storage. -type Disk struct { - Path string -} - -// CS3 is the cs3 implementation of the storage. -type CS3 struct { - ProviderAddr string - DataURL string - DataPrefix string - JWTSecret string -} - -// Index defines config for indexes. -type Index struct { - UID, GID Bound -} - -// Bound defines a lower and upper bound. -type Bound struct { - Lower, Upper int64 -} - -// Config merges all Account config parameters. -type Config struct { - Repo Repo - Index Index - ServiceUser config.ServiceUser -} - -// New returns a new config. -func New() *Config { - return &Config{} -} diff --git a/ocis-pkg/indexer/errors/errors.go b/ocis-pkg/indexer/errors/errors.go deleted file mode 100644 index f96213f42..000000000 --- a/ocis-pkg/indexer/errors/errors.go +++ /dev/null @@ -1,35 +0,0 @@ -package errors - -import ( - "fmt" -) - -// AlreadyExistsErr implements the Error interface. -type AlreadyExistsErr struct { - TypeName, Key, Value string -} - -func (e *AlreadyExistsErr) Error() string { - return fmt.Sprintf("%s with %s=%s does already exist", e.TypeName, e.Key, e.Value) -} - -// IsAlreadyExistsErr checks whether an error is of type AlreadyExistsErr. -func IsAlreadyExistsErr(e error) bool { - _, ok := e.(*AlreadyExistsErr) - return ok -} - -// NotFoundErr implements the Error interface. -type NotFoundErr struct { - TypeName, Key, Value string -} - -func (e *NotFoundErr) Error() string { - return fmt.Sprintf("%s with %s=%s not found", e.TypeName, e.Key, e.Value) -} - -// IsNotFoundErr checks whether an error is of type IsNotFoundErr. -func IsNotFoundErr(e error) bool { - _, ok := e.(*NotFoundErr) - return ok -} diff --git a/ocis-pkg/indexer/helper.go b/ocis-pkg/indexer/helper.go deleted file mode 100644 index 8591beb1d..000000000 --- a/ocis-pkg/indexer/helper.go +++ /dev/null @@ -1,15 +0,0 @@ -package indexer - -// dedup removes duplicate values in given slice -func dedup(s []string) []string { - for i := 0; i < len(s); i++ { - for i2 := i + 1; i2 < len(s); i2++ { - if s[i] == s[i2] { - // delete - s = append(s[:i2], s[i2+1:]...) - i2-- - } - } - } - return s -} diff --git a/ocis-pkg/indexer/index/cs3/autoincrement.go b/ocis-pkg/indexer/index/cs3/autoincrement.go deleted file mode 100644 index 95ac79834..000000000 --- a/ocis-pkg/indexer/index/cs3/autoincrement.go +++ /dev/null @@ -1,375 +0,0 @@ -package cs3 - -import ( - "context" - "os" - "path" - "path/filepath" - "sort" - "strconv" - "strings" - - "github.com/owncloud/ocis/v2/extensions/accounts/pkg/storage" - - idxerrs "github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors" - - v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - revactx "github.com/cs3org/reva/v2/pkg/ctx" - "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/v2/pkg/token" - "github.com/cs3org/reva/v2/pkg/token/manager/jwt" - "github.com/cs3org/reva/v2/pkg/utils" - "google.golang.org/grpc/metadata" - - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/registry" - metadatastorage "github.com/owncloud/ocis/v2/ocis-pkg/metadata_storage" -) - -// Autoincrement are fields for an index of type autoincrement. -type Autoincrement struct { - indexBy string - typeName string - filesDir string - indexBaseDir string - indexRootDir string - - tokenManager token.Manager - storageProvider provider.ProviderAPIClient - metadataStorage *metadatastorage.MetadataStorage - - cs3conf *Config - bound *option.Bound -} - -func init() { - registry.IndexConstructorRegistry["cs3"]["autoincrement"] = NewAutoincrementIndex -} - -// NewAutoincrementIndex instantiates a new AutoincrementIndex instance. -func NewAutoincrementIndex(o ...option.Option) index.Index { - opts := &option.Options{} - for _, opt := range o { - opt(opts) - } - - u := &Autoincrement{ - indexBy: opts.IndexBy, - typeName: opts.TypeName, - filesDir: opts.FilesDir, - bound: opts.Bound, - indexBaseDir: path.Join(opts.DataDir, "index.cs3"), - indexRootDir: path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"autoincrement", opts.TypeName, opts.IndexBy}, ".")), - cs3conf: &Config{ - ProviderAddr: opts.ProviderAddr, - JWTSecret: opts.JWTSecret, - ServiceUser: opts.ServiceUser, - }, - } - - return u -} - -// Init initializes an autoincrement index. -func (idx *Autoincrement) Init() error { - tokenManager, err := jwt.New(map[string]interface{}{ - "secret": idx.cs3conf.JWTSecret, - }) - if err != nil { - return err - } - idx.tokenManager = tokenManager - - client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr) - if err != nil { - return err - } - idx.storageProvider = client - - m, err := metadatastorage.NewMetadataStorage(idx.cs3conf.ProviderAddr) - if err != nil { - return err - } - idx.metadataStorage = &m - - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return err - } - if err := idx.metadataStorage.Init(ctx, idx.cs3conf.ServiceUser); err != nil { - return err - } - - if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil { - return err - } - - if err := idx.makeDirIfNotExists(idx.indexRootDir); err != nil { - return err - } - - return nil -} - -// Lookup exact lookup by value. -func (idx *Autoincrement) Lookup(v string) ([]string, error) { - searchPath := path.Join(idx.indexRootDir, v) - oldname, err := idx.resolveSymlink(searchPath) - if err != nil { - if os.IsNotExist(err) { - err = &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v} - } - - return nil, err - } - - return []string{oldname}, nil -} - -// Add a new value to the index. -func (idx *Autoincrement) Add(id, v string) (string, error) { - var newName string - if v == "" { - next, err := idx.next() - if err != nil { - return "", err - } - newName = path.Join(idx.indexRootDir, strconv.Itoa(next)) - } else { - newName = path.Join(idx.indexRootDir, v) - } - if err := idx.createSymlink(id, newName); err != nil { - if os.IsExist(err) { - return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v} - } - - return "", err - } - - return newName, nil -} - -// Remove a value v from an index. -func (idx *Autoincrement) Remove(id string, v string) error { - if v == "" { - return nil - } - searchPath := path.Join(idx.indexRootDir, v) - _, err := idx.resolveSymlink(searchPath) - if err != nil { - if os.IsNotExist(err) { - err = &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v} - } - - return err - } - - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return err - } - - deletePath := path.Join("/", idx.indexRootDir, v) - resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ - Ref: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: idx.cs3conf.ServiceUser.UUID, - OpaqueId: idx.cs3conf.ServiceUser.UUID, - }, - Path: utils.MakeRelativePath(deletePath), - }, - }) - - if err != nil { - return err - } - - // TODO Handle other error codes? - if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND { - return &idxerrs.NotFoundErr{} - } - - return err -} - -// Update index from to . -func (idx *Autoincrement) Update(id, oldV, newV string) error { - if err := idx.Remove(id, oldV); err != nil { - return err - } - - if _, err := idx.Add(id, newV); err != nil { - return err - } - - return nil -} - -// Search allows for glob search on the index. -func (idx *Autoincrement) Search(pattern string) ([]string, error) { - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return nil, err - } - - res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ - Ref: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: idx.cs3conf.ServiceUser.UUID, - OpaqueId: idx.cs3conf.ServiceUser.UUID, - }, - Path: utils.MakeRelativePath(idx.indexRootDir), - }, - }) - - if err != nil { - return nil, err - } - - searchPath := idx.indexRootDir - matches := make([]string, 0) - for _, i := range res.GetInfos() { - if found, err := filepath.Match(pattern, path.Base(i.Path)); found { - if err != nil { - return nil, err - } - - oldPath, err := idx.resolveSymlink(path.Join(searchPath, path.Base(i.Path))) - if err != nil { - return nil, err - } - matches = append(matches, oldPath) - } - } - - return matches, nil -} - -// CaseInsensitive undocumented. -func (idx *Autoincrement) CaseInsensitive() bool { - return false -} - -// IndexBy undocumented. -func (idx *Autoincrement) IndexBy() string { - return idx.indexBy -} - -// TypeName undocumented. -func (idx *Autoincrement) TypeName() string { - return idx.typeName -} - -// FilesDir undocumented. -func (idx *Autoincrement) FilesDir() string { - return idx.filesDir -} - -func (idx *Autoincrement) createSymlink(oldname, newname string) error { - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return err - } - - if _, err := idx.resolveSymlink(newname); err == nil { - return os.ErrExist - } - - err = idx.metadataStorage.SimpleUpload(ctx, newname, []byte(oldname)) - if err != nil { - return err - } - return nil -} - -func (idx *Autoincrement) resolveSymlink(name string) (string, error) { - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return "", err - } - - b, err := idx.metadataStorage.SimpleDownload(ctx, name) - if err != nil { - if metadatastorage.IsNotFoundErr(err) { - return "", os.ErrNotExist - } - return "", err - } - - return string(b), err -} - -func (idx *Autoincrement) makeDirIfNotExists(folder string) error { - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return err - } - return storage.MakeDirIfNotExist(ctx, idx.storageProvider, &provider.ResourceId{ - StorageId: idx.cs3conf.ServiceUser.UUID, - OpaqueId: idx.cs3conf.ServiceUser.UUID, - }, folder) -} - -func (idx *Autoincrement) next() (int, error) { - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return -1, err - } - - res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ - Ref: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: idx.cs3conf.ServiceUser.UUID, - OpaqueId: idx.cs3conf.ServiceUser.UUID, - }, - Path: utils.MakeRelativePath(idx.indexRootDir), - }, - }) - - if err != nil { - return -1, err - } - - if len(res.GetInfos()) == 0 { - return 0, nil - } - - infos := res.GetInfos() - sort.Slice(infos, func(i, j int) bool { - a, _ := strconv.Atoi(path.Base(infos[i].Path)) - b, _ := strconv.Atoi(path.Base(infos[j].Path)) - return a < b - }) - - latest, err := strconv.Atoi(path.Base(infos[len(infos)-1].Path)) // would returning a string be a better interface? - if err != nil { - return -1, err - } - - if int64(latest) < idx.bound.Lower { - return int(idx.bound.Lower), nil - } - - return latest + 1, nil -} - -func (idx *Autoincrement) getAuthenticatedContext(ctx context.Context) (context.Context, error) { - t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager) - if err != nil { - return nil, err - } - ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) - return ctx, nil -} - -// Delete deletes the index folder from its storage. -func (idx *Autoincrement) Delete() error { - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return err - } - - return deleteIndexRoot(ctx, idx.storageProvider, idx.cs3conf.ServiceUser.UUID, idx.indexRootDir) -} diff --git a/ocis-pkg/indexer/index/cs3/autoincrement_test.go b/ocis-pkg/indexer/index/cs3/autoincrement_test.go deleted file mode 100644 index 207aa69fc..000000000 --- a/ocis-pkg/indexer/index/cs3/autoincrement_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package cs3 - -// -//import ( -// "os" -// "testing" -// -// "github.com/owncloud/ocis/v2/accounts/pkg/config" -// "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option" -// . "github.com/owncloud/ocis/v2/ocis-pkg/indexer/test" -// "github.com/stretchr/testify/assert" -//) -// -//const cs3RootFolder = "/tmp/ocis/storage/users/data" -// -//func TestAutoincrementIndexAdd(t *testing.T) { -// dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder) -// assert.NoError(t, err) -// cfg := generateConfig() -// -// sut := NewAutoincrementIndex( -// option.WithTypeName(GetTypeFQN(User{})), -// option.WithIndexBy("UID"), -// option.WithDataURL(cfg.Repo.CS3.DataURL), -// option.WithDataPrefix(cfg.Repo.CS3.DataPrefix), -// option.WithJWTSecret(cfg.Repo.CS3.JWTSecret), -// option.WithProviderAddr(cfg.Repo.CS3.ProviderAddr), -// ) -// -// assert.NoError(t, sut.Init()) -// -// for i := 0; i < 5; i++ { -// res, err := sut.Add("abcdefg-123", "") -// assert.NoError(t, err) -// t.Log(res) -// } -// -// _ = os.RemoveAll(dataDir) -//} -// -//func BenchmarkAutoincrementIndexAdd(b *testing.B) { -// dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder) -// assert.NoError(b, err) -// cfg := generateConfig() -// -// sut := NewAutoincrementIndex( -// option.WithTypeName(GetTypeFQN(User{})), -// option.WithIndexBy("UID"), -// option.WithDataURL(cfg.Repo.CS3.DataURL), -// option.WithDataPrefix(cfg.Repo.CS3.DataPrefix), -// option.WithJWTSecret(cfg.Repo.CS3.JWTSecret), -// option.WithProviderAddr(cfg.Repo.CS3.ProviderAddr), -// ) -// -// err = sut.Init() -// assert.NoError(b, err) -// -// for n := 0; n < b.N; n++ { -// _, err := sut.Add("abcdefg-123", "") -// if err != nil { -// b.Error(err) -// } -// assert.NoError(b, err) -// } -// -// _ = os.RemoveAll(dataDir) -//} -// -//func generateConfig() config.Config { -// return config.Config{ -// Repo: config.Repo{ -// Disk: config.Disk{ -// Path: "", -// }, -// CS3: config.CS3{ -// ProviderAddr: "0.0.0.0:9215", -// DataURL: "http://localhost:9216", -// DataPrefix: "data", -// JWTSecret: "Pive-Fumkiu4", -// }, -// }, -// } -//} diff --git a/ocis-pkg/indexer/index/cs3/config.go b/ocis-pkg/indexer/index/cs3/config.go deleted file mode 100644 index 87ceffc60..000000000 --- a/ocis-pkg/indexer/index/cs3/config.go +++ /dev/null @@ -1,12 +0,0 @@ -package cs3 - -import ( - acccfg "github.com/owncloud/ocis/v2/extensions/accounts/pkg/config" -) - -// Config represents cs3conf. Should be deprecated in favor of config.Config. -type Config struct { - ProviderAddr string - JWTSecret string - ServiceUser acccfg.ServiceUser -} diff --git a/ocis-pkg/indexer/index/cs3/helper.go b/ocis-pkg/indexer/index/cs3/helper.go deleted file mode 100644 index 6d7af96eb..000000000 --- a/ocis-pkg/indexer/index/cs3/helper.go +++ /dev/null @@ -1,30 +0,0 @@ -package cs3 - -import ( - "context" - "fmt" - - rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/v2/pkg/utils" -) - -func deleteIndexRoot(ctx context.Context, storageProvider provider.ProviderAPIClient, spaceid, indexRootDir string) error { - res, err := storageProvider.Delete(ctx, &provider.DeleteRequest{ - Ref: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: spaceid, - OpaqueId: spaceid, - }, - Path: utils.MakeRelativePath(indexRootDir), - }, - }) - if err != nil { - return err - } - if res.Status.Code != rpc.Code_CODE_OK { - return fmt.Errorf("error deleting index root dir: %v", indexRootDir) - } - - return nil -} diff --git a/ocis-pkg/indexer/index/cs3/non_unique.go b/ocis-pkg/indexer/index/cs3/non_unique.go deleted file mode 100644 index 34978105a..000000000 --- a/ocis-pkg/indexer/index/cs3/non_unique.go +++ /dev/null @@ -1,396 +0,0 @@ -package cs3 - -import ( - "context" - "os" - "path" - "path/filepath" - "strings" - - "github.com/owncloud/ocis/v2/extensions/accounts/pkg/storage" - - v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - revactx "github.com/cs3org/reva/v2/pkg/ctx" - "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/v2/pkg/token" - "github.com/cs3org/reva/v2/pkg/token/manager/jwt" - "github.com/cs3org/reva/v2/pkg/utils" - idxerrs "github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/registry" - metadatastorage "github.com/owncloud/ocis/v2/ocis-pkg/metadata_storage" - "google.golang.org/grpc/metadata" -) - -func init() { - registry.IndexConstructorRegistry["cs3"]["non_unique"] = NewNonUniqueIndexWithOptions -} - -// NonUnique are fields for an index of type non_unique. -type NonUnique struct { - caseInsensitive bool - indexBy string - typeName string - filesDir string - indexBaseDir string - indexRootDir string - - tokenManager token.Manager - storageProvider provider.ProviderAPIClient - metadataStorage *metadatastorage.MetadataStorage - - cs3conf *Config -} - -// NewNonUniqueIndexWithOptions instantiates a new NonUniqueIndex instance. -// /tmp/ocis/accounts/index.cs3/Pets/Bro* -// ├── Brown/ -// │ └── rebef-123 -> /tmp/testfiles-395764020/pets/rebef-123 -// ├── Green/ -// │ ├── goefe-789 -> /tmp/testfiles-395764020/pets/goefe-789 -// │ └── xadaf-189 -> /tmp/testfiles-395764020/pets/xadaf-189 -// └── White/ -// └── wefwe-456 -> /tmp/testfiles-395764020/pets/wefwe-456 -func NewNonUniqueIndexWithOptions(o ...option.Option) index.Index { - opts := &option.Options{} - for _, opt := range o { - opt(opts) - } - - return &NonUnique{ - caseInsensitive: opts.CaseInsensitive, - indexBy: opts.IndexBy, - typeName: opts.TypeName, - filesDir: opts.FilesDir, - indexBaseDir: path.Join(opts.DataDir, "index.cs3"), - indexRootDir: path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"non_unique", opts.TypeName, opts.IndexBy}, ".")), - cs3conf: &Config{ - ProviderAddr: opts.ProviderAddr, - JWTSecret: opts.JWTSecret, - ServiceUser: opts.ServiceUser, - }, - } -} - -// Init initializes a non_unique index. -func (idx *NonUnique) Init() error { - tokenManager, err := jwt.New(map[string]interface{}{ - "secret": idx.cs3conf.JWTSecret, - }) - if err != nil { - return err - } - idx.tokenManager = tokenManager - - client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr) - if err != nil { - return err - } - idx.storageProvider = client - - m, err := metadatastorage.NewMetadataStorage(idx.cs3conf.ProviderAddr) - if err != nil { - return err - } - idx.metadataStorage = &m - - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return err - } - if err := idx.metadataStorage.Init(ctx, idx.cs3conf.ServiceUser); err != nil { - return err - } - - if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil { - return err - } - - if err := idx.makeDirIfNotExists(idx.indexRootDir); err != nil { - return err - } - - return nil -} - -// Lookup exact lookup by value. -func (idx *NonUnique) Lookup(v string) ([]string, error) { - if idx.caseInsensitive { - v = strings.ToLower(v) - } - var matches = make([]string, 0) - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return nil, err - } - - res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ - Ref: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: idx.cs3conf.ServiceUser.UUID, - OpaqueId: idx.cs3conf.ServiceUser.UUID, - }, - Path: utils.MakeRelativePath(path.Join("/", idx.indexRootDir, v)), - }, - }) - - if err != nil { - return nil, err - } - - for _, info := range res.Infos { - matches = append(matches, path.Base(info.Path)) - } - - return matches, nil -} - -// Add a new value to the index. -func (idx *NonUnique) Add(id, v string) (string, error) { - if v == "" { - return "", nil - } - if idx.caseInsensitive { - v = strings.ToLower(v) - } - - newName := path.Join(idx.indexRootDir, v) - if err := idx.makeDirIfNotExists(newName); err != nil { - return "", err - } - - if err := idx.createSymlink(id, path.Join(newName, id)); err != nil { - if os.IsExist(err) { - return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v} - } - - return "", err - } - - return newName, nil -} - -// Remove a value v from an index. -func (idx *NonUnique) Remove(id string, v string) error { - if v == "" { - return nil - } - if idx.caseInsensitive { - v = strings.ToLower(v) - } - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return err - } - - deletePath := path.Join("/", idx.indexRootDir, v, id) - resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ - Ref: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: idx.cs3conf.ServiceUser.UUID, - OpaqueId: idx.cs3conf.ServiceUser.UUID, - }, - Path: utils.MakeRelativePath(deletePath), - }, - }) - - if err != nil { - return err - } - - if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND { - return &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v} - } - - toStat := path.Join("/", idx.indexRootDir, v) - lcResp, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ - Ref: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: idx.cs3conf.ServiceUser.UUID, - OpaqueId: idx.cs3conf.ServiceUser.UUID, - }, - Path: utils.MakeRelativePath(toStat), - }, - }) - if err != nil { - return err - } - - if len(lcResp.Infos) == 0 { - deletePath = path.Join("/", idx.indexRootDir, v) - _, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ - Ref: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: idx.cs3conf.ServiceUser.UUID, - OpaqueId: idx.cs3conf.ServiceUser.UUID, - }, - Path: utils.MakeRelativePath(deletePath), - }, - }) - if err != nil { - return err - } - } - - return nil -} - -// Update index from to . -func (idx *NonUnique) Update(id, oldV, newV string) error { - if idx.caseInsensitive { - oldV = strings.ToLower(oldV) - newV = strings.ToLower(newV) - } - - if err := idx.Remove(id, oldV); err != nil { - return err - } - - if _, err := idx.Add(id, newV); err != nil { - return err - } - - return nil -} - -// Search allows for glob search on the index. -func (idx *NonUnique) Search(pattern string) ([]string, error) { - if idx.caseInsensitive { - pattern = strings.ToLower(pattern) - } - - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return nil, err - } - - foldersMatched := make([]string, 0) - matches := make([]string, 0) - res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ - Ref: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: idx.cs3conf.ServiceUser.UUID, - OpaqueId: idx.cs3conf.ServiceUser.UUID, - }, - Path: utils.MakeRelativePath(idx.indexRootDir), - }, - }) - - if err != nil { - return nil, err - } - - for _, i := range res.Infos { - if found, err := filepath.Match(pattern, path.Base(i.Path)); found { - if err != nil { - return nil, err - } - - foldersMatched = append(foldersMatched, i.Path) - } - } - - for i := range foldersMatched { - res, _ := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ - Ref: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: idx.cs3conf.ServiceUser.UUID, - OpaqueId: idx.cs3conf.ServiceUser.UUID, - }, - Path: utils.MakeRelativePath(foldersMatched[i]), - }, - }) - - for _, info := range res.Infos { - matches = append(matches, path.Base(info.Path)) - } - } - - return matches, nil -} - -// CaseInsensitive undocumented. -func (idx *NonUnique) CaseInsensitive() bool { - return idx.caseInsensitive -} - -// IndexBy undocumented. -func (idx *NonUnique) IndexBy() string { - return idx.indexBy -} - -// TypeName undocumented. -func (idx *NonUnique) TypeName() string { - return idx.typeName -} - -// FilesDir undocumented. -func (idx *NonUnique) FilesDir() string { - return idx.filesDir -} - -func (idx *NonUnique) makeDirIfNotExists(folder string) error { - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return err - } - return storage.MakeDirIfNotExist(ctx, idx.storageProvider, &provider.ResourceId{ - StorageId: idx.cs3conf.ServiceUser.UUID, - OpaqueId: idx.cs3conf.ServiceUser.UUID, - }, folder) -} - -func (idx *NonUnique) createSymlink(oldname, newname string) error { - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return err - } - - if _, err := idx.resolveSymlink(newname); err == nil { - return os.ErrExist - } - - err = idx.metadataStorage.SimpleUpload(ctx, newname, []byte(oldname)) - if err != nil { - return err - } - return nil -} - -func (idx *NonUnique) resolveSymlink(name string) (string, error) { - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return "", err - } - - b, err := idx.metadataStorage.SimpleDownload(ctx, name) - if err != nil { - if metadatastorage.IsNotFoundErr(err) { - return "", os.ErrNotExist - } - return "", err - } - - return string(b), err -} - -func (idx *NonUnique) getAuthenticatedContext(ctx context.Context) (context.Context, error) { - t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager) - if err != nil { - return nil, err - } - ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) - return ctx, nil -} - -// Delete deletes the index folder from its storage. -func (idx *NonUnique) Delete() error { - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return err - } - - return deleteIndexRoot(ctx, idx.storageProvider, idx.cs3conf.ServiceUser.UUID, idx.indexRootDir) -} diff --git a/ocis-pkg/indexer/index/cs3/non_unique_test.go b/ocis-pkg/indexer/index/cs3/non_unique_test.go deleted file mode 100644 index ec82b3aba..000000000 --- a/ocis-pkg/indexer/index/cs3/non_unique_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package cs3 - -// -//import ( -// "os" -// "path" -// "testing" -// -// "github.com/owncloud/ocis/v2/accounts/pkg/config" -// "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option" -// . "github.com/owncloud/ocis/v2/ocis-pkg/indexer/test" -// "github.com/stretchr/testify/assert" -//) -// -//func TestCS3NonUniqueIndex_FakeSymlink(t *testing.T) { -// dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder) -// assert.NoError(t, err) -// cfg := config.Config{ -// Repo: config.Repo{ -// Disk: config.Disk{ -// Path: "", -// }, -// CS3: config.CS3{ -// ProviderAddr: "0.0.0.0:9215", -// DataURL: "http://localhost:9216", -// DataPrefix: "data", -// JWTSecret: "Pive-Fumkiu4", -// }, -// }, -// } -// -// sut := NewNonUniqueIndexWithOptions( -// option.WithTypeName(GetTypeFQN(User{})), -// option.WithIndexBy("UserName"), -// option.WithFilesDir(path.Join(cfg.Repo.Disk.Path, "/")), -// option.WithDataDir(cfg.Repo.Disk.Path), -// option.WithDataURL(cfg.Repo.CS3.DataURL), -// option.WithDataPrefix(cfg.Repo.CS3.DataPrefix), -// option.WithJWTSecret(cfg.Repo.CS3.JWTSecret), -// option.WithProviderAddr(cfg.Repo.CS3.ProviderAddr), -// ) -// -// err = sut.Init() -// assert.NoError(t, err) -// -// res, err := sut.Add("abcdefg-123", "mikey") -// assert.NoError(t, err) -// t.Log(res) -// -// resLookup, err := sut.Lookup("mikey") -// assert.NoError(t, err) -// t.Log(resLookup) -// -// err = sut.Update("abcdefg-123", "mikey", "mickeyX") -// assert.NoError(t, err) -// -// searchRes, err := sut.Search("m*") -// assert.NoError(t, err) -// assert.Len(t, searchRes, 1) -// assert.Equal(t, searchRes[0], "abcdefg-123") -// -// resp, err := sut.Lookup("mikey") -// assert.Len(t, resp, 0) -// assert.NoError(t, err) -// -// _ = os.RemoveAll(dataDir) -// -//} diff --git a/ocis-pkg/indexer/index/cs3/unique.go b/ocis-pkg/indexer/index/cs3/unique.go deleted file mode 100644 index c249ea63b..000000000 --- a/ocis-pkg/indexer/index/cs3/unique.go +++ /dev/null @@ -1,342 +0,0 @@ -package cs3 - -import ( - "context" - "os" - "path" - "path/filepath" - "strings" - - "github.com/owncloud/ocis/v2/extensions/accounts/pkg/storage" - - v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - revactx "github.com/cs3org/reva/v2/pkg/ctx" - "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/v2/pkg/token" - "github.com/cs3org/reva/v2/pkg/token/manager/jwt" - "github.com/cs3org/reva/v2/pkg/utils" - idxerrs "github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/registry" - metadatastorage "github.com/owncloud/ocis/v2/ocis-pkg/metadata_storage" - "google.golang.org/grpc/metadata" -) - -// Unique are fields for an index of type non_unique. -type Unique struct { - caseInsensitive bool - indexBy string - typeName string - filesDir string - indexBaseDir string - indexRootDir string - - tokenManager token.Manager - storageProvider provider.ProviderAPIClient - metadataStorage *metadatastorage.MetadataStorage - - cs3conf *Config -} - -func init() { - registry.IndexConstructorRegistry["cs3"]["unique"] = NewUniqueIndexWithOptions -} - -// NewUniqueIndexWithOptions instantiates a new UniqueIndex instance. Init() should be -// called afterward to ensure correct on-disk structure. -func NewUniqueIndexWithOptions(o ...option.Option) index.Index { - opts := &option.Options{} - for _, opt := range o { - opt(opts) - } - - u := &Unique{ - caseInsensitive: opts.CaseInsensitive, - indexBy: opts.IndexBy, - typeName: opts.TypeName, - filesDir: opts.FilesDir, - indexBaseDir: path.Join(opts.DataDir, "index.cs3"), - indexRootDir: path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"unique", opts.TypeName, opts.IndexBy}, ".")), - cs3conf: &Config{ - ProviderAddr: opts.ProviderAddr, - JWTSecret: opts.JWTSecret, - ServiceUser: opts.ServiceUser, - }, - } - - return u -} - -// Init initializes a unique index. -func (idx *Unique) Init() error { - tokenManager, err := jwt.New(map[string]interface{}{ - "secret": idx.cs3conf.JWTSecret, - }) - if err != nil { - return err - } - idx.tokenManager = tokenManager - - client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr) - if err != nil { - return err - } - idx.storageProvider = client - - m, err := metadatastorage.NewMetadataStorage(idx.cs3conf.ProviderAddr) - if err != nil { - return err - } - idx.metadataStorage = &m - - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return err - } - if err := idx.metadataStorage.Init(ctx, idx.cs3conf.ServiceUser); err != nil { - return err - } - - if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil { - return err - } - - if err := idx.makeDirIfNotExists(idx.indexRootDir); err != nil { - return err - } - - return nil -} - -// Lookup exact lookup by value. -func (idx *Unique) Lookup(v string) ([]string, error) { - if idx.caseInsensitive { - v = strings.ToLower(v) - } - searchPath := path.Join(idx.indexRootDir, v) - oldname, err := idx.resolveSymlink(searchPath) - if err != nil { - if os.IsNotExist(err) { - err = &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v} - } - - return nil, err - } - - return []string{oldname}, nil -} - -// Add adds a value to the index, returns the path to the root-document -func (idx *Unique) Add(id, v string) (string, error) { - if v == "" { - return "", nil - } - if idx.caseInsensitive { - v = strings.ToLower(v) - } - newName := path.Join(idx.indexRootDir, v) - if err := idx.createSymlink(id, newName); err != nil { - if os.IsExist(err) { - return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v} - } - - return "", err - } - - return newName, nil -} - -// Remove a value v from an index. -func (idx *Unique) Remove(id string, v string) error { - if v == "" { - return nil - } - if idx.caseInsensitive { - v = strings.ToLower(v) - } - searchPath := path.Join(idx.indexRootDir, v) - _, err := idx.resolveSymlink(searchPath) - if err != nil { - if os.IsNotExist(err) { - err = &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v} - } - - return err - } - - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return err - } - - deletePath := path.Join("/", idx.indexRootDir, v) - resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ - Ref: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: idx.cs3conf.ServiceUser.UUID, - OpaqueId: idx.cs3conf.ServiceUser.UUID, - }, - Path: utils.MakeRelativePath(deletePath), - }, - }) - - if err != nil { - return err - } - - // TODO Handle other error codes? - if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND { - return &idxerrs.NotFoundErr{} - } - - return err -} - -// Update index from to . -func (idx *Unique) Update(id, oldV, newV string) error { - if idx.caseInsensitive { - oldV = strings.ToLower(oldV) - newV = strings.ToLower(newV) - } - - if err := idx.Remove(id, oldV); err != nil { - return err - } - - if _, err := idx.Add(id, newV); err != nil { - return err - } - - return nil -} - -// Search allows for glob search on the index. -func (idx *Unique) Search(pattern string) ([]string, error) { - if idx.caseInsensitive { - pattern = strings.ToLower(pattern) - } - - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return nil, err - } - - res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ - Ref: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: idx.cs3conf.ServiceUser.UUID, - OpaqueId: idx.cs3conf.ServiceUser.UUID, - }, - Path: utils.MakeRelativePath(idx.indexRootDir), - }, - }) - - if err != nil { - return nil, err - } - - searchPath := idx.indexRootDir - matches := make([]string, 0) - for _, i := range res.GetInfos() { - if found, err := filepath.Match(pattern, path.Base(i.Path)); found { - if err != nil { - return nil, err - } - - oldPath, err := idx.resolveSymlink(path.Join(searchPath, path.Base(i.Path))) - if err != nil { - return nil, err - } - matches = append(matches, oldPath) - } - } - - return matches, nil -} - -// CaseInsensitive undocumented. -func (idx *Unique) CaseInsensitive() bool { - return idx.caseInsensitive -} - -// IndexBy undocumented. -func (idx *Unique) IndexBy() string { - return idx.indexBy -} - -// TypeName undocumented. -func (idx *Unique) TypeName() string { - return idx.typeName -} - -// FilesDir undocumented. -func (idx *Unique) FilesDir() string { - return idx.filesDir -} - -func (idx *Unique) createSymlink(oldname, newname string) error { - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return err - } - - if _, err := idx.resolveSymlink(newname); err == nil { - return os.ErrExist - } - - err = idx.metadataStorage.SimpleUpload(ctx, newname, []byte(oldname)) - if err != nil { - return err - } - - return nil -} - -func (idx *Unique) resolveSymlink(name string) (string, error) { - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return "", err - } - - b, err := idx.metadataStorage.SimpleDownload(ctx, name) - if err != nil { - if metadatastorage.IsNotFoundErr(err) { - return "", os.ErrNotExist - } - return "", err - } - - return string(b), err -} - -func (idx *Unique) makeDirIfNotExists(folder string) error { - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return err - } - return storage.MakeDirIfNotExist(ctx, idx.storageProvider, &provider.ResourceId{ - StorageId: idx.cs3conf.ServiceUser.UUID, - OpaqueId: idx.cs3conf.ServiceUser.UUID, - }, folder) -} - -func (idx *Unique) getAuthenticatedContext(ctx context.Context) (context.Context, error) { - t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager) - if err != nil { - return nil, err - } - ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t) - return ctx, nil -} - -// Delete deletes the index folder from its storage. -func (idx *Unique) Delete() error { - ctx, err := idx.getAuthenticatedContext(context.Background()) - if err != nil { - return err - } - - return deleteIndexRoot(ctx, idx.storageProvider, idx.cs3conf.ServiceUser.UUID, idx.indexRootDir) -} diff --git a/ocis-pkg/indexer/index/cs3/unique_test.go b/ocis-pkg/indexer/index/cs3/unique_test.go deleted file mode 100644 index 2a883ccea..000000000 --- a/ocis-pkg/indexer/index/cs3/unique_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package cs3 - -// -//import ( -// "os" -// "path" -// "testing" -// -// "github.com/owncloud/ocis/v2/accounts/pkg/config" -// "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option" -// . "github.com/owncloud/ocis/v2/ocis-pkg/indexer/test" -// "github.com/stretchr/testify/assert" -//) -// -//func TestCS3UniqueIndex_FakeSymlink(t *testing.T) { -// dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder) -// assert.NoError(t, err) -// cfg := config.Config{ -// Repo: config.Repo{ -// Disk: config.Disk{ -// Path: "", -// }, -// CS3: config.CS3{ -// ProviderAddr: "0.0.0.0:9215", -// DataURL: "http://localhost:9216", -// DataPrefix: "data", -// JWTSecret: "Pive-Fumkiu4", -// }, -// }, -// } -// -// sut := NewUniqueIndexWithOptions( -// option.WithTypeName(GetTypeFQN(User{})), -// option.WithIndexBy("UserName"), -// option.WithFilesDir(path.Join(cfg.Repo.Disk.Path, "/")), -// option.WithDataDir(cfg.Repo.Disk.Path), -// option.WithDataURL(cfg.Repo.CS3.DataURL), -// option.WithDataPrefix(cfg.Repo.CS3.DataPrefix), -// option.WithJWTSecret(cfg.Repo.CS3.JWTSecret), -// option.WithProviderAddr(cfg.Repo.CS3.ProviderAddr), -// ) -// -// err = sut.Init() -// assert.NoError(t, err) -// -// res, err := sut.Add("abcdefg-123", "mikey") -// assert.NoError(t, err) -// t.Log(res) -// -// resLookup, err := sut.Lookup("mikey") -// assert.NoError(t, err) -// t.Log(resLookup) -// -// err = sut.Update("abcdefg-123", "mikey", "mickeyX") -// assert.NoError(t, err) -// -// searchRes, err := sut.Search("m*") -// assert.NoError(t, err) -// assert.Len(t, searchRes, 1) -// assert.Equal(t, searchRes[0], "abcdefg-123") -// -// _ = os.RemoveAll(dataDir) -//} -// -//func TestCS3UniqueIndexSearch(t *testing.T) { -// dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder) -// assert.NoError(t, err) -// cfg := config.Config{ -// Repo: config.Repo{ -// Disk: config.Disk{ -// Path: "", -// }, -// CS3: config.CS3{ -// ProviderAddr: "0.0.0.0:9215", -// DataURL: "http://localhost:9216", -// DataPrefix: "data", -// JWTSecret: "Pive-Fumkiu4", -// }, -// }, -// } -// -// sut := NewUniqueIndexWithOptions( -// option.WithTypeName(GetTypeFQN(User{})), -// option.WithIndexBy("UserName"), -// option.WithFilesDir(path.Join(cfg.Repo.Disk.Path, "/")), -// option.WithDataDir(cfg.Repo.Disk.Path), -// option.WithDataURL(cfg.Repo.CS3.DataURL), -// option.WithDataPrefix(cfg.Repo.CS3.DataPrefix), -// option.WithJWTSecret(cfg.Repo.CS3.JWTSecret), -// option.WithProviderAddr(cfg.Repo.CS3.ProviderAddr), -// ) -// -// err = sut.Init() -// assert.NoError(t, err) -// -// _, err = sut.Add("hijklmn-456", "mikey") -// assert.NoError(t, err) -// -// _, err = sut.Add("ewf4ofk-555", "jacky") -// assert.NoError(t, err) -// -// res, err := sut.Search("*y") -// assert.NoError(t, err) -// t.Log(res) -// -// _ = os.RemoveAll(dataDir) -//} diff --git a/ocis-pkg/indexer/index/disk/autoincrement.go b/ocis-pkg/indexer/index/disk/autoincrement.go deleted file mode 100644 index 25892109c..000000000 --- a/ocis-pkg/indexer/index/disk/autoincrement.go +++ /dev/null @@ -1,226 +0,0 @@ -package disk - -import ( - "errors" - "os" - "path" - "path/filepath" - "strconv" - "strings" - - idxerrs "github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors" - - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/registry" -) - -// Autoincrement are fields for an index of type autoincrement. -type Autoincrement struct { - indexBy string - typeName string - filesDir string - indexBaseDir string - indexRootDir string - - bound *option.Bound -} - -func init() { - registry.IndexConstructorRegistry["disk"]["autoincrement"] = NewAutoincrementIndex -} - -// NewAutoincrementIndex instantiates a new AutoincrementIndex instance. Init() MUST be called upon instantiation. -func NewAutoincrementIndex(o ...option.Option) index.Index { - opts := &option.Options{} - for _, opt := range o { - opt(opts) - } - - if opts.Entity == nil { - panic("invalid autoincrement index: configured without entity") - } - - k, err := getKind(opts.Entity, opts.IndexBy) - if !isValidKind(k) || err != nil { - panic("invalid autoincrement index: configured on non-numeric field") - } - - return &Autoincrement{ - indexBy: opts.IndexBy, - typeName: opts.TypeName, - filesDir: opts.FilesDir, - bound: opts.Bound, - indexBaseDir: path.Join(opts.DataDir, "index.disk"), - indexRootDir: path.Join(path.Join(opts.DataDir, "index.disk"), strings.Join([]string{"autoincrement", opts.TypeName, opts.IndexBy}, ".")), - } -} - -// Init initializes an autoincrement index. -func (idx *Autoincrement) Init() error { - if _, err := os.Stat(idx.filesDir); err != nil { - return err - } - - if err := os.MkdirAll(idx.indexRootDir, 0777); err != nil { - return err - } - - return nil -} - -// Lookup exact lookup by value. -func (idx *Autoincrement) Lookup(v string) ([]string, error) { - searchPath := path.Join(idx.indexRootDir, v) - if err := isValidSymlink(searchPath); err != nil { - if os.IsNotExist(err) { - err = &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v} - } - - return nil, err - } - - p, err := os.Readlink(searchPath) - if err != nil { - return []string{}, nil - } - - return []string{p}, err -} - -// Add a new value to the index. -func (idx *Autoincrement) Add(id, v string) (string, error) { - nextID, err := idx.next() - if err != nil { - return "", err - } - oldName := filepath.Join(idx.filesDir, id) - var newName string - if v == "" { - newName = filepath.Join(idx.indexRootDir, strconv.Itoa(nextID)) - } else { - newName = filepath.Join(idx.indexRootDir, v) - } - err = os.Symlink(oldName, newName) - if errors.Is(err, os.ErrExist) { - return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v} - } - - return newName, err -} - -// Remove a value v from an index. -func (idx *Autoincrement) Remove(id string, v string) error { - if v == "" { - return nil - } - searchPath := path.Join(idx.indexRootDir, v) - return os.Remove(searchPath) -} - -// Update index from to . -func (idx *Autoincrement) Update(id, oldV, newV string) error { - oldPath := path.Join(idx.indexRootDir, oldV) - if err := isValidSymlink(oldPath); err != nil { - if os.IsNotExist(err) { - return &idxerrs.NotFoundErr{TypeName: idx.TypeName(), Key: idx.IndexBy(), Value: oldV} - } - - return err - } - - newPath := path.Join(idx.indexRootDir, newV) - err := isValidSymlink(newPath) - if err == nil { - return &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: newV} - } - - if os.IsNotExist(err) { - err = os.Rename(oldPath, newPath) - } - - return err -} - -// Search allows for glob search on the index. -func (idx *Autoincrement) Search(pattern string) ([]string, error) { - paths, err := filepath.Glob(path.Join(idx.indexRootDir, pattern)) - if err != nil { - return nil, err - } - - if len(paths) == 0 { - return nil, &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: pattern} - } - - res := make([]string, 0) - for _, p := range paths { - if err := isValidSymlink(p); err != nil { - return nil, err - } - - src, err := os.Readlink(p) - if err != nil { - return nil, err - } - - res = append(res, src) - } - - return res, nil -} - -// CaseInsensitive undocumented. -func (idx *Autoincrement) CaseInsensitive() bool { - return false -} - -// IndexBy undocumented. -func (idx *Autoincrement) IndexBy() string { - return idx.indexBy -} - -// TypeName undocumented. -func (idx *Autoincrement) TypeName() string { - return idx.typeName -} - -// FilesDir undocumented. -func (idx *Autoincrement) FilesDir() string { - return idx.filesDir -} - -func (idx *Autoincrement) next() (int, error) { - files, err := readDir(idx.indexRootDir) - if err != nil { - return -1, err - } - - if len(files) == 0 { - return int(idx.bound.Lower), nil - } - - latest, err := lastValueFromTree(files) - if err != nil { - return -1, err - } - - if int64(latest) < idx.bound.Lower { - return int(idx.bound.Lower), nil - } - - return latest + 1, nil -} - -// Delete deletes the index root folder from the configured storage. -func (idx *Autoincrement) Delete() error { - return os.RemoveAll(idx.indexRootDir) -} - -func lastValueFromTree(files []os.FileInfo) (int, error) { - latest, err := strconv.Atoi(path.Base(files[len(files)-1].Name())) - if err != nil { - return -1, err - } - return latest, nil -} diff --git a/ocis-pkg/indexer/index/disk/autoincrement_test.go b/ocis-pkg/indexer/index/disk/autoincrement_test.go deleted file mode 100644 index 5d4b38dd3..000000000 --- a/ocis-pkg/indexer/index/disk/autoincrement_test.go +++ /dev/null @@ -1,299 +0,0 @@ -package disk - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option" - //. "github.com/owncloud/ocis/v2/ocis-pkg/indexer/test" - accountsmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/accounts/v0" - "github.com/stretchr/testify/assert" -) - -func TestIsValidKind(t *testing.T) { - scenarios := []struct { - panics bool - name string - indexBy string - entity struct { - Number int - Name string - NumberFloat float32 - } - }{ - { - name: "valid autoincrement index", - panics: false, - indexBy: "Number", - entity: struct { - Number int - Name string - NumberFloat float32 - }{ - Name: "tesy-mc-testace", - }, - }, - { - name: "create autoincrement index on a non-existing field", - panics: true, - indexBy: "Age", - entity: struct { - Number int - Name string - NumberFloat float32 - }{ - Name: "tesy-mc-testace", - }, - }, - { - name: "attempt to create an autoincrement index with no entity", - panics: true, - indexBy: "Age", - }, - { - name: "create autoincrement index on a non-numeric field", - panics: true, - indexBy: "Name", - entity: struct { - Number int - Name string - NumberFloat float32 - }{ - Name: "tesy-mc-testace", - }, - }, - } - - for _, scenario := range scenarios { - t.Run(scenario.name, func(t *testing.T) { - if scenario.panics { - assert.Panics(t, func() { - _ = NewAutoincrementIndex( - option.WithEntity(scenario.entity), - option.WithIndexBy(scenario.indexBy), - ) - }) - } else { - assert.NotPanics(t, func() { - _ = NewAutoincrementIndex( - option.WithEntity(scenario.entity), - option.WithIndexBy(scenario.indexBy), - ) - }) - } - }) - } -} - -func TestNext(t *testing.T) { - scenarios := []struct { - name string - expected int - indexBy string - entity interface{} - }{ - { - name: "get next value", - expected: 0, - indexBy: "Number", - entity: struct { - Number int - Name string - NumberFloat float32 - }{ - Name: "tesy-mc-testace", - }, - }, - } - - for _, scenario := range scenarios { - t.Run(scenario.name, func(t *testing.T) { - tmpDir, err := createTmpDirStr() - assert.NoError(t, err) - - err = os.MkdirAll(filepath.Join(tmpDir, "data"), 0777) - assert.NoError(t, err) - - i := NewAutoincrementIndex( - option.WithBounds(&option.Bound{ - Lower: 0, - Upper: 0, - }), - option.WithDataDir(tmpDir), - option.WithFilesDir(filepath.Join(tmpDir, "data")), - option.WithEntity(scenario.entity), - option.WithTypeName("LambdaType"), - option.WithIndexBy(scenario.indexBy), - ) - - err = i.Init() - assert.NoError(t, err) - - tmpFile, err := os.Create(filepath.Join(tmpDir, "data", "test-example")) - assert.NoError(t, err) - assert.NoError(t, tmpFile.Close()) - - oldName, err := i.Add("test-example", "") - assert.NoError(t, err) - assert.Equal(t, "0", filepath.Base(oldName)) - - oldName, err = i.Add("test-example", "") - assert.NoError(t, err) - assert.Equal(t, "1", filepath.Base(oldName)) - - oldName, err = i.Add("test-example", "") - assert.NoError(t, err) - assert.Equal(t, "2", filepath.Base(oldName)) - t.Log(oldName) - - _ = os.RemoveAll(tmpDir) - }) - } -} - -func TestLowerBound(t *testing.T) { - scenarios := []struct { - name string - expected int - indexBy string - entity interface{} - }{ - { - name: "get next value with a lower bound specified", - expected: 0, - indexBy: "Number", - entity: struct { - Number int - Name string - NumberFloat float32 - }{ - Name: "tesy-mc-testace", - }, - }, - } - - for _, scenario := range scenarios { - t.Run(scenario.name, func(t *testing.T) { - tmpDir, err := createTmpDirStr() - assert.NoError(t, err) - - err = os.MkdirAll(filepath.Join(tmpDir, "data"), 0777) - assert.NoError(t, err) - - i := NewAutoincrementIndex( - option.WithBounds(&option.Bound{ - Lower: 1000, - }), - option.WithDataDir(tmpDir), - option.WithFilesDir(filepath.Join(tmpDir, "data")), - option.WithEntity(scenario.entity), - option.WithTypeName("LambdaType"), - option.WithIndexBy(scenario.indexBy), - ) - - err = i.Init() - assert.NoError(t, err) - - tmpFile, err := os.Create(filepath.Join(tmpDir, "data", "test-example")) - assert.NoError(t, err) - assert.NoError(t, tmpFile.Close()) - - oldName, err := i.Add("test-example", "") - assert.NoError(t, err) - assert.Equal(t, "1000", filepath.Base(oldName)) - - oldName, err = i.Add("test-example", "") - assert.NoError(t, err) - assert.Equal(t, "1001", filepath.Base(oldName)) - - oldName, err = i.Add("test-example", "") - assert.NoError(t, err) - assert.Equal(t, "1002", filepath.Base(oldName)) - t.Log(oldName) - - _ = os.RemoveAll(tmpDir) - }) - } -} - -func TestAdd(t *testing.T) { - tmpDir, err := createTmpDirStr() - assert.NoError(t, err) - - err = os.MkdirAll(filepath.Join(tmpDir, "data"), 0777) - assert.NoError(t, err) - - tmpFile, err := os.Create(filepath.Join(tmpDir, "data", "test-example")) - assert.NoError(t, err) - assert.NoError(t, tmpFile.Close()) - - i := NewAutoincrementIndex( - option.WithBounds(&option.Bound{ - Lower: 0, - Upper: 0, - }), - option.WithDataDir(tmpDir), - option.WithFilesDir(filepath.Join(tmpDir, "data")), - option.WithEntity(&accountsmsg.Account{}), - option.WithTypeName("owncloud.Account"), - option.WithIndexBy("UidNumber"), - ) - - err = i.Init() - assert.NoError(t, err) - - _, err = i.Add("test-example", "") - if err != nil { - t.Error(err) - } -} - -func BenchmarkAdd(b *testing.B) { - tmpDir, err := createTmpDirStr() - assert.NoError(b, err) - - err = os.MkdirAll(filepath.Join(tmpDir, "data"), 0777) - assert.NoError(b, err) - - tmpFile, err := os.Create(filepath.Join(tmpDir, "data", "test-example")) - assert.NoError(b, err) - assert.NoError(b, tmpFile.Close()) - - i := NewAutoincrementIndex( - option.WithBounds(&option.Bound{ - Lower: 0, - Upper: 0, - }), - option.WithDataDir(tmpDir), - option.WithFilesDir(filepath.Join(tmpDir, "data")), - option.WithEntity(struct { - Number int - Name string - NumberFloat float32 - }{}), - option.WithTypeName("LambdaType"), - option.WithIndexBy("Number"), - ) - - err = i.Init() - assert.NoError(b, err) - - for n := 0; n < b.N; n++ { - _, err := i.Add("test-example", "") - if err != nil { - b.Error(err) - } - assert.NoError(b, err) - } -} - -func createTmpDirStr() (string, error) { - name, err := ioutil.TempDir("/tmp", "testfiles-*") - if err != nil { - return "", err - } - - return name, nil -} diff --git a/ocis-pkg/indexer/index/disk/helper.go b/ocis-pkg/indexer/index/disk/helper.go deleted file mode 100644 index 50eafd413..000000000 --- a/ocis-pkg/indexer/index/disk/helper.go +++ /dev/null @@ -1,52 +0,0 @@ -package disk - -import ( - "os" - "reflect" - "sort" - "strconv" -) - -var ( - validKinds = []reflect.Kind{ - reflect.Int, - reflect.Int8, - reflect.Int16, - reflect.Int32, - reflect.Int64, - } -) - -// verifies an autoincrement field kind on the target struct. -func isValidKind(k reflect.Kind) bool { - for _, v := range validKinds { - if k == v { - return true - } - } - return false -} - -func getKind(i interface{}, field string) (reflect.Kind, error) { - r := reflect.ValueOf(i) - return reflect.Indirect(r).FieldByName(field).Kind(), nil -} - -// readDir is an implementation of os.ReadDir but with different sorting. -func readDir(dirname string) ([]os.FileInfo, error) { - f, err := os.Open(dirname) - if err != nil { - return nil, err - } - list, err := f.Readdir(-1) - f.Close() - if err != nil { - return nil, err - } - sort.Slice(list, func(i, j int) bool { - a, _ := strconv.Atoi(list[i].Name()) - b, _ := strconv.Atoi(list[j].Name()) - return a < b - }) - return list, nil -} diff --git a/ocis-pkg/indexer/index/disk/non_unique.go b/ocis-pkg/indexer/index/disk/non_unique.go deleted file mode 100644 index cbf759637..000000000 --- a/ocis-pkg/indexer/index/disk/non_unique.go +++ /dev/null @@ -1,240 +0,0 @@ -package disk - -import ( - "errors" - "io/ioutil" - "os" - "path" - "path/filepath" - "strings" - - idxerrs "github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/registry" -) - -// NonUnique is able to index an document by a key which might contain non-unique values -// -// /tmp/testfiles-395764020/index.disk/PetByColor/ -// ├── Brown -// │ └── rebef-123 -> /tmp/testfiles-395764020/pets/rebef-123 -// ├── Green -// │ ├── goefe-789 -> /tmp/testfiles-395764020/pets/goefe-789 -// │ └── xadaf-189 -> /tmp/testfiles-395764020/pets/xadaf-189 -// └── White -// └── wefwe-456 -> /tmp/testfiles-395764020/pets/wefwe-456 -type NonUnique struct { - caseInsensitive bool - indexBy string - typeName string - filesDir string - indexBaseDir string - indexRootDir string -} - -func init() { - registry.IndexConstructorRegistry["disk"]["non_unique"] = NewNonUniqueIndexWithOptions -} - -// NewNonUniqueIndexWithOptions instantiates a new UniqueIndex instance. Init() should be -// called afterward to ensure correct on-disk structure. -func NewNonUniqueIndexWithOptions(o ...option.Option) index.Index { - opts := &option.Options{} - for _, opt := range o { - opt(opts) - } - - return &NonUnique{ - caseInsensitive: opts.CaseInsensitive, - indexBy: opts.IndexBy, - typeName: opts.TypeName, - filesDir: opts.FilesDir, - indexBaseDir: path.Join(opts.DataDir, "index.disk"), - indexRootDir: path.Join(path.Join(opts.DataDir, "index.disk"), strings.Join([]string{"non_unique", opts.TypeName, opts.IndexBy}, ".")), - } -} - -// Init initializes a unique index. -func (idx *NonUnique) Init() error { - if _, err := os.Stat(idx.filesDir); err != nil { - return err - } - - if err := os.MkdirAll(idx.indexRootDir, 0777); err != nil { - return err - } - - return nil -} - -// Lookup exact lookup by value. -func (idx *NonUnique) Lookup(v string) ([]string, error) { - if idx.caseInsensitive { - v = strings.ToLower(v) - } - searchPath := path.Join(idx.indexRootDir, v) - fi, err := ioutil.ReadDir(searchPath) - if os.IsNotExist(err) { - return []string{}, &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v} - } - - if err != nil { - return []string{}, err - } - - ids := make([]string, 0, len(fi)) - for _, f := range fi { - ids = append(ids, f.Name()) - } - - if len(ids) == 0 { - return []string{}, &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v} - } - - return ids, nil -} - -// Add adds a value to the index, returns the path to the root-document -func (idx *NonUnique) Add(id, v string) (string, error) { - if v == "" { - return "", nil - } - if idx.caseInsensitive { - v = strings.ToLower(v) - } - oldName := path.Join(idx.filesDir, id) - newName := path.Join(idx.indexRootDir, v, id) - - if err := os.MkdirAll(path.Join(idx.indexRootDir, v), 0777); err != nil { - return "", err - } - - err := os.Symlink(oldName, newName) - if errors.Is(err, os.ErrExist) { - return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v} - } - - return newName, err - -} - -// Remove a value v from an index. -func (idx *NonUnique) Remove(id string, v string) error { - if v == "" { - return nil - } - if idx.caseInsensitive { - v = strings.ToLower(v) - } - res, err := filepath.Glob(path.Join(idx.indexRootDir, "/*/", id)) - if err != nil { - return err - } - - for _, p := range res { - if err := os.Remove(p); err != nil { - return err - } - } - - // Remove value directory if it is empty - valueDir := path.Join(idx.indexRootDir, v) - fi, err := ioutil.ReadDir(valueDir) - if err != nil { - return err - } - - if len(fi) == 0 { - if err := os.RemoveAll(valueDir); err != nil { - return err - } - } - - return nil -} - -// Update index from to . -func (idx *NonUnique) Update(id, oldV, newV string) (err error) { - if idx.caseInsensitive { - oldV = strings.ToLower(oldV) - newV = strings.ToLower(newV) - } - oldDir := path.Join(idx.indexRootDir, oldV) - oldPath := path.Join(oldDir, id) - newDir := path.Join(idx.indexRootDir, newV) - newPath := path.Join(newDir, id) - - if _, err = os.Stat(oldPath); os.IsNotExist(err) { - return &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: oldV} - } - - if err != nil { - return - } - - if err = os.MkdirAll(newDir, 0777); err != nil { - return - } - - if err = os.Rename(oldPath, newPath); err != nil { - return - } - - di, err := ioutil.ReadDir(oldDir) - if err != nil { - return err - } - - if len(di) == 0 { - err = os.RemoveAll(oldDir) - if err != nil { - return - } - } - - return - -} - -// Search allows for glob search on the index. -func (idx *NonUnique) Search(pattern string) ([]string, error) { - if idx.caseInsensitive { - pattern = strings.ToLower(pattern) - } - paths, err := filepath.Glob(path.Join(idx.indexRootDir, pattern, "*")) - if err != nil { - return nil, err - } - - if len(paths) == 0 { - return nil, &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: pattern} - } - - return paths, nil -} - -// CaseInsensitive undocumented. -func (idx *NonUnique) CaseInsensitive() bool { - return idx.caseInsensitive -} - -// IndexBy undocumented. -func (idx *NonUnique) IndexBy() string { - return idx.indexBy -} - -// TypeName undocumented. -func (idx *NonUnique) TypeName() string { - return idx.typeName -} - -// FilesDir undocumented. -func (idx *NonUnique) FilesDir() string { - return idx.filesDir -} - -// Delete deletes the index folder from its storage. -func (idx *NonUnique) Delete() error { - return os.RemoveAll(idx.indexRootDir) -} diff --git a/ocis-pkg/indexer/index/disk/non_unique_test.go b/ocis-pkg/indexer/index/disk/non_unique_test.go deleted file mode 100644 index eac3da199..000000000 --- a/ocis-pkg/indexer/index/disk/non_unique_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package disk - -import ( - "fmt" - "os" - "path" - "testing" - - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/config" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option" - . "github.com/owncloud/ocis/v2/ocis-pkg/indexer/test" - "github.com/stretchr/testify/assert" -) - -func TestNonUniqueIndexAdd(t *testing.T) { - sut, dataPath := getNonUniqueIdxSut(t, Pet{}, "Color") - - ids, err := sut.Lookup("Green") - assert.NoError(t, err) - assert.EqualValues(t, []string{"goefe-789", "xadaf-189"}, ids) - - ids, err = sut.Lookup("White") - assert.NoError(t, err) - assert.EqualValues(t, []string{"wefwe-456"}, ids) - - ids, err = sut.Lookup("Cyan") - assert.Error(t, err) - assert.EqualValues(t, []string{}, ids) - - _ = os.RemoveAll(dataPath) - -} - -func TestNonUniqueIndexUpdate(t *testing.T) { - sut, dataPath := getNonUniqueIdxSut(t, Pet{}, "Color") - - err := sut.Update("goefe-789", "Green", "Black") - assert.NoError(t, err) - - err = sut.Update("xadaf-189", "Green", "Black") - assert.NoError(t, err) - - assert.DirExists(t, path.Join(dataPath, fmt.Sprintf("index.disk/non_unique.%v.Color/Black", GetTypeFQN(Pet{})))) - assert.NoDirExists(t, path.Join(dataPath, fmt.Sprintf("index.disk/non_unique.%v.Color/Green", GetTypeFQN(Pet{})))) - - _ = os.RemoveAll(dataPath) -} - -func TestNonUniqueIndexDelete(t *testing.T) { - sut, dataPath := getNonUniqueIdxSut(t, Pet{}, "Color") - assert.FileExists(t, path.Join(dataPath, fmt.Sprintf("index.disk/non_unique.%v.Color/Green/goefe-789", GetTypeFQN(Pet{})))) - - err := sut.Remove("goefe-789", "Green") - assert.NoError(t, err) - assert.NoFileExists(t, path.Join(dataPath, fmt.Sprintf("index.disk/non_unique.%v.Color/Green/goefe-789", GetTypeFQN(Pet{})))) - assert.FileExists(t, path.Join(dataPath, fmt.Sprintf("index.disk/non_unique.%v.Color/Green/xadaf-189", GetTypeFQN(Pet{})))) - - _ = os.RemoveAll(dataPath) -} - -func TestNonUniqueIndexSearch(t *testing.T) { - sut, dataPath := getNonUniqueIdxSut(t, Pet{}, "Email") - - res, err := sut.Search("Gr*") - - assert.NoError(t, err) - assert.Len(t, res, 2) - - assert.Equal(t, "goefe-789", path.Base(res[0])) - assert.Equal(t, "xadaf-189", path.Base(res[1])) - - _, err = sut.Search("does-not-exist@example.com") - assert.Error(t, err) - assert.IsType(t, &errors.NotFoundErr{}, err) - - _ = os.RemoveAll(dataPath) -} - -// entity: used to get the fully qualified name for the index root path. -func getNonUniqueIdxSut(t *testing.T, entity interface{}, indexBy string) (index.Index, string) { - dataPath, _ := WriteIndexTestData(Data, "ID", "") - cfg := config.Config{ - Repo: config.Repo{ - Backend: "disk", - Disk: config.Disk{ - Path: dataPath, - }, - }, - } - - sut := NewNonUniqueIndexWithOptions( - option.WithTypeName(GetTypeFQN(entity)), - option.WithIndexBy(indexBy), - option.WithFilesDir(path.Join(cfg.Repo.Disk.Path, "pets")), - option.WithDataDir(cfg.Repo.Disk.Path), - ) - err := sut.Init() - if err != nil { - t.Fatal(err) - } - - for _, u := range Data["pets"] { - pkVal := ValueOf(u, "ID") - idxByVal := ValueOf(u, "Color") - _, err := sut.Add(pkVal, idxByVal) - if err != nil { - t.Fatal(err) - } - } - - return sut, dataPath -} diff --git a/ocis-pkg/indexer/index/disk/unique.go b/ocis-pkg/indexer/index/disk/unique.go deleted file mode 100644 index afce71295..000000000 --- a/ocis-pkg/indexer/index/disk/unique.go +++ /dev/null @@ -1,227 +0,0 @@ -package disk - -import ( - "errors" - "fmt" - "os" - "path" - "path/filepath" - "strings" - - idxerrs "github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/registry" -) - -// Unique ensures that only one document of the same type and key-value combination can exist in the index. -// -// Modeled by creating a indexer-folder per entity and key with symlinks which point to respective documents which contain -// the link-filename as value. -// -// Directory Layout -// -// /var/data/index.disk/UniqueUserByEmail/ -// ├── jacky@example.com -> /var/data/users/ewf4ofk-555 -// ├── jones@example.com -> /var/data/users/rulan54-777 -// └── mikey@example.com -> /var/data/users/abcdefg-123 -// -// Example user -// -// { -// "Id": "ewf4ofk-555", -// "UserName": "jacky", -// "Email": "jacky@example.com" -// } -// -type Unique struct { - caseInsensitive bool - indexBy string - typeName string - filesDir string - indexBaseDir string - indexRootDir string -} - -func init() { - registry.IndexConstructorRegistry["disk"]["unique"] = NewUniqueIndexWithOptions -} - -// NewUniqueIndexWithOptions instantiates a new UniqueIndex instance. Init() should be -// called afterward to ensure correct on-disk structure. -func NewUniqueIndexWithOptions(o ...option.Option) index.Index { - opts := &option.Options{} - for _, opt := range o { - opt(opts) - } - - return &Unique{ - caseInsensitive: opts.CaseInsensitive, - indexBy: opts.IndexBy, - typeName: opts.TypeName, - filesDir: opts.FilesDir, - indexBaseDir: path.Join(opts.DataDir, "index.disk"), - indexRootDir: path.Join(path.Join(opts.DataDir, "index.disk"), strings.Join([]string{"unique", opts.TypeName, opts.IndexBy}, ".")), - } -} - -// Init initializes a unique index. -func (idx *Unique) Init() error { - if _, err := os.Stat(idx.filesDir); err != nil { - return err - } - - if err := os.MkdirAll(idx.indexRootDir, 0777); err != nil { - return err - } - - return nil -} - -// Add adds a value to the index, returns the path to the root-document -func (idx *Unique) Add(id, v string) (string, error) { - if v == "" { - return "", nil - } - if idx.caseInsensitive { - v = strings.ToLower(v) - } - oldName := path.Join(idx.filesDir, id) - newName := path.Join(idx.indexRootDir, v) - err := os.Symlink(oldName, newName) - if errors.Is(err, os.ErrExist) { - return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v} - } - - return newName, err -} - -// Remove a value v from an index. -func (idx *Unique) Remove(id string, v string) (err error) { - if v == "" { - return nil - } - if idx.caseInsensitive { - v = strings.ToLower(v) - } - searchPath := path.Join(idx.indexRootDir, v) - return os.Remove(searchPath) -} - -// Lookup exact lookup by value. -func (idx *Unique) Lookup(v string) (resultPath []string, err error) { - if idx.caseInsensitive { - v = strings.ToLower(v) - } - searchPath := path.Join(idx.indexRootDir, v) - if err = isValidSymlink(searchPath); err != nil { - if os.IsNotExist(err) { - err = &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v} - } - return - } - - p, err := os.Readlink(searchPath) - if err != nil { - return []string{}, nil - } - - return []string{p}, err -} - -// Update index from to . -func (idx *Unique) Update(id, oldV, newV string) (err error) { - if idx.caseInsensitive { - oldV = strings.ToLower(oldV) - newV = strings.ToLower(newV) - } - oldPath := path.Join(idx.indexRootDir, oldV) - if err = isValidSymlink(oldPath); err != nil { - if os.IsNotExist(err) { - return &idxerrs.NotFoundErr{TypeName: idx.TypeName(), Key: idx.IndexBy(), Value: oldV} - } - - return - } - - newPath := path.Join(idx.indexRootDir, newV) - if err = isValidSymlink(newPath); err == nil { - return &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: newV} - } - - if os.IsNotExist(err) { - err = os.Rename(oldPath, newPath) - } - - return -} - -// Search allows for glob search on the index. -func (idx *Unique) Search(pattern string) ([]string, error) { - if idx.caseInsensitive { - pattern = strings.ToLower(pattern) - } - paths, err := filepath.Glob(path.Join(idx.indexRootDir, pattern)) - if err != nil { - return nil, err - } - - if len(paths) == 0 { - return nil, &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: pattern} - } - - res := make([]string, 0) - for _, p := range paths { - if err := isValidSymlink(p); err != nil { - return nil, err - } - - src, err := os.Readlink(p) - if err != nil { - return nil, err - } - - res = append(res, src) - } - - return res, nil -} - -// CaseInsensitive undocumented. -func (idx *Unique) CaseInsensitive() bool { - return idx.caseInsensitive -} - -// IndexBy undocumented. -func (idx *Unique) IndexBy() string { - return idx.indexBy -} - -// TypeName undocumented. -func (idx *Unique) TypeName() string { - return idx.typeName -} - -// FilesDir undocumented. -func (idx *Unique) FilesDir() string { - return idx.filesDir -} - -func isValidSymlink(path string) (err error) { - var symInfo os.FileInfo - if symInfo, err = os.Lstat(path); err != nil { - return - } - - if symInfo.Mode()&os.ModeSymlink == 0 { - err = fmt.Errorf("%s is not a valid symlink (bug/corruption?)", path) - return - } - - return -} - -// Delete deletes the index folder from its storage. -func (idx *Unique) Delete() error { - return os.RemoveAll(idx.indexRootDir) -} diff --git a/ocis-pkg/indexer/index/disk/unique_test.go b/ocis-pkg/indexer/index/disk/unique_test.go deleted file mode 100644 index fbd9d6a50..000000000 --- a/ocis-pkg/indexer/index/disk/unique_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package disk - -import ( - "os" - "path" - "testing" - - "github.com/owncloud/ocis/v2/extensions/accounts/pkg/config" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option" - . "github.com/owncloud/ocis/v2/ocis-pkg/indexer/test" - "github.com/stretchr/testify/assert" -) - -func TestUniqueLookupSingleEntry(t *testing.T) { - uniq, dataDir := getUniqueIdxSut(t, "Email", User{}) - filesDir := path.Join(dataDir, "users") - - t.Log("existing lookup") - resultPath, err := uniq.Lookup("mikey@example.com") - assert.NoError(t, err) - - assert.Equal(t, []string{path.Join(filesDir, "abcdefg-123")}, resultPath) - - t.Log("non-existing lookup") - resultPath, err = uniq.Lookup("doesnotExists@example.com") - assert.Error(t, err) - assert.IsType(t, &errors.NotFoundErr{}, err) - assert.Empty(t, resultPath) - - _ = os.RemoveAll(dataDir) - -} - -func TestUniqueUniqueConstraint(t *testing.T) { - uniq, dataDir := getUniqueIdxSut(t, "Email", User{}) - - _, err := uniq.Add("abcdefg-123", "mikey@example.com") - assert.Error(t, err) - assert.IsType(t, &errors.AlreadyExistsErr{}, err) - - _ = os.RemoveAll(dataDir) -} - -func TestUniqueRemove(t *testing.T) { - uniq, dataDir := getUniqueIdxSut(t, "Email", User{}) - - err := uniq.Remove("", "mikey@example.com") - assert.NoError(t, err) - - _, err = uniq.Lookup("mikey@example.com") - assert.Error(t, err) - assert.IsType(t, &errors.NotFoundErr{}, err) - - _ = os.RemoveAll(dataDir) -} - -func TestUniqueUpdate(t *testing.T) { - uniq, dataDir := getUniqueIdxSut(t, "Email", User{}) - - t.Log("successful update") - err := uniq.Update("", "mikey@example.com", "mikey2@example.com") - assert.NoError(t, err) - - t.Log("failed update because already exists") - err = uniq.Update("", "mikey2@example.com", "mikey2@example.com") - assert.Error(t, err) - assert.IsType(t, &errors.AlreadyExistsErr{}, err) - - t.Log("failed update because not found") - err = uniq.Update("", "nonexisting@example.com", "something2@example.com") - assert.Error(t, err) - assert.IsType(t, &errors.NotFoundErr{}, err) - - _ = os.RemoveAll(dataDir) -} - -func TestUniqueIndexSearch(t *testing.T) { - sut, dataDir := getUniqueIdxSut(t, "Email", User{}) - - res, err := sut.Search("j*@example.com") - - assert.NoError(t, err) - assert.Len(t, res, 2) - - assert.Equal(t, "ewf4ofk-555", path.Base(res[0])) - assert.Equal(t, "rulan54-777", path.Base(res[1])) - - _, err = sut.Search("does-not-exist@example.com") - assert.Error(t, err) - assert.IsType(t, &errors.NotFoundErr{}, err) - - _ = os.RemoveAll(dataDir) -} - -func TestErrors(t *testing.T) { - assert.True(t, errors.IsAlreadyExistsErr(&errors.AlreadyExistsErr{})) - assert.True(t, errors.IsNotFoundErr(&errors.NotFoundErr{})) -} - -func getUniqueIdxSut(t *testing.T, indexBy string, entityType interface{}) (index.Index, string) { - dataPath, _ := WriteIndexTestData(Data, "ID", "") - cfg := config.Config{ - Repo: config.Repo{ - Backend: "disk", - Disk: config.Disk{ - Path: dataPath, - }, - }, - } - - sut := NewUniqueIndexWithOptions( - option.WithTypeName(GetTypeFQN(entityType)), - option.WithIndexBy(indexBy), - option.WithFilesDir(path.Join(cfg.Repo.Disk.Path, "users")), - option.WithDataDir(cfg.Repo.Disk.Path), - ) - err := sut.Init() - if err != nil { - t.Fatal(err) - } - - for _, u := range Data["users"] { - pkVal := ValueOf(u, "ID") - idxByVal := ValueOf(u, "Email") - _, err := sut.Add(pkVal, idxByVal) - if err != nil { - t.Fatal(err) - } - } - - return sut, dataPath -} diff --git a/ocis-pkg/indexer/index/index.go b/ocis-pkg/indexer/index/index.go deleted file mode 100644 index 9ad7aee7b..000000000 --- a/ocis-pkg/indexer/index/index.go +++ /dev/null @@ -1,17 +0,0 @@ -package index - -// Index can be implemented to create new indexer-strategies. See Unique for example. -// Each indexer implementation is bound to one data-column (IndexBy) and a data-type (TypeName) -type Index interface { - Init() error - Lookup(v string) ([]string, error) - Add(id, v string) (string, error) - Remove(id string, v string) error - Update(id, oldV, newV string) error - Search(pattern string) ([]string, error) - CaseInsensitive() bool - IndexBy() string - TypeName() string - FilesDir() string - Delete() error // Delete deletes the index folder from its storage. -} diff --git a/ocis-pkg/indexer/indexer.go b/ocis-pkg/indexer/indexer.go deleted file mode 100644 index 3e406b86d..000000000 --- a/ocis-pkg/indexer/indexer.go +++ /dev/null @@ -1,403 +0,0 @@ -// Package indexer provides symlink-based indexer for on-disk document-directories. -package indexer - -import ( - "context" - "fmt" - "path" - "strings" - - "github.com/owncloud/ocis/v2/ocis-pkg/sync" - - "github.com/CiscoM31/godata" - "github.com/iancoleman/strcase" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/config" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index" - _ "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index/cs3" // to populate index - _ "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index/disk" // to populate index - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/registry" -) - -// Indexer is a facade to configure and query over multiple indices. -type Indexer struct { - config *config.Config - indices typeMap - mu sync.NamedRWMutex -} - -// IdxAddResult represents the result of an Add call on an index -type IdxAddResult struct { - Field, Value string -} - -// CreateIndexer creates a new Indexer. -func CreateIndexer(cfg *config.Config) *Indexer { - return &Indexer{ - config: cfg, - indices: typeMap{}, - mu: sync.NewNamedRWMutex(), - } -} - -// Reset takes care of deleting all indices from storage and from the internal map of indices -func (i *Indexer) Reset() error { - for j := range i.indices { - for _, indices := range i.indices[j].IndicesByField { - for _, idx := range indices { - err := idx.Delete() - if err != nil { - return err - } - } - } - delete(i.indices, j) - } - - return nil -} - -// AddIndex adds a new index to the indexer receiver. -func (i *Indexer) AddIndex(t interface{}, indexBy, pkName, entityDirName, indexType string, bound *option.Bound, caseInsensitive bool) error { - f := registry.IndexConstructorRegistry[i.config.Repo.Backend][indexType] - var idx index.Index - - if i.config.Repo.Backend == "cs3" { - idx = f( - option.CaseInsensitive(caseInsensitive), - option.WithEntity(t), - option.WithBounds(bound), - option.WithTypeName(getTypeFQN(t)), - option.WithIndexBy(indexBy), - option.WithDataURL(i.config.Repo.CS3.DataURL), - option.WithDataPrefix(i.config.Repo.CS3.DataPrefix), - option.WithJWTSecret(i.config.Repo.CS3.JWTSecret), - option.WithProviderAddr(i.config.Repo.CS3.ProviderAddr), - option.WithServiceUser(i.config.ServiceUser), - ) - } else { - idx = f( - option.CaseInsensitive(caseInsensitive), - option.WithEntity(t), - option.WithBounds(bound), - option.WithTypeName(getTypeFQN(t)), - option.WithIndexBy(indexBy), - option.WithFilesDir(path.Join(i.config.Repo.Disk.Path, entityDirName)), - option.WithDataDir(i.config.Repo.Disk.Path), - ) - } - - i.indices.addIndex(getTypeFQN(t), pkName, idx) - return idx.Init() -} - -// Add a new entry to the indexer -func (i *Indexer) Add(t interface{}) ([]IdxAddResult, error) { - typeName := getTypeFQN(t) - - i.mu.Lock(typeName) - defer i.mu.Unlock(typeName) - - var results []IdxAddResult - if fields, ok := i.indices[typeName]; ok { - for _, indices := range fields.IndicesByField { - for _, idx := range indices { - pkVal := valueOf(t, fields.PKFieldName) - idxByVal := valueOf(t, idx.IndexBy()) - value, err := idx.Add(pkVal, idxByVal) - if err != nil { - return []IdxAddResult{}, err - } - if value == "" { - continue - } - results = append(results, IdxAddResult{Field: idx.IndexBy(), Value: value}) - } - } - } - - return results, nil -} - -// FindBy finds a value on an index by field and value. -func (i *Indexer) FindBy(t interface{}, field string, val string) ([]string, error) { - typeName := getTypeFQN(t) - - i.mu.RLock(typeName) - defer i.mu.RUnlock(typeName) - - resultPaths := make([]string, 0) - if fields, ok := i.indices[typeName]; ok { - for _, idx := range fields.IndicesByField[strcase.ToCamel(field)] { - idxVal := val - res, err := idx.Lookup(idxVal) - if err != nil { - if errors.IsNotFoundErr(err) { - continue - } - - if err != nil { - return nil, err - } - } - - resultPaths = append(resultPaths, res...) - - } - } - - result := make([]string, 0, len(resultPaths)) - for _, v := range resultPaths { - result = append(result, path.Base(v)) - } - - return result, nil -} - -// Delete deletes all indexed fields of a given type t on the Indexer. -func (i *Indexer) Delete(t interface{}) error { - typeName := getTypeFQN(t) - - i.mu.Lock(typeName) - defer i.mu.Unlock(typeName) - - if fields, ok := i.indices[typeName]; ok { - for _, indices := range fields.IndicesByField { - for _, idx := range indices { - pkVal := valueOf(t, fields.PKFieldName) - idxByVal := valueOf(t, idx.IndexBy()) - if err := idx.Remove(pkVal, idxByVal); err != nil { - return err - } - } - } - } - - return nil -} - -// FindByPartial allows for glob search across all indexes. -func (i *Indexer) FindByPartial(t interface{}, field string, pattern string) ([]string, error) { - typeName := getTypeFQN(t) - - i.mu.RLock(typeName) - defer i.mu.RUnlock(typeName) - - resultPaths := make([]string, 0) - if fields, ok := i.indices[typeName]; ok { - for _, idx := range fields.IndicesByField[strcase.ToCamel(field)] { - res, err := idx.Search(pattern) - if err != nil { - if errors.IsNotFoundErr(err) { - continue - } - - if err != nil { - return nil, err - } - } - - resultPaths = append(resultPaths, res...) - - } - } - - result := make([]string, 0, len(resultPaths)) - for _, v := range resultPaths { - result = append(result, path.Base(v)) - } - - return result, nil - -} - -// Update updates all indexes on a value to a value . -func (i *Indexer) Update(from, to interface{}) error { - typeNameFrom := getTypeFQN(from) - - i.mu.Lock(typeNameFrom) - defer i.mu.Unlock(typeNameFrom) - - if typeNameTo := getTypeFQN(to); typeNameFrom != typeNameTo { - return fmt.Errorf("update types do not match: from %v to %v", typeNameFrom, typeNameTo) - } - - if fields, ok := i.indices[typeNameFrom]; ok { - for fName, indices := range fields.IndicesByField { - oldV := valueOf(from, fName) - newV := valueOf(to, fName) - pkVal := valueOf(from, fields.PKFieldName) - for _, idx := range indices { - if oldV == newV { - continue - } - if oldV == "" { - if _, err := idx.Add(pkVal, newV); err != nil { - return err - } - continue - } - if newV == "" { - if err := idx.Remove(pkVal, oldV); err != nil { - return err - } - continue - } - if err := idx.Update(pkVal, oldV, newV); err != nil { - return err - } - } - } - } - - return nil -} - -// Query parses an OData query into something our indexer.Index understands and resolves it. -func (i *Indexer) Query(ctx context.Context, t interface{}, q string) ([]string, error) { - query, err := godata.ParseFilterString(ctx, q) - if err != nil { - return nil, err - } - - tree := newQueryTree() - if err := buildTreeFromOdataQuery(query.Tree, &tree); err != nil { - return nil, err - } - - results := make([]string, 0) - if err := i.resolveTree(t, &tree, &results); err != nil { - return nil, err - } - - return results, nil -} - -// t is used to infer the indexed field names. When building an index search query, field names have to respect Golang -// conventions and be in PascalCase. For a better overview on this contemplate reading the reflection package under the -// indexer directory. Traversal of the tree happens in a pre-order fashion. -// TODO implement logic for `and` operators. -func (i *Indexer) resolveTree(t interface{}, tree *queryTree, partials *[]string) error { - if partials == nil { - return fmt.Errorf("return value cannot be nil: partials") - } - - if tree.left != nil { - _ = i.resolveTree(t, tree.left, partials) - } - - if tree.right != nil { - _ = i.resolveTree(t, tree.right, partials) - } - - // by the time we're here we reached a leaf node. - if tree.token != nil { - switch tree.token.filterType { - case "FindBy": - operand, err := sanitizeInput(tree.token.operands) - if err != nil { - return err - } - - r, err := i.FindBy(t, operand.field, operand.value) - if err != nil { - return err - } - - *partials = append(*partials, r...) - case "FindByPartial": - operand, err := sanitizeInput(tree.token.operands) - if err != nil { - return err - } - - r, err := i.FindByPartial(t, operand.field, fmt.Sprintf("%v*", operand.value)) - if err != nil { - return err - } - - *partials = append(*partials, r...) - default: - return fmt.Errorf("unsupported filter: %v", tree.token.filterType) - } - } - - *partials = dedup(*partials) - return nil -} - -type indexerTuple struct { - field, value string -} - -// sanitizeInput returns a tuple of fieldName + value to be applied on indexer.Index filters. -func sanitizeInput(operands []string) (*indexerTuple, error) { - if len(operands) != 2 { - return nil, fmt.Errorf("invalid number of operands for filter function: got %v expected 2", len(operands)) - } - - // field names are Go public types and by design they are in PascalCase, therefore we need to adhere to this rules. - // for further information on this have a look at the reflection package. - f := strcase.ToCamel(operands[0]) - - // remove single quotes from value. - v := strings.ReplaceAll(operands[1], "'", "") - return &indexerTuple{ - field: f, - value: v, - }, nil -} - -// buildTreeFromOdataQuery builds an indexer.queryTree out of a GOData ParseNode. The purpose of this intermediate tree -// is to transform godata operators and functions into supported operations on our index. At the time of this writing -// we only support `FindBy` and `FindByPartial` queries as these are the only implemented filters on indexer.Index(es). -func buildTreeFromOdataQuery(root *godata.ParseNode, tree *queryTree) error { - if root.Token.Type == godata.ExpressionTokenFunc { // i.e "startswith", "contains" - switch root.Token.Value { - case "startswith": - token := token{ - operator: root.Token.Value, - filterType: "FindByPartial", - // TODO sanitize the number of operands it the expected one. - operands: []string{ - root.Children[0].Token.Value, // field name, i.e: Name - root.Children[1].Token.Value, // field value, i.e: Jac - }, - } - - tree.insert(&token) - default: - return fmt.Errorf("operation not supported") - } - } - - if root.Token.Type == godata.ExpressionTokenLogical { - switch root.Token.Value { - case "or": - tree.insert(&token{operator: root.Token.Value}) - for _, child := range root.Children { - if err := buildTreeFromOdataQuery(child, tree.left); err != nil { - return err - } - } - case "eq": - tree.insert(&token{ - operator: root.Token.Value, - filterType: "FindBy", - operands: []string{ - root.Children[0].Token.Value, - root.Children[1].Token.Value, - }, - }) - for _, child := range root.Children { - if err := buildTreeFromOdataQuery(child, tree.left); err != nil { - return err - } - } - default: - return fmt.Errorf("operator not supported") - } - } - return nil -} diff --git a/ocis-pkg/indexer/indexer_test.go b/ocis-pkg/indexer/indexer_test.go deleted file mode 100644 index 25ac2f8da..000000000 --- a/ocis-pkg/indexer/indexer_test.go +++ /dev/null @@ -1,311 +0,0 @@ -package indexer - -import ( - "context" - "os" - "path" - "testing" - - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option" - - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/config" - _ "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index/cs3" - _ "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index/disk" - . "github.com/owncloud/ocis/v2/ocis-pkg/indexer/test" - "github.com/stretchr/testify/assert" -) - -//const cs3RootFolder = "/tmp/ocis/storage/users/data" -// -//func TestIndexer_CS3_AddWithUniqueIndex(t *testing.T) { -// dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder) -// assert.NoError(t, err) -// indexer := createCs3Indexer() -// -// err = indexer.AddIndex(&User{}, "UserName", "ID", "users", "unique", nil, false) -// assert.NoError(t, err) -// -// u := &User{ID: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"} -// _, err = indexer.Add(u) -// assert.NoError(t, err) -// -// _ = os.RemoveAll(dataDir) -//} -// -//func TestIndexer_CS3_AddWithNonUniqueIndex(t *testing.T) { -// dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder) -// assert.NoError(t, err) -// indexer := createCs3Indexer() -// -// err = indexer.AddIndex(&User{}, "UserName", "ID", "users", "non_unique", nil, false) -// assert.NoError(t, err) -// -// u := &User{ID: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"} -// _, err = indexer.Add(u) -// assert.NoError(t, err) -// -// _ = os.RemoveAll(dataDir) -//} - -func TestIndexer_Disk_FindByWithUniqueIndex(t *testing.T) { - dataDir, err := WriteIndexTestData(Data, "ID", "") - assert.NoError(t, err) - indexer := createDiskIndexer(dataDir) - - err = indexer.AddIndex(&User{}, "UserName", "ID", "users", "unique", nil, false) - assert.NoError(t, err) - - u := &User{ID: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"} - _, err = indexer.Add(u) - assert.NoError(t, err) - - res, err := indexer.FindBy(User{}, "UserName", "mikey") - assert.NoError(t, err) - t.Log(res) - - _ = os.RemoveAll(dataDir) -} - -func TestIndexer_Disk_AddWithUniqueIndex(t *testing.T) { - dataDir, err := WriteIndexTestData(Data, "ID", "") - assert.NoError(t, err) - indexer := createDiskIndexer(dataDir) - - err = indexer.AddIndex(&User{}, "UserName", "ID", "users", "unique", nil, false) - assert.NoError(t, err) - - u := &User{ID: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"} - _, err = indexer.Add(u) - assert.NoError(t, err) - - _ = os.RemoveAll(dataDir) -} - -func TestIndexer_Disk_AddWithNonUniqueIndex(t *testing.T) { - dataDir, err := WriteIndexTestData(Data, "ID", "") - assert.NoError(t, err) - indexer := createDiskIndexer(dataDir) - - err = indexer.AddIndex(&Pet{}, "Kind", "ID", "pets", "non_unique", nil, false) - assert.NoError(t, err) - - pet1 := Pet{ID: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"} - pet2 := Pet{ID: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"} - - _, err = indexer.Add(pet1) - assert.NoError(t, err) - - _, err = indexer.Add(pet2) - assert.NoError(t, err) - - res, err := indexer.FindBy(Pet{}, "Kind", "Hog") - assert.NoError(t, err) - - t.Log(res) - - _ = os.RemoveAll(dataDir) -} - -func TestIndexer_Disk_AddWithAutoincrementIndex(t *testing.T) { - dataDir, err := WriteIndexTestData(Data, "ID", "") - assert.NoError(t, err) - indexer := createDiskIndexer(dataDir) - - err = indexer.AddIndex(&User{}, "UID", "ID", "users", "autoincrement", &option.Bound{Lower: 5}, false) - assert.NoError(t, err) - - res1, err := indexer.Add(Data["users"][0]) - assert.NoError(t, err) - assert.Equal(t, "UID", res1[0].Field) - assert.Equal(t, "5", path.Base(res1[0].Value)) - - res2, err := indexer.Add(Data["users"][1]) - assert.NoError(t, err) - assert.Equal(t, "UID", res2[0].Field) - assert.Equal(t, "6", path.Base(res2[0].Value)) - - resFindBy, err := indexer.FindBy(User{}, "UID", "6") - assert.NoError(t, err) - assert.Equal(t, "hijklmn-456", resFindBy[0]) - t.Log(resFindBy) - - _ = os.RemoveAll(dataDir) -} - -func TestIndexer_Disk_DeleteWithNonUniqueIndex(t *testing.T) { - dataDir, err := WriteIndexTestData(Data, "ID", "") - assert.NoError(t, err) - indexer := createDiskIndexer(dataDir) - - err = indexer.AddIndex(&Pet{}, "Kind", "ID", "pets", "non_unique", nil, false) - assert.NoError(t, err) - - pet1 := Pet{ID: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"} - pet2 := Pet{ID: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"} - - _, err = indexer.Add(pet1) - assert.NoError(t, err) - - _, err = indexer.Add(pet2) - assert.NoError(t, err) - - err = indexer.Delete(pet2) - assert.NoError(t, err) - - _ = os.RemoveAll(dataDir) -} - -func TestIndexer_Disk_SearchWithNonUniqueIndex(t *testing.T) { - dataDir, err := WriteIndexTestData(Data, "ID", "") - assert.NoError(t, err) - indexer := createDiskIndexer(dataDir) - - err = indexer.AddIndex(&Pet{}, "Name", "ID", "pets", "non_unique", nil, false) - assert.NoError(t, err) - - pet1 := Pet{ID: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"} - pet2 := Pet{ID: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"} - - _, err = indexer.Add(pet1) - assert.NoError(t, err) - - _, err = indexer.Add(pet2) - assert.NoError(t, err) - - res, err := indexer.FindByPartial(pet2, "Name", "*ky") - assert.NoError(t, err) - - t.Log(res) - _ = os.RemoveAll(dataDir) -} - -func TestIndexer_Disk_UpdateWithUniqueIndex(t *testing.T) { - dataDir, err := WriteIndexTestData(Data, "ID", "") - assert.NoError(t, err) - indexer := createDiskIndexer(dataDir) - - err = indexer.AddIndex(&User{}, "UserName", "ID", "users", "unique", nil, false) - assert.NoError(t, err) - - err = indexer.AddIndex(&User{}, "Email", "ID", "users", "unique", nil, false) - assert.NoError(t, err) - - user1 := &User{ID: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"} - user2 := &User{ID: "hijklmn-456", UserName: "frank", Email: "frank@example.com"} - - _, err = indexer.Add(user1) - assert.NoError(t, err) - - _, err = indexer.Add(user2) - assert.NoError(t, err) - - err = indexer.Update(user1, &User{ - ID: "abcdefg-123", - UserName: "mikey-new", - Email: "mikey@example.com", - }) - assert.NoError(t, err) - v, err1 := indexer.FindBy(&User{}, "UserName", "mikey-new") - assert.NoError(t, err1) - assert.Len(t, v, 1) - v, err2 := indexer.FindBy(&User{}, "UserName", "mikey") - assert.NoError(t, err2) - assert.Len(t, v, 0) - - err1 = indexer.Update(&User{ - ID: "abcdefg-123", - UserName: "mikey-new", - Email: "mikey@example.com", - }, &User{ - ID: "abcdefg-123", - UserName: "mikey-newest", - Email: "mikey-new@example.com", - }) - assert.NoError(t, err1) - fbUserName, err2 := indexer.FindBy(&User{}, "UserName", "mikey-newest") - assert.NoError(t, err2) - assert.Len(t, fbUserName, 1) - fbEmail, err3 := indexer.FindBy(&User{}, "Email", "mikey-new@example.com") - assert.NoError(t, err3) - assert.Len(t, fbEmail, 1) - - _ = os.RemoveAll(dataDir) -} - -func TestIndexer_Disk_UpdateWithNonUniqueIndex(t *testing.T) { - dataDir, err := WriteIndexTestData(Data, "ID", "") - assert.NoError(t, err) - indexer := createDiskIndexer(dataDir) - - err = indexer.AddIndex(&Pet{}, "Name", "ID", "pets", "non_unique", nil, false) - assert.NoError(t, err) - - pet1 := Pet{ID: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"} - pet2 := Pet{ID: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"} - - _, err = indexer.Add(pet1) - assert.NoError(t, err) - - _, err = indexer.Add(pet2) - assert.NoError(t, err) - - _ = os.RemoveAll(dataDir) -} - -func TestQueryDiskImpl(t *testing.T) { - dataDir, err := WriteIndexTestData(Data, "ID", "") - assert.NoError(t, err) - indexer := createDiskIndexer(dataDir) - ctx := context.Background() - - err = indexer.AddIndex(&Account{}, "OnPremisesSamAccountName", "ID", "accounts", "non_unique", nil, false) - assert.NoError(t, err) - - err = indexer.AddIndex(&Account{}, "Mail", "ID", "accounts", "non_unique", nil, false) - assert.NoError(t, err) - - err = indexer.AddIndex(&Account{}, "ID", "ID", "accounts", "non_unique", nil, false) - assert.NoError(t, err) - - acc := Account{ - ID: "ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2", - Mail: "spooky@skeletons.org", - OnPremisesSamAccountName: "MrDootDoot", - } - - _, err = indexer.Add(acc) - assert.NoError(t, err) - - r, err := indexer.Query(ctx, &Account{}, "on_premises_sam_account_name eq 'MrDootDoot'") // this query will match both pets. - assert.NoError(t, err) - assert.Equal(t, []string{"ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2"}, r) - - r, err = indexer.Query(ctx, &Account{}, "mail eq 'spooky@skeletons.org'") // this query will match both pets. - assert.NoError(t, err) - assert.Equal(t, []string{"ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2"}, r) - - r, err = indexer.Query(ctx, &Account{}, "on_premises_sam_account_name eq 'MrDootDoot' or mail eq 'spooky@skeletons.org'") // this query will match both pets. - assert.NoError(t, err) - assert.Equal(t, []string{"ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2"}, r) - - r, err = indexer.Query(ctx, &Account{}, "startswith(on_premises_sam_account_name,'MrDoo')") // this query will match both pets. - assert.NoError(t, err) - assert.Equal(t, []string{"ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2"}, r) - - r, err = indexer.Query(ctx, &Account{}, "id eq 'ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2' or on_premises_sam_account_name eq 'MrDootDoot'") // this query will match both pets. - assert.NoError(t, err) - assert.Equal(t, []string{"ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2"}, r) - - _ = os.RemoveAll(dataDir) -} - -func createDiskIndexer(dataDir string) *Indexer { - return CreateIndexer(&config.Config{ - Repo: config.Repo{ - Backend: "disk", - Disk: config.Disk{ - Path: dataDir, - }, - }, - }) -} diff --git a/ocis-pkg/indexer/map.go b/ocis-pkg/indexer/map.go deleted file mode 100644 index f58a4eafc..000000000 --- a/ocis-pkg/indexer/map.go +++ /dev/null @@ -1,27 +0,0 @@ -package indexer - -import "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index" - -// typeMap stores the indexer layout at runtime. - -type typeMap map[tName]typeMapping -type tName = string -type fieldName = string - -type typeMapping struct { - PKFieldName string - IndicesByField map[fieldName][]index.Index -} - -func (m typeMap) addIndex(typeName string, pkName string, idx index.Index) { - if val, ok := m[typeName]; ok { - val.IndicesByField[idx.IndexBy()] = append(val.IndicesByField[idx.IndexBy()], idx) - return - } - m[typeName] = typeMapping{ - PKFieldName: pkName, - IndicesByField: map[string][]index.Index{ - idx.IndexBy(): {idx}, - }, - } -} diff --git a/ocis-pkg/indexer/option/option.go b/ocis-pkg/indexer/option/option.go deleted file mode 100644 index 36d68ac84..000000000 --- a/ocis-pkg/indexer/option/option.go +++ /dev/null @@ -1,126 +0,0 @@ -package option - -import "github.com/owncloud/ocis/v2/extensions/accounts/pkg/config" - -// Option defines a single option function. -type Option func(o *Options) - -// Bound represents a lower and upper bound range for an index. -// todo: if we would like to provide an upper bound then we would need to deal with ranges, in which case this is why the -// upper bound attribute is here. -type Bound struct { - Lower, Upper int64 -} - -// Options defines the available options for this package. -type Options struct { - CaseInsensitive bool - Bound *Bound - - // Disk Options - TypeName string - IndexBy string - FilesDir string - IndexBaseDir string - DataDir string - EntityDirName string - Entity interface{} - - // CS3 options - DataURL string - DataPrefix string - JWTSecret string - ProviderAddr string - ServiceUser config.ServiceUser -} - -// CaseInsensitive sets the CaseInsensitive field. -func CaseInsensitive(val bool) Option { - return func(o *Options) { - o.CaseInsensitive = val - } -} - -// WithBounds sets the Bounds field. -func WithBounds(val *Bound) Option { - return func(o *Options) { - o.Bound = val - } -} - -// WithEntity sets the Entity field. -func WithEntity(val interface{}) Option { - return func(o *Options) { - o.Entity = val - } -} - -// WithJWTSecret sets the JWTSecret field. -func WithJWTSecret(val string) Option { - return func(o *Options) { - o.JWTSecret = val - } -} - -// WithDataURL sets the DataURl field. -func WithDataURL(val string) Option { - return func(o *Options) { - o.DataURL = val - } -} - -// WithDataPrefix sets the DataPrefix field. -func WithDataPrefix(val string) Option { - return func(o *Options) { - o.DataPrefix = val - } -} - -// WithEntityDirName sets the EntityDirName field. -func WithEntityDirName(val string) Option { - return func(o *Options) { - o.EntityDirName = val - } -} - -// WithDataDir sets the DataDir option. -func WithDataDir(val string) Option { - return func(o *Options) { - o.DataDir = val - } -} - -// WithTypeName sets the TypeName option. -func WithTypeName(val string) Option { - return func(o *Options) { - o.TypeName = val - } -} - -// WithIndexBy sets the option IndexBy. -func WithIndexBy(val string) Option { - return func(o *Options) { - o.IndexBy = val - } -} - -// WithFilesDir sets the option FilesDir. -func WithFilesDir(val string) Option { - return func(o *Options) { - o.FilesDir = val - } -} - -// WithProviderAddr sets the option ProviderAddr. -func WithProviderAddr(val string) Option { - return func(o *Options) { - o.ProviderAddr = val - } -} - -// WithServiceUser sets the option ServiceUser. -func WithServiceUser(val config.ServiceUser) Option { - return func(o *Options) { - o.ServiceUser = val - } -} diff --git a/ocis-pkg/indexer/query_tree.go b/ocis-pkg/indexer/query_tree.go deleted file mode 100644 index 1177cc085..000000000 --- a/ocis-pkg/indexer/query_tree.go +++ /dev/null @@ -1,40 +0,0 @@ -package indexer - -type queryTree struct { - token *token - root bool - left *queryTree - right *queryTree -} - -// token to be resolved by the index -type token struct { - operator string // original OData operator. i.e: 'startswith', `or`, `and`. - filterType string // equivalent operator from OData -> indexer i.e FindByPartial or FindBy. - operands []string -} - -// newQueryTree constructs a new tree with a root node. -func newQueryTree() queryTree { - return queryTree{ - root: true, - } -} - -// insert populates first the LHS of the tree first, if this is not possible it fills the RHS. -func (t *queryTree) insert(tkn *token) { - if t != nil && t.root { - t.left = &queryTree{token: tkn} - return - } - - if t.left == nil { - t.left = &queryTree{token: tkn} - return - } - - if t.left != nil && t.right == nil { - t.right = &queryTree{token: tkn} - return - } -} diff --git a/ocis-pkg/indexer/reflect.go b/ocis-pkg/indexer/reflect.go deleted file mode 100644 index 137175292..000000000 --- a/ocis-pkg/indexer/reflect.go +++ /dev/null @@ -1,41 +0,0 @@ -package indexer - -import ( - "errors" - "path" - "reflect" - "strconv" - "strings" -) - -func getType(v interface{}) (reflect.Value, error) { - rv := reflect.ValueOf(v) - for rv.Kind() == reflect.Ptr || rv.Kind() == reflect.Interface { - rv = rv.Elem() - } - if !rv.IsValid() { - return reflect.Value{}, errors.New("failed to read value via reflection") - } - - return rv, nil -} - -func getTypeFQN(t interface{}) string { - typ, _ := getType(t) - typeName := path.Join(typ.Type().PkgPath(), typ.Type().Name()) - typeName = strings.ReplaceAll(typeName, "/", ".") - return typeName -} - -func valueOf(v interface{}, field string) string { - r := reflect.ValueOf(v) - f := reflect.Indirect(r).FieldByName(field) - - if f.Kind() == reflect.String { - return f.String() - } - if f.IsZero() { - return "" - } - return strconv.Itoa(int(f.Int())) -} diff --git a/ocis-pkg/indexer/reflect_test.go b/ocis-pkg/indexer/reflect_test.go deleted file mode 100644 index e9cd4ab4f..000000000 --- a/ocis-pkg/indexer/reflect_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package indexer - -import ( - "testing" -) - -func Test_getTypeFQN(t *testing.T) { - type someT struct{} - - type args struct { - t interface{} - } - tests := []struct { - name string - args args - want string - }{ - {name: "ByValue", args: args{&someT{}}, want: "github.com.owncloud.ocis.v2.ocis-pkg.indexer.someT"}, - {name: "ByRef", args: args{someT{}}, want: "github.com.owncloud.ocis.v2.ocis-pkg.indexer.someT"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := getTypeFQN(tt.args.t); got != tt.want { - t.Errorf("getTypeFQN() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_valueOf(t *testing.T) { - type someT struct { - val string - } - type args struct { - v interface{} - field string - } - tests := []struct { - name string - args args - want string - }{ - {name: "ByValue", args: args{v: someT{val: "hello"}, field: "val"}, want: "hello"}, - {name: "ByRef", args: args{v: &someT{val: "hello"}, field: "val"}, want: "hello"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := valueOf(tt.args.v, tt.args.field); got != tt.want { - t.Errorf("valueOf() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/ocis-pkg/indexer/registry/registry.go b/ocis-pkg/indexer/registry/registry.go deleted file mode 100644 index 1760dc7c5..000000000 --- a/ocis-pkg/indexer/registry/registry.go +++ /dev/null @@ -1,15 +0,0 @@ -package registry - -import ( - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index" - "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option" -) - -// IndexConstructor is a constructor function for creating index.Index. -type IndexConstructor func(o ...option.Option) index.Index - -// IndexConstructorRegistry undocumented. -var IndexConstructorRegistry = map[string]map[string]IndexConstructor{ - "disk": {}, - "cs3": {}, -} diff --git a/ocis-pkg/indexer/test/data.go b/ocis-pkg/indexer/test/data.go deleted file mode 100644 index 85534dab8..000000000 --- a/ocis-pkg/indexer/test/data.go +++ /dev/null @@ -1,102 +0,0 @@ -package test - -import ( - "encoding/json" - "io/ioutil" - "os" - "path" -) - -// User is a user. -type User struct { - ID, UserName, Email string - UID int -} - -// Pet is a pet. -type Pet struct { - ID, Kind, Color, Name string - UID int -} - -// Account mocks an ocis account. -type Account struct { - ID string - OnPremisesSamAccountName string - Mail string -} - -// Data mock data. -var Data = map[string][]interface{}{ - "users": { - User{ID: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"}, - User{ID: "hijklmn-456", UserName: "frank", Email: "frank@example.com"}, - User{ID: "ewf4ofk-555", UserName: "jacky", Email: "jacky@example.com"}, - User{ID: "rulan54-777", UserName: "jones", Email: "jones@example.com"}, - }, - "pets": { - Pet{ID: "rebef-123", Kind: "Dog", Color: "Brown", Name: "Waldo"}, - Pet{ID: "wefwe-456", Kind: "Cat", Color: "White", Name: "Snowy"}, - Pet{ID: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"}, - Pet{ID: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"}, - }, - "accounts": { - Account{ID: "ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2", Mail: "spooky@skeletons.org", OnPremisesSamAccountName: "MrDootDoot"}, - }, -} - -// WriteIndexTestData writes mock data to disk. -func WriteIndexTestData(m map[string][]interface{}, privateKey, dir string) (string, error) { - rootDir, err := getRootDir(dir) - if err != nil { - return "", err - } - - err = writeData(m, privateKey, rootDir) - if err != nil { - return "", err - } - - return rootDir, nil -} - -// getRootDir allows for some minimal behavior on destination on disk. Testing the cs3 api behavior locally means -// keeping track of where the cs3 data lives on disk, this function allows for multiplexing whether or not to use a -// temporary folder or an already defined one. -func getRootDir(dir string) (string, error) { - var rootDir string - var err error - - if dir != "" { - rootDir = dir - } else { - rootDir, err = CreateTmpDir() - if err != nil { - return "", err - } - } - return rootDir, nil -} - -// writeData writes test data to disk on rootDir location Marshaled as json. -func writeData(m map[string][]interface{}, privateKey string, rootDir string) error { - for dirName := range m { - fileTypePath := path.Join(rootDir, dirName) - - if err := os.MkdirAll(fileTypePath, 0755); err != nil { - return err - } - for _, u := range m[dirName] { - data, err := json.Marshal(u) - if err != nil { - return err - } - - pkVal := ValueOf(u, privateKey) - if err := ioutil.WriteFile(path.Join(fileTypePath, pkVal), data, 0600); err != nil { - return err - } - } - } - return nil -} diff --git a/ocis-pkg/indexer/test/helpers.go b/ocis-pkg/indexer/test/helpers.go deleted file mode 100644 index b939aae6e..000000000 --- a/ocis-pkg/indexer/test/helpers.go +++ /dev/null @@ -1,48 +0,0 @@ -package test - -import ( - "errors" - "io/ioutil" - "path" - "reflect" - "strings" -) - -// CreateTmpDir creates a temporary dir for tests data. -func CreateTmpDir() (string, error) { - name, err := ioutil.TempDir("/tmp", "testfiles-") - if err != nil { - return "", err - } - - return name, nil -} - -// ValueOf gets the value of a type v on a given field . -func ValueOf(v interface{}, field string) string { - r := reflect.ValueOf(v) - f := reflect.Indirect(r).FieldByName(field) - - return f.String() -} - -func getType(v interface{}) (reflect.Value, error) { - rv := reflect.ValueOf(v) - for rv.Kind() == reflect.Ptr || rv.Kind() == reflect.Interface { - rv = rv.Elem() - } - if !rv.IsValid() { - return reflect.Value{}, errors.New("failed to read value via reflection") - } - - return rv, nil -} - -// GetTypeFQN formats a valid name from a type . This is a duplication of the already existing function in the -// indexer package, but since there is a circular dependency we chose to duplicate it. -func GetTypeFQN(t interface{}) string { - typ, _ := getType(t) - typeName := path.Join(typ.Type().PkgPath(), typ.Type().Name()) - typeName = strings.ReplaceAll(typeName, "/", ".") - return typeName -}