From d2d865038d74bf740926c49793bf821236d4eb97 Mon Sep 17 00:00:00 2001 From: "A.Unger" Date: Mon, 5 Oct 2020 13:06:53 +0200 Subject: [PATCH] fix indexer folder --- accounts/pkg/indexer/index/cs3/unique.go | 202 +++++++++- .../{indexer => }/index/cs3/unique_test.go | 0 accounts/pkg/indexer/indexer/errors/errors.go | 31 -- .../pkg/indexer/indexer/index/cs3/unique.go | 347 ------------------ .../indexer/indexer/index/disk/non_unique.go | 171 --------- .../indexer/index/disk/non_unique_test.go | 111 ------ .../pkg/indexer/indexer/index/disk/unique.go | 180 --------- .../indexer/indexer/index/disk/unique_test.go | 136 ------- accounts/pkg/indexer/indexer/index/index.go | 15 - accounts/pkg/indexer/indexer/indexer.go | 158 -------- accounts/pkg/indexer/indexer/indexer_test.go | 255 ------------- accounts/pkg/indexer/indexer/map.go | 27 -- accounts/pkg/indexer/indexer/reflect.go | 34 -- accounts/pkg/indexer/indexer/reflect_test.go | 53 --- accounts/pkg/indexer/indexer/test/data.go | 56 --- accounts/pkg/indexer/indexer/test/helpers.go | 23 -- 16 files changed, 190 insertions(+), 1609 deletions(-) rename accounts/pkg/indexer/{indexer => }/index/cs3/unique_test.go (100%) delete mode 100644 accounts/pkg/indexer/indexer/errors/errors.go delete mode 100644 accounts/pkg/indexer/indexer/index/cs3/unique.go delete mode 100644 accounts/pkg/indexer/indexer/index/disk/non_unique.go delete mode 100644 accounts/pkg/indexer/indexer/index/disk/non_unique_test.go delete mode 100644 accounts/pkg/indexer/indexer/index/disk/unique.go delete mode 100644 accounts/pkg/indexer/indexer/index/disk/unique_test.go delete mode 100644 accounts/pkg/indexer/indexer/index/index.go delete mode 100644 accounts/pkg/indexer/indexer/indexer.go delete mode 100644 accounts/pkg/indexer/indexer/indexer_test.go delete mode 100644 accounts/pkg/indexer/indexer/map.go delete mode 100644 accounts/pkg/indexer/indexer/reflect.go delete mode 100644 accounts/pkg/indexer/indexer/reflect_test.go delete mode 100644 accounts/pkg/indexer/indexer/test/data.go delete mode 100644 accounts/pkg/indexer/indexer/test/helpers.go diff --git a/accounts/pkg/indexer/index/cs3/unique.go b/accounts/pkg/indexer/index/cs3/unique.go index bf91e0ab3f..4d118484ef 100644 --- a/accounts/pkg/indexer/index/cs3/unique.go +++ b/accounts/pkg/indexer/index/cs3/unique.go @@ -2,14 +2,21 @@ package cs3 import ( "context" + "fmt" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/pkg/rgrpc/todo/pool" "github.com/cs3org/reva/pkg/token" "github.com/cs3org/reva/pkg/token/manager/jwt" + idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors" + "google.golang.org/grpc/metadata" "io" + "io/ioutil" "net/http" + "os" "path" + "path/filepath" "strings" ) @@ -72,48 +79,219 @@ func (idx *Unique) Init() error { idx.storageProvider = client + ctx := context.Background() + tk, err := idx.authenticate(ctx) + if err != nil { + return err + } + ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, tk) + + if err := idx.makeDirIfNotExists(ctx, idx.indexBaseDir); err != nil { + return err + } + + if err := idx.makeDirIfNotExists(ctx, idx.indexRootDir); err != nil { + return err + } + return nil } +// Add adds a value to the index, returns the path to the root-document func (idx *Unique) Add(id, v string) (string, error) { - //oldName := path.Join(idx.filesDir, id) - //newName := path.Join(idx.indexRootDir, v) + newName := idx.indexURL(v) + if err := idx.createSymlink(id, newName); err != nil { + if os.IsExist(err) { + return "", &idxerrs.AlreadyExistsErr{idx.typeName, idx.indexBy, v} + } - panic("implement me") + return "", err + } + return newName, nil } func (idx *Unique) Lookup(v string) ([]string, error) { - panic("implement me") + searchPath := singleJoiningSlash(idx.cs3conf.DataURL, path.Join(idx.cs3conf.DataPrefix, idx.indexRootDir, v)) + oldname, err := idx.resolveSymlink(searchPath) + if err != nil { + if os.IsNotExist(err) { + err = &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, v} + } + + return nil, err + } + + return []string{oldname}, nil } +// 97d28b57 func (idx *Unique) Remove(id string, v string) error { - panic("implement me") + searchPath := singleJoiningSlash(idx.cs3conf.DataURL, path.Join(idx.cs3conf.DataPrefix, idx.indexRootDir, v)) + _, err := idx.resolveSymlink(searchPath) + if err != nil { + if os.IsNotExist(err) { + err = &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, v} + } + + return err + } + + ctx := context.Background() + t, err := idx.authenticate(ctx) + if err != nil { + return err + } + + deletePath := path.Join("/meta", idx.indexRootDir, v) + ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) + resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ + Ref: &provider.Reference{ + Spec: &provider.Reference_Path{Path: deletePath}, + }, + }) + + if err != nil { + return err + } + + // TODO Handle other error codes? + if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND { + return &idxerrs.NotFoundErr{} + } + + return err } func (idx *Unique) Update(id, oldV, newV string) error { - panic("implement me") + if err := idx.Remove(id, oldV); err != nil { + return err + } + + if _, err := idx.Add(id, newV); err != nil { + return err + } + + return nil } func (idx *Unique) Search(pattern string) ([]string, error) { - panic("implement me") + ctx := context.Background() + t, err := idx.authenticate(ctx) + if err != nil { + return nil, err + } + + ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) + res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + Ref: &provider.Reference{ + Spec: &provider.Reference_Path{Path: path.Join("/meta", idx.indexRootDir)}, + }, + }) + + searchPath := singleJoiningSlash(idx.cs3conf.DataURL, path.Join(idx.cs3conf.DataPrefix, idx.indexRootDir)) + + matches := []string{} + for _, i := range res.GetInfos() { + if found, err := filepath.Match(pattern, path.Base(i.Path)); found { + if err != nil { + return nil, err + } + + oldPath, err := idx.resolveSymlink(singleJoiningSlash(searchPath, path.Base(i.Path))) + if err != nil { + return nil, err + } + matches = append(matches, oldPath) + } + } + + return matches, nil + } func (idx *Unique) IndexBy() string { - panic("implement me") + return idx.indexBy } func (idx *Unique) TypeName() string { - panic("implement me") + return idx.typeName } func (idx *Unique) FilesDir() string { - panic("implement me") + return idx.filesDir } -func (idx *Unique) fakeSymlink(oldname, newname string) { - //idx.dataProvider.put() +func (idx *Unique) createSymlink(oldname, newname string) error { + t, err := idx.authenticate(context.TODO()) + if err != nil { + return err + } + if _, err := idx.resolveSymlink(newname); err == nil { + return os.ErrExist + } + + _, err = idx.dataProvider.put(newname, strings.NewReader(oldname), t) + if err != nil { + return err + } + + return nil + +} + +func (idx *Unique) resolveSymlink(name string) (string, error) { + t, err := idx.authenticate(context.TODO()) + if err != nil { + return "", err + } + + resp, err := idx.dataProvider.get(name, t) + if err != nil { + return "", err + } + + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return "", os.ErrNotExist + } + + return "", fmt.Errorf("could not resolve symlink %s, got status %v", name, resp.StatusCode) + } + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + + } + return string(b), err +} + +func (idx *Unique) makeDirIfNotExists(ctx context.Context, folder string) error { + var rootPathRef = &provider.Reference{ + Spec: &provider.Reference_Path{Path: fmt.Sprintf("/meta/%v", folder)}, + } + + resp, err := idx.storageProvider.Stat(ctx, &provider.StatRequest{ + Ref: rootPathRef, + }) + + if err != nil { + return err + } + + if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND { + _, err := idx.storageProvider.CreateContainer(ctx, &provider.CreateContainerRequest{ + Ref: rootPathRef, + }) + + if err != nil { + return err + } + } + + return nil } func (idx *Unique) indexURL(id string) string { diff --git a/accounts/pkg/indexer/indexer/index/cs3/unique_test.go b/accounts/pkg/indexer/index/cs3/unique_test.go similarity index 100% rename from accounts/pkg/indexer/indexer/index/cs3/unique_test.go rename to accounts/pkg/indexer/index/cs3/unique_test.go diff --git a/accounts/pkg/indexer/indexer/errors/errors.go b/accounts/pkg/indexer/indexer/errors/errors.go deleted file mode 100644 index 85e785c892..0000000000 --- a/accounts/pkg/indexer/indexer/errors/errors.go +++ /dev/null @@ -1,31 +0,0 @@ -package errors - -import ( - "fmt" -) - -type AlreadyExistsErr struct { - TypeName, Key, Value string -} - -func (e *AlreadyExistsErr) Error() string { - return fmt.Sprintf("%s with %s=%s does already exist", e.TypeName, e.Key, e.Value) -} - -func IsAlreadyExistsErr(e error) bool { - _, ok := e.(*AlreadyExistsErr) - return ok -} - -type NotFoundErr struct { - TypeName, Key, Value string -} - -func (e *NotFoundErr) Error() string { - return fmt.Sprintf("%s with %s=%s not found", e.TypeName, e.Key, e.Value) -} - -func IsNotFoundErr(e error) bool { - _, ok := e.(*NotFoundErr) - return ok -} diff --git a/accounts/pkg/indexer/indexer/index/cs3/unique.go b/accounts/pkg/indexer/indexer/index/cs3/unique.go deleted file mode 100644 index 4d118484ef..0000000000 --- a/accounts/pkg/indexer/indexer/index/cs3/unique.go +++ /dev/null @@ -1,347 +0,0 @@ -package cs3 - -import ( - "context" - "fmt" - user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/cs3org/reva/pkg/rgrpc/todo/pool" - "github.com/cs3org/reva/pkg/token" - "github.com/cs3org/reva/pkg/token/manager/jwt" - idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors" - "google.golang.org/grpc/metadata" - "io" - "io/ioutil" - "net/http" - "os" - "path" - "path/filepath" - "strings" -) - -type Unique struct { - indexBy string - typeName string - filesDir string - indexBaseDir string - indexRootDir string - - tokenManager token.Manager - storageProvider provider.ProviderAPIClient - dataProvider dataProviderClient // Used to create and download data via http, bypassing reva upload protocol - - cs3conf *Config -} - -type Config struct { - ProviderAddr string - DataURL string - DataPrefix string - JWTSecret string - ServiceUserName string - ServiceUserUUID string -} - -// NewUniqueIndex instantiates a new UniqueIndex instance. Init() should be -// called afterward to ensure correct on-disk structure. -func NewUniqueIndex(typeName, indexBy, filesDir, indexBaseDir string, cfg *Config) Unique { - return Unique{ - indexBy: indexBy, - typeName: typeName, - filesDir: filesDir, - indexBaseDir: indexBaseDir, - indexRootDir: path.Join(indexBaseDir, strings.Join([]string{"unique", typeName, indexBy}, ".")), - cs3conf: cfg, - dataProvider: dataProviderClient{ - client: http.Client{ - Transport: http.DefaultTransport, - }, - }, - } -} - -func (idx *Unique) Init() error { - tokenManager, err := jwt.New(map[string]interface{}{ - "secret": idx.cs3conf.JWTSecret, - }) - - if err != nil { - return err - } - - idx.tokenManager = tokenManager - - client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr) - if err != nil { - return err - } - - idx.storageProvider = client - - ctx := context.Background() - tk, err := idx.authenticate(ctx) - if err != nil { - return err - } - ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, tk) - - if err := idx.makeDirIfNotExists(ctx, idx.indexBaseDir); err != nil { - return err - } - - if err := idx.makeDirIfNotExists(ctx, idx.indexRootDir); err != nil { - return err - } - - return nil -} - -// Add adds a value to the index, returns the path to the root-document -func (idx *Unique) Add(id, v string) (string, error) { - newName := idx.indexURL(v) - if err := idx.createSymlink(id, newName); err != nil { - if os.IsExist(err) { - return "", &idxerrs.AlreadyExistsErr{idx.typeName, idx.indexBy, v} - } - - return "", err - } - - return newName, nil -} - -func (idx *Unique) Lookup(v string) ([]string, error) { - searchPath := singleJoiningSlash(idx.cs3conf.DataURL, path.Join(idx.cs3conf.DataPrefix, idx.indexRootDir, v)) - oldname, err := idx.resolveSymlink(searchPath) - if err != nil { - if os.IsNotExist(err) { - err = &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, v} - } - - return nil, err - } - - return []string{oldname}, nil -} - -// 97d28b57 -func (idx *Unique) Remove(id string, v string) error { - searchPath := singleJoiningSlash(idx.cs3conf.DataURL, path.Join(idx.cs3conf.DataPrefix, idx.indexRootDir, v)) - _, err := idx.resolveSymlink(searchPath) - if err != nil { - if os.IsNotExist(err) { - err = &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, v} - } - - return err - } - - ctx := context.Background() - t, err := idx.authenticate(ctx) - if err != nil { - return err - } - - deletePath := path.Join("/meta", idx.indexRootDir, v) - ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) - resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ - Ref: &provider.Reference{ - Spec: &provider.Reference_Path{Path: deletePath}, - }, - }) - - if err != nil { - return err - } - - // TODO Handle other error codes? - if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND { - return &idxerrs.NotFoundErr{} - } - - return err -} - -func (idx *Unique) Update(id, oldV, newV string) error { - if err := idx.Remove(id, oldV); err != nil { - return err - } - - if _, err := idx.Add(id, newV); err != nil { - return err - } - - return nil -} - -func (idx *Unique) Search(pattern string) ([]string, error) { - ctx := context.Background() - t, err := idx.authenticate(ctx) - if err != nil { - return nil, err - } - - ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) - res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ - Ref: &provider.Reference{ - Spec: &provider.Reference_Path{Path: path.Join("/meta", idx.indexRootDir)}, - }, - }) - - searchPath := singleJoiningSlash(idx.cs3conf.DataURL, path.Join(idx.cs3conf.DataPrefix, idx.indexRootDir)) - - matches := []string{} - for _, i := range res.GetInfos() { - if found, err := filepath.Match(pattern, path.Base(i.Path)); found { - if err != nil { - return nil, err - } - - oldPath, err := idx.resolveSymlink(singleJoiningSlash(searchPath, path.Base(i.Path))) - if err != nil { - return nil, err - } - matches = append(matches, oldPath) - } - } - - return matches, nil - -} - -func (idx *Unique) IndexBy() string { - return idx.indexBy -} - -func (idx *Unique) TypeName() string { - return idx.typeName -} - -func (idx *Unique) FilesDir() string { - return idx.filesDir -} - -func (idx *Unique) createSymlink(oldname, newname string) error { - t, err := idx.authenticate(context.TODO()) - if err != nil { - return err - } - - if _, err := idx.resolveSymlink(newname); err == nil { - return os.ErrExist - } - - _, err = idx.dataProvider.put(newname, strings.NewReader(oldname), t) - if err != nil { - return err - } - - return nil - -} - -func (idx *Unique) resolveSymlink(name string) (string, error) { - t, err := idx.authenticate(context.TODO()) - if err != nil { - return "", err - } - - resp, err := idx.dataProvider.get(name, t) - if err != nil { - return "", err - } - - if resp.StatusCode != http.StatusOK { - if resp.StatusCode == http.StatusNotFound { - return "", os.ErrNotExist - } - - return "", fmt.Errorf("could not resolve symlink %s, got status %v", name, resp.StatusCode) - } - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - - } - return string(b), err -} - -func (idx *Unique) makeDirIfNotExists(ctx context.Context, folder string) error { - var rootPathRef = &provider.Reference{ - Spec: &provider.Reference_Path{Path: fmt.Sprintf("/meta/%v", folder)}, - } - - resp, err := idx.storageProvider.Stat(ctx, &provider.StatRequest{ - Ref: rootPathRef, - }) - - if err != nil { - return err - } - - if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND { - _, err := idx.storageProvider.CreateContainer(ctx, &provider.CreateContainerRequest{ - Ref: rootPathRef, - }) - - if err != nil { - return err - } - } - - return nil -} - -func (idx *Unique) indexURL(id string) string { - return singleJoiningSlash(idx.cs3conf.DataURL, path.Join(idx.cs3conf.DataPrefix, idx.indexRootDir, id)) -} - -// TODO: this is copied from proxy. Find a better solution or move it to ocis-pkg -func singleJoiningSlash(a, b string) string { - aslash := strings.HasSuffix(a, "/") - bslash := strings.HasPrefix(b, "/") - switch { - case aslash && bslash: - return a + b[1:] - case !aslash && !bslash: - return a + "/" + b - } - return a + b -} - -func (idx *Unique) authenticate(ctx context.Context) (token string, err error) { - u := &user.User{ - Id: &user.UserId{}, - Groups: []string{}, - } - if idx.cs3conf.ServiceUserName != "" { - u.Id.OpaqueId = idx.cs3conf.ServiceUserUUID - } - return idx.tokenManager.MintToken(ctx, u) -} - -type dataProviderClient struct { - client http.Client -} - -func (d dataProviderClient) put(url string, body io.Reader, token string) (*http.Response, error) { - req, err := http.NewRequest(http.MethodPut, url, body) - if err != nil { - return nil, err - } - - req.Header.Add("x-access-token", token) - return d.client.Do(req) -} - -func (d dataProviderClient) get(url string, token string) (*http.Response, error) { - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return nil, err - } - - req.Header.Add("x-access-token", token) - return d.client.Do(req) -} diff --git a/accounts/pkg/indexer/indexer/index/disk/non_unique.go b/accounts/pkg/indexer/indexer/index/disk/non_unique.go deleted file mode 100644 index d3194043cd..0000000000 --- a/accounts/pkg/indexer/indexer/index/disk/non_unique.go +++ /dev/null @@ -1,171 +0,0 @@ -package disk - -import ( - "errors" - "fmt" - idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors" - "io/ioutil" - "os" - "path" - "path/filepath" -) - -// NonUniqueIndex is able to index an document by a key which might contain non-unique values -// -// /var/tmp/testfiles-395764020/index.disk/PetByColor/ -// ├── Brown -// │ └── rebef-123 -> /var/tmp/testfiles-395764020/pets/rebef-123 -// ├── Green -// │ ├── goefe-789 -> /var/tmp/testfiles-395764020/pets/goefe-789 -// │ └── xadaf-189 -> /var/tmp/testfiles-395764020/pets/xadaf-189 -// └── White -// └── wefwe-456 -> /var/tmp/testfiles-395764020/pets/wefwe-456 -type NonUniqueIndex struct { - indexBy string - typeName string - filesDir string - indexBaseDir string - indexRootDir string -} - -// NewNonUniqueIndex instantiates a new NonUniqueIndex instance. Init() should be -// called afterward to ensure correct on-disk structure. -func NewNonUniqueIndex(typeName, indexBy, filesDir, indexBaseDir string) NonUniqueIndex { - return NonUniqueIndex{ - indexBy: indexBy, - typeName: typeName, - filesDir: filesDir, - indexBaseDir: indexBaseDir, - indexRootDir: path.Join(indexBaseDir, fmt.Sprintf("%sBy%s", typeName, indexBy)), - } -} - -func (idx NonUniqueIndex) Init() error { - if _, err := os.Stat(idx.filesDir); err != nil { - return err - } - - if err := os.MkdirAll(idx.indexRootDir, 0777); err != nil { - return err - } - - return nil -} - -func (idx NonUniqueIndex) Lookup(v string) ([]string, error) { - searchPath := path.Join(idx.indexRootDir, v) - fi, err := ioutil.ReadDir(searchPath) - if os.IsNotExist(err) { - return []string{}, &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, v} - } - - if err != nil { - return []string{}, err - } - - var ids []string = nil - for _, f := range fi { - ids = append(ids, f.Name()) - } - - if len(ids) == 0 { - return []string{}, &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, v} - } - - return ids, nil -} - -func (idx NonUniqueIndex) Add(id, v string) (string, error) { - oldName := path.Join(idx.filesDir, id) - newName := path.Join(idx.indexRootDir, v, id) - - if err := os.MkdirAll(path.Join(idx.indexRootDir, v), 0777); err != nil { - return "", err - } - - err := os.Symlink(oldName, newName) - if errors.Is(err, os.ErrExist) { - return "", &idxerrs.AlreadyExistsErr{idx.typeName, idx.indexBy, v} - } - - return newName, err - -} - -func (idx NonUniqueIndex) Remove(id string, v string) error { - res, err := filepath.Glob(path.Join(idx.indexRootDir, "/*/", id)) - if err != nil { - return err - } - - for _, p := range res { - if err := os.Remove(p); err != nil { - return err - } - } - - return nil -} - -func (idx NonUniqueIndex) Update(id, oldV, newV string) (err error) { - oldDir := path.Join(idx.indexRootDir, oldV) - oldPath := path.Join(oldDir, id) - newDir := path.Join(idx.indexRootDir, newV) - newPath := path.Join(newDir, id) - - if _, err = os.Stat(oldPath); os.IsNotExist(err) { - return &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, oldV} - } - - if err != nil { - return - } - - if err = os.MkdirAll(newDir, 0777); err != nil { - return - } - - if err = os.Rename(oldPath, newPath); err != nil { - return - } - - di, err := ioutil.ReadDir(oldDir) - if err != nil { - return err - } - - if len(di) == 0 { - err = os.RemoveAll(oldDir) - if err != nil { - return - } - } - - return - -} - -func (idx NonUniqueIndex) Search(pattern string) ([]string, error) { - paths, err := filepath.Glob(path.Join(idx.indexRootDir, pattern, "*")) - if err != nil { - return nil, err - } - - if len(paths) == 0 { - return nil, &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, pattern} - } - - return paths, nil -} - -func (idx NonUniqueIndex) IndexBy() string { - return idx.indexBy -} - -func (idx NonUniqueIndex) TypeName() string { - return idx.typeName -} - -func (idx NonUniqueIndex) FilesDir() string { - return idx.filesDir -} diff --git a/accounts/pkg/indexer/indexer/index/disk/non_unique_test.go b/accounts/pkg/indexer/indexer/index/disk/non_unique_test.go deleted file mode 100644 index 4eb5c42e01..0000000000 --- a/accounts/pkg/indexer/indexer/index/disk/non_unique_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package disk - -import ( - "github.com/owncloud/ocis/accounts/pkg/indexer/errors" - "github.com/owncloud/ocis/accounts/pkg/indexer/index" - . "github.com/owncloud/ocis/accounts/pkg/indexer/test" - "github.com/stretchr/testify/assert" - "os" - "path" - "testing" -) - -func TestNonUniqueIndexAdd(t *testing.T) { - sut, dataPath := getNonUniqueIdxSut(t) - - ids, err := sut.Lookup("Green") - assert.NoError(t, err) - assert.EqualValues(t, []string{"goefe-789", "xadaf-189"}, ids) - - ids, err = sut.Lookup("White") - assert.NoError(t, err) - assert.EqualValues(t, []string{"wefwe-456"}, ids) - - ids, err = sut.Lookup("Cyan") - assert.Error(t, err) - assert.EqualValues(t, []string{}, ids) - - _ = os.RemoveAll(dataPath) - -} - -func TestNonUniqueIndexUpdate(t *testing.T) { - sut, dataPath := getNonUniqueIdxSut(t) - - err := sut.Update("goefe-789", "", "Black") - assert.NoError(t, err) - - err = sut.Update("xadaf-189", "", "Black") - assert.NoError(t, err) - - assert.DirExists(t, path.Join(dataPath, "index.disk/PetByColor/Black")) - assert.NoDirExists(t, path.Join(dataPath, "index.disk/PetByColor/Green")) - - _ = os.RemoveAll(dataPath) -} - -func TestNonUniqueIndexDelete(t *testing.T) { - sut, dataPath := getNonUniqueIdxSut(t) - assert.FileExists(t, path.Join(dataPath, "index.disk/PetByColor/Green/goefe-789")) - err := sut.Remove("goefe-789", "") - assert.NoError(t, err) - assert.NoFileExists(t, path.Join(dataPath, "index.disk/PetByColor/Green/goefe-789")) - _ = os.RemoveAll(dataPath) -} - -func TestNonUniqueIndexInit(t *testing.T) { - dataDir := CreateTmpDir(t) - indexRootDir := path.Join(dataDir, "index.disk") - filesDir := path.Join(dataDir, "users") - - uniq := NewNonUniqueIndex("User", "DisplayName", filesDir, indexRootDir) - assert.Error(t, uniq.Init(), "Init should return an error about missing files-dir") - - if err := os.Mkdir(filesDir, 0777); err != nil { - t.Fatalf("Could not create test data-dir %s", err) - } - - assert.NoError(t, uniq.Init(), "Init shouldn't return an error") - assert.DirExists(t, indexRootDir) - assert.DirExists(t, path.Join(indexRootDir, "UserByDisplayName")) - - _ = os.RemoveAll(dataDir) -} - -func TestNonUniqueIndexSearch(t *testing.T) { - sut, dataPath := getNonUniqueIdxSut(t) - - res, err := sut.Search("Gr*") - - assert.NoError(t, err) - assert.Len(t, res, 2) - - assert.Equal(t, "goefe-789", path.Base(res[0])) - assert.Equal(t, "xadaf-189", path.Base(res[1])) - - res, err = sut.Search("does-not-exist@example.com") - assert.Error(t, err) - assert.IsType(t, &errors.NotFoundErr{}, err) - - _ = os.RemoveAll(dataPath) -} - -func getNonUniqueIdxSut(t *testing.T) (sut index.Index, dataPath string) { - dataPath = WriteIndexTestData(t, TestData, "Id") - sut = NewNonUniqueIndex("Pet", "Color", path.Join(dataPath, "pets"), path.Join(dataPath, "index.disk")) - err := sut.Init() - if err != nil { - t.Fatal(err) - } - - for _, u := range TestData["pets"] { - pkVal := ValueOf(u, "Id") - idxByVal := ValueOf(u, "Color") - _, err := sut.Add(pkVal, idxByVal) - if err != nil { - t.Fatal(err) - } - } - - return -} diff --git a/accounts/pkg/indexer/indexer/index/disk/unique.go b/accounts/pkg/indexer/indexer/index/disk/unique.go deleted file mode 100644 index 7cb78766f4..0000000000 --- a/accounts/pkg/indexer/indexer/index/disk/unique.go +++ /dev/null @@ -1,180 +0,0 @@ -package disk - -import ( - "errors" - "fmt" - idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors" - "os" - "path" - "path/filepath" - "strings" -) - -// Unique ensures that only one document of the same type and key-value combination can exist in the index. -// -// Modeled by creating a indexer-folder per entity and key with symlinks which point to respective documents which contain -// the link-filename as value. -// -// Directory Layout -// -// /var/data/index.disk/UniqueUserByEmail/ -// ├── jacky@example.com -> /var/data/users/ewf4ofk-555 -// ├── jones@example.com -> /var/data/users/rulan54-777 -// └── mikey@example.com -> /var/data/users/abcdefg-123 -// -// Example user -// -// { -// "Id": "ewf4ofk-555", -// "UserName": "jacky", -// "Email": "jacky@example.com" -// } -// -type Unique struct { - indexBy string - typeName string - filesDir string - indexBaseDir string - indexRootDir string -} - -// NewUniqueIndex instantiates a new UniqueIndex instance. Init() should be -// called afterward to ensure correct on-disk structure. -func NewUniqueIndex(typeName, indexBy, filesDir, indexBaseDir string) Unique { - return Unique{ - indexBy: indexBy, - typeName: typeName, - filesDir: filesDir, - indexBaseDir: indexBaseDir, - indexRootDir: path.Join(indexBaseDir, strings.Join([]string{"unique", typeName, indexBy}, ".")), - } -} - -func (idx Unique) Init() error { - if _, err := os.Stat(idx.filesDir); err != nil { - return err - } - - if err := os.MkdirAll(idx.indexRootDir, 0777); err != nil { - return err - } - - return nil -} - -func (idx Unique) Add(id, v string) (string, error) { - oldName := path.Join(idx.filesDir, id) - newName := path.Join(idx.indexRootDir, v) - err := os.Symlink(oldName, newName) - if errors.Is(err, os.ErrExist) { - return "", &idxerrs.AlreadyExistsErr{idx.typeName, idx.indexBy, v} - } - - return newName, err -} - -func (idx Unique) Remove(id string, v string) (err error) { - searchPath := path.Join(idx.indexRootDir, v) - if err = isValidSymlink(searchPath); err != nil { - return - } - - return os.Remove(searchPath) -} - -// unique.github.com.owncloud.ocis.accounts.pkg.indexer.User.UserName -// unique.github.com.owncloud.ocis.accounts.pkg.indexer.User.UserName/UserName -func (idx Unique) Lookup(v string) (resultPath []string, err error) { - searchPath := path.Join(idx.indexRootDir, v) - if err = isValidSymlink(searchPath); err != nil { - if os.IsNotExist(err) { - err = &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, v} - } - - return - } - - p, err := os.Readlink(searchPath) - if err != nil { - return []string{}, nil - } - - return []string{p}, err - -} - -func (idx Unique) Update(id, oldV, newV string) (err error) { - oldPath := path.Join(idx.indexRootDir, oldV) - if err = isValidSymlink(oldPath); err != nil { - if os.IsNotExist(err) { - return &idxerrs.NotFoundErr{idx.TypeName(), idx.IndexBy(), oldV} - } - - return - } - - newPath := path.Join(idx.indexRootDir, newV) - if err = isValidSymlink(newPath); err == nil { - return &idxerrs.AlreadyExistsErr{idx.typeName, idx.indexBy, newV} - } - - if os.IsNotExist(err) { - err = os.Rename(oldPath, newPath) - } - - return -} - -func (idx Unique) Search(pattern string) ([]string, error) { - paths, err := filepath.Glob(path.Join(idx.indexRootDir, pattern)) - if err != nil { - return nil, err - } - - if len(paths) == 0 { - return nil, &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, pattern} - } - - res := make([]string, 0, 0) - for _, p := range paths { - if err := isValidSymlink(p); err != nil { - return nil, err - } - - src, err := os.Readlink(p) - if err != nil { - return nil, err - } - - res = append(res, src) - } - - return res, nil -} - -func (idx Unique) IndexBy() string { - return idx.indexBy -} - -func (idx Unique) TypeName() string { - return idx.typeName -} - -func (idx Unique) FilesDir() string { - return idx.filesDir -} - -func isValidSymlink(path string) (err error) { - var symInfo os.FileInfo - if symInfo, err = os.Lstat(path); err != nil { - return - } - - if symInfo.Mode()&os.ModeSymlink == 0 { - err = fmt.Errorf("%s is not a valid symlink (bug/corruption?)", path) - return - } - - return - -} diff --git a/accounts/pkg/indexer/indexer/index/disk/unique_test.go b/accounts/pkg/indexer/indexer/index/disk/unique_test.go deleted file mode 100644 index ee47d45970..0000000000 --- a/accounts/pkg/indexer/indexer/index/disk/unique_test.go +++ /dev/null @@ -1,136 +0,0 @@ -package disk - -import ( - "github.com/owncloud/ocis/accounts/pkg/indexer/errors" - "github.com/owncloud/ocis/accounts/pkg/indexer/index" - . "github.com/owncloud/ocis/accounts/pkg/indexer/test" - "github.com/stretchr/testify/assert" - "os" - "path" - "testing" -) - -func TestUniqueLookupSingleEntry(t *testing.T) { - uniq, dataDir := getUniqueIdxSut(t) - filesDir := path.Join(dataDir, "users") - - t.Log("existing lookup") - resultPath, err := uniq.Lookup("mikey@example.com") - assert.NoError(t, err) - - assert.Equal(t, []string{path.Join(filesDir, "abcdefg-123")}, resultPath) - - t.Log("non-existing lookup") - resultPath, err = uniq.Lookup("doesnotExists@example.com") - assert.Error(t, err) - assert.IsType(t, &errors.NotFoundErr{}, err) - assert.Empty(t, resultPath) - - _ = os.RemoveAll(dataDir) - -} - -func TestUniqueUniqueConstraint(t *testing.T) { - uniq, dataDir := getUniqueIdxSut(t) - - _, err := uniq.Add("abcdefg-123", "mikey@example.com") - assert.Error(t, err) - assert.IsType(t, &errors.AlreadyExistsErr{}, err) - - _ = os.RemoveAll(dataDir) -} - -func TestUniqueRemove(t *testing.T) { - uniq, dataDir := getUniqueIdxSut(t) - - err := uniq.Remove("", "mikey@example.com") - assert.NoError(t, err) - - _, err = uniq.Lookup("mikey@example.com") - assert.Error(t, err) - assert.IsType(t, &errors.NotFoundErr{}, err) - - _ = os.RemoveAll(dataDir) -} - -func TestUniqueUpdate(t *testing.T) { - uniq, dataDir := getUniqueIdxSut(t) - - t.Log("successful update") - err := uniq.Update("", "", "mikey2@example.com") - assert.NoError(t, err) - - t.Log("failed update because already exists") - err = uniq.Update("", "", "mikey2@example.com") - assert.Error(t, err) - assert.IsType(t, &errors.AlreadyExistsErr{}, err) - - t.Log("failed update because not found") - err = uniq.Update("", "", "something2@example.com") - assert.Error(t, err) - assert.IsType(t, &errors.NotFoundErr{}, err) - - _ = os.RemoveAll(dataDir) -} - -func TestUniqueInit(t *testing.T) { - dataDir := CreateTmpDir(t) - indexRootDir := path.Join(dataDir, "index.disk") - filesDir := path.Join(dataDir, "users") - - uniq := NewUniqueIndex("User", "Email", filesDir, indexRootDir) - assert.Error(t, uniq.Init(), "Init should return an error about missing files-dir") - - if err := os.Mkdir(filesDir, 0777); err != nil { - t.Fatalf("Could not create test data-dir %s", err) - } - - assert.NoError(t, uniq.Init(), "Init shouldn't return an error") - assert.DirExists(t, indexRootDir) - assert.DirExists(t, path.Join(indexRootDir, "UniqueUserByEmail")) - - _ = os.RemoveAll(dataDir) -} - -func TestUniqueIndexSearch(t *testing.T) { - sut, dataPath := getUniqueIdxSut(t) - - res, err := sut.Search("j*@example.com") - - assert.NoError(t, err) - assert.Len(t, res, 2) - - assert.Equal(t, "ewf4ofk-555", path.Base(res[0])) - assert.Equal(t, "rulan54-777", path.Base(res[1])) - - res, err = sut.Search("does-not-exist@example.com") - assert.Error(t, err) - assert.IsType(t, &errors.NotFoundErr{}, err) - - _ = os.RemoveAll(dataPath) -} - -func TestErrors(t *testing.T) { - assert.True(t, errors.IsAlreadyExistsErr(&errors.AlreadyExistsErr{})) - assert.True(t, errors.IsNotFoundErr(&errors.NotFoundErr{})) -} - -func getUniqueIdxSut(t *testing.T) (sut index.Index, dataPath string) { - dataPath = WriteIndexTestData(t, TestData, "Id") - sut = NewUniqueIndex("User", "Email", path.Join(dataPath, "users"), path.Join(dataPath, "indexer.disk")) - err := sut.Init() - if err != nil { - t.Fatal(err) - } - - for _, u := range TestData["users"] { - pkVal := ValueOf(u, "Id") - idxByVal := ValueOf(u, "Email") - _, err := sut.Add(pkVal, idxByVal) - if err != nil { - t.Fatal(err) - } - } - - return -} diff --git a/accounts/pkg/indexer/indexer/index/index.go b/accounts/pkg/indexer/indexer/index/index.go deleted file mode 100644 index 151a891a39..0000000000 --- a/accounts/pkg/indexer/indexer/index/index.go +++ /dev/null @@ -1,15 +0,0 @@ -package index - -// Index can be implemented to create new indexer-strategies. See Unique for example. -// Each indexer implementation is bound to one data-column (IndexBy) and a data-type (TypeName) -type Index interface { - Init() error - Lookup(v string) ([]string, error) - Add(id, v string) (string, error) - Remove(id string, v string) error - Update(id, oldV, newV string) error - Search(pattern string) ([]string, error) - IndexBy() string - TypeName() string - FilesDir() string -} diff --git a/accounts/pkg/indexer/indexer/indexer.go b/accounts/pkg/indexer/indexer/indexer.go deleted file mode 100644 index 76189a8db1..0000000000 --- a/accounts/pkg/indexer/indexer/indexer.go +++ /dev/null @@ -1,158 +0,0 @@ -// Package indexer provides symlink-based indexer for on-disk document-directories. -package indexer - -import ( - "github.com/owncloud/ocis/accounts/pkg/indexer/errors" - "github.com/owncloud/ocis/accounts/pkg/indexer/index/disk" - "github.com/rs/zerolog" - "path" -) - -// Indexer is a facade to configure and query over multiple indices. -type Indexer struct { - config *Config - indices typeMap -} - -type Config struct { - DataDir string - IndexRootDirName string - Log zerolog.Logger -} - -func NewIndex(cfg *Config) *Indexer { - return &Indexer{ - config: cfg, - indices: typeMap{}, - } -} - -func (i Indexer) AddUniqueIndex(t interface{}, indexBy, pkName, entityDirName string) error { - typeName := getTypeFQN(t) - fullDataPath := path.Join(i.config.DataDir, entityDirName) - indexPath := path.Join(i.config.DataDir, i.config.IndexRootDirName) - - idx := disk.NewUniqueIndex(typeName, indexBy, fullDataPath, indexPath) - - i.indices.addIndex(typeName, pkName, idx) - return idx.Init() -} - -func (i Indexer) AddNonUniqueIndex(t interface{}, indexBy, pkName, entityDirName string) error { - typeName := getTypeFQN(t) - fullDataPath := path.Join(i.config.DataDir, entityDirName) - indexPath := path.Join(i.config.DataDir, i.config.IndexRootDirName) - - idx := disk.NewNonUniqueIndex(typeName, indexBy, fullDataPath, indexPath) - - i.indices.addIndex(typeName, pkName, idx) - return idx.Init() -} - -// Add a new entry to the indexer -func (i Indexer) Add(t interface{}) error { - typeName := getTypeFQN(t) - if fields, ok := i.indices[typeName]; ok { - for _, indices := range fields.IndicesByField { - for _, idx := range indices { - pkVal := valueOf(t, fields.PKFieldName) - idxByVal := valueOf(t, idx.IndexBy()) - if _, err := idx.Add(pkVal, idxByVal); err != nil { - return err - } - } - } - } - - return nil -} - -func (i Indexer) FindBy(t interface{}, field string, val string) ([]string, error) { - typeName := getTypeFQN(t) - resultPaths := make([]string, 0) - if fields, ok := i.indices[typeName]; ok { - for _, idx := range fields.IndicesByField[field] { - res, err := idx.Lookup(val) - if err != nil { - if errors.IsNotFoundErr(err) { - continue - } - - if err != nil { - return nil, err - } - } - - resultPaths = append(resultPaths, res...) - - } - } - - result := make([]string, 0, len(resultPaths)) - for _, v := range resultPaths { - result = append(result, path.Base(v)) - } - - return result, nil -} - -func (i Indexer) Delete(t interface{}) error { - typeName := getTypeFQN(t) - if fields, ok := i.indices[typeName]; ok { - for _, indices := range fields.IndicesByField { - for _, idx := range indices { - pkVal := valueOf(t, fields.PKFieldName) - idxByVal := valueOf(t, idx.IndexBy()) - if err := idx.Remove(pkVal, idxByVal); err != nil { - return err - } - } - } - } - - return nil -} - -func (i Indexer) FindByPartial(t interface{}, field string, pattern string) ([]string, error) { - typeName := getTypeFQN(t) - resultPaths := make([]string, 0) - if fields, ok := i.indices[typeName]; ok { - for _, idx := range fields.IndicesByField[field] { - res, err := idx.Search(pattern) - if err != nil { - if errors.IsNotFoundErr(err) { - continue - } - - if err != nil { - return nil, err - } - } - - resultPaths = append(resultPaths, res...) - - } - } - - result := make([]string, 0, len(resultPaths)) - for _, v := range resultPaths { - result = append(result, path.Base(v)) - } - - return result, nil - -} - -func (i Indexer) Update(t interface{}, field, oldVal, newVal string) error { - typeName := getTypeFQN(t) - if fields, ok := i.indices[typeName]; ok { - for _, idx := range fields.IndicesByField[field] { - pkVal := valueOf(t, fields.PKFieldName) - if err := idx.Update(pkVal, oldVal, newVal); err != nil { - return err - } - } - } - - return nil -} diff --git a/accounts/pkg/indexer/indexer/indexer_test.go b/accounts/pkg/indexer/indexer/indexer_test.go deleted file mode 100644 index 618ff6e0a9..0000000000 --- a/accounts/pkg/indexer/indexer/indexer_test.go +++ /dev/null @@ -1,255 +0,0 @@ -package indexer - -import ( - "github.com/owncloud/ocis/accounts/pkg/indexer/errors" - . "github.com/owncloud/ocis/accounts/pkg/indexer/test" - "github.com/rs/zerolog" - "github.com/stretchr/testify/assert" - "testing" -) - -func TestIndexer_AddWithUniqueIndex(t *testing.T) { - dataDir := WriteIndexTestData(t, TestData, "Id") - indexer := NewIndex(&Config{ - DataDir: dataDir, - IndexRootDirName: "index.disk", - Log: zerolog.Logger{}, - }) - - indexer.AddUniqueIndex(&User{}, "UserName", "Id", "users") - - u := &User{Id: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"} - err := indexer.Add(u) - assert.NoError(t, err) - -} - -func TestIndexer_FindByWithUniqueIndex(t *testing.T) { - dataDir := WriteIndexTestData(t, TestData, "Id") - indexer := NewIndex(&Config{ - DataDir: dataDir, - IndexRootDirName: "index.disk", - Log: zerolog.Logger{}, - }) - - indexer.AddUniqueIndex(&User{}, "UserName", "Id", "users") - - u := &User{Id: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"} - err := indexer.Add(u) - assert.NoError(t, err) - - res, err := indexer.FindBy(User{}, "UserName", "mikey") - assert.NoError(t, err) - t.Log(res) -} - -func TestIndexer_AddWithNonUniqueIndex(t *testing.T) { - dataDir := WriteIndexTestData(t, TestData, "Id") - indexer := NewIndex(&Config{ - DataDir: dataDir, - IndexRootDirName: "index.disk", - Log: zerolog.Logger{}, - }) - - indexer.AddNonUniqueIndex(&TestPet{}, "Kind", "Id", "pets") - - pet1 := TestPet{Id: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"} - pet2 := TestPet{Id: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"} - - err := indexer.Add(pet1) - assert.NoError(t, err) - - err = indexer.Add(pet2) - assert.NoError(t, err) - - res, err := indexer.FindBy(TestPet{}, "Kind", "Hog") - assert.NoError(t, err) - - t.Log(res) -} - -func TestIndexer_DeleteWithNonUniqueIndex(t *testing.T) { - dataDir := WriteIndexTestData(t, TestData, "Id") - indexer := NewIndex(&Config{ - DataDir: dataDir, - IndexRootDirName: "index.disk", - Log: zerolog.Logger{}, - }) - - indexer.AddNonUniqueIndex(&TestPet{}, "Kind", "Id", "pets") - - pet1 := TestPet{Id: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"} - pet2 := TestPet{Id: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"} - - err := indexer.Add(pet1) - assert.NoError(t, err) - - err = indexer.Add(pet2) - assert.NoError(t, err) - - err = indexer.Delete(pet2) - assert.NoError(t, err) -} - -func TestIndexer_SearchWithNonUniqueIndex(t *testing.T) { - dataDir := WriteIndexTestData(t, TestData, "Id") - indexer := NewIndex(&Config{ - DataDir: dataDir, - IndexRootDirName: "index.disk", - Log: zerolog.Logger{}, - }) - - indexer.AddNonUniqueIndex(&TestPet{}, "Name", "Id", "pets") - - pet1 := TestPet{Id: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"} - pet2 := TestPet{Id: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"} - - err := indexer.Add(pet1) - assert.NoError(t, err) - - err = indexer.Add(pet2) - assert.NoError(t, err) - - res, err := indexer.FindByPartial(pet2, "Name", "*ky") - assert.NoError(t, err) - - t.Log(res) -} - -func TestIndexer_UpdateWithUniqueIndex(t *testing.T) { - dataDir := WriteIndexTestData(t, TestData, "Id") - indexer := NewIndex(&Config{ - DataDir: dataDir, - IndexRootDirName: "index.disk", - Log: zerolog.Logger{}, - }) - - indexer.AddUniqueIndex(&User{}, "UserName", "Id", "users") - - user1 := &User{Id: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"} - user2 := &User{Id: "hijklmn-456", UserName: "frank", Email: "frank@example.com"} - - err := indexer.Add(user1) - assert.NoError(t, err) - - err = indexer.Add(user2) - assert.NoError(t, err) - - // Update to non existing value - err = indexer.Update(user2, "UserName", "frank", "jane") - assert.NoError(t, err) - - // Update to non existing value - err = indexer.Update(user2, "UserName", "mikey", "jane") - assert.Error(t, err) - assert.IsType(t, &errors.AlreadyExistsErr{}, err) -} - -func TestIndexer_UpdateWithNonUniqueIndex(t *testing.T) { - dataDir := WriteIndexTestData(t, TestData, "Id") - indexer := NewIndex(&Config{ - DataDir: dataDir, - IndexRootDirName: "index.disk", - Log: zerolog.Logger{}, - }) - - indexer.AddNonUniqueIndex(&TestPet{}, "Name", "Id", "pets") - - pet1 := TestPet{Id: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"} - pet2 := TestPet{Id: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"} - - err := indexer.Add(pet1) - assert.NoError(t, err) - - err = indexer.Add(pet2) - assert.NoError(t, err) - - err = indexer.Update(pet2, "Name", "Ricky", "Jonny") - assert.NoError(t, err) -} - -/* -func TestManagerQueryMultipleIndices(t *testing.T) { - dataDir := writeIndexTestData(t, testData, "Id") - man := NewIndex(&Config{ - DataDir: dataDir, - IndexRootDirName: "index.disk", - Log: zerolog.Logger{}, - }) - - err := man.AddUniqueIndex("User", "Email", "users") - assert.NoError(t, err) - - err = man.AddUniqueIndex("User", "UserName", "users") - assert.NoError(t, err) - - err = man.AddNormalIndex("TestPet", "Color", "pets") - assert.NoError(t, err) - - err = man.AddUniqueIndex("TestPet", "Name", "pets") - assert.NoError(t, err) - - for path := range testData { - for _, entity := range testData[path] { - err := man.Add(valueOf(entity, "Id"), entity) - assert.NoError(t, err) - } - } - - type test struct { - typeName, key, value, wantRes string - wantErr error - } - - tests := []test{ - {typeName: "User", key: "Email", value: "jacky@example.com", wantRes: "ewf4ofk-555"}, - {typeName: "User", key: "UserName", value: "jacky", wantRes: "ewf4ofk-555"}, - {typeName: "TestPet", key: "Color", value: "Brown", wantRes: "rebef-123"}, - {typeName: "TestPet", key: "Color", value: "Cyan", wantRes: "", wantErr: ¬FoundErr{}}, - } - - for _, tc := range tests { - name := fmt.Sprintf("Query%sBy%s=%s", tc.typeName, tc.key, tc.value) - t.Run(name, func(t *testing.T) { - pk, err := man.Find(tc.typeName, tc.key, tc.value) - assert.Equal(t, tc.wantRes, pk) - assert.IsType(t, tc.wantErr, err) - }) - } - - _ = os.RemoveAll(dataDir) -} - -*/ - -/* -func TestManagerDelete(t *testing.T) { - dataDir := writeIndexTestData(t, testData, "Id") - man := NewIndex(&Config{ - DataDir: dataDir, - IndexRootDirName: "index.disk", - Log: zerolog.Logger{}, - }) - - err := man.AddUniqueIndex("User", "Email", "users") - assert.NoError(t, err) - - err = man.AddUniqueIndex("User", "UserName", "users") - assert.NoError(t, err) - - err = man.AddUniqueIndex("TestPet", "Name", "pets") - assert.NoError(t, err) - - for path := range testData { - for _, entity := range testData[path] { - err := man.Add(valueOf(entity, "Id"), entity) - assert.NoError(t, err) - } - } - - err = man.Delete("User", "hijklmn-456") - _ = os.RemoveAll(dataDir) - -} - -*/ diff --git a/accounts/pkg/indexer/indexer/map.go b/accounts/pkg/indexer/indexer/map.go deleted file mode 100644 index 77e8e07770..0000000000 --- a/accounts/pkg/indexer/indexer/map.go +++ /dev/null @@ -1,27 +0,0 @@ -package indexer - -import "github.com/owncloud/ocis/accounts/pkg/indexer/index" - -// typeMap stores the indexer layout at runtime. - -type typeMap map[tName]typeMapping -type tName = string -type fieldName = string - -type typeMapping struct { - PKFieldName string - IndicesByField map[fieldName][]index.Index -} - -func (m typeMap) addIndex(typeName string, pkName string, idx index.Index) { - if val, ok := m[typeName]; ok { - val.IndicesByField[idx.IndexBy()] = append(val.IndicesByField[idx.IndexBy()], idx) - return - } - m[typeName] = typeMapping{ - PKFieldName: pkName, - IndicesByField: map[string][]index.Index{ - idx.IndexBy(): {idx}, - }, - } -} diff --git a/accounts/pkg/indexer/indexer/reflect.go b/accounts/pkg/indexer/indexer/reflect.go deleted file mode 100644 index 90036b3e19..0000000000 --- a/accounts/pkg/indexer/indexer/reflect.go +++ /dev/null @@ -1,34 +0,0 @@ -package indexer - -import ( - "errors" - "path" - "reflect" - "strings" -) - -func getType(v interface{}) (reflect.Value, error) { - rv := reflect.ValueOf(v) - for rv.Kind() == reflect.Ptr || rv.Kind() == reflect.Interface { - rv = rv.Elem() - } - if !rv.IsValid() { - return reflect.Value{}, errors.New("failed to read value via reflection") - } - - return rv, nil -} - -func getTypeFQN(t interface{}) string { - typ, _ := getType(t) - typeName := path.Join(typ.Type().PkgPath(), typ.Type().Name()) - typeName = strings.ReplaceAll(typeName, "/", ".") - return typeName -} - -func valueOf(v interface{}, field string) string { - r := reflect.ValueOf(v) - f := reflect.Indirect(r).FieldByName(field) - - return f.String() -} diff --git a/accounts/pkg/indexer/indexer/reflect_test.go b/accounts/pkg/indexer/indexer/reflect_test.go deleted file mode 100644 index 0da505d2ee..0000000000 --- a/accounts/pkg/indexer/indexer/reflect_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package indexer - -import ( - "testing" -) - -func Test_getTypeFQN(t *testing.T) { - type someT struct{} - - type args struct { - t interface{} - } - tests := []struct { - name string - args args - want string - }{ - {name: "ByValue", args: args{&someT{}}, want: "github.com.owncloud.ocis.accounts.pkg.indexer.someT"}, - {name: "ByRef", args: args{someT{}}, want: "github.com.owncloud.ocis.accounts.pkg.indexer.someT"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := getTypeFQN(tt.args.t); got != tt.want { - t.Errorf("getTypeFQN() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_valueOf(t *testing.T) { - type someT struct { - val string - } - type args struct { - v interface{} - field string - } - tests := []struct { - name string - args args - want string - }{ - {name: "ByValue", args: args{v: someT{val: "hello"}, field: "val"}, want: "hello"}, - {name: "ByRef", args: args{v: &someT{val: "hello"}, field: "val"}, want: "hello"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := valueOf(tt.args.v, tt.args.field); got != tt.want { - t.Errorf("valueOf() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/accounts/pkg/indexer/indexer/test/data.go b/accounts/pkg/indexer/indexer/test/data.go deleted file mode 100644 index 2b14d62ff9..0000000000 --- a/accounts/pkg/indexer/indexer/test/data.go +++ /dev/null @@ -1,56 +0,0 @@ -package test - -import ( - "encoding/json" - "io/ioutil" - "os" - "path" - "testing" -) - -type User struct { - Id, UserName, Email string -} - -type TestPet struct { - Id, Kind, Color, Name string -} - -var TestData = map[string][]interface{}{ - "users": { - User{Id: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"}, - User{Id: "hijklmn-456", UserName: "frank", Email: "frank@example.com"}, - User{Id: "ewf4ofk-555", UserName: "jacky", Email: "jacky@example.com"}, - User{Id: "rulan54-777", UserName: "jones", Email: "jones@example.com"}, - }, - "pets": { - TestPet{Id: "rebef-123", Kind: "Dog", Color: "Brown", Name: "Waldo"}, - TestPet{Id: "wefwe-456", Kind: "Cat", Color: "White", Name: "Snowy"}, - TestPet{Id: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"}, - TestPet{Id: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"}, - }, -} - -func WriteIndexTestData(t *testing.T, m map[string][]interface{}, pk string) string { - rootDir := CreateTmpDir(t) - for dirName := range m { - fileTypePath := path.Join(rootDir, dirName) - - if err := os.MkdirAll(fileTypePath, 0777); err != nil { - t.Fatal(err) - } - for _, u := range m[dirName] { - data, err := json.Marshal(u) - if err != nil { - t.Fatal(err) - } - - pkVal := ValueOf(u, pk) - if err := ioutil.WriteFile(path.Join(fileTypePath, pkVal), data, 0777); err != nil { - t.Fatal(err) - } - } - } - - return rootDir -} diff --git a/accounts/pkg/indexer/indexer/test/helpers.go b/accounts/pkg/indexer/indexer/test/helpers.go deleted file mode 100644 index f6ccb9be4d..0000000000 --- a/accounts/pkg/indexer/indexer/test/helpers.go +++ /dev/null @@ -1,23 +0,0 @@ -package test - -import ( - "io/ioutil" - "reflect" - "testing" -) - -func CreateTmpDir(t *testing.T) string { - name, err := ioutil.TempDir("/var/tmp", "testfiles-*") - if err != nil { - t.Fatal(err) - } - - return name -} - -func ValueOf(v interface{}, field string) string { - r := reflect.ValueOf(v) - f := reflect.Indirect(r).FieldByName(field) - - return f.String() -}