mirror/opencloud
1
0
Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-01-30 17:01:17 -05:00
Code Issues Packages Projects Releases Wiki Activity

Remove ocis-pkg/indexer it has been moved to reva a while ago

Browse Source
This commit is contained in:
Ralf Haferkamp
2022-05-05 08:31:51 +02:00
committed by Ralf Haferkamp
parent d25aa7b20f
commit 9c66eed29d
29 changed files with 0 additions and 3986 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
ocis-pkg/indexer/config/config.go
View File

@@ -1,48 +0,0 @@
// Package config should be moved to internal
package config
import (
"github.com/owncloud/ocis/v2/extensions/accounts/pkg/config"
)
// Repo defines which storage implementation is to be used.
type Repo struct {
Backend string
Disk Disk
CS3 CS3
}
// Disk is the local disk implementation of the storage.
type Disk struct {
Path string
}
// CS3 is the cs3 implementation of the storage.
type CS3 struct {
ProviderAddr string
DataURL string
DataPrefix string
JWTSecret string
}
// Index defines config for indexes.
type Index struct {
UID, GID Bound
}
// Bound defines a lower and upper bound.
type Bound struct {
Lower, Upper int64
}
// Config merges all Account config parameters.
type Config struct {
Repo Repo
Index Index
ServiceUser config.ServiceUser
}
// New returns a new config.
func New() *Config {
return &Config{}
}

35
ocis-pkg/indexer/errors/errors.go
View File

@@ -1,35 +0,0 @@
package errors
import (
"fmt"
)
// AlreadyExistsErr implements the Error interface.
type AlreadyExistsErr struct {
TypeName, Key, Value string
}
func (e *AlreadyExistsErr) Error() string {
return fmt.Sprintf("%s with %s=%s does already exist", e.TypeName, e.Key, e.Value)
}
// IsAlreadyExistsErr checks whether an error is of type AlreadyExistsErr.
func IsAlreadyExistsErr(e error) bool {
_, ok := e.(*AlreadyExistsErr)
return ok
}
// NotFoundErr implements the Error interface.
type NotFoundErr struct {
TypeName, Key, Value string
}
func (e *NotFoundErr) Error() string {
return fmt.Sprintf("%s with %s=%s not found", e.TypeName, e.Key, e.Value)
}
// IsNotFoundErr checks whether an error is of type IsNotFoundErr.
func IsNotFoundErr(e error) bool {
_, ok := e.(*NotFoundErr)
return ok
}

15
ocis-pkg/indexer/helper.go
View File

@@ -1,15 +0,0 @@
package indexer
// dedup removes duplicate values in given slice
func dedup(s []string) []string {
for i := 0; i < len(s); i++ {
for i2 := i + 1; i2 < len(s); i2++ {
if s[i] == s[i2] {
// delete
s = append(s[:i2], s[i2+1:]...)
i2--
}
}
}
return s
}

375
ocis-pkg/indexer/index/cs3/autoincrement.go
View File

@@ -1,375 +0,0 @@
package cs3
import (
"context"
"os"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"github.com/owncloud/ocis/v2/extensions/accounts/pkg/storage"
idxerrs "github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors"
v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
revactx "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/v2/pkg/token"
"github.com/cs3org/reva/v2/pkg/token/manager/jwt"
"github.com/cs3org/reva/v2/pkg/utils"
"google.golang.org/grpc/metadata"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/index"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/option"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/registry"
metadatastorage "github.com/owncloud/ocis/v2/ocis-pkg/metadata_storage"
)
// Autoincrement are fields for an index of type autoincrement.
type Autoincrement struct {
indexBy string
typeName string
filesDir string
indexBaseDir string
indexRootDir string
tokenManager token.Manager
storageProvider provider.ProviderAPIClient
metadataStorage *metadatastorage.MetadataStorage
cs3conf *Config
bound *option.Bound
}
func init() {
registry.IndexConstructorRegistry["cs3"]["autoincrement"] = NewAutoincrementIndex
}
// NewAutoincrementIndex instantiates a new AutoincrementIndex instance.
func NewAutoincrementIndex(o ...option.Option) index.Index {
opts := &option.Options{}
for _, opt := range o {
opt(opts)
}
u := &Autoincrement{
indexBy: opts.IndexBy,
typeName: opts.TypeName,
filesDir: opts.FilesDir,
bound: opts.Bound,
indexBaseDir: path.Join(opts.DataDir, "index.cs3"),
indexRootDir: path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"autoincrement", opts.TypeName, opts.IndexBy}, ".")),
cs3conf: &Config{
ProviderAddr: opts.ProviderAddr,
JWTSecret: opts.JWTSecret,
ServiceUser: opts.ServiceUser,
},
}
return u
}
// Init initializes an autoincrement index.
func (idx *Autoincrement) Init() error {
tokenManager, err := jwt.New(map[string]interface{}{
"secret": idx.cs3conf.JWTSecret,
})
if err != nil {
return err
}
idx.tokenManager = tokenManager
client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr)
if err != nil {
return err
}
idx.storageProvider = client
m, err := metadatastorage.NewMetadataStorage(idx.cs3conf.ProviderAddr)
if err != nil {
return err
}
idx.metadataStorage = &m
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
if err := idx.metadataStorage.Init(ctx, idx.cs3conf.ServiceUser); err != nil {
return err
}
if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil {
return err
}
if err := idx.makeDirIfNotExists(idx.indexRootDir); err != nil {
return err
}
return nil
}
// Lookup exact lookup by value.
func (idx *Autoincrement) Lookup(v string) ([]string, error) {
searchPath := path.Join(idx.indexRootDir, v)
oldname, err := idx.resolveSymlink(searchPath)
if err != nil {
if os.IsNotExist(err) {
err = &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
}
return nil, err
}
return []string{oldname}, nil
}
// Add a new value to the index.
func (idx *Autoincrement) Add(id, v string) (string, error) {
var newName string
if v == "" {
next, err := idx.next()
if err != nil {
return "", err
}
newName = path.Join(idx.indexRootDir, strconv.Itoa(next))
} else {
newName = path.Join(idx.indexRootDir, v)
}
if err := idx.createSymlink(id, newName); err != nil {
if os.IsExist(err) {
return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
}
return "", err
}
return newName, nil
}
// Remove a value v from an index.
func (idx *Autoincrement) Remove(id string, v string) error {
if v == "" {
return nil
}
searchPath := path.Join(idx.indexRootDir, v)
_, err := idx.resolveSymlink(searchPath)
if err != nil {
if os.IsNotExist(err) {
err = &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
}
return err
}
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
deletePath := path.Join("/", idx.indexRootDir, v)
resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{
Ref: &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
},
Path: utils.MakeRelativePath(deletePath),
},
})
if err != nil {
return err
}
// TODO Handle other error codes?
if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND {
return &idxerrs.NotFoundErr{}
}
return err
}
// Update index from <oldV> to <newV>.
func (idx *Autoincrement) Update(id, oldV, newV string) error {
if err := idx.Remove(id, oldV); err != nil {
return err
}
if _, err := idx.Add(id, newV); err != nil {
return err
}
return nil
}
// Search allows for glob search on the index.
func (idx *Autoincrement) Search(pattern string) ([]string, error) {
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return nil, err
}
res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
Ref: &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
},
Path: utils.MakeRelativePath(idx.indexRootDir),
},
})
if err != nil {
return nil, err
}
searchPath := idx.indexRootDir
matches := make([]string, 0)
for _, i := range res.GetInfos() {
if found, err := filepath.Match(pattern, path.Base(i.Path)); found {
if err != nil {
return nil, err
}
oldPath, err := idx.resolveSymlink(path.Join(searchPath, path.Base(i.Path)))
if err != nil {
return nil, err
}
matches = append(matches, oldPath)
}
}
return matches, nil
}
// CaseInsensitive undocumented.
func (idx *Autoincrement) CaseInsensitive() bool {
return false
}
// IndexBy undocumented.
func (idx *Autoincrement) IndexBy() string {
return idx.indexBy
}
// TypeName undocumented.
func (idx *Autoincrement) TypeName() string {
return idx.typeName
}
// FilesDir undocumented.
func (idx *Autoincrement) FilesDir() string {
return idx.filesDir
}
func (idx *Autoincrement) createSymlink(oldname, newname string) error {
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
if _, err := idx.resolveSymlink(newname); err == nil {
return os.ErrExist
}
err = idx.metadataStorage.SimpleUpload(ctx, newname, []byte(oldname))
if err != nil {
return err
}
return nil
}
func (idx *Autoincrement) resolveSymlink(name string) (string, error) {
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return "", err
}
b, err := idx.metadataStorage.SimpleDownload(ctx, name)
if err != nil {
if metadatastorage.IsNotFoundErr(err) {
return "", os.ErrNotExist
}
return "", err
}
return string(b), err
}
func (idx *Autoincrement) makeDirIfNotExists(folder string) error {
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
return storage.MakeDirIfNotExist(ctx, idx.storageProvider, &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
}, folder)
}
func (idx *Autoincrement) next() (int, error) {
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return -1, err
}
res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
Ref: &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
},
Path: utils.MakeRelativePath(idx.indexRootDir),
},
})
if err != nil {
return -1, err
}
if len(res.GetInfos()) == 0 {
return 0, nil
}
infos := res.GetInfos()
sort.Slice(infos, func(i, j int) bool {
a, _ := strconv.Atoi(path.Base(infos[i].Path))
b, _ := strconv.Atoi(path.Base(infos[j].Path))
return a < b
})
latest, err := strconv.Atoi(path.Base(infos[len(infos)-1].Path)) // would returning a string be a better interface?
if err != nil {
return -1, err
}
if int64(latest) < idx.bound.Lower {
return int(idx.bound.Lower), nil
}
return latest + 1, nil
}
func (idx *Autoincrement) getAuthenticatedContext(ctx context.Context) (context.Context, error) {
t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager)
if err != nil {
return nil, err
}
ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t)
return ctx, nil
}
// Delete deletes the index folder from its storage.
func (idx *Autoincrement) Delete() error {
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
return deleteIndexRoot(ctx, idx.storageProvider, idx.cs3conf.ServiceUser.UUID, idx.indexRootDir)
}

83
ocis-pkg/indexer/index/cs3/autoincrement_test.go
View File

@@ -1,83 +0,0 @@
package cs3
//
//import (
// "os"
// "testing"
//
// "github.com/owncloud/ocis/v2/accounts/pkg/config"
// "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option"
// . "github.com/owncloud/ocis/v2/ocis-pkg/indexer/test"
// "github.com/stretchr/testify/assert"
//)
//
//const cs3RootFolder = "/tmp/ocis/storage/users/data"
//
//func TestAutoincrementIndexAdd(t *testing.T) {
// dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder)
// assert.NoError(t, err)
// cfg := generateConfig()
//
// sut := NewAutoincrementIndex(
// option.WithTypeName(GetTypeFQN(User{})),
// option.WithIndexBy("UID"),
// option.WithDataURL(cfg.Repo.CS3.DataURL),
// option.WithDataPrefix(cfg.Repo.CS3.DataPrefix),
// option.WithJWTSecret(cfg.Repo.CS3.JWTSecret),
// option.WithProviderAddr(cfg.Repo.CS3.ProviderAddr),
// )
//
// assert.NoError(t, sut.Init())
//
// for i := 0; i < 5; i++ {
// res, err := sut.Add("abcdefg-123", "")
// assert.NoError(t, err)
// t.Log(res)
// }
//
// _ = os.RemoveAll(dataDir)
//}
//
//func BenchmarkAutoincrementIndexAdd(b *testing.B) {
// dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder)
// assert.NoError(b, err)
// cfg := generateConfig()
//
// sut := NewAutoincrementIndex(
// option.WithTypeName(GetTypeFQN(User{})),
// option.WithIndexBy("UID"),
// option.WithDataURL(cfg.Repo.CS3.DataURL),
// option.WithDataPrefix(cfg.Repo.CS3.DataPrefix),
// option.WithJWTSecret(cfg.Repo.CS3.JWTSecret),
// option.WithProviderAddr(cfg.Repo.CS3.ProviderAddr),
// )
//
// err = sut.Init()
// assert.NoError(b, err)
//
// for n := 0; n < b.N; n++ {
// _, err := sut.Add("abcdefg-123", "")
// if err != nil {
// b.Error(err)
// }
// assert.NoError(b, err)
// }
//
// _ = os.RemoveAll(dataDir)
//}
//
//func generateConfig() config.Config {
// return config.Config{
// Repo: config.Repo{
// Disk: config.Disk{
// Path: "",
// },
// CS3: config.CS3{
// ProviderAddr: "0.0.0.0:9215",
// DataURL: "http://localhost:9216",
// DataPrefix: "data",
// JWTSecret: "Pive-Fumkiu4",
// },
// },
// }
//}

12
ocis-pkg/indexer/index/cs3/config.go
View File

@@ -1,12 +0,0 @@
package cs3
import (
acccfg "github.com/owncloud/ocis/v2/extensions/accounts/pkg/config"
)
// Config represents cs3conf. Should be deprecated in favor of config.Config.
type Config struct {
ProviderAddr string
JWTSecret string
ServiceUser acccfg.ServiceUser
}

30
ocis-pkg/indexer/index/cs3/helper.go
View File

@@ -1,30 +0,0 @@
package cs3
import (
"context"
"fmt"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/cs3org/reva/v2/pkg/utils"
)
func deleteIndexRoot(ctx context.Context, storageProvider provider.ProviderAPIClient, spaceid, indexRootDir string) error {
res, err := storageProvider.Delete(ctx, &provider.DeleteRequest{
Ref: &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: spaceid,
OpaqueId: spaceid,
},
Path: utils.MakeRelativePath(indexRootDir),
},
})
if err != nil {
return err
}
if res.Status.Code != rpc.Code_CODE_OK {
return fmt.Errorf("error deleting index root dir: %v", indexRootDir)
}
return nil
}

396
ocis-pkg/indexer/index/cs3/non_unique.go
View File

@@ -1,396 +0,0 @@
package cs3
import (
"context"
"os"
"path"
"path/filepath"
"strings"
"github.com/owncloud/ocis/v2/extensions/accounts/pkg/storage"
v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
revactx "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/v2/pkg/token"
"github.com/cs3org/reva/v2/pkg/token/manager/jwt"
"github.com/cs3org/reva/v2/pkg/utils"
idxerrs "github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/index"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/option"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/registry"
metadatastorage "github.com/owncloud/ocis/v2/ocis-pkg/metadata_storage"
"google.golang.org/grpc/metadata"
)
func init() {
registry.IndexConstructorRegistry["cs3"]["non_unique"] = NewNonUniqueIndexWithOptions
}
// NonUnique are fields for an index of type non_unique.
type NonUnique struct {
caseInsensitive bool
indexBy string
typeName string
filesDir string
indexBaseDir string
indexRootDir string
tokenManager token.Manager
storageProvider provider.ProviderAPIClient
metadataStorage *metadatastorage.MetadataStorage
cs3conf *Config
}
// NewNonUniqueIndexWithOptions instantiates a new NonUniqueIndex instance.
// /tmp/ocis/accounts/index.cs3/Pets/Bro*
// ├── Brown/
// │ └── rebef-123 -> /tmp/testfiles-395764020/pets/rebef-123
// ├── Green/
// │ ├── goefe-789 -> /tmp/testfiles-395764020/pets/goefe-789
// │ └── xadaf-189 -> /tmp/testfiles-395764020/pets/xadaf-189
// └── White/
// └── wefwe-456 -> /tmp/testfiles-395764020/pets/wefwe-456
func NewNonUniqueIndexWithOptions(o ...option.Option) index.Index {
opts := &option.Options{}
for _, opt := range o {
opt(opts)
}
return &NonUnique{
caseInsensitive: opts.CaseInsensitive,
indexBy: opts.IndexBy,
typeName: opts.TypeName,
filesDir: opts.FilesDir,
indexBaseDir: path.Join(opts.DataDir, "index.cs3"),
indexRootDir: path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"non_unique", opts.TypeName, opts.IndexBy}, ".")),
cs3conf: &Config{
ProviderAddr: opts.ProviderAddr,
JWTSecret: opts.JWTSecret,
ServiceUser: opts.ServiceUser,
},
}
}
// Init initializes a non_unique index.
func (idx *NonUnique) Init() error {
tokenManager, err := jwt.New(map[string]interface{}{
"secret": idx.cs3conf.JWTSecret,
})
if err != nil {
return err
}
idx.tokenManager = tokenManager
client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr)
if err != nil {
return err
}
idx.storageProvider = client
m, err := metadatastorage.NewMetadataStorage(idx.cs3conf.ProviderAddr)
if err != nil {
return err
}
idx.metadataStorage = &m
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
if err := idx.metadataStorage.Init(ctx, idx.cs3conf.ServiceUser); err != nil {
return err
}
if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil {
return err
}
if err := idx.makeDirIfNotExists(idx.indexRootDir); err != nil {
return err
}
return nil
}
// Lookup exact lookup by value.
func (idx *NonUnique) Lookup(v string) ([]string, error) {
if idx.caseInsensitive {
v = strings.ToLower(v)
}
var matches = make([]string, 0)
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return nil, err
}
res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
Ref: &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
},
Path: utils.MakeRelativePath(path.Join("/", idx.indexRootDir, v)),
},
})
if err != nil {
return nil, err
}
for _, info := range res.Infos {
matches = append(matches, path.Base(info.Path))
}
return matches, nil
}
// Add a new value to the index.
func (idx *NonUnique) Add(id, v string) (string, error) {
if v == "" {
return "", nil
}
if idx.caseInsensitive {
v = strings.ToLower(v)
}
newName := path.Join(idx.indexRootDir, v)
if err := idx.makeDirIfNotExists(newName); err != nil {
return "", err
}
if err := idx.createSymlink(id, path.Join(newName, id)); err != nil {
if os.IsExist(err) {
return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
}
return "", err
}
return newName, nil
}
// Remove a value v from an index.
func (idx *NonUnique) Remove(id string, v string) error {
if v == "" {
return nil
}
if idx.caseInsensitive {
v = strings.ToLower(v)
}
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
deletePath := path.Join("/", idx.indexRootDir, v, id)
resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{
Ref: &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
},
Path: utils.MakeRelativePath(deletePath),
},
})
if err != nil {
return err
}
if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND {
return &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
}
toStat := path.Join("/", idx.indexRootDir, v)
lcResp, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
Ref: &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
},
Path: utils.MakeRelativePath(toStat),
},
})
if err != nil {
return err
}
if len(lcResp.Infos) == 0 {
deletePath = path.Join("/", idx.indexRootDir, v)
_, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{
Ref: &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
},
Path: utils.MakeRelativePath(deletePath),
},
})
if err != nil {
return err
}
}
return nil
}
// Update index from <oldV> to <newV>.
func (idx *NonUnique) Update(id, oldV, newV string) error {
if idx.caseInsensitive {
oldV = strings.ToLower(oldV)
newV = strings.ToLower(newV)
}
if err := idx.Remove(id, oldV); err != nil {
return err
}
if _, err := idx.Add(id, newV); err != nil {
return err
}
return nil
}
// Search allows for glob search on the index.
func (idx *NonUnique) Search(pattern string) ([]string, error) {
if idx.caseInsensitive {
pattern = strings.ToLower(pattern)
}
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return nil, err
}
foldersMatched := make([]string, 0)
matches := make([]string, 0)
res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
Ref: &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
},
Path: utils.MakeRelativePath(idx.indexRootDir),
},
})
if err != nil {
return nil, err
}
for _, i := range res.Infos {
if found, err := filepath.Match(pattern, path.Base(i.Path)); found {
if err != nil {
return nil, err
}
foldersMatched = append(foldersMatched, i.Path)
}
}
for i := range foldersMatched {
res, _ := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
Ref: &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
},
Path: utils.MakeRelativePath(foldersMatched[i]),
},
})
for _, info := range res.Infos {
matches = append(matches, path.Base(info.Path))
}
}
return matches, nil
}
// CaseInsensitive undocumented.
func (idx *NonUnique) CaseInsensitive() bool {
return idx.caseInsensitive
}
// IndexBy undocumented.
func (idx *NonUnique) IndexBy() string {
return idx.indexBy
}
// TypeName undocumented.
func (idx *NonUnique) TypeName() string {
return idx.typeName
}
// FilesDir undocumented.
func (idx *NonUnique) FilesDir() string {
return idx.filesDir
}
func (idx *NonUnique) makeDirIfNotExists(folder string) error {
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
return storage.MakeDirIfNotExist(ctx, idx.storageProvider, &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
}, folder)
}
func (idx *NonUnique) createSymlink(oldname, newname string) error {
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
if _, err := idx.resolveSymlink(newname); err == nil {
return os.ErrExist
}
err = idx.metadataStorage.SimpleUpload(ctx, newname, []byte(oldname))
if err != nil {
return err
}
return nil
}
func (idx *NonUnique) resolveSymlink(name string) (string, error) {
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return "", err
}
b, err := idx.metadataStorage.SimpleDownload(ctx, name)
if err != nil {
if metadatastorage.IsNotFoundErr(err) {
return "", os.ErrNotExist
}
return "", err
}
return string(b), err
}
func (idx *NonUnique) getAuthenticatedContext(ctx context.Context) (context.Context, error) {
t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager)
if err != nil {
return nil, err
}
ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t)
return ctx, nil
}
// Delete deletes the index folder from its storage.
func (idx *NonUnique) Delete() error {
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
return deleteIndexRoot(ctx, idx.storageProvider, idx.cs3conf.ServiceUser.UUID, idx.indexRootDir)
}

68
ocis-pkg/indexer/index/cs3/non_unique_test.go
View File

@@ -1,68 +0,0 @@
package cs3
//
//import (
// "os"
// "path"
// "testing"
//
// "github.com/owncloud/ocis/v2/accounts/pkg/config"
// "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option"
// . "github.com/owncloud/ocis/v2/ocis-pkg/indexer/test"
// "github.com/stretchr/testify/assert"
//)
//
//func TestCS3NonUniqueIndex_FakeSymlink(t *testing.T) {
// dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder)
// assert.NoError(t, err)
// cfg := config.Config{
// Repo: config.Repo{
// Disk: config.Disk{
// Path: "",
// },
// CS3: config.CS3{
// ProviderAddr: "0.0.0.0:9215",
// DataURL: "http://localhost:9216",
// DataPrefix: "data",
// JWTSecret: "Pive-Fumkiu4",
// },
// },
// }
//
// sut := NewNonUniqueIndexWithOptions(
// option.WithTypeName(GetTypeFQN(User{})),
// option.WithIndexBy("UserName"),
// option.WithFilesDir(path.Join(cfg.Repo.Disk.Path, "/")),
// option.WithDataDir(cfg.Repo.Disk.Path),
// option.WithDataURL(cfg.Repo.CS3.DataURL),
// option.WithDataPrefix(cfg.Repo.CS3.DataPrefix),
// option.WithJWTSecret(cfg.Repo.CS3.JWTSecret),
// option.WithProviderAddr(cfg.Repo.CS3.ProviderAddr),
// )
//
// err = sut.Init()
// assert.NoError(t, err)
//
// res, err := sut.Add("abcdefg-123", "mikey")
// assert.NoError(t, err)
// t.Log(res)
//
// resLookup, err := sut.Lookup("mikey")
// assert.NoError(t, err)
// t.Log(resLookup)
//
// err = sut.Update("abcdefg-123", "mikey", "mickeyX")
// assert.NoError(t, err)
//
// searchRes, err := sut.Search("m*")
// assert.NoError(t, err)
// assert.Len(t, searchRes, 1)
// assert.Equal(t, searchRes[0], "abcdefg-123")
//
// resp, err := sut.Lookup("mikey")
// assert.Len(t, resp, 0)
// assert.NoError(t, err)
//
// _ = os.RemoveAll(dataDir)
//
//}

342
ocis-pkg/indexer/index/cs3/unique.go
View File

@@ -1,342 +0,0 @@
package cs3
import (
"context"
"os"
"path"
"path/filepath"
"strings"
"github.com/owncloud/ocis/v2/extensions/accounts/pkg/storage"
v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
revactx "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/v2/pkg/token"
"github.com/cs3org/reva/v2/pkg/token/manager/jwt"
"github.com/cs3org/reva/v2/pkg/utils"
idxerrs "github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/index"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/option"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/registry"
metadatastorage "github.com/owncloud/ocis/v2/ocis-pkg/metadata_storage"
"google.golang.org/grpc/metadata"
)
// Unique are fields for an index of type non_unique.
type Unique struct {
caseInsensitive bool
indexBy string
typeName string
filesDir string
indexBaseDir string
indexRootDir string
tokenManager token.Manager
storageProvider provider.ProviderAPIClient
metadataStorage *metadatastorage.MetadataStorage
cs3conf *Config
}
func init() {
registry.IndexConstructorRegistry["cs3"]["unique"] = NewUniqueIndexWithOptions
}
// NewUniqueIndexWithOptions instantiates a new UniqueIndex instance. Init() should be
// called afterward to ensure correct on-disk structure.
func NewUniqueIndexWithOptions(o ...option.Option) index.Index {
opts := &option.Options{}
for _, opt := range o {
opt(opts)
}
u := &Unique{
caseInsensitive: opts.CaseInsensitive,
indexBy: opts.IndexBy,
typeName: opts.TypeName,
filesDir: opts.FilesDir,
indexBaseDir: path.Join(opts.DataDir, "index.cs3"),
indexRootDir: path.Join(path.Join(opts.DataDir, "index.cs3"), strings.Join([]string{"unique", opts.TypeName, opts.IndexBy}, ".")),
cs3conf: &Config{
ProviderAddr: opts.ProviderAddr,
JWTSecret: opts.JWTSecret,
ServiceUser: opts.ServiceUser,
},
}
return u
}
// Init initializes a unique index.
func (idx *Unique) Init() error {
tokenManager, err := jwt.New(map[string]interface{}{
"secret": idx.cs3conf.JWTSecret,
})
if err != nil {
return err
}
idx.tokenManager = tokenManager
client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr)
if err != nil {
return err
}
idx.storageProvider = client
m, err := metadatastorage.NewMetadataStorage(idx.cs3conf.ProviderAddr)
if err != nil {
return err
}
idx.metadataStorage = &m
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
if err := idx.metadataStorage.Init(ctx, idx.cs3conf.ServiceUser); err != nil {
return err
}
if err := idx.makeDirIfNotExists(idx.indexBaseDir); err != nil {
return err
}
if err := idx.makeDirIfNotExists(idx.indexRootDir); err != nil {
return err
}
return nil
}
// Lookup exact lookup by value.
func (idx *Unique) Lookup(v string) ([]string, error) {
if idx.caseInsensitive {
v = strings.ToLower(v)
}
searchPath := path.Join(idx.indexRootDir, v)
oldname, err := idx.resolveSymlink(searchPath)
if err != nil {
if os.IsNotExist(err) {
err = &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
}
return nil, err
}
return []string{oldname}, nil
}
// Add adds a value to the index, returns the path to the root-document
func (idx *Unique) Add(id, v string) (string, error) {
if v == "" {
return "", nil
}
if idx.caseInsensitive {
v = strings.ToLower(v)
}
newName := path.Join(idx.indexRootDir, v)
if err := idx.createSymlink(id, newName); err != nil {
if os.IsExist(err) {
return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
}
return "", err
}
return newName, nil
}
// Remove a value v from an index.
func (idx *Unique) Remove(id string, v string) error {
if v == "" {
return nil
}
if idx.caseInsensitive {
v = strings.ToLower(v)
}
searchPath := path.Join(idx.indexRootDir, v)
_, err := idx.resolveSymlink(searchPath)
if err != nil {
if os.IsNotExist(err) {
err = &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
}
return err
}
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
deletePath := path.Join("/", idx.indexRootDir, v)
resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{
Ref: &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
},
Path: utils.MakeRelativePath(deletePath),
},
})
if err != nil {
return err
}
// TODO Handle other error codes?
if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND {
return &idxerrs.NotFoundErr{}
}
return err
}
// Update index from <oldV> to <newV>.
func (idx *Unique) Update(id, oldV, newV string) error {
if idx.caseInsensitive {
oldV = strings.ToLower(oldV)
newV = strings.ToLower(newV)
}
if err := idx.Remove(id, oldV); err != nil {
return err
}
if _, err := idx.Add(id, newV); err != nil {
return err
}
return nil
}
// Search allows for glob search on the index.
func (idx *Unique) Search(pattern string) ([]string, error) {
if idx.caseInsensitive {
pattern = strings.ToLower(pattern)
}
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return nil, err
}
res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
Ref: &provider.Reference{
ResourceId: &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
},
Path: utils.MakeRelativePath(idx.indexRootDir),
},
})
if err != nil {
return nil, err
}
searchPath := idx.indexRootDir
matches := make([]string, 0)
for _, i := range res.GetInfos() {
if found, err := filepath.Match(pattern, path.Base(i.Path)); found {
if err != nil {
return nil, err
}
oldPath, err := idx.resolveSymlink(path.Join(searchPath, path.Base(i.Path)))
if err != nil {
return nil, err
}
matches = append(matches, oldPath)
}
}
return matches, nil
}
// CaseInsensitive undocumented.
func (idx *Unique) CaseInsensitive() bool {
return idx.caseInsensitive
}
// IndexBy undocumented.
func (idx *Unique) IndexBy() string {
return idx.indexBy
}
// TypeName undocumented.
func (idx *Unique) TypeName() string {
return idx.typeName
}
// FilesDir undocumented.
func (idx *Unique) FilesDir() string {
return idx.filesDir
}
func (idx *Unique) createSymlink(oldname, newname string) error {
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
if _, err := idx.resolveSymlink(newname); err == nil {
return os.ErrExist
}
err = idx.metadataStorage.SimpleUpload(ctx, newname, []byte(oldname))
if err != nil {
return err
}
return nil
}
func (idx *Unique) resolveSymlink(name string) (string, error) {
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return "", err
}
b, err := idx.metadataStorage.SimpleDownload(ctx, name)
if err != nil {
if metadatastorage.IsNotFoundErr(err) {
return "", os.ErrNotExist
}
return "", err
}
return string(b), err
}
func (idx *Unique) makeDirIfNotExists(folder string) error {
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
return storage.MakeDirIfNotExist(ctx, idx.storageProvider, &provider.ResourceId{
StorageId: idx.cs3conf.ServiceUser.UUID,
OpaqueId: idx.cs3conf.ServiceUser.UUID,
}, folder)
}
func (idx *Unique) getAuthenticatedContext(ctx context.Context) (context.Context, error) {
t, err := storage.AuthenticateCS3(ctx, idx.cs3conf.ServiceUser, idx.tokenManager)
if err != nil {
return nil, err
}
ctx = metadata.AppendToOutgoingContext(ctx, revactx.TokenHeader, t)
return ctx, nil
}
// Delete deletes the index folder from its storage.
func (idx *Unique) Delete() error {
ctx, err := idx.getAuthenticatedContext(context.Background())
if err != nil {
return err
}
return deleteIndexRoot(ctx, idx.storageProvider, idx.cs3conf.ServiceUser.UUID, idx.indexRootDir)
}

107
ocis-pkg/indexer/index/cs3/unique_test.go
View File

@@ -1,107 +0,0 @@
package cs3
//
//import (
// "os"
// "path"
// "testing"
//
// "github.com/owncloud/ocis/v2/accounts/pkg/config"
// "github.com/owncloud/ocis/v2/ocis-pkg/indexer/option"
// . "github.com/owncloud/ocis/v2/ocis-pkg/indexer/test"
// "github.com/stretchr/testify/assert"
//)
//
//func TestCS3UniqueIndex_FakeSymlink(t *testing.T) {
// dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder)
// assert.NoError(t, err)
// cfg := config.Config{
// Repo: config.Repo{
// Disk: config.Disk{
// Path: "",
// },
// CS3: config.CS3{
// ProviderAddr: "0.0.0.0:9215",
// DataURL: "http://localhost:9216",
// DataPrefix: "data",
// JWTSecret: "Pive-Fumkiu4",
// },
// },
// }
//
// sut := NewUniqueIndexWithOptions(
// option.WithTypeName(GetTypeFQN(User{})),
// option.WithIndexBy("UserName"),
// option.WithFilesDir(path.Join(cfg.Repo.Disk.Path, "/")),
// option.WithDataDir(cfg.Repo.Disk.Path),
// option.WithDataURL(cfg.Repo.CS3.DataURL),
// option.WithDataPrefix(cfg.Repo.CS3.DataPrefix),
// option.WithJWTSecret(cfg.Repo.CS3.JWTSecret),
// option.WithProviderAddr(cfg.Repo.CS3.ProviderAddr),
// )
//
// err = sut.Init()
// assert.NoError(t, err)
//
// res, err := sut.Add("abcdefg-123", "mikey")
// assert.NoError(t, err)
// t.Log(res)
//
// resLookup, err := sut.Lookup("mikey")
// assert.NoError(t, err)
// t.Log(resLookup)
//
// err = sut.Update("abcdefg-123", "mikey", "mickeyX")
// assert.NoError(t, err)
//
// searchRes, err := sut.Search("m*")
// assert.NoError(t, err)
// assert.Len(t, searchRes, 1)
// assert.Equal(t, searchRes[0], "abcdefg-123")
//
// _ = os.RemoveAll(dataDir)
//}
//
//func TestCS3UniqueIndexSearch(t *testing.T) {
// dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder)
// assert.NoError(t, err)
// cfg := config.Config{
// Repo: config.Repo{
// Disk: config.Disk{
// Path: "",
// },
// CS3: config.CS3{
// ProviderAddr: "0.0.0.0:9215",
// DataURL: "http://localhost:9216",
// DataPrefix: "data",
// JWTSecret: "Pive-Fumkiu4",
// },
// },
// }
//
// sut := NewUniqueIndexWithOptions(
// option.WithTypeName(GetTypeFQN(User{})),
// option.WithIndexBy("UserName"),
// option.WithFilesDir(path.Join(cfg.Repo.Disk.Path, "/")),
// option.WithDataDir(cfg.Repo.Disk.Path),
// option.WithDataURL(cfg.Repo.CS3.DataURL),
// option.WithDataPrefix(cfg.Repo.CS3.DataPrefix),
// option.WithJWTSecret(cfg.Repo.CS3.JWTSecret),
// option.WithProviderAddr(cfg.Repo.CS3.ProviderAddr),
// )
//
// err = sut.Init()
// assert.NoError(t, err)
//
// _, err = sut.Add("hijklmn-456", "mikey")
// assert.NoError(t, err)
//
// _, err = sut.Add("ewf4ofk-555", "jacky")
// assert.NoError(t, err)
//
// res, err := sut.Search("*y")
// assert.NoError(t, err)
// t.Log(res)
//
// _ = os.RemoveAll(dataDir)
//}

226
ocis-pkg/indexer/index/disk/autoincrement.go
View File

@@ -1,226 +0,0 @@
package disk
import (
"errors"
"os"
"path"
"path/filepath"
"strconv"
"strings"
idxerrs "github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/index"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/option"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/registry"
)
// Autoincrement are fields for an index of type autoincrement.
type Autoincrement struct {
indexBy string
typeName string
filesDir string
indexBaseDir string
indexRootDir string
bound *option.Bound
}
func init() {
registry.IndexConstructorRegistry["disk"]["autoincrement"] = NewAutoincrementIndex
}
// NewAutoincrementIndex instantiates a new AutoincrementIndex instance. Init() MUST be called upon instantiation.
func NewAutoincrementIndex(o ...option.Option) index.Index {
opts := &option.Options{}
for _, opt := range o {
opt(opts)
}
if opts.Entity == nil {
panic("invalid autoincrement index: configured without entity")
}
k, err := getKind(opts.Entity, opts.IndexBy)
if !isValidKind(k) || err != nil {
panic("invalid autoincrement index: configured on non-numeric field")
}
return &Autoincrement{
indexBy: opts.IndexBy,
typeName: opts.TypeName,
filesDir: opts.FilesDir,
bound: opts.Bound,
indexBaseDir: path.Join(opts.DataDir, "index.disk"),
indexRootDir: path.Join(path.Join(opts.DataDir, "index.disk"), strings.Join([]string{"autoincrement", opts.TypeName, opts.IndexBy}, ".")),
}
}
// Init initializes an autoincrement index.
func (idx *Autoincrement) Init() error {
if _, err := os.Stat(idx.filesDir); err != nil {
return err
}
if err := os.MkdirAll(idx.indexRootDir, 0777); err != nil {
return err
}
return nil
}
// Lookup exact lookup by value.
func (idx *Autoincrement) Lookup(v string) ([]string, error) {
searchPath := path.Join(idx.indexRootDir, v)
if err := isValidSymlink(searchPath); err != nil {
if os.IsNotExist(err) {
err = &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
}
return nil, err
}
p, err := os.Readlink(searchPath)
if err != nil {
return []string{}, nil
}
return []string{p}, err
}
// Add a new value to the index.
func (idx *Autoincrement) Add(id, v string) (string, error) {
nextID, err := idx.next()
if err != nil {
return "", err
}
oldName := filepath.Join(idx.filesDir, id)
var newName string
if v == "" {
newName = filepath.Join(idx.indexRootDir, strconv.Itoa(nextID))
} else {
newName = filepath.Join(idx.indexRootDir, v)
}
err = os.Symlink(oldName, newName)
if errors.Is(err, os.ErrExist) {
return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
}
return newName, err
}
// Remove a value v from an index.
func (idx *Autoincrement) Remove(id string, v string) error {
if v == "" {
return nil
}
searchPath := path.Join(idx.indexRootDir, v)
return os.Remove(searchPath)
}
// Update index from <oldV> to <newV>.
func (idx *Autoincrement) Update(id, oldV, newV string) error {
oldPath := path.Join(idx.indexRootDir, oldV)
if err := isValidSymlink(oldPath); err != nil {
if os.IsNotExist(err) {
return &idxerrs.NotFoundErr{TypeName: idx.TypeName(), Key: idx.IndexBy(), Value: oldV}
}
return err
}
newPath := path.Join(idx.indexRootDir, newV)
err := isValidSymlink(newPath)
if err == nil {
return &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: newV}
}
if os.IsNotExist(err) {
err = os.Rename(oldPath, newPath)
}
return err
}
// Search allows for glob search on the index.
func (idx *Autoincrement) Search(pattern string) ([]string, error) {
paths, err := filepath.Glob(path.Join(idx.indexRootDir, pattern))
if err != nil {
return nil, err
}
if len(paths) == 0 {
return nil, &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: pattern}
}
res := make([]string, 0)
for _, p := range paths {
if err := isValidSymlink(p); err != nil {
return nil, err
}
src, err := os.Readlink(p)
if err != nil {
return nil, err
}
res = append(res, src)
}
return res, nil
}
// CaseInsensitive undocumented.
func (idx *Autoincrement) CaseInsensitive() bool {
return false
}
// IndexBy undocumented.
func (idx *Autoincrement) IndexBy() string {
return idx.indexBy
}
// TypeName undocumented.
func (idx *Autoincrement) TypeName() string {
return idx.typeName
}
// FilesDir undocumented.
func (idx *Autoincrement) FilesDir() string {
return idx.filesDir
}
func (idx *Autoincrement) next() (int, error) {
files, err := readDir(idx.indexRootDir)
if err != nil {
return -1, err
}
if len(files) == 0 {
return int(idx.bound.Lower), nil
}
latest, err := lastValueFromTree(files)
if err != nil {
return -1, err
}
if int64(latest) < idx.bound.Lower {
return int(idx.bound.Lower), nil
}
return latest + 1, nil
}
// Delete deletes the index root folder from the configured storage.
func (idx *Autoincrement) Delete() error {
return os.RemoveAll(idx.indexRootDir)
}
func lastValueFromTree(files []os.FileInfo) (int, error) {
latest, err := strconv.Atoi(path.Base(files[len(files)-1].Name()))
if err != nil {
return -1, err
}
return latest, nil
}

299
ocis-pkg/indexer/index/disk/autoincrement_test.go
View File

@@ -1,299 +0,0 @@
package disk
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/option"
//. "github.com/owncloud/ocis/v2/ocis-pkg/indexer/test"
accountsmsg "github.com/owncloud/ocis/v2/protogen/gen/ocis/messages/accounts/v0"
"github.com/stretchr/testify/assert"
)
func TestIsValidKind(t *testing.T) {
scenarios := []struct {
panics bool
name string
indexBy string
entity struct {
Number int
Name string
NumberFloat float32
}
}{
{
name: "valid autoincrement index",
panics: false,
indexBy: "Number",
entity: struct {
Number int
Name string
NumberFloat float32
}{
Name: "tesy-mc-testace",
},
},
{
name: "create autoincrement index on a non-existing field",
panics: true,
indexBy: "Age",
entity: struct {
Number int
Name string
NumberFloat float32
}{
Name: "tesy-mc-testace",
},
},
{
name: "attempt to create an autoincrement index with no entity",
panics: true,
indexBy: "Age",
},
{
name: "create autoincrement index on a non-numeric field",
panics: true,
indexBy: "Name",
entity: struct {
Number int
Name string
NumberFloat float32
}{
Name: "tesy-mc-testace",
},
},
}
for _, scenario := range scenarios {
t.Run(scenario.name, func(t *testing.T) {
if scenario.panics {
assert.Panics(t, func() {
_ = NewAutoincrementIndex(
option.WithEntity(scenario.entity),
option.WithIndexBy(scenario.indexBy),
)
})
} else {
assert.NotPanics(t, func() {
_ = NewAutoincrementIndex(
option.WithEntity(scenario.entity),
option.WithIndexBy(scenario.indexBy),
)
})
}
})
}
}
func TestNext(t *testing.T) {
scenarios := []struct {
name string
expected int
indexBy string
entity interface{}
}{
{
name: "get next value",
expected: 0,
indexBy: "Number",
entity: struct {
Number int
Name string
NumberFloat float32
}{
Name: "tesy-mc-testace",
},
},
}
for _, scenario := range scenarios {
t.Run(scenario.name, func(t *testing.T) {
tmpDir, err := createTmpDirStr()
assert.NoError(t, err)
err = os.MkdirAll(filepath.Join(tmpDir, "data"), 0777)
assert.NoError(t, err)
i := NewAutoincrementIndex(
option.WithBounds(&option.Bound{
Lower: 0,
Upper: 0,
}),
option.WithDataDir(tmpDir),
option.WithFilesDir(filepath.Join(tmpDir, "data")),
option.WithEntity(scenario.entity),
option.WithTypeName("LambdaType"),
option.WithIndexBy(scenario.indexBy),
)
err = i.Init()
assert.NoError(t, err)
tmpFile, err := os.Create(filepath.Join(tmpDir, "data", "test-example"))
assert.NoError(t, err)
assert.NoError(t, tmpFile.Close())
oldName, err := i.Add("test-example", "")
assert.NoError(t, err)
assert.Equal(t, "0", filepath.Base(oldName))
oldName, err = i.Add("test-example", "")
assert.NoError(t, err)
assert.Equal(t, "1", filepath.Base(oldName))
oldName, err = i.Add("test-example", "")
assert.NoError(t, err)
assert.Equal(t, "2", filepath.Base(oldName))
t.Log(oldName)
_ = os.RemoveAll(tmpDir)
})
}
}
func TestLowerBound(t *testing.T) {
scenarios := []struct {
name string
expected int
indexBy string
entity interface{}
}{
{
name: "get next value with a lower bound specified",
expected: 0,
indexBy: "Number",
entity: struct {
Number int
Name string
NumberFloat float32
}{
Name: "tesy-mc-testace",
},
},
}
for _, scenario := range scenarios {
t.Run(scenario.name, func(t *testing.T) {
tmpDir, err := createTmpDirStr()
assert.NoError(t, err)
err = os.MkdirAll(filepath.Join(tmpDir, "data"), 0777)
assert.NoError(t, err)
i := NewAutoincrementIndex(
option.WithBounds(&option.Bound{
Lower: 1000,
}),
option.WithDataDir(tmpDir),
option.WithFilesDir(filepath.Join(tmpDir, "data")),
option.WithEntity(scenario.entity),
option.WithTypeName("LambdaType"),
option.WithIndexBy(scenario.indexBy),
)
err = i.Init()
assert.NoError(t, err)
tmpFile, err := os.Create(filepath.Join(tmpDir, "data", "test-example"))
assert.NoError(t, err)
assert.NoError(t, tmpFile.Close())
oldName, err := i.Add("test-example", "")
assert.NoError(t, err)
assert.Equal(t, "1000", filepath.Base(oldName))
oldName, err = i.Add("test-example", "")
assert.NoError(t, err)
assert.Equal(t, "1001", filepath.Base(oldName))
oldName, err = i.Add("test-example", "")
assert.NoError(t, err)
assert.Equal(t, "1002", filepath.Base(oldName))
t.Log(oldName)
_ = os.RemoveAll(tmpDir)
})
}
}
func TestAdd(t *testing.T) {
tmpDir, err := createTmpDirStr()
assert.NoError(t, err)
err = os.MkdirAll(filepath.Join(tmpDir, "data"), 0777)
assert.NoError(t, err)
tmpFile, err := os.Create(filepath.Join(tmpDir, "data", "test-example"))
assert.NoError(t, err)
assert.NoError(t, tmpFile.Close())
i := NewAutoincrementIndex(
option.WithBounds(&option.Bound{
Lower: 0,
Upper: 0,
}),
option.WithDataDir(tmpDir),
option.WithFilesDir(filepath.Join(tmpDir, "data")),
option.WithEntity(&accountsmsg.Account{}),
option.WithTypeName("owncloud.Account"),
option.WithIndexBy("UidNumber"),
)
err = i.Init()
assert.NoError(t, err)
_, err = i.Add("test-example", "")
if err != nil {
t.Error(err)
}
}
func BenchmarkAdd(b *testing.B) {
tmpDir, err := createTmpDirStr()
assert.NoError(b, err)
err = os.MkdirAll(filepath.Join(tmpDir, "data"), 0777)
assert.NoError(b, err)
tmpFile, err := os.Create(filepath.Join(tmpDir, "data", "test-example"))
assert.NoError(b, err)
assert.NoError(b, tmpFile.Close())
i := NewAutoincrementIndex(
option.WithBounds(&option.Bound{
Lower: 0,
Upper: 0,
}),
option.WithDataDir(tmpDir),
option.WithFilesDir(filepath.Join(tmpDir, "data")),
option.WithEntity(struct {
Number int
Name string
NumberFloat float32
}{}),
option.WithTypeName("LambdaType"),
option.WithIndexBy("Number"),
)
err = i.Init()
assert.NoError(b, err)
for n := 0; n < b.N; n++ {
_, err := i.Add("test-example", "")
if err != nil {
b.Error(err)
}
assert.NoError(b, err)
}
}
func createTmpDirStr() (string, error) {
name, err := ioutil.TempDir("/tmp", "testfiles-*")
if err != nil {
return "", err
}
return name, nil
}

52
ocis-pkg/indexer/index/disk/helper.go
View File

@@ -1,52 +0,0 @@
package disk
import (
"os"
"reflect"
"sort"
"strconv"
)
var (
validKinds = []reflect.Kind{
reflect.Int,
reflect.Int8,
reflect.Int16,
reflect.Int32,
reflect.Int64,
}
)
// verifies an autoincrement field kind on the target struct.
func isValidKind(k reflect.Kind) bool {
for _, v := range validKinds {
if k == v {
return true
}
}
return false
}
func getKind(i interface{}, field string) (reflect.Kind, error) {
r := reflect.ValueOf(i)
return reflect.Indirect(r).FieldByName(field).Kind(), nil
}
// readDir is an implementation of os.ReadDir but with different sorting.
func readDir(dirname string) ([]os.FileInfo, error) {
f, err := os.Open(dirname)
if err != nil {
return nil, err
}
list, err := f.Readdir(-1)
f.Close()
if err != nil {
return nil, err
}
sort.Slice(list, func(i, j int) bool {
a, _ := strconv.Atoi(list[i].Name())
b, _ := strconv.Atoi(list[j].Name())
return a < b
})
return list, nil
}

240
ocis-pkg/indexer/index/disk/non_unique.go
View File

@@ -1,240 +0,0 @@
package disk
import (
"errors"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
idxerrs "github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/index"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/option"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/registry"
)
// NonUnique is able to index an document by a key which might contain non-unique values
//
// /tmp/testfiles-395764020/index.disk/PetByColor/
// ├── Brown
// │ └── rebef-123 -> /tmp/testfiles-395764020/pets/rebef-123
// ├── Green
// │ ├── goefe-789 -> /tmp/testfiles-395764020/pets/goefe-789
// │ └── xadaf-189 -> /tmp/testfiles-395764020/pets/xadaf-189
// └── White
// └── wefwe-456 -> /tmp/testfiles-395764020/pets/wefwe-456
type NonUnique struct {
caseInsensitive bool
indexBy string
typeName string
filesDir string
indexBaseDir string
indexRootDir string
}
func init() {
registry.IndexConstructorRegistry["disk"]["non_unique"] = NewNonUniqueIndexWithOptions
}
// NewNonUniqueIndexWithOptions instantiates a new UniqueIndex instance. Init() should be
// called afterward to ensure correct on-disk structure.
func NewNonUniqueIndexWithOptions(o ...option.Option) index.Index {
opts := &option.Options{}
for _, opt := range o {
opt(opts)
}
return &NonUnique{
caseInsensitive: opts.CaseInsensitive,
indexBy: opts.IndexBy,
typeName: opts.TypeName,
filesDir: opts.FilesDir,
indexBaseDir: path.Join(opts.DataDir, "index.disk"),
indexRootDir: path.Join(path.Join(opts.DataDir, "index.disk"), strings.Join([]string{"non_unique", opts.TypeName, opts.IndexBy}, ".")),
}
}
// Init initializes a unique index.
func (idx *NonUnique) Init() error {
if _, err := os.Stat(idx.filesDir); err != nil {
return err
}
if err := os.MkdirAll(idx.indexRootDir, 0777); err != nil {
return err
}
return nil
}
// Lookup exact lookup by value.
func (idx *NonUnique) Lookup(v string) ([]string, error) {
if idx.caseInsensitive {
v = strings.ToLower(v)
}
searchPath := path.Join(idx.indexRootDir, v)
fi, err := ioutil.ReadDir(searchPath)
if os.IsNotExist(err) {
return []string{}, &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
}
if err != nil {
return []string{}, err
}
ids := make([]string, 0, len(fi))
for _, f := range fi {
ids = append(ids, f.Name())
}
if len(ids) == 0 {
return []string{}, &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
}
return ids, nil
}
// Add adds a value to the index, returns the path to the root-document
func (idx *NonUnique) Add(id, v string) (string, error) {
if v == "" {
return "", nil
}
if idx.caseInsensitive {
v = strings.ToLower(v)
}
oldName := path.Join(idx.filesDir, id)
newName := path.Join(idx.indexRootDir, v, id)
if err := os.MkdirAll(path.Join(idx.indexRootDir, v), 0777); err != nil {
return "", err
}
err := os.Symlink(oldName, newName)
if errors.Is(err, os.ErrExist) {
return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
}
return newName, err
}
// Remove a value v from an index.
func (idx *NonUnique) Remove(id string, v string) error {
if v == "" {
return nil
}
if idx.caseInsensitive {
v = strings.ToLower(v)
}
res, err := filepath.Glob(path.Join(idx.indexRootDir, "/*/", id))
if err != nil {
return err
}
for _, p := range res {
if err := os.Remove(p); err != nil {
return err
}
}
// Remove value directory if it is empty
valueDir := path.Join(idx.indexRootDir, v)
fi, err := ioutil.ReadDir(valueDir)
if err != nil {
return err
}
if len(fi) == 0 {
if err := os.RemoveAll(valueDir); err != nil {
return err
}
}
return nil
}
// Update index from <oldV> to <newV>.
func (idx *NonUnique) Update(id, oldV, newV string) (err error) {
if idx.caseInsensitive {
oldV = strings.ToLower(oldV)
newV = strings.ToLower(newV)
}
oldDir := path.Join(idx.indexRootDir, oldV)
oldPath := path.Join(oldDir, id)
newDir := path.Join(idx.indexRootDir, newV)
newPath := path.Join(newDir, id)
if _, err = os.Stat(oldPath); os.IsNotExist(err) {
return &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: oldV}
}
if err != nil {
return
}
if err = os.MkdirAll(newDir, 0777); err != nil {
return
}
if err = os.Rename(oldPath, newPath); err != nil {
return
}
di, err := ioutil.ReadDir(oldDir)
if err != nil {
return err
}
if len(di) == 0 {
err = os.RemoveAll(oldDir)
if err != nil {
return
}
}
return
}
// Search allows for glob search on the index.
func (idx *NonUnique) Search(pattern string) ([]string, error) {
if idx.caseInsensitive {
pattern = strings.ToLower(pattern)
}
paths, err := filepath.Glob(path.Join(idx.indexRootDir, pattern, "*"))
if err != nil {
return nil, err
}
if len(paths) == 0 {
return nil, &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: pattern}
}
return paths, nil
}
// CaseInsensitive undocumented.
func (idx *NonUnique) CaseInsensitive() bool {
return idx.caseInsensitive
}
// IndexBy undocumented.
func (idx *NonUnique) IndexBy() string {
return idx.indexBy
}
// TypeName undocumented.
func (idx *NonUnique) TypeName() string {
return idx.typeName
}
// FilesDir undocumented.
func (idx *NonUnique) FilesDir() string {
return idx.filesDir
}
// Delete deletes the index folder from its storage.
func (idx *NonUnique) Delete() error {
return os.RemoveAll(idx.indexRootDir)
}

114
ocis-pkg/indexer/index/disk/non_unique_test.go
View File

@@ -1,114 +0,0 @@
package disk
import (
"fmt"
"os"
"path"
"testing"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/config"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/index"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/option"
. "github.com/owncloud/ocis/v2/ocis-pkg/indexer/test"
"github.com/stretchr/testify/assert"
)
func TestNonUniqueIndexAdd(t *testing.T) {
sut, dataPath := getNonUniqueIdxSut(t, Pet{}, "Color")
ids, err := sut.Lookup("Green")
assert.NoError(t, err)
assert.EqualValues(t, []string{"goefe-789", "xadaf-189"}, ids)
ids, err = sut.Lookup("White")
assert.NoError(t, err)
assert.EqualValues(t, []string{"wefwe-456"}, ids)
ids, err = sut.Lookup("Cyan")
assert.Error(t, err)
assert.EqualValues(t, []string{}, ids)
_ = os.RemoveAll(dataPath)
}
func TestNonUniqueIndexUpdate(t *testing.T) {
sut, dataPath := getNonUniqueIdxSut(t, Pet{}, "Color")
err := sut.Update("goefe-789", "Green", "Black")
assert.NoError(t, err)
err = sut.Update("xadaf-189", "Green", "Black")
assert.NoError(t, err)
assert.DirExists(t, path.Join(dataPath, fmt.Sprintf("index.disk/non_unique.%v.Color/Black", GetTypeFQN(Pet{}))))
assert.NoDirExists(t, path.Join(dataPath, fmt.Sprintf("index.disk/non_unique.%v.Color/Green", GetTypeFQN(Pet{}))))
_ = os.RemoveAll(dataPath)
}
func TestNonUniqueIndexDelete(t *testing.T) {
sut, dataPath := getNonUniqueIdxSut(t, Pet{}, "Color")
assert.FileExists(t, path.Join(dataPath, fmt.Sprintf("index.disk/non_unique.%v.Color/Green/goefe-789", GetTypeFQN(Pet{}))))
err := sut.Remove("goefe-789", "Green")
assert.NoError(t, err)
assert.NoFileExists(t, path.Join(dataPath, fmt.Sprintf("index.disk/non_unique.%v.Color/Green/goefe-789", GetTypeFQN(Pet{}))))
assert.FileExists(t, path.Join(dataPath, fmt.Sprintf("index.disk/non_unique.%v.Color/Green/xadaf-189", GetTypeFQN(Pet{}))))
_ = os.RemoveAll(dataPath)
}
func TestNonUniqueIndexSearch(t *testing.T) {
sut, dataPath := getNonUniqueIdxSut(t, Pet{}, "Email")
res, err := sut.Search("Gr*")
assert.NoError(t, err)
assert.Len(t, res, 2)
assert.Equal(t, "goefe-789", path.Base(res[0]))
assert.Equal(t, "xadaf-189", path.Base(res[1]))
_, err = sut.Search("does-not-exist@example.com")
assert.Error(t, err)
assert.IsType(t, &errors.NotFoundErr{}, err)
_ = os.RemoveAll(dataPath)
}
// entity: used to get the fully qualified name for the index root path.
func getNonUniqueIdxSut(t *testing.T, entity interface{}, indexBy string) (index.Index, string) {
dataPath, _ := WriteIndexTestData(Data, "ID", "")
cfg := config.Config{
Repo: config.Repo{
Backend: "disk",
Disk: config.Disk{
Path: dataPath,
},
},
}
sut := NewNonUniqueIndexWithOptions(
option.WithTypeName(GetTypeFQN(entity)),
option.WithIndexBy(indexBy),
option.WithFilesDir(path.Join(cfg.Repo.Disk.Path, "pets")),
option.WithDataDir(cfg.Repo.Disk.Path),
)
err := sut.Init()
if err != nil {
t.Fatal(err)
}
for _, u := range Data["pets"] {
pkVal := ValueOf(u, "ID")
idxByVal := ValueOf(u, "Color")
_, err := sut.Add(pkVal, idxByVal)
if err != nil {
t.Fatal(err)
}
}
return sut, dataPath
}

227
ocis-pkg/indexer/index/disk/unique.go
View File

@@ -1,227 +0,0 @@
package disk
import (
"errors"
"fmt"
"os"
"path"
"path/filepath"
"strings"
idxerrs "github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/index"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/option"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/registry"
)
// Unique ensures that only one document of the same type and key-value combination can exist in the index.
//
// Modeled by creating a indexer-folder per entity and key with symlinks which point to respective documents which contain
// the link-filename as value.
//
// Directory Layout
//
// /var/data/index.disk/UniqueUserByEmail/
// ├── jacky@example.com -> /var/data/users/ewf4ofk-555
// ├── jones@example.com -> /var/data/users/rulan54-777
// └── mikey@example.com -> /var/data/users/abcdefg-123
//
// Example user
//
// {
// "Id": "ewf4ofk-555",
// "UserName": "jacky",
// "Email": "jacky@example.com"
// }
//
type Unique struct {
caseInsensitive bool
indexBy string
typeName string
filesDir string
indexBaseDir string
indexRootDir string
}
func init() {
registry.IndexConstructorRegistry["disk"]["unique"] = NewUniqueIndexWithOptions
}
// NewUniqueIndexWithOptions instantiates a new UniqueIndex instance. Init() should be
// called afterward to ensure correct on-disk structure.
func NewUniqueIndexWithOptions(o ...option.Option) index.Index {
opts := &option.Options{}
for _, opt := range o {
opt(opts)
}
return &Unique{
caseInsensitive: opts.CaseInsensitive,
indexBy: opts.IndexBy,
typeName: opts.TypeName,
filesDir: opts.FilesDir,
indexBaseDir: path.Join(opts.DataDir, "index.disk"),
indexRootDir: path.Join(path.Join(opts.DataDir, "index.disk"), strings.Join([]string{"unique", opts.TypeName, opts.IndexBy}, ".")),
}
}
// Init initializes a unique index.
func (idx *Unique) Init() error {
if _, err := os.Stat(idx.filesDir); err != nil {
return err
}
if err := os.MkdirAll(idx.indexRootDir, 0777); err != nil {
return err
}
return nil
}
// Add adds a value to the index, returns the path to the root-document
func (idx *Unique) Add(id, v string) (string, error) {
if v == "" {
return "", nil
}
if idx.caseInsensitive {
v = strings.ToLower(v)
}
oldName := path.Join(idx.filesDir, id)
newName := path.Join(idx.indexRootDir, v)
err := os.Symlink(oldName, newName)
if errors.Is(err, os.ErrExist) {
return "", &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
}
return newName, err
}
// Remove a value v from an index.
func (idx *Unique) Remove(id string, v string) (err error) {
if v == "" {
return nil
}
if idx.caseInsensitive {
v = strings.ToLower(v)
}
searchPath := path.Join(idx.indexRootDir, v)
return os.Remove(searchPath)
}
// Lookup exact lookup by value.
func (idx *Unique) Lookup(v string) (resultPath []string, err error) {
if idx.caseInsensitive {
v = strings.ToLower(v)
}
searchPath := path.Join(idx.indexRootDir, v)
if err = isValidSymlink(searchPath); err != nil {
if os.IsNotExist(err) {
err = &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: v}
}
return
}
p, err := os.Readlink(searchPath)
if err != nil {
return []string{}, nil
}
return []string{p}, err
}
// Update index from <oldV> to <newV>.
func (idx *Unique) Update(id, oldV, newV string) (err error) {
if idx.caseInsensitive {
oldV = strings.ToLower(oldV)
newV = strings.ToLower(newV)
}
oldPath := path.Join(idx.indexRootDir, oldV)
if err = isValidSymlink(oldPath); err != nil {
if os.IsNotExist(err) {
return &idxerrs.NotFoundErr{TypeName: idx.TypeName(), Key: idx.IndexBy(), Value: oldV}
}
return
}
newPath := path.Join(idx.indexRootDir, newV)
if err = isValidSymlink(newPath); err == nil {
return &idxerrs.AlreadyExistsErr{TypeName: idx.typeName, Key: idx.indexBy, Value: newV}
}
if os.IsNotExist(err) {
err = os.Rename(oldPath, newPath)
}
return
}
// Search allows for glob search on the index.
func (idx *Unique) Search(pattern string) ([]string, error) {
if idx.caseInsensitive {
pattern = strings.ToLower(pattern)
}
paths, err := filepath.Glob(path.Join(idx.indexRootDir, pattern))
if err != nil {
return nil, err
}
if len(paths) == 0 {
return nil, &idxerrs.NotFoundErr{TypeName: idx.typeName, Key: idx.indexBy, Value: pattern}
}
res := make([]string, 0)
for _, p := range paths {
if err := isValidSymlink(p); err != nil {
return nil, err
}
src, err := os.Readlink(p)
if err != nil {
return nil, err
}
res = append(res, src)
}
return res, nil
}
// CaseInsensitive undocumented.
func (idx *Unique) CaseInsensitive() bool {
return idx.caseInsensitive
}
// IndexBy undocumented.
func (idx *Unique) IndexBy() string {
return idx.indexBy
}
// TypeName undocumented.
func (idx *Unique) TypeName() string {
return idx.typeName
}
// FilesDir undocumented.
func (idx *Unique) FilesDir() string {
return idx.filesDir
}
func isValidSymlink(path string) (err error) {
var symInfo os.FileInfo
if symInfo, err = os.Lstat(path); err != nil {
return
}
if symInfo.Mode()&os.ModeSymlink == 0 {
err = fmt.Errorf("%s is not a valid symlink (bug/corruption?)", path)
return
}
return
}
// Delete deletes the index folder from its storage.
func (idx *Unique) Delete() error {
return os.RemoveAll(idx.indexRootDir)
}

134
ocis-pkg/indexer/index/disk/unique_test.go
View File

@@ -1,134 +0,0 @@
package disk
import (
"os"
"path"
"testing"
"github.com/owncloud/ocis/v2/extensions/accounts/pkg/config"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/index"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/option"
. "github.com/owncloud/ocis/v2/ocis-pkg/indexer/test"
"github.com/stretchr/testify/assert"
)
func TestUniqueLookupSingleEntry(t *testing.T) {
uniq, dataDir := getUniqueIdxSut(t, "Email", User{})
filesDir := path.Join(dataDir, "users")
t.Log("existing lookup")
resultPath, err := uniq.Lookup("mikey@example.com")
assert.NoError(t, err)
assert.Equal(t, []string{path.Join(filesDir, "abcdefg-123")}, resultPath)
t.Log("non-existing lookup")
resultPath, err = uniq.Lookup("doesnotExists@example.com")
assert.Error(t, err)
assert.IsType(t, &errors.NotFoundErr{}, err)
assert.Empty(t, resultPath)
_ = os.RemoveAll(dataDir)
}
func TestUniqueUniqueConstraint(t *testing.T) {
uniq, dataDir := getUniqueIdxSut(t, "Email", User{})
_, err := uniq.Add("abcdefg-123", "mikey@example.com")
assert.Error(t, err)
assert.IsType(t, &errors.AlreadyExistsErr{}, err)
_ = os.RemoveAll(dataDir)
}
func TestUniqueRemove(t *testing.T) {
uniq, dataDir := getUniqueIdxSut(t, "Email", User{})
err := uniq.Remove("", "mikey@example.com")
assert.NoError(t, err)
_, err = uniq.Lookup("mikey@example.com")
assert.Error(t, err)
assert.IsType(t, &errors.NotFoundErr{}, err)
_ = os.RemoveAll(dataDir)
}
func TestUniqueUpdate(t *testing.T) {
uniq, dataDir := getUniqueIdxSut(t, "Email", User{})
t.Log("successful update")
err := uniq.Update("", "mikey@example.com", "mikey2@example.com")
assert.NoError(t, err)
t.Log("failed update because already exists")
err = uniq.Update("", "mikey2@example.com", "mikey2@example.com")
assert.Error(t, err)
assert.IsType(t, &errors.AlreadyExistsErr{}, err)
t.Log("failed update because not found")
err = uniq.Update("", "nonexisting@example.com", "something2@example.com")
assert.Error(t, err)
assert.IsType(t, &errors.NotFoundErr{}, err)
_ = os.RemoveAll(dataDir)
}
func TestUniqueIndexSearch(t *testing.T) {
sut, dataDir := getUniqueIdxSut(t, "Email", User{})
res, err := sut.Search("j*@example.com")
assert.NoError(t, err)
assert.Len(t, res, 2)
assert.Equal(t, "ewf4ofk-555", path.Base(res[0]))
assert.Equal(t, "rulan54-777", path.Base(res[1]))
_, err = sut.Search("does-not-exist@example.com")
assert.Error(t, err)
assert.IsType(t, &errors.NotFoundErr{}, err)
_ = os.RemoveAll(dataDir)
}
func TestErrors(t *testing.T) {
assert.True(t, errors.IsAlreadyExistsErr(&errors.AlreadyExistsErr{}))
assert.True(t, errors.IsNotFoundErr(&errors.NotFoundErr{}))
}
func getUniqueIdxSut(t *testing.T, indexBy string, entityType interface{}) (index.Index, string) {
dataPath, _ := WriteIndexTestData(Data, "ID", "")
cfg := config.Config{
Repo: config.Repo{
Backend: "disk",
Disk: config.Disk{
Path: dataPath,
},
},
}
sut := NewUniqueIndexWithOptions(
option.WithTypeName(GetTypeFQN(entityType)),
option.WithIndexBy(indexBy),
option.WithFilesDir(path.Join(cfg.Repo.Disk.Path, "users")),
option.WithDataDir(cfg.Repo.Disk.Path),
)
err := sut.Init()
if err != nil {
t.Fatal(err)
}
for _, u := range Data["users"] {
pkVal := ValueOf(u, "ID")
idxByVal := ValueOf(u, "Email")
_, err := sut.Add(pkVal, idxByVal)
if err != nil {
t.Fatal(err)
}
}
return sut, dataPath
}

17
ocis-pkg/indexer/index/index.go
View File

@@ -1,17 +0,0 @@
package index
// Index can be implemented to create new indexer-strategies. See Unique for example.
// Each indexer implementation is bound to one data-column (IndexBy) and a data-type (TypeName)
type Index interface {
Init() error
Lookup(v string) ([]string, error)
Add(id, v string) (string, error)
Remove(id string, v string) error
Update(id, oldV, newV string) error
Search(pattern string) ([]string, error)
CaseInsensitive() bool
IndexBy() string
TypeName() string
FilesDir() string
Delete() error // Delete deletes the index folder from its storage.
}

403
ocis-pkg/indexer/indexer.go
View File

@@ -1,403 +0,0 @@
// Package indexer provides symlink-based indexer for on-disk document-directories.
package indexer
import (
"context"
"fmt"
"path"
"strings"
"github.com/owncloud/ocis/v2/ocis-pkg/sync"
"github.com/CiscoM31/godata"
"github.com/iancoleman/strcase"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/config"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/errors"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/index"
_ "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index/cs3" // to populate index
_ "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index/disk" // to populate index
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/option"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/registry"
)
// Indexer is a facade to configure and query over multiple indices.
type Indexer struct {
config *config.Config
indices typeMap
mu sync.NamedRWMutex
}
// IdxAddResult represents the result of an Add call on an index
type IdxAddResult struct {
Field, Value string
}
// CreateIndexer creates a new Indexer.
func CreateIndexer(cfg *config.Config) *Indexer {
return &Indexer{
config: cfg,
indices: typeMap{},
mu: sync.NewNamedRWMutex(),
}
}
// Reset takes care of deleting all indices from storage and from the internal map of indices
func (i *Indexer) Reset() error {
for j := range i.indices {
for _, indices := range i.indices[j].IndicesByField {
for _, idx := range indices {
err := idx.Delete()
if err != nil {
return err
}
}
}
delete(i.indices, j)
}
return nil
}
// AddIndex adds a new index to the indexer receiver.
func (i *Indexer) AddIndex(t interface{}, indexBy, pkName, entityDirName, indexType string, bound *option.Bound, caseInsensitive bool) error {
f := registry.IndexConstructorRegistry[i.config.Repo.Backend][indexType]
var idx index.Index
if i.config.Repo.Backend == "cs3" {
idx = f(
option.CaseInsensitive(caseInsensitive),
option.WithEntity(t),
option.WithBounds(bound),
option.WithTypeName(getTypeFQN(t)),
option.WithIndexBy(indexBy),
option.WithDataURL(i.config.Repo.CS3.DataURL),
option.WithDataPrefix(i.config.Repo.CS3.DataPrefix),
option.WithJWTSecret(i.config.Repo.CS3.JWTSecret),
option.WithProviderAddr(i.config.Repo.CS3.ProviderAddr),
option.WithServiceUser(i.config.ServiceUser),
)
} else {
idx = f(
option.CaseInsensitive(caseInsensitive),
option.WithEntity(t),
option.WithBounds(bound),
option.WithTypeName(getTypeFQN(t)),
option.WithIndexBy(indexBy),
option.WithFilesDir(path.Join(i.config.Repo.Disk.Path, entityDirName)),
option.WithDataDir(i.config.Repo.Disk.Path),
)
}
i.indices.addIndex(getTypeFQN(t), pkName, idx)
return idx.Init()
}
// Add a new entry to the indexer
func (i *Indexer) Add(t interface{}) ([]IdxAddResult, error) {
typeName := getTypeFQN(t)
i.mu.Lock(typeName)
defer i.mu.Unlock(typeName)
var results []IdxAddResult
if fields, ok := i.indices[typeName]; ok {
for _, indices := range fields.IndicesByField {
for _, idx := range indices {
pkVal := valueOf(t, fields.PKFieldName)
idxByVal := valueOf(t, idx.IndexBy())
value, err := idx.Add(pkVal, idxByVal)
if err != nil {
return []IdxAddResult{}, err
}
if value == "" {
continue
}
results = append(results, IdxAddResult{Field: idx.IndexBy(), Value: value})
}
}
}
return results, nil
}
// FindBy finds a value on an index by field and value.
func (i *Indexer) FindBy(t interface{}, field string, val string) ([]string, error) {
typeName := getTypeFQN(t)
i.mu.RLock(typeName)
defer i.mu.RUnlock(typeName)
resultPaths := make([]string, 0)
if fields, ok := i.indices[typeName]; ok {
for _, idx := range fields.IndicesByField[strcase.ToCamel(field)] {
idxVal := val
res, err := idx.Lookup(idxVal)
if err != nil {
if errors.IsNotFoundErr(err) {
continue
}
if err != nil {
return nil, err
}
}
resultPaths = append(resultPaths, res...)
}
}
result := make([]string, 0, len(resultPaths))
for _, v := range resultPaths {
result = append(result, path.Base(v))
}
return result, nil
}
// Delete deletes all indexed fields of a given type t on the Indexer.
func (i *Indexer) Delete(t interface{}) error {
typeName := getTypeFQN(t)
i.mu.Lock(typeName)
defer i.mu.Unlock(typeName)
if fields, ok := i.indices[typeName]; ok {
for _, indices := range fields.IndicesByField {
for _, idx := range indices {
pkVal := valueOf(t, fields.PKFieldName)
idxByVal := valueOf(t, idx.IndexBy())
if err := idx.Remove(pkVal, idxByVal); err != nil {
return err
}
}
}
}
return nil
}
// FindByPartial allows for glob search across all indexes.
func (i *Indexer) FindByPartial(t interface{}, field string, pattern string) ([]string, error) {
typeName := getTypeFQN(t)
i.mu.RLock(typeName)
defer i.mu.RUnlock(typeName)
resultPaths := make([]string, 0)
if fields, ok := i.indices[typeName]; ok {
for _, idx := range fields.IndicesByField[strcase.ToCamel(field)] {
res, err := idx.Search(pattern)
if err != nil {
if errors.IsNotFoundErr(err) {
continue
}
if err != nil {
return nil, err
}
}
resultPaths = append(resultPaths, res...)
}
}
result := make([]string, 0, len(resultPaths))
for _, v := range resultPaths {
result = append(result, path.Base(v))
}
return result, nil
}
// Update updates all indexes on a value <from> to a value <to>.
func (i *Indexer) Update(from, to interface{}) error {
typeNameFrom := getTypeFQN(from)
i.mu.Lock(typeNameFrom)
defer i.mu.Unlock(typeNameFrom)
if typeNameTo := getTypeFQN(to); typeNameFrom != typeNameTo {
return fmt.Errorf("update types do not match: from %v to %v", typeNameFrom, typeNameTo)
}
if fields, ok := i.indices[typeNameFrom]; ok {
for fName, indices := range fields.IndicesByField {
oldV := valueOf(from, fName)
newV := valueOf(to, fName)
pkVal := valueOf(from, fields.PKFieldName)
for _, idx := range indices {
if oldV == newV {
continue
}
if oldV == "" {
if _, err := idx.Add(pkVal, newV); err != nil {
return err
}
continue
}
if newV == "" {
if err := idx.Remove(pkVal, oldV); err != nil {
return err
}
continue
}
if err := idx.Update(pkVal, oldV, newV); err != nil {
return err
}
}
}
}
return nil
}
// Query parses an OData query into something our indexer.Index understands and resolves it.
func (i *Indexer) Query(ctx context.Context, t interface{}, q string) ([]string, error) {
query, err := godata.ParseFilterString(ctx, q)
if err != nil {
return nil, err
}
tree := newQueryTree()
if err := buildTreeFromOdataQuery(query.Tree, &tree); err != nil {
return nil, err
}
results := make([]string, 0)
if err := i.resolveTree(t, &tree, &results); err != nil {
return nil, err
}
return results, nil
}
// t is used to infer the indexed field names. When building an index search query, field names have to respect Golang
// conventions and be in PascalCase. For a better overview on this contemplate reading the reflection package under the
// indexer directory. Traversal of the tree happens in a pre-order fashion.
// TODO implement logic for `and` operators.
func (i *Indexer) resolveTree(t interface{}, tree *queryTree, partials *[]string) error {
if partials == nil {
return fmt.Errorf("return value cannot be nil: partials")
}
if tree.left != nil {
_ = i.resolveTree(t, tree.left, partials)
}
if tree.right != nil {
_ = i.resolveTree(t, tree.right, partials)
}
// by the time we're here we reached a leaf node.
if tree.token != nil {
switch tree.token.filterType {
case "FindBy":
operand, err := sanitizeInput(tree.token.operands)
if err != nil {
return err
}
r, err := i.FindBy(t, operand.field, operand.value)
if err != nil {
return err
}
*partials = append(*partials, r...)
case "FindByPartial":
operand, err := sanitizeInput(tree.token.operands)
if err != nil {
return err
}
r, err := i.FindByPartial(t, operand.field, fmt.Sprintf("%v*", operand.value))
if err != nil {
return err
}
*partials = append(*partials, r...)
default:
return fmt.Errorf("unsupported filter: %v", tree.token.filterType)
}
}
*partials = dedup(*partials)
return nil
}
type indexerTuple struct {
field, value string
}
// sanitizeInput returns a tuple of fieldName + value to be applied on indexer.Index filters.
func sanitizeInput(operands []string) (*indexerTuple, error) {
if len(operands) != 2 {
return nil, fmt.Errorf("invalid number of operands for filter function: got %v expected 2", len(operands))
}
// field names are Go public types and by design they are in PascalCase, therefore we need to adhere to this rules.
// for further information on this have a look at the reflection package.
f := strcase.ToCamel(operands[0])
// remove single quotes from value.
v := strings.ReplaceAll(operands[1], "'", "")
return &indexerTuple{
field: f,
value: v,
}, nil
}
// buildTreeFromOdataQuery builds an indexer.queryTree out of a GOData ParseNode. The purpose of this intermediate tree
// is to transform godata operators and functions into supported operations on our index. At the time of this writing
// we only support `FindBy` and `FindByPartial` queries as these are the only implemented filters on indexer.Index(es).
func buildTreeFromOdataQuery(root *godata.ParseNode, tree *queryTree) error {
if root.Token.Type == godata.ExpressionTokenFunc { // i.e "startswith", "contains"
switch root.Token.Value {
case "startswith":
token := token{
operator: root.Token.Value,
filterType: "FindByPartial",
// TODO sanitize the number of operands it the expected one.
operands: []string{
root.Children[0].Token.Value, // field name, i.e: Name
root.Children[1].Token.Value, // field value, i.e: Jac
},
}
tree.insert(&token)
default:
return fmt.Errorf("operation not supported")
}
}
if root.Token.Type == godata.ExpressionTokenLogical {
switch root.Token.Value {
case "or":
tree.insert(&token{operator: root.Token.Value})
for _, child := range root.Children {
if err := buildTreeFromOdataQuery(child, tree.left); err != nil {
return err
}
}
case "eq":
tree.insert(&token{
operator: root.Token.Value,
filterType: "FindBy",
operands: []string{
root.Children[0].Token.Value,
root.Children[1].Token.Value,
},
})
for _, child := range root.Children {
if err := buildTreeFromOdataQuery(child, tree.left); err != nil {
return err
}
}
default:
return fmt.Errorf("operator not supported")
}
}
return nil
}

311
ocis-pkg/indexer/indexer_test.go
View File

@@ -1,311 +0,0 @@
package indexer
import (
"context"
"os"
"path"
"testing"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/option"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/config"
_ "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index/cs3"
_ "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index/disk"
. "github.com/owncloud/ocis/v2/ocis-pkg/indexer/test"
"github.com/stretchr/testify/assert"
)
//const cs3RootFolder = "/tmp/ocis/storage/users/data"
//
//func TestIndexer_CS3_AddWithUniqueIndex(t *testing.T) {
// dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder)
// assert.NoError(t, err)
// indexer := createCs3Indexer()
//
// err = indexer.AddIndex(&User{}, "UserName", "ID", "users", "unique", nil, false)
// assert.NoError(t, err)
//
// u := &User{ID: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"}
// _, err = indexer.Add(u)
// assert.NoError(t, err)
//
// _ = os.RemoveAll(dataDir)
//}
//
//func TestIndexer_CS3_AddWithNonUniqueIndex(t *testing.T) {
// dataDir, err := WriteIndexTestData(Data, "ID", cs3RootFolder)
// assert.NoError(t, err)
// indexer := createCs3Indexer()
//
// err = indexer.AddIndex(&User{}, "UserName", "ID", "users", "non_unique", nil, false)
// assert.NoError(t, err)
//
// u := &User{ID: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"}
// _, err = indexer.Add(u)
// assert.NoError(t, err)
//
// _ = os.RemoveAll(dataDir)
//}
func TestIndexer_Disk_FindByWithUniqueIndex(t *testing.T) {
dataDir, err := WriteIndexTestData(Data, "ID", "")
assert.NoError(t, err)
indexer := createDiskIndexer(dataDir)
err = indexer.AddIndex(&User{}, "UserName", "ID", "users", "unique", nil, false)
assert.NoError(t, err)
u := &User{ID: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"}
_, err = indexer.Add(u)
assert.NoError(t, err)
res, err := indexer.FindBy(User{}, "UserName", "mikey")
assert.NoError(t, err)
t.Log(res)
_ = os.RemoveAll(dataDir)
}
func TestIndexer_Disk_AddWithUniqueIndex(t *testing.T) {
dataDir, err := WriteIndexTestData(Data, "ID", "")
assert.NoError(t, err)
indexer := createDiskIndexer(dataDir)
err = indexer.AddIndex(&User{}, "UserName", "ID", "users", "unique", nil, false)
assert.NoError(t, err)
u := &User{ID: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"}
_, err = indexer.Add(u)
assert.NoError(t, err)
_ = os.RemoveAll(dataDir)
}
func TestIndexer_Disk_AddWithNonUniqueIndex(t *testing.T) {
dataDir, err := WriteIndexTestData(Data, "ID", "")
assert.NoError(t, err)
indexer := createDiskIndexer(dataDir)
err = indexer.AddIndex(&Pet{}, "Kind", "ID", "pets", "non_unique", nil, false)
assert.NoError(t, err)
pet1 := Pet{ID: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"}
pet2 := Pet{ID: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"}
_, err = indexer.Add(pet1)
assert.NoError(t, err)
_, err = indexer.Add(pet2)
assert.NoError(t, err)
res, err := indexer.FindBy(Pet{}, "Kind", "Hog")
assert.NoError(t, err)
t.Log(res)
_ = os.RemoveAll(dataDir)
}
func TestIndexer_Disk_AddWithAutoincrementIndex(t *testing.T) {
dataDir, err := WriteIndexTestData(Data, "ID", "")
assert.NoError(t, err)
indexer := createDiskIndexer(dataDir)
err = indexer.AddIndex(&User{}, "UID", "ID", "users", "autoincrement", &option.Bound{Lower: 5}, false)
assert.NoError(t, err)
res1, err := indexer.Add(Data["users"][0])
assert.NoError(t, err)
assert.Equal(t, "UID", res1[0].Field)
assert.Equal(t, "5", path.Base(res1[0].Value))
res2, err := indexer.Add(Data["users"][1])
assert.NoError(t, err)
assert.Equal(t, "UID", res2[0].Field)
assert.Equal(t, "6", path.Base(res2[0].Value))
resFindBy, err := indexer.FindBy(User{}, "UID", "6")
assert.NoError(t, err)
assert.Equal(t, "hijklmn-456", resFindBy[0])
t.Log(resFindBy)
_ = os.RemoveAll(dataDir)
}
func TestIndexer_Disk_DeleteWithNonUniqueIndex(t *testing.T) {
dataDir, err := WriteIndexTestData(Data, "ID", "")
assert.NoError(t, err)
indexer := createDiskIndexer(dataDir)
err = indexer.AddIndex(&Pet{}, "Kind", "ID", "pets", "non_unique", nil, false)
assert.NoError(t, err)
pet1 := Pet{ID: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"}
pet2 := Pet{ID: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"}
_, err = indexer.Add(pet1)
assert.NoError(t, err)
_, err = indexer.Add(pet2)
assert.NoError(t, err)
err = indexer.Delete(pet2)
assert.NoError(t, err)
_ = os.RemoveAll(dataDir)
}
func TestIndexer_Disk_SearchWithNonUniqueIndex(t *testing.T) {
dataDir, err := WriteIndexTestData(Data, "ID", "")
assert.NoError(t, err)
indexer := createDiskIndexer(dataDir)
err = indexer.AddIndex(&Pet{}, "Name", "ID", "pets", "non_unique", nil, false)
assert.NoError(t, err)
pet1 := Pet{ID: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"}
pet2 := Pet{ID: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"}
_, err = indexer.Add(pet1)
assert.NoError(t, err)
_, err = indexer.Add(pet2)
assert.NoError(t, err)
res, err := indexer.FindByPartial(pet2, "Name", "*ky")
assert.NoError(t, err)
t.Log(res)
_ = os.RemoveAll(dataDir)
}
func TestIndexer_Disk_UpdateWithUniqueIndex(t *testing.T) {
dataDir, err := WriteIndexTestData(Data, "ID", "")
assert.NoError(t, err)
indexer := createDiskIndexer(dataDir)
err = indexer.AddIndex(&User{}, "UserName", "ID", "users", "unique", nil, false)
assert.NoError(t, err)
err = indexer.AddIndex(&User{}, "Email", "ID", "users", "unique", nil, false)
assert.NoError(t, err)
user1 := &User{ID: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"}
user2 := &User{ID: "hijklmn-456", UserName: "frank", Email: "frank@example.com"}
_, err = indexer.Add(user1)
assert.NoError(t, err)
_, err = indexer.Add(user2)
assert.NoError(t, err)
err = indexer.Update(user1, &User{
ID: "abcdefg-123",
UserName: "mikey-new",
Email: "mikey@example.com",
})
assert.NoError(t, err)
v, err1 := indexer.FindBy(&User{}, "UserName", "mikey-new")
assert.NoError(t, err1)
assert.Len(t, v, 1)
v, err2 := indexer.FindBy(&User{}, "UserName", "mikey")
assert.NoError(t, err2)
assert.Len(t, v, 0)
err1 = indexer.Update(&User{
ID: "abcdefg-123",
UserName: "mikey-new",
Email: "mikey@example.com",
}, &User{
ID: "abcdefg-123",
UserName: "mikey-newest",
Email: "mikey-new@example.com",
})
assert.NoError(t, err1)
fbUserName, err2 := indexer.FindBy(&User{}, "UserName", "mikey-newest")
assert.NoError(t, err2)
assert.Len(t, fbUserName, 1)
fbEmail, err3 := indexer.FindBy(&User{}, "Email", "mikey-new@example.com")
assert.NoError(t, err3)
assert.Len(t, fbEmail, 1)
_ = os.RemoveAll(dataDir)
}
func TestIndexer_Disk_UpdateWithNonUniqueIndex(t *testing.T) {
dataDir, err := WriteIndexTestData(Data, "ID", "")
assert.NoError(t, err)
indexer := createDiskIndexer(dataDir)
err = indexer.AddIndex(&Pet{}, "Name", "ID", "pets", "non_unique", nil, false)
assert.NoError(t, err)
pet1 := Pet{ID: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"}
pet2 := Pet{ID: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"}
_, err = indexer.Add(pet1)
assert.NoError(t, err)
_, err = indexer.Add(pet2)
assert.NoError(t, err)
_ = os.RemoveAll(dataDir)
}
func TestQueryDiskImpl(t *testing.T) {
dataDir, err := WriteIndexTestData(Data, "ID", "")
assert.NoError(t, err)
indexer := createDiskIndexer(dataDir)
ctx := context.Background()
err = indexer.AddIndex(&Account{}, "OnPremisesSamAccountName", "ID", "accounts", "non_unique", nil, false)
assert.NoError(t, err)
err = indexer.AddIndex(&Account{}, "Mail", "ID", "accounts", "non_unique", nil, false)
assert.NoError(t, err)
err = indexer.AddIndex(&Account{}, "ID", "ID", "accounts", "non_unique", nil, false)
assert.NoError(t, err)
acc := Account{
ID: "ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2",
Mail: "spooky@skeletons.org",
OnPremisesSamAccountName: "MrDootDoot",
}
_, err = indexer.Add(acc)
assert.NoError(t, err)
r, err := indexer.Query(ctx, &Account{}, "on_premises_sam_account_name eq 'MrDootDoot'") // this query will match both pets.
assert.NoError(t, err)
assert.Equal(t, []string{"ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2"}, r)
r, err = indexer.Query(ctx, &Account{}, "mail eq 'spooky@skeletons.org'") // this query will match both pets.
assert.NoError(t, err)
assert.Equal(t, []string{"ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2"}, r)
r, err = indexer.Query(ctx, &Account{}, "on_premises_sam_account_name eq 'MrDootDoot' or mail eq 'spooky@skeletons.org'") // this query will match both pets.
assert.NoError(t, err)
assert.Equal(t, []string{"ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2"}, r)
r, err = indexer.Query(ctx, &Account{}, "startswith(on_premises_sam_account_name,'MrDoo')") // this query will match both pets.
assert.NoError(t, err)
assert.Equal(t, []string{"ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2"}, r)
r, err = indexer.Query(ctx, &Account{}, "id eq 'ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2' or on_premises_sam_account_name eq 'MrDootDoot'") // this query will match both pets.
assert.NoError(t, err)
assert.Equal(t, []string{"ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2"}, r)
_ = os.RemoveAll(dataDir)
}
func createDiskIndexer(dataDir string) *Indexer {
return CreateIndexer(&config.Config{
Repo: config.Repo{
Backend: "disk",
Disk: config.Disk{
Path: dataDir,
},
},
})
}

27
ocis-pkg/indexer/map.go
View File

@@ -1,27 +0,0 @@
package indexer
import "github.com/owncloud/ocis/v2/ocis-pkg/indexer/index"
// typeMap stores the indexer layout at runtime.
type typeMap map[tName]typeMapping
type tName = string
type fieldName = string
type typeMapping struct {
PKFieldName string
IndicesByField map[fieldName][]index.Index
}
func (m typeMap) addIndex(typeName string, pkName string, idx index.Index) {
if val, ok := m[typeName]; ok {
val.IndicesByField[idx.IndexBy()] = append(val.IndicesByField[idx.IndexBy()], idx)
return
}
m[typeName] = typeMapping{
PKFieldName: pkName,
IndicesByField: map[string][]index.Index{
idx.IndexBy(): {idx},
},
}
}

126
ocis-pkg/indexer/option/option.go
View File

@@ -1,126 +0,0 @@
package option
import "github.com/owncloud/ocis/v2/extensions/accounts/pkg/config"
// Option defines a single option function.
type Option func(o *Options)
// Bound represents a lower and upper bound range for an index.
// todo: if we would like to provide an upper bound then we would need to deal with ranges, in which case this is why the
// upper bound attribute is here.
type Bound struct {
Lower, Upper int64
}
// Options defines the available options for this package.
type Options struct {
CaseInsensitive bool
Bound *Bound
// Disk Options
TypeName string
IndexBy string
FilesDir string
IndexBaseDir string
DataDir string
EntityDirName string
Entity interface{}
// CS3 options
DataURL string
DataPrefix string
JWTSecret string
ProviderAddr string
ServiceUser config.ServiceUser
}
// CaseInsensitive sets the CaseInsensitive field.
func CaseInsensitive(val bool) Option {
return func(o *Options) {
o.CaseInsensitive = val
}
}
// WithBounds sets the Bounds field.
func WithBounds(val *Bound) Option {
return func(o *Options) {
o.Bound = val
}
}
// WithEntity sets the Entity field.
func WithEntity(val interface{}) Option {
return func(o *Options) {
o.Entity = val
}
}
// WithJWTSecret sets the JWTSecret field.
func WithJWTSecret(val string) Option {
return func(o *Options) {
o.JWTSecret = val
}
}
// WithDataURL sets the DataURl field.
func WithDataURL(val string) Option {
return func(o *Options) {
o.DataURL = val
}
}
// WithDataPrefix sets the DataPrefix field.
func WithDataPrefix(val string) Option {
return func(o *Options) {
o.DataPrefix = val
}
}
// WithEntityDirName sets the EntityDirName field.
func WithEntityDirName(val string) Option {
return func(o *Options) {
o.EntityDirName = val
}
}
// WithDataDir sets the DataDir option.
func WithDataDir(val string) Option {
return func(o *Options) {
o.DataDir = val
}
}
// WithTypeName sets the TypeName option.
func WithTypeName(val string) Option {
return func(o *Options) {
o.TypeName = val
}
}
// WithIndexBy sets the option IndexBy.
func WithIndexBy(val string) Option {
return func(o *Options) {
o.IndexBy = val
}
}
// WithFilesDir sets the option FilesDir.
func WithFilesDir(val string) Option {
return func(o *Options) {
o.FilesDir = val
}
}
// WithProviderAddr sets the option ProviderAddr.
func WithProviderAddr(val string) Option {
return func(o *Options) {
o.ProviderAddr = val
}
}
// WithServiceUser sets the option ServiceUser.
func WithServiceUser(val config.ServiceUser) Option {
return func(o *Options) {
o.ServiceUser = val
}
}

40
ocis-pkg/indexer/query_tree.go
View File

@@ -1,40 +0,0 @@
package indexer
type queryTree struct {
token *token
root bool
left *queryTree
right *queryTree
}
// token to be resolved by the index
type token struct {
operator string // original OData operator. i.e: 'startswith', `or`, `and`.
filterType string // equivalent operator from OData -> indexer i.e FindByPartial or FindBy.
operands []string
}
// newQueryTree constructs a new tree with a root node.
func newQueryTree() queryTree {
return queryTree{
root: true,
}
}
// insert populates first the LHS of the tree first, if this is not possible it fills the RHS.
func (t *queryTree) insert(tkn *token) {
if t != nil && t.root {
t.left = &queryTree{token: tkn}
return
}
if t.left == nil {
t.left = &queryTree{token: tkn}
return
}
if t.left != nil && t.right == nil {
t.right = &queryTree{token: tkn}
return
}
}

41
ocis-pkg/indexer/reflect.go
View File

@@ -1,41 +0,0 @@
package indexer
import (
"errors"
"path"
"reflect"
"strconv"
"strings"
)
func getType(v interface{}) (reflect.Value, error) {
rv := reflect.ValueOf(v)
for rv.Kind() == reflect.Ptr || rv.Kind() == reflect.Interface {
rv = rv.Elem()
}
if !rv.IsValid() {
return reflect.Value{}, errors.New("failed to read value via reflection")
}
return rv, nil
}
func getTypeFQN(t interface{}) string {
typ, _ := getType(t)
typeName := path.Join(typ.Type().PkgPath(), typ.Type().Name())
typeName = strings.ReplaceAll(typeName, "/", ".")
return typeName
}
func valueOf(v interface{}, field string) string {
r := reflect.ValueOf(v)
f := reflect.Indirect(r).FieldByName(field)
if f.Kind() == reflect.String {
return f.String()
}
if f.IsZero() {
return ""
}
return strconv.Itoa(int(f.Int()))
}

53
ocis-pkg/indexer/reflect_test.go
View File

@@ -1,53 +0,0 @@
package indexer
import (
"testing"
)
func Test_getTypeFQN(t *testing.T) {
type someT struct{}
type args struct {
t interface{}
}
tests := []struct {
name string
args args
want string
}{
{name: "ByValue", args: args{&someT{}}, want: "github.com.owncloud.ocis.v2.ocis-pkg.indexer.someT"},
{name: "ByRef", args: args{someT{}}, want: "github.com.owncloud.ocis.v2.ocis-pkg.indexer.someT"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := getTypeFQN(tt.args.t); got != tt.want {
t.Errorf("getTypeFQN() = %v, want %v", got, tt.want)
}
})
}
}
func Test_valueOf(t *testing.T) {
type someT struct {
val string
}
type args struct {
v interface{}
field string
}
tests := []struct {
name string
args args
want string
}{
{name: "ByValue", args: args{v: someT{val: "hello"}, field: "val"}, want: "hello"},
{name: "ByRef", args: args{v: &someT{val: "hello"}, field: "val"}, want: "hello"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := valueOf(tt.args.v, tt.args.field); got != tt.want {
t.Errorf("valueOf() = %v, want %v", got, tt.want)
}
})
}
}

15
ocis-pkg/indexer/registry/registry.go
View File

@@ -1,15 +0,0 @@
package registry
import (
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/index"
"github.com/owncloud/ocis/v2/ocis-pkg/indexer/option"
)
// IndexConstructor is a constructor function for creating index.Index.
type IndexConstructor func(o ...option.Option) index.Index
// IndexConstructorRegistry undocumented.
var IndexConstructorRegistry = map[string]map[string]IndexConstructor{
"disk": {},
"cs3": {},
}

102
ocis-pkg/indexer/test/data.go
View File

@@ -1,102 +0,0 @@
package test
import (
"encoding/json"
"io/ioutil"
"os"
"path"
)
// User is a user.
type User struct {
ID, UserName, Email string
UID int
}
// Pet is a pet.
type Pet struct {
ID, Kind, Color, Name string
UID int
}
// Account mocks an ocis account.
type Account struct {
ID string
OnPremisesSamAccountName string
Mail string
}
// Data mock data.
var Data = map[string][]interface{}{
"users": {
User{ID: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"},
User{ID: "hijklmn-456", UserName: "frank", Email: "frank@example.com"},
User{ID: "ewf4ofk-555", UserName: "jacky", Email: "jacky@example.com"},
User{ID: "rulan54-777", UserName: "jones", Email: "jones@example.com"},
},
"pets": {
Pet{ID: "rebef-123", Kind: "Dog", Color: "Brown", Name: "Waldo"},
Pet{ID: "wefwe-456", Kind: "Cat", Color: "White", Name: "Snowy"},
Pet{ID: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"},
Pet{ID: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"},
},
"accounts": {
Account{ID: "ba5b6e54-e29d-4b2b-8cc4-0a0b958140d2", Mail: "spooky@skeletons.org", OnPremisesSamAccountName: "MrDootDoot"},
},
}
// WriteIndexTestData writes mock data to disk.
func WriteIndexTestData(m map[string][]interface{}, privateKey, dir string) (string, error) {
rootDir, err := getRootDir(dir)
if err != nil {
return "", err
}
err = writeData(m, privateKey, rootDir)
if err != nil {
return "", err
}
return rootDir, nil
}
// getRootDir allows for some minimal behavior on destination on disk. Testing the cs3 api behavior locally means
// keeping track of where the cs3 data lives on disk, this function allows for multiplexing whether or not to use a
// temporary folder or an already defined one.
func getRootDir(dir string) (string, error) {
var rootDir string
var err error
if dir != "" {
rootDir = dir
} else {
rootDir, err = CreateTmpDir()
if err != nil {
return "", err
}
}
return rootDir, nil
}
// writeData writes test data to disk on rootDir location Marshaled as json.
func writeData(m map[string][]interface{}, privateKey string, rootDir string) error {
for dirName := range m {
fileTypePath := path.Join(rootDir, dirName)
if err := os.MkdirAll(fileTypePath, 0755); err != nil {
return err
}
for _, u := range m[dirName] {
data, err := json.Marshal(u)
if err != nil {
return err
}
pkVal := ValueOf(u, privateKey)
if err := ioutil.WriteFile(path.Join(fileTypePath, pkVal), data, 0600); err != nil {
return err
}
}
}
return nil
}

48
ocis-pkg/indexer/test/helpers.go
View File

@@ -1,48 +0,0 @@
package test
import (
"errors"
"io/ioutil"
"path"
"reflect"
"strings"
)
// CreateTmpDir creates a temporary dir for tests data.
func CreateTmpDir() (string, error) {
name, err := ioutil.TempDir("/tmp", "testfiles-")
if err != nil {
return "", err
}
return name, nil
}
// ValueOf gets the value of a type v on a given field <field>.
func ValueOf(v interface{}, field string) string {
r := reflect.ValueOf(v)
f := reflect.Indirect(r).FieldByName(field)
return f.String()
}
func getType(v interface{}) (reflect.Value, error) {
rv := reflect.ValueOf(v)
for rv.Kind() == reflect.Ptr || rv.Kind() == reflect.Interface {
rv = rv.Elem()
}
if !rv.IsValid() {
return reflect.Value{}, errors.New("failed to read value via reflection")
}
return rv, nil
}
// GetTypeFQN formats a valid name from a type <t>. This is a duplication of the already existing function in the
// indexer package, but since there is a circular dependency we chose to duplicate it.
func GetTypeFQN(t interface{}) string {
typ, _ := getType(t)
typeName := path.Join(typ.Type().PkgPath(), typ.Type().Name())
typeName = strings.ReplaceAll(typeName, "/", ".")
return typeName
}