From ca12c2faa23ff43cb25661cc0d203607995b3eed Mon Sep 17 00:00:00 2001 From: jkoberg Date: Thu, 23 May 2024 12:03:31 +0200 Subject: [PATCH] feat(ocis): improve consistency command Signed-off-by: jkoberg Co-authored-by: dragonchaser Signed-off-by: jkoberg --- ocis/pkg/backup/backup.go | 264 ++++++++++++++++++++++++------------- ocis/pkg/command/backup.go | 42 ++++-- 2 files changed, 206 insertions(+), 100 deletions(-) diff --git a/ocis/pkg/backup/backup.go b/ocis/pkg/backup/backup.go index 8027984a3..a70608097 100644 --- a/ocis/pkg/backup/backup.go +++ b/ocis/pkg/backup/backup.go @@ -7,28 +7,45 @@ import ( "io/fs" "os" "path/filepath" + "regexp" "strings" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/shamaton/msgpack/v2" ) -// Inconsistency describes the type of incosistency +// Inconsistency describes the type of inconsistency type Inconsistency string var ( - InconsistencyBlobMissing Inconsistency = "blob missing" - InconsistencyBlobOrphaned Inconsistency = "blob orphaned" - InconsistencyNodeMissing Inconsistency = "node missing" + // InconsistencyBlobMissing is an inconsistency where a blob is missing in the blobstore + InconsistencyBlobMissing Inconsistency = "blob missing" + // InconsistencyBlobOrphaned is an inconsistency where a blob in the blobstore has no reference + InconsistencyBlobOrphaned Inconsistency = "blob orphaned" + // InconsistencyNodeMissing is an inconsistency where a symlink points to a non-existing node + InconsistencyNodeMissing Inconsistency = "node missing" + // InconsistencyMetadataMissing is an inconsistency where a node is missing metadata InconsistencyMetadataMissing Inconsistency = "metadata missing" - InconsistencySymlinkMissing Inconsistency = "symlink missing" + // InconsistencySymlinkMissing is an inconsistency where a node is missing a symlink + InconsistencySymlinkMissing Inconsistency = "symlink missing" + // InconsistencyFilesMissing is an inconsistency where a node is missing metadata files like .mpk or .mlock + InconsistencyFilesMissing Inconsistency = "files missing" + // InconsistencyMalformedFile is an inconsistency where a node has a malformed metadata file + InconsistencyMalformedFile Inconsistency = "malformed file" + + // regex to determine if a node is trashed or versioned. + // 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z + _versionRegex = regexp.MustCompile(`\.REV\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z$`) + // 9113a718-8285-4b32-9042-f930f1a58ac2.T.2024-05-23T08:25:20.006571811Z <- this HAS a symlink + _trashRegex = regexp.MustCompile(`\.T\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z$`) ) // Consistency holds the node and blob data of a space type Consistency struct { - Nodes map[string]Inconsistency - Links map[string]Inconsistency - Blobs map[string]Inconsistency + Nodes map[string][]Inconsistency + Links map[string][]Inconsistency + BlobReferences map[string][]Inconsistency + Blobs map[string][]Inconsistency fsys fs.FS discpath string @@ -45,9 +62,10 @@ type ListBlobstore interface { // New creates a new Consistency object func New(fsys fs.FS, discpath string) *Consistency { return &Consistency{ - Nodes: make(map[string]Inconsistency), - Links: make(map[string]Inconsistency), - Blobs: make(map[string]Inconsistency), + Nodes: make(map[string][]Inconsistency), + Links: make(map[string][]Inconsistency), + BlobReferences: make(map[string][]Inconsistency), + Blobs: make(map[string][]Inconsistency), fsys: fsys, discpath: discpath, @@ -55,71 +73,24 @@ func New(fsys fs.FS, discpath string) *Consistency { } // CheckSpaceConsistency checks the consistency of a space -func CheckSpaceConsistency(pathToSpace string, lbs ListBlobstore) error { - fsys := os.DirFS(pathToSpace) +func CheckSpaceConsistency(storagepath string, lbs ListBlobstore) error { + fsys := os.DirFS(storagepath) - con := New(fsys, pathToSpace) - err := con.Initialize() - if err != nil { + c := New(fsys, storagepath) + if err := c.Initialize(); err != nil { return err } - for n := range con.Nodes { - if _, ok := con.Links[n]; !ok { - // TODO: This is no inconsistency if - // * this is the spaceroot - // * this is a trashed node - con.Nodes[n] = InconsistencySymlinkMissing - continue - } - delete(con.Links, n) - delete(con.Nodes, n) - } - - for l := range con.Links { - con.Links[l] = InconsistencyNodeMissing - } - - blobs, err := lbs.List() - if err != nil { + if err := c.Evaluate(lbs); err != nil { return err } - deadBlobs := make(map[string]Inconsistency) - for _, b := range blobs { - if _, ok := con.Blobs[b]; !ok { - deadBlobs[b] = InconsistencyBlobOrphaned - continue - } - delete(con.Blobs, b) - delete(deadBlobs, b) - } - - for b := range con.Blobs { - con.Blobs[b] = InconsistencyBlobMissing - } - // get blobs from blobstore. - // initialize blobstore (s3, local) - // compare con.Blobs with blobstore.GetBlobs() - - for n := range con.Nodes { - fmt.Println("Inconsistency", n, con.Nodes[n]) - } - for l := range con.Links { - fmt.Println("Inconsistency", l, con.Links[l]) - } - for b := range con.Blobs { - fmt.Println("Inconsistency", b, con.Blobs[b]) - } - for b := range deadBlobs { - fmt.Println("Inconsistency", b, deadBlobs[b]) - } - - return nil + return c.PrintResults() } +// Initialize initializes the Consistency object func (c *Consistency) Initialize() error { - dirs, err := fs.Glob(c.fsys, "nodes/*/*/*/*") + dirs, err := fs.Glob(c.fsys, "spaces/*/*/nodes/*/*/*/*") if err != nil { return err } @@ -129,6 +100,7 @@ func (c *Consistency) Initialize() error { if err != nil { return err } + for _, e := range entries { switch { case e.IsDir(): @@ -138,52 +110,162 @@ func (c *Consistency) Initialize() error { continue } for _, l := range ls { - p, err := filepath.EvalSymlinks(filepath.Join(c.discpath, d, e.Name(), l.Name())) + p, err := os.Readlink(filepath.Join(c.discpath, d, e.Name(), l.Name())) if err != nil { - fmt.Println("error evaluating symlink", filepath.Join(d, e.Name(), l.Name()), err) - continue + fmt.Println("error reading symlink", err) } - c.Links[p] = "" + p = filepath.Join(c.discpath, d, e.Name(), p) + c.Links[p] = []Inconsistency{} } - case filepath.Ext(e.Name()) == ".mpk": - inc, err := c.checkNode(filepath.Join(d, e.Name())) - if err != nil { - fmt.Println("error checking node", err) - continue + fallthrough + case filepath.Ext(e.Name()) == "" || _versionRegex.MatchString(e.Name()) || _trashRegex.MatchString(e.Name()): + if !c.filesExist(filepath.Join(d, e.Name())) { + dp := filepath.Join(c.discpath, d, e.Name()) + c.Nodes[dp] = append(c.Nodes[dp], InconsistencyFilesMissing) + } + inc := c.checkNode(filepath.Join(d, e.Name()+".mpk")) + dp := filepath.Join(c.discpath, d, e.Name()) + if inc != "" { + c.Nodes[dp] = append(c.Nodes[dp], inc) + } else if len(c.Nodes[dp]) == 0 { + c.Nodes[dp] = []Inconsistency{} } - c.Nodes[filepath.Join(c.discpath, d, strings.TrimSuffix(e.Name(), ".mpk"))] = inc - default: - // fmt.Println("unknown", e.Name()) } } } + + links, err := fs.Glob(c.fsys, "spaces/*/*/trash/*/*/*/*/*") + if err != nil { + return err + } + for _, l := range links { + p, err := os.Readlink(filepath.Join(c.discpath, l)) + if err != nil { + fmt.Println("error reading symlink", err) + } + p = filepath.Join(c.discpath, l, "..", p) + c.Links[p] = []Inconsistency{} + } return nil } -func (c *Consistency) checkNode(path string) (Inconsistency, error) { +// Evaluate evaluates inconsistencies +func (c *Consistency) Evaluate(lbs ListBlobstore) error { + for n := range c.Nodes { + if _, ok := c.Links[n]; !ok && c.requiresSymlink(n) { + c.Nodes[n] = append(c.Nodes[n], InconsistencySymlinkMissing) + continue + } + + deleteInconsistency(c.Links, n) + deleteInconsistency(c.Nodes, n) + } + + for l := range c.Links { + c.Links[l] = append(c.Links[l], InconsistencyNodeMissing) + } + + blobs, err := lbs.List() + if err != nil { + return err + } + + for _, b := range blobs { + if _, ok := c.BlobReferences[b]; !ok { + c.Blobs[b] = append(c.Blobs[b], InconsistencyBlobOrphaned) + continue + } + deleteInconsistency(c.BlobReferences, b) + } + + for b := range c.BlobReferences { + c.BlobReferences[b] = append(c.BlobReferences[b], InconsistencyBlobMissing) + } + + return nil +} + +// PrintResults prints the results of the evaluation +func (c *Consistency) PrintResults() error { + if len(c.Nodes) != 0 { + fmt.Println("šŸšØ Inconsistent Nodes:") + } + for n := range c.Nodes { + fmt.Printf("\tšŸ‘‰ļø %v\tpath: %s\n", c.Nodes[n], n) + } + if len(c.Links) != 0 { + fmt.Println("šŸšØ Inconsistent Links:") + } + for l := range c.Links { + fmt.Printf("\tšŸ‘‰ļø %v\tpath: %s\n", c.Links[l], l) + } + if len(c.Blobs) != 0 { + fmt.Println("šŸšØ Inconsistent Blobs:") + } + for b := range c.Blobs { + fmt.Printf("\tšŸ‘‰ļø %v\tblob: %s\n", c.Blobs[b], b) + } + if len(c.BlobReferences) != 0 { + fmt.Println("šŸšØ Inconsistent BlobReferences:") + } + for b := range c.BlobReferences { + fmt.Printf("\tšŸ‘‰ļø %v\tblob: %s\n", c.BlobReferences[b], b) + } + if len(c.Nodes) == 0 && len(c.Links) == 0 && len(c.Blobs) == 0 && len(c.BlobReferences) == 0 { + fmt.Printf("šŸ’š No inconsistency found. The backup in '%s' seems to be valid.\n", c.discpath) + } + return nil + +} + +func (c *Consistency) checkNode(path string) Inconsistency { b, err := fs.ReadFile(c.fsys, path) if err != nil { - return "", err + return InconsistencyFilesMissing } m := map[string][]byte{} if err := msgpack.Unmarshal(b, &m); err != nil { - return "", err + return InconsistencyMalformedFile } if bid := m["user.ocis.blobid"]; string(bid) != "" { - c.Blobs[string(bid)] = "" + c.BlobReferences[string(bid)] = []Inconsistency{} } - return "", nil + return "" } -func iterate(fsys fs.FS, path string, d *Consistency) ([]string, error) { - // open symlink -> NodeMissing - // remove node from data.Nodes - // check blob -> BlobMissing - // remove blob from data.Blobs - // list children (symlinks!) - // return children (symlinks!) - return nil, nil +func (c *Consistency) requiresSymlink(path string) bool { + rawIDs := strings.Split(path, "/nodes/") + if len(rawIDs) != 2 { + return true + } + + s := strings.Split(rawIDs[0], "/spaces/") + if len(s) != 2 { + return true + } + + spaceID := strings.Replace(s[1], "/", "", -1) + nodeID := strings.Replace(rawIDs[1], "/", "", -1) + if spaceID == nodeID || _versionRegex.MatchString(nodeID) { + return false + } + + return true +} + +func (c *Consistency) filesExist(path string) bool { + check := func(p string) bool { + _, err := fs.Stat(c.fsys, p) + return err == nil + } + return check(path) && check(path+".mpk") +} + +func deleteInconsistency(incs map[string][]Inconsistency, path string) { + if len(incs[path]) == 0 { + delete(incs, path) + } } diff --git a/ocis/pkg/command/backup.go b/ocis/pkg/command/backup.go index 3353b9995..4f594579d 100644 --- a/ocis/pkg/command/backup.go +++ b/ocis/pkg/command/backup.go @@ -2,9 +2,9 @@ package command import ( "fmt" - "path/filepath" - "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore" + ocisbs "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore" + s3bs "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore" "github.com/owncloud/ocis/v2/ocis-pkg/config" "github.com/owncloud/ocis/v2/ocis/pkg/backup" "github.com/owncloud/ocis/v2/ocis/pkg/register" @@ -19,7 +19,7 @@ func BackupCommand(cfg *config.Config) *cli.Command { Subcommands: []*cli.Command{ ConsistencyCommand(cfg), }, - Action: func(c *cli.Context) error { + Action: func(_ *cli.Context) error { fmt.Println("Read the docs") return nil }, @@ -27,11 +27,17 @@ func BackupCommand(cfg *config.Config) *cli.Command { } // ConsistencyCommand is the entrypoint for the consistency Command -func ConsistencyCommand(_ *config.Config) *cli.Command { +func ConsistencyCommand(cfg *config.Config) *cli.Command { return &cli.Command{ Name: "consistency", Usage: "check backup consistency", Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "basepath", + Aliases: []string{"p"}, + Usage: "the basepath of the decomposedfs (e.g. /var/tmp/ocis/storage/users)", + Required: true, + }, &cli.StringFlag{ Name: "blobstore", Aliases: []string{"b"}, @@ -39,16 +45,34 @@ func ConsistencyCommand(_ *config.Config) *cli.Command { }, }, Action: func(c *cli.Context) error { - basePath := "/home/jkoberg/.ocis/storage/users" + basePath := c.String("basepath") + if basePath == "" { + fmt.Println("basepath is required") + return cli.ShowCommandHelp(c, "consistency") + } - // TODO: switch for s3ng blobstore - bs, err := blobstore.New(basePath) + var ( + bs backup.ListBlobstore + err error + ) + switch c.String("blobstore") { + case "s3ng": + bs, err = s3bs.New( + cfg.StorageUsers.Drivers.S3NG.Endpoint, + cfg.StorageUsers.Drivers.S3NG.Region, + cfg.StorageUsers.Drivers.S3NG.Bucket, + cfg.StorageUsers.Drivers.S3NG.AccessKey, + cfg.StorageUsers.Drivers.S3NG.SecretKey, + s3bs.Options{}, + ) + case "ocis": + bs, err = ocisbs.New(basePath) + } if err != nil { fmt.Println(err) return err } - - if err := backup.CheckSpaceConsistency(filepath.Join(basePath, "spaces/23/ebf113-76d4-43c0-8594-df974b02cd74"), bs); err != nil { + if err := backup.CheckSpaceConsistency(basePath, bs); err != nil { fmt.Println(err) return err }