From 8a08c9f9b871c32c84f394abe513a8f6562297f0 Mon Sep 17 00:00:00 2001 From: jkoberg Date: Wed, 22 May 2024 16:08:51 +0200 Subject: [PATCH] feat(ocis): Add backup consistency command Signed-off-by: jkoberg Co-authored-by: dragonchaser --- .../add-consistency-check-command.md | 5 + ocis/pkg/backup/backup.go | 189 ++++++++++++++++++ ocis/pkg/command/backup.go | 63 ++++++ 3 files changed, 257 insertions(+) create mode 100644 changelog/unreleased/add-consistency-check-command.md create mode 100644 ocis/pkg/backup/backup.go create mode 100644 ocis/pkg/command/backup.go diff --git a/changelog/unreleased/add-consistency-check-command.md b/changelog/unreleased/add-consistency-check-command.md new file mode 100644 index 0000000000..1f44a71751 --- /dev/null +++ b/changelog/unreleased/add-consistency-check-command.md @@ -0,0 +1,5 @@ +Enhancement: Add command to check ocis backup consistency + +Adds a command that checks the consistency of an ocis backup. + +https://github.com/owncloud/ocis/pull/9238 diff --git a/ocis/pkg/backup/backup.go b/ocis/pkg/backup/backup.go new file mode 100644 index 0000000000..8027984a35 --- /dev/null +++ b/ocis/pkg/backup/backup.go @@ -0,0 +1,189 @@ +// Package backup contains ocis backup functionality. +package backup + +import ( + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" + "github.com/shamaton/msgpack/v2" +) + +// Inconsistency describes the type of incosistency +type Inconsistency string + +var ( + InconsistencyBlobMissing Inconsistency = "blob missing" + InconsistencyBlobOrphaned Inconsistency = "blob orphaned" + InconsistencyNodeMissing Inconsistency = "node missing" + InconsistencyMetadataMissing Inconsistency = "metadata missing" + InconsistencySymlinkMissing Inconsistency = "symlink missing" +) + +// Consistency holds the node and blob data of a space +type Consistency struct { + Nodes map[string]Inconsistency + Links map[string]Inconsistency + Blobs map[string]Inconsistency + + fsys fs.FS + discpath string +} + +// ListBlobstore required to check blob consistency +type ListBlobstore interface { + Upload(node *node.Node, source string) error + Download(node *node.Node) (io.ReadCloser, error) + Delete(node *node.Node) error + List() ([]string, error) +} + +// New creates a new Consistency object +func New(fsys fs.FS, discpath string) *Consistency { + return &Consistency{ + Nodes: make(map[string]Inconsistency), + Links: make(map[string]Inconsistency), + Blobs: make(map[string]Inconsistency), + + fsys: fsys, + discpath: discpath, + } +} + +// CheckSpaceConsistency checks the consistency of a space +func CheckSpaceConsistency(pathToSpace string, lbs ListBlobstore) error { + fsys := os.DirFS(pathToSpace) + + con := New(fsys, pathToSpace) + err := con.Initialize() + if err != nil { + return err + } + + for n := range con.Nodes { + if _, ok := con.Links[n]; !ok { + // TODO: This is no inconsistency if + // * this is the spaceroot + // * this is a trashed node + con.Nodes[n] = InconsistencySymlinkMissing + continue + } + delete(con.Links, n) + delete(con.Nodes, n) + } + + for l := range con.Links { + con.Links[l] = InconsistencyNodeMissing + } + + blobs, err := lbs.List() + if err != nil { + return err + } + + deadBlobs := make(map[string]Inconsistency) + for _, b := range blobs { + if _, ok := con.Blobs[b]; !ok { + deadBlobs[b] = InconsistencyBlobOrphaned + continue + } + delete(con.Blobs, b) + delete(deadBlobs, b) + } + + for b := range con.Blobs { + con.Blobs[b] = InconsistencyBlobMissing + } + // get blobs from blobstore. + // initialize blobstore (s3, local) + // compare con.Blobs with blobstore.GetBlobs() + + for n := range con.Nodes { + fmt.Println("Inconsistency", n, con.Nodes[n]) + } + for l := range con.Links { + fmt.Println("Inconsistency", l, con.Links[l]) + } + for b := range con.Blobs { + fmt.Println("Inconsistency", b, con.Blobs[b]) + } + for b := range deadBlobs { + fmt.Println("Inconsistency", b, deadBlobs[b]) + } + + return nil +} + +func (c *Consistency) Initialize() error { + dirs, err := fs.Glob(c.fsys, "nodes/*/*/*/*") + if err != nil { + return err + } + + for _, d := range dirs { + entries, err := fs.ReadDir(c.fsys, d) + if err != nil { + return err + } + for _, e := range entries { + switch { + case e.IsDir(): + ls, err := fs.ReadDir(c.fsys, filepath.Join(d, e.Name())) + if err != nil { + fmt.Println("error reading dir", err) + continue + } + for _, l := range ls { + p, err := filepath.EvalSymlinks(filepath.Join(c.discpath, d, e.Name(), l.Name())) + if err != nil { + fmt.Println("error evaluating symlink", filepath.Join(d, e.Name(), l.Name()), err) + continue + } + c.Links[p] = "" + } + case filepath.Ext(e.Name()) == ".mpk": + inc, err := c.checkNode(filepath.Join(d, e.Name())) + if err != nil { + fmt.Println("error checking node", err) + continue + } + c.Nodes[filepath.Join(c.discpath, d, strings.TrimSuffix(e.Name(), ".mpk"))] = inc + default: + // fmt.Println("unknown", e.Name()) + } + } + } + return nil +} + +func (c *Consistency) checkNode(path string) (Inconsistency, error) { + b, err := fs.ReadFile(c.fsys, path) + if err != nil { + return "", err + } + + m := map[string][]byte{} + if err := msgpack.Unmarshal(b, &m); err != nil { + return "", err + } + + if bid := m["user.ocis.blobid"]; string(bid) != "" { + c.Blobs[string(bid)] = "" + } + + return "", nil +} + +func iterate(fsys fs.FS, path string, d *Consistency) ([]string, error) { + // open symlink -> NodeMissing + // remove node from data.Nodes + // check blob -> BlobMissing + // remove blob from data.Blobs + // list children (symlinks!) + // return children (symlinks!) + return nil, nil +} diff --git a/ocis/pkg/command/backup.go b/ocis/pkg/command/backup.go new file mode 100644 index 0000000000..3353b9995b --- /dev/null +++ b/ocis/pkg/command/backup.go @@ -0,0 +1,63 @@ +package command + +import ( + "fmt" + "path/filepath" + + "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore" + "github.com/owncloud/ocis/v2/ocis-pkg/config" + "github.com/owncloud/ocis/v2/ocis/pkg/backup" + "github.com/owncloud/ocis/v2/ocis/pkg/register" + "github.com/urfave/cli/v2" +) + +// BackupCommand is the entrypoint for the backup command +func BackupCommand(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "backup", + Usage: "ocis backup functionality", + Subcommands: []*cli.Command{ + ConsistencyCommand(cfg), + }, + Action: func(c *cli.Context) error { + fmt.Println("Read the docs") + return nil + }, + } +} + +// ConsistencyCommand is the entrypoint for the consistency Command +func ConsistencyCommand(_ *config.Config) *cli.Command { + return &cli.Command{ + Name: "consistency", + Usage: "check backup consistency", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "blobstore", + Aliases: []string{"b"}, + Usage: "the blobstore type. Can be (ocis, s3ng). Default ocis", + }, + }, + Action: func(c *cli.Context) error { + basePath := "/home/jkoberg/.ocis/storage/users" + + // TODO: switch for s3ng blobstore + bs, err := blobstore.New(basePath) + if err != nil { + fmt.Println(err) + return err + } + + if err := backup.CheckSpaceConsistency(filepath.Join(basePath, "spaces/23/ebf113-76d4-43c0-8594-df974b02cd74"), bs); err != nil { + fmt.Println(err) + return err + } + + return nil + }, + } +} + +func init() { + register.AddCommand(BackupCommand) +}