// Package backup contains ocis backup functionality. package backup import ( "fmt" "os" "regexp" ) // Inconsistency describes the type of inconsistency type Inconsistency string var ( // InconsistencyBlobMissing is an inconsistency where a blob is missing in the blobstore InconsistencyBlobMissing Inconsistency = "blob missing" // InconsistencyBlobOrphaned is an inconsistency where a blob in the blobstore has no reference InconsistencyBlobOrphaned Inconsistency = "blob orphaned" // InconsistencyNodeMissing is an inconsistency where a symlink points to a non-existing node InconsistencyNodeMissing Inconsistency = "node missing" // InconsistencyMetadataMissing is an inconsistency where a node is missing metadata InconsistencyMetadataMissing Inconsistency = "metadata missing" // InconsistencySymlinkMissing is an inconsistency where a node is missing a symlink InconsistencySymlinkMissing Inconsistency = "symlink missing" // InconsistencyFilesMissing is an inconsistency where a node is missing metadata files like .mpk or .mlock InconsistencyFilesMissing Inconsistency = "files missing" // InconsistencyMalformedFile is an inconsistency where a node has a malformed metadata file InconsistencyMalformedFile Inconsistency = "malformed file" // regex to determine if a node is trashed or versioned. // 9113a718-8285-4b32-9042-f930f1a58ac2.REV.2024-05-22T07:32:53.89969726Z _versionRegex = regexp.MustCompile(`\.REV\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z$`) // 9113a718-8285-4b32-9042-f930f1a58ac2.T.2024-05-23T08:25:20.006571811Z <- this HAS a symlink _trashRegex = regexp.MustCompile(`\.T\.[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]+Z$`) ) // Consistency holds the node and blob data of a storage provider type Consistency struct { // Storing the data like this might take a lot of memory // we might need to optimize this if we run into memory issues Nodes map[string][]Inconsistency LinkedNodes map[string][]Inconsistency BlobReferences map[string][]Inconsistency Blobs map[string][]Inconsistency nodeToLink map[string]string blobToNode map[string]string } // NewConsistency creates a new Consistency object func NewConsistency() *Consistency { return &Consistency{ Nodes: make(map[string][]Inconsistency), LinkedNodes: make(map[string][]Inconsistency), BlobReferences: make(map[string][]Inconsistency), Blobs: make(map[string][]Inconsistency), nodeToLink: make(map[string]string), blobToNode: make(map[string]string), } } // CheckProviderConsistency checks the consistency of a space func CheckProviderConsistency(storagepath string, lbs ListBlobstore, fail bool) error { fsys := os.DirFS(storagepath) p := NewProvider(fsys, storagepath, lbs) if err := p.ProduceData(); err != nil { return err } c := NewConsistency() c.GatherData(p.Events) return c.PrintResults(storagepath, fail) } // GatherData gathers and evaluates data produced by the DataProvider func (c *Consistency) GatherData(events <-chan interface{}) { for ev := range events { switch d := ev.(type) { case NodeData: // does it have inconsistencies? if len(d.Inconsistencies) != 0 { c.Nodes[d.NodePath] = append(c.Nodes[d.NodePath], d.Inconsistencies...) } // is it linked? if _, ok := c.LinkedNodes[d.NodePath]; ok { deleteInconsistency(c.LinkedNodes, d.NodePath) } else if d.RequiresSymlink && c.Nodes[d.NodePath] == nil { c.Nodes[d.NodePath] = []Inconsistency{} } // does it have a blob? if d.BlobPath != "" { if _, ok := c.Blobs[d.BlobPath]; ok { deleteInconsistency(c.Blobs, d.BlobPath) } else { c.BlobReferences[d.BlobPath] = []Inconsistency{} c.blobToNode[d.BlobPath] = d.NodePath } } case LinkData: // does it have a node? if _, ok := c.Nodes[d.NodePath]; ok { deleteInconsistency(c.Nodes, d.NodePath) } else { c.LinkedNodes[d.NodePath] = []Inconsistency{} c.nodeToLink[d.NodePath] = d.LinkPath } case BlobData: // does it have a reference? if _, ok := c.BlobReferences[d.BlobPath]; ok { deleteInconsistency(c.BlobReferences, d.BlobPath) } else { c.Blobs[d.BlobPath] = []Inconsistency{} } } } for n := range c.Nodes { if len(c.Nodes[n]) == 0 { c.Nodes[n] = append(c.Nodes[n], InconsistencySymlinkMissing) } } for l := range c.LinkedNodes { c.LinkedNodes[l] = append(c.LinkedNodes[l], InconsistencyNodeMissing) } for b := range c.Blobs { c.Blobs[b] = append(c.Blobs[b], InconsistencyBlobOrphaned) } for b := range c.BlobReferences { c.BlobReferences[b] = append(c.BlobReferences[b], InconsistencyBlobMissing) } } // PrintResults prints the results of the evaluation func (c *Consistency) PrintResults(discpath string, fail bool) error { if len(c.Nodes) != 0 { fmt.Println("\n🚨 Inconsistent Nodes:") } for n := range c.Nodes { fmt.Printf("\t👉️ %v\tpath: %s\n", c.Nodes[n], n) } if len(c.LinkedNodes) != 0 { fmt.Println("\n🚨 Inconsistent Links:") } for l := range c.LinkedNodes { fmt.Printf("\t👉️ %v\tpath: %s\n\t\t\t\tmissing node:%s\n", c.LinkedNodes[l], c.nodeToLink[l], l) } if len(c.Blobs) != 0 { fmt.Println("\n🚨 Inconsistent Blobs:") } for b := range c.Blobs { fmt.Printf("\t👉️ %v\tblob: %s\n", c.Blobs[b], b) } if len(c.BlobReferences) != 0 { fmt.Println("\n🚨 Inconsistent BlobReferences:") } for b := range c.BlobReferences { fmt.Printf("\t👉️ %v\tblob: %s\n\t\t\t\treferencing node:%s\n", c.BlobReferences[b], b, c.blobToNode[b]) } if len(c.Nodes) == 0 && len(c.LinkedNodes) == 0 && len(c.Blobs) == 0 && len(c.BlobReferences) == 0 { fmt.Printf("💚 No inconsistency found. The backup in '%s' seems to be valid.\n", discpath) } else if fail { os.Exit(1) } return nil } func deleteInconsistency(incs map[string][]Inconsistency, path string) { if len(incs[path]) == 0 { delete(incs, path) } }