diff --git a/opencloud/pkg/command/posixfs.go b/opencloud/pkg/command/posixfs.go index e38a400afb..751e3db1be 100644 --- a/opencloud/pkg/command/posixfs.go +++ b/opencloud/pkg/command/posixfs.go @@ -1,18 +1,15 @@ package command import ( - "bufio" "fmt" "os" "path/filepath" - "strings" "time" "github.com/opencloud-eu/opencloud/opencloud/pkg/register" "github.com/opencloud-eu/opencloud/pkg/config" "github.com/opencloud-eu/opencloud/pkg/config/configlog" "github.com/opencloud-eu/opencloud/pkg/config/parser" - oclog "github.com/opencloud-eu/opencloud/pkg/log" "github.com/opencloud-eu/opencloud/pkg/x/path/filepathx" storageUsersParser "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config/parser" "github.com/opencloud-eu/opencloud/services/storage-users/pkg/event" @@ -21,27 +18,10 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/ignore" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" "github.com/pkg/xattr" - "github.com/rs/zerolog" "github.com/spf13/cobra" - "github.com/theckman/yacspin" - "github.com/vmihailenco/msgpack/v5" -) - -// Define the names of the extended attributes we are working with. -const ( - parentIDAttrName = "user.oc.parentid" - idAttrName = "user.oc.id" - nameAttrName = "user.oc.name" - spaceIDAttrName = "user.oc.space.id" - ownerIDAttrName = "user.oc.owner.id" -) - -var ( - spinner *yacspin.Spinner - restartRequired = false - ignorer *ignore.Ignorer ) type IDCacher interface { @@ -140,11 +120,7 @@ func scanCmd(ocCfg *config.Config) *cobra.Command { fmt.Fprintf(os.Stderr, "Failed to create event stream for posix driver: %v\n", err) os.Exit(1) } - log := oclog.NewLogger( - oclog.Name("posixfs scan"), - oclog.Level("error"), - oclog.Pretty(true), - oclog.Color(false)).Logger + log := logger("posixfs") if !defaultRoot { log = log.With().Str("basepath", root).Logger() @@ -188,403 +164,93 @@ func scanCmd(ocCfg *config.Config) *cobra.Command { } // consistencyCmd returns a command to check the consistency of the posixfs storage. -func consistencyCmd(cfg *config.Config) *cobra.Command { +func consistencyCmd(ocCfg *config.Config) *cobra.Command { consCmd := &cobra.Command{ - Use: "consistency", - Short: "check the consistency of the posixfs storage", + Use: "consistency [path ...]", + Short: "Check the consistency of the posixfs storage", + Long: `Check the consistency of the posixfs storage. + +You can specify one or more paths to limit the scope of the check. +If no path is provided, the whole storage is checked. + +The provided arguments determines the scope of the check: + - a storage root: the whole storage (all personal and project spaces) is checked + - a space root: only that space is checked + - a file or directory: only that single entity is checked (and its children, if it is a directory)`, + Args: cobra.ArbitraryArgs, + PreRunE: func(cmd *cobra.Command, args []string) error { + + if err := parser.ParseConfig(ocCfg, true); err != nil { + return configlog.ReturnError(err) + } + + // Parse storage users config + ocCfg.StorageUsers.Commons = ocCfg.Commons + + return configlog.ReturnFatal(storageUsersParser.ParseConfig(ocCfg.StorageUsers)) + }, RunE: func(cmd *cobra.Command, args []string) error { - return checkPosixfsConsistency(cmd, cfg) + cfg := ocCfg.StorageUsers + if len(args) == 0 { + args = []string{cfg.Drivers.Posix.Root} + } + log := logger("posixfs") + recalculateChecksums, _ := cmd.Flags().GetBool("fix-checksums") + + drivers := revaconfig.StorageProviderDrivers(cfg) + drivers["posix"] = revaconfig.Posix(cfg, false, false) + opts, err := options.New(drivers["posix"].(map[string]any)) + if err != nil { + return err + } + ignorer := ignore.NewIgnorer(opts, &log) + + checker := &consistencyChecker{ + cfg: cfg, + ignorer: ignorer, + recalculateChecksums: recalculateChecksums, + } + + return checker.Check(args) }, } - consCmd.Flags().StringP("root", "r", "", "Path to the root directory of the posixfs storage") - _ = consCmd.MarkFlagRequired("root") + consCmd.Flags().Bool("fix-checksums", false, "Recalculate and fix the file checksums. This reads every file and can be slow on large storages.") return consCmd } -// checkPosixfsConsistency checks the consistency of the posixfs storage. -func checkPosixfsConsistency(cmd *cobra.Command, cfg *config.Config) error { - rootPath, _ := cmd.Flags().GetString("root") - indexesPath := filepath.Join(rootPath, "indexes") - - opt, _ := options.New(map[string]interface{}{ - "root": rootPath, - }) - log := zerolog.Nop() - ignorer = ignore.NewIgnorer(opt, &log) - - _, err := os.Stat(indexesPath) - if err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("consistency check failed: '%s' is not a posixfs root", rootPath) - } - return fmt.Errorf("error accessing '%s': %w", indexesPath, err) - } - - spinnerCfg := yacspin.Config{ - Frequency: 100 * time.Millisecond, - CharSet: yacspin.CharSets[11], - StopCharacter: "✓", - StopColors: []string{"fgGreen"}, - StopFailCharacter: "✗", - StopFailColors: []string{"fgRed"}, - } - - spinner, err = yacspin.New(spinnerCfg) - err = spinner.Start() - if err != nil { - return fmt.Errorf("error creating spinner: %w", err) - } - - checkSpaces(filepath.Join(rootPath, "users")) - spinner.Suffix(" Personal spaces check ") - spinner.StopMessage("completed\n") - spinner.Stop() - - checkSpaces(filepath.Join(rootPath, "projects")) - spinner.Suffix(" Project spaces check ") - spinner.StopMessage("completed") - spinner.Stop() - - if restartRequired { - fmt.Println("\n\n ⚠️ Please restart your openCloud instance to apply changes.") - } - return nil -} - -func checkSpaces(basePath string) { - dirEntries, err := os.ReadDir(basePath) - if err != nil { - spinner.Message(fmt.Sprintf("Error reading spaces directory '%s'\n", basePath)) - spinner.StopFail() - return - } - - for _, entry := range dirEntries { - if entry.IsDir() { - fullPath := filepath.Join(basePath, entry.Name()) - checkSpace(fullPath) - } - } -} - -func checkSpace(spacePath string) { - spinner.Message("") - spinner.Suffix(fmt.Sprintf(" Checking space '%s'", spacePath)) - - info, err := os.Stat(spacePath) - if err != nil { - logFailure("Error accessing path '%s': %v", spacePath, err) - return - } - if !info.IsDir() { - logFailure("Error: The provided path '%s' is not a directory\n", spacePath) - return - } - - spaceID, err := xattr.Get(spacePath, spaceIDAttrName) - if err != nil || len(spaceID) == 0 { - logFailure("Error: The directory '%s' does not seem to be a space root, it's missing the '%s' attribute\n", spacePath, spaceIDAttrName) - return - } - - checkSpaceID(spacePath) - checkNodeIDs(spacePath) -} - -func checkSpaceID(spacePath string) { - spinner.Message(" - checking space ID uniqueness") - - entries, uniqueIDs, oldestEntry, err := gatherAttributes(spacePath) - if err != nil { - logFailure("Failed to gather attributes: %v", err) - return - } - - if len(entries) == 0 { - return - } - - if len(uniqueIDs) > 1 { - spinner.Pause() - fmt.Println("\n ⚠ Multiple space IDs found:") - for id := range uniqueIDs { - fmt.Printf(" - %s\n", id) - } - - fmt.Printf("\n ⏳ Oldest entry is '%s' (modified on %s).\n", - filepath.Base(oldestEntry.Path), oldestEntry.ModTime.Format(time.RFC1123)) - - targetID := oldestEntry.ParentID - fmt.Printf(" ✅ Proposed target Parent ID: %s\n", targetID) - - fmt.Printf("\n Do you want to unify all parent IDs to '%s'? This will modify %d entries, the directory, and the user index. (y/N): ", targetID, len(entries)) - - reader := bufio.NewReader(os.Stdin) - input, _ := reader.ReadString('\n') - input = strings.TrimSpace(strings.ToLower(input)) - - if input != "y" { - spinner.Unpause() - logFailure("Operation cancelled by user.") - return - } - restartRequired = true - - obsoleteIDs := []string{} - for id := range uniqueIDs { - if id != targetID { - obsoleteIDs = append(obsoleteIDs, id) - } - } - fixSpaceID(spacePath, obsoleteIDs, targetID, entries) - spinner.Unpause() - } -} - -func walkNodes(dir string, parentID string) int { - fixes := 0 - entries, err := os.ReadDir(dir) - if err != nil { - logFailure("Error reading directory '%s': %v", dir, err) - return 0 - } - - for _, entry := range entries { - fullPath := filepath.Join(dir, entry.Name()) - - if ignorer.IsIgnored(fullPath) { - continue - } - - // Check if the parent ID attribute matches the expected parent ID, if not, fix it. - actualParentID, err := xattr.Get(fullPath, parentIDAttrName) - if err != nil || string(actualParentID) != parentID { - err = xattr.Set(fullPath, parentIDAttrName, []byte(parentID)) - if err != nil { - logFailure("Failed to fix parent ID for '%s': %v", fullPath, err) - } else { - spinner.Pause() - fmt.Printf(" + Fixed parent ID for '%s'", fullPath) - spinner.Unpause() - fixes++ - restartRequired = true - } - } - - // Check that the name attribute matches the actual name of the file/directory, if not, fix it. - nameAttr, err := xattr.Get(fullPath, nameAttrName) - if err != nil || string(nameAttr) != entry.Name() { - err = xattr.Set(fullPath, nameAttrName, []byte(entry.Name())) - if err != nil { - logFailure("Failed to fix name attribute for '%s': %v", fullPath, err) - } else { - spinner.Pause() - fmt.Printf(" + Fixed name attribute for '%s'", fullPath) - spinner.Unpause() - fixes++ - restartRequired = true - } - } - - if entry.IsDir() { - nodeID, err := xattr.Get(fullPath, idAttrName) - if err != nil || len(nodeID) == 0 { - logFailure("Directory '%s' missing '%s', skipping its children", fullPath, idAttrName) - continue - } - walkNodes(fullPath, string(nodeID)) - } - } - return fixes -} - -func checkNodeIDs(spacePath string) { - spinner.Message(" - checking nodes") - - rootID, err := xattr.Get(spacePath, idAttrName) - if err != nil || len(rootID) == 0 { - logFailure("Space root '%s' missing '%s' attribute", spacePath, idAttrName) - return - } - - fixes := walkNodes(spacePath, string(rootID)) - - if fixes > 0 { - spinner.Pause() - fmt.Printf("\n ✓ Fixed %d incorrect node attributes in %s\n", fixes, filepath.Base(spacePath)) - spinner.Unpause() - } -} - -func fixSpaceID(spacePath string, obsoleteIDs []string, targetID string, entries []EntryInfo) { - // Set all parentid attributes to the proper space ID - err := setAllParentIDAttributes(entries, targetID) - if err != nil { - logFailure("an error occurred during file attribute update: %v", err) - return - } - - // Update space ID itself - fmt.Printf(" Updating directory '%s' with attribute '%s' -> %s\n", filepath.Base(spacePath), idAttrName, targetID) - err = xattr.Set(spacePath, idAttrName, []byte(targetID)) - if err != nil { - logFailure("Failed to set attribute on directory '%s': %v", spacePath, err) - return - } - err = xattr.Set(spacePath, spaceIDAttrName, []byte(targetID)) - if err != nil { - logFailure("Failed to set attribute on directory '%s': %v", spacePath, err) - return - } - - // update the index - err = updateOwnerIndexFile(spacePath, obsoleteIDs) - if err != nil { - logFailure("Could not update the owner index file: %v", err) - } -} - -func gatherAttributes(path string) ([]EntryInfo, map[string]struct{}, EntryInfo, error) { - dirEntries, err := os.ReadDir(path) - if err != nil { - return nil, nil, EntryInfo{}, fmt.Errorf("failed to read directory: %w", err) - } - - var allEntries []EntryInfo - uniqueIDs := make(map[string]struct{}) - var oldestEntry EntryInfo - oldestTime := time.Now().Add(100 * 365 * 24 * time.Hour) // Set to a future date to find the oldest entry - - for _, entry := range dirEntries { - fullPath := filepath.Join(path, entry.Name()) - if ignorer.IsIgnored(fullPath) { - continue - } - info, err := os.Stat(fullPath) - if err != nil { - fmt.Printf(" - Warning: could not stat %s: %v\n", entry.Name(), err) - continue - } - - parentID, err := xattr.Get(fullPath, parentIDAttrName) - if err != nil { - continue // Skip if attribute doesn't exist or can't be read - } - - entryInfo := EntryInfo{ - Path: fullPath, - ModTime: info.ModTime(), - ParentID: string(parentID), - } - - allEntries = append(allEntries, entryInfo) - uniqueIDs[string(parentID)] = struct{}{} - - if entryInfo.ModTime.Before(oldestTime) { - oldestTime = entryInfo.ModTime - oldestEntry = entryInfo - } - } - - return allEntries, uniqueIDs, oldestEntry, nil -} - -func setAllParentIDAttributes(entries []EntryInfo, targetID string) error { - fmt.Printf(" Setting all parent IDs to '%s':\n", targetID) - - for _, entry := range entries { - if entry.ParentID == targetID { - fmt.Printf(" - Skipping '%s' (already has target ID).\n", filepath.Base(entry.Path)) - continue - } - - fmt.Printf(" - Removing all attributes from '%s'. It will be re-assimilated\n", filepath.Base(entry.Path)) - filepath.WalkDir(entry.Path, func(path string, d os.DirEntry, err error) error { - if err != nil { - return fmt.Errorf("error walking path '%s': %w", path, err) - } - - // Remove all attributes from the file. - if err := removeAttributes(path); err != nil { - fmt.Printf("failed to remove attributes from '%s': %v", path, err) - } - return nil - }) - } - return nil -} - -// updateOwnerIndexFile handles the logic of reading, modifying, and writing the MessagePack index file. -func updateOwnerIndexFile(basePath string, obsoleteIDs []string) error { - fmt.Printf(" Rewriting index file '%s'\n", basePath) - - ownerID, err := xattr.Get(basePath, ownerIDAttrName) - if err != nil { - return fmt.Errorf("could not get owner ID from oldest entry '%s' to find index: %w", basePath, err) - } - - indexPath := filepath.Join(basePath, "../../indexes/by-user-id", string(ownerID)+".mpk") - indexPath = filepath.Clean(indexPath) - - // Read the MessagePack file - fileData, err := os.ReadFile(indexPath) - if err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("index file does not exist, skipping update") - } - return fmt.Errorf("could not read index file: %w", err) - } - var indexMap map[string]string - if err := msgpack.Unmarshal(fileData, &indexMap); err != nil { - return fmt.Errorf("failed to parse MessagePack index file (is it corrupt?): %w", err) - } - - // Remove obsolete IDs from the map - itemsRemoved := 0 - for _, id := range obsoleteIDs { - if _, exists := indexMap[id]; exists { - fmt.Printf(" - Removing obsolete ID '%s' from index.\n", id) - delete(indexMap, id) - itemsRemoved++ - } else { - fmt.Printf(" - Obsolete ID '%s' not found in index\n", id) - } - } - - if itemsRemoved == 0 { - return nil - } - - // Write the data back to the file - updatedData, err := msgpack.Marshal(&indexMap) - if err != nil { - return fmt.Errorf("failed to marshal updated index map: %w", err) - } - if err := os.WriteFile(indexPath, updatedData, 0644); err != nil { - return fmt.Errorf("failed to write updated index file: %w", err) - } - - fmt.Printf(" ✓ Successfully removed %d item(s) and saved index file.\n", itemsRemoved) - return nil -} - -func removeAttributes(path string) error { - attrNames, err := xattr.List(path) - if err != nil { - return fmt.Errorf("failed to list attributes for '%s': %w", path, err) - } - - for _, attrName := range attrNames { - if err := xattr.Remove(path, attrName); err != nil { - return fmt.Errorf("failed to remove attribute '%s' from '%s': %w", attrName, path, err) - } - } - return nil -} - func logFailure(message string, args ...any) { - spinner.StopFailMessage(fmt.Sprintf("\n"+message, args...)) - spinner.StopFail() - spinner.Start() + fmt.Fprintf(os.Stderr, message+"\n", args...) +} + +// findStorageRoot walks up the directory tree starting at path until it finds a +// directory that contains an "indexes" subdirectory which marks the root of a +// posixfs storage. A user directory inside a space might also be named "indexes", +// so to disambiguate we require that the "indexes" directory is an internal +// directory: the storage's own indexes directory is skipped during assimilation +// and therefore never receives a node ID attribute, whereas a regular user +// directory would have one. +func findStorageRoot(path string) (string, error) { + current := path + for { + indexesPath := filepath.Join(current, "indexes") + if info, err := os.Stat(indexesPath); err == nil && info.IsDir() { + if id, err := xattr.Get(indexesPath, prefixes.IDAttr); err != nil || len(id) == 0 { + return current, nil + } + } + + parent := filepath.Dir(current) + if parent == current { + return "", fmt.Errorf("'%s' does not appear to be inside a posixfs storage (no 'indexes' directory found)", path) + } + current = parent + } +} + +// isSpaceRoot reports whether the given path is a space root, which is +// identified by the presence of the space ID attribute. +func isSpaceRoot(path string) bool { + spaceID, err := xattr.Get(path, prefixes.SpaceIDAttr) + return err == nil && len(spaceID) > 0 } diff --git a/opencloud/pkg/command/posixfs_consistency.go b/opencloud/pkg/command/posixfs_consistency.go new file mode 100644 index 0000000000..f822143bbc --- /dev/null +++ b/opencloud/pkg/command/posixfs_consistency.go @@ -0,0 +1,494 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package command + +import ( + "bufio" + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/opencloud-eu/opencloud/pkg/x/path/filepathx" + storageUsersConfig "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config" + "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/ignore" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" + "github.com/pkg/xattr" + "github.com/shamaton/msgpack/v2" +) + +type consistencyChecker struct { + cfg *storageUsersConfig.Config + ignorer *ignore.Ignorer + recalculateChecksums bool + + restartRequired bool +} + +// checkPosixfsConsistency checks the consistency of the posixfs storage. The +// given path determines the scope of the check: the whole storage, a single +// space or a single entity within a space. +func (c *consistencyChecker) Check(paths []string) error { + for _, path := range paths { + rootPath, err := findStorageRoot(path) + if err != nil { + return err + } + path = filepath.Clean(path) + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("error accessing '%s': %w", path, err) + } + contained, _ := filepathx.IsSameOrContainedBy(rootPath, path) + + switch { + case path == rootPath: + fmt.Println("Checking personal spaces...") + c.checkSpaces(filepath.Join(path, "users")) + + fmt.Println("Checking project spaces...") + c.checkSpaces(filepath.Join(path, "projects")) + case isSpaceRoot(path): + fmt.Printf("Checking space '%s'...\n", path) + c.checkSpace(path) + case contained: + fmt.Printf("Checking '%s'...\n", path) + c.checkEntity(path) + default: + return fmt.Errorf("the provided path '%s' is neither a space root nor contained by the storage root '%s'", path, rootPath) + } + } + + if c.restartRequired { + fmt.Println("\n\n ⚠️ Please restart your openCloud instance to apply changes.") + } + return nil +} + +func (c *consistencyChecker) checkSpaces(basePath string) { + dirEntries, err := os.ReadDir(basePath) + if err != nil { + logFailure("Error reading spaces directory '%s': %v", basePath, err) + return + } + + for _, entry := range dirEntries { + if entry.IsDir() { + fullPath := filepath.Join(basePath, entry.Name()) + c.checkSpace(fullPath) + } + } +} + +func (c *consistencyChecker) checkSpace(spacePath string) { + info, err := os.Stat(spacePath) + if err != nil { + logFailure("Error accessing path '%s': %v", spacePath, err) + return + } + if !info.IsDir() { + logFailure("Error: The provided path '%s' is not a directory\n", spacePath) + return + } + + spaceID, err := xattr.Get(spacePath, prefixes.SpaceIDAttr) + if err != nil || len(spaceID) == 0 { + logFailure("Error: The directory '%s' does not seem to be a space root, it's missing the '%s' attribute\n", spacePath, prefixes.SpaceIDAttr) + return + } + + c.checkSpaceID(spacePath) + c.checkNodes(spacePath) +} + +func (c *consistencyChecker) checkSpaceID(spacePath string) { + entries, uniqueIDs, oldestEntry, err := c.gatherAttributes(spacePath) + if err != nil { + logFailure("Failed to gather attributes: %v", err) + return + } + + if len(entries) == 0 { + return + } + + if len(uniqueIDs) > 1 { + fmt.Println("\n ⚠ Multiple space IDs found:") + for id := range uniqueIDs { + fmt.Printf(" - %s\n", id) + } + + fmt.Printf("\n ⏳ Oldest entry is '%s' (modified on %s).\n", + filepath.Base(oldestEntry.Path), oldestEntry.ModTime.Format(time.RFC1123)) + + targetID := oldestEntry.ParentID + fmt.Printf(" ✅ Proposed target Parent ID: %s\n", targetID) + + fmt.Printf("\n Do you want to unify all parent IDs to '%s'? This will modify %d entries, the directory, and the user index. (y/N): ", targetID, len(entries)) + + reader := bufio.NewReader(os.Stdin) + input, _ := reader.ReadString('\n') + input = strings.TrimSpace(strings.ToLower(input)) + + if input != "y" { + logFailure("Operation cancelled by user.") + return + } + c.restartRequired = true + + obsoleteIDs := []string{} + for id := range uniqueIDs { + if id != targetID { + obsoleteIDs = append(obsoleteIDs, id) + } + } + c.fixSpaceID(spacePath, obsoleteIDs, targetID, entries) + } +} + +func (c *consistencyChecker) walkNodes(dir string, parentID string) int { + fixes := 0 + entries, err := os.ReadDir(dir) + if err != nil { + logFailure("Error reading directory '%s': %v", dir, err) + return 0 + } + + for _, entry := range entries { + fullPath := filepath.Join(dir, entry.Name()) + + if c.ignorer.IsIgnored(fullPath) { + continue + } + + fixes += c.checkNodeAttributes(fullPath, entry.Name(), parentID, entry.IsDir()) + + if entry.IsDir() { + nodeID, err := xattr.Get(fullPath, prefixes.IDAttr) + if err != nil || len(nodeID) == 0 { + logFailure("Directory '%s' missing '%s', skipping its children", fullPath, prefixes.IDAttr) + continue + } + fixes += c.walkNodes(fullPath, string(nodeID)) + } + } + return fixes +} + +// checkNodeAttributes checks and fixes the parent ID and name attributes of a +// single node. For files it additionally checks the blobsize and, when +// requested, the checksums. It returns the number of fixes applied. +func (c *consistencyChecker) checkNodeAttributes(path, name, parentID string, isDir bool) int { + fixes := 0 + + // Check if the parent ID attribute matches the expected parent ID, if not, fix it. + actualParentID, err := xattr.Get(path, prefixes.ParentidAttr) + if err != nil || string(actualParentID) != parentID { + if err := xattr.Set(path, prefixes.ParentidAttr, []byte(parentID)); err != nil { + logFailure("Failed to fix parent ID for '%s': %v", path, err) + } else { + fmt.Printf(" + Fixed parent ID for '%s'\n", path) + fixes++ + c.restartRequired = true + } + } + + // Check that the name attribute matches the actual name of the file/directory, if not, fix it. + nameAttr, err := xattr.Get(path, prefixes.NameAttr) + if err != nil || string(nameAttr) != name { + if err := xattr.Set(path, prefixes.NameAttr, []byte(name)); err != nil { + logFailure("Failed to fix name attribute for '%s': %v", path, err) + } else { + fmt.Printf(" + Fixed name attribute for '%s'\n", path) + fixes++ + c.restartRequired = true + } + } + + if !isDir { + fixes += c.checkBlobsize(path) + if c.recalculateChecksums { + fixes += c.fixChecksums(path) + } + } + + return fixes +} + +// checkEntity checks a single file or directory within a space, including its own +// parent ID, name and (for files) blobsize/checksums. If the entity is a directory +// its children are checked recursively. +func (c *consistencyChecker) checkEntity(path string) { + info, err := os.Stat(path) + if err != nil { + logFailure("Error accessing path '%s': %v", path, err) + return + } + + // The expected parent ID is the ID attribute of the containing directory. + parentDir := filepath.Dir(path) + parentID, err := xattr.Get(parentDir, prefixes.IDAttr) + if err != nil || len(parentID) == 0 { + logFailure("Parent directory '%s' is missing the '%s' attribute", parentDir, prefixes.IDAttr) + return + } + + fixes := c.checkNodeAttributes(path, info.Name(), string(parentID), info.IsDir()) + + if info.IsDir() { + nodeID, err := xattr.Get(path, prefixes.IDAttr) + if err != nil || len(nodeID) == 0 { + logFailure("Directory '%s' missing '%s' attribute", path, prefixes.IDAttr) + } else { + fixes += c.walkNodes(path, string(nodeID)) + } + } + + if fixes > 0 { + fmt.Printf(" ✓ Fixed %d incorrect node attributes for %s\n", fixes, filepath.Base(path)) + } +} + +// checkBlobsize verifies that the stored blobsize attribute matches the actual +// file size and fixes it if it doesn't. It returns the number of fixes applied. +func (c *consistencyChecker) checkBlobsize(path string) int { + info, err := os.Stat(path) + if err != nil { + logFailure("Error accessing file '%s': %v", path, err) + return 0 + } + + expectedSize := strconv.FormatInt(info.Size(), 10) + blobsize, err := xattr.Get(path, prefixes.BlobsizeAttr) + if err == nil && string(blobsize) == expectedSize { + return 0 + } + + if err := xattr.Set(path, prefixes.BlobsizeAttr, []byte(expectedSize)); err != nil { + logFailure("Failed to fix blobsize for '%s': %v", path, err) + return 0 + } + + fmt.Printf(" + Fixed blobsize for '%s'\n", path) + c.restartRequired = true + return 1 +} + +// fixChecksums recalculates the sha1, md5 and adler32 checksums of the file and +// updates the stored attributes if they differ. It returns the number of fixes applied. +func (c *consistencyChecker) fixChecksums(path string) int { + sha1h, md5h, adler32h, err := node.CalculateChecksums(context.Background(), path) + if err != nil { + logFailure("Failed to calculate checksums for '%s': %v", path, err) + return 0 + } + + checksums := map[string][]byte{ + prefixes.ChecksumPrefix + "sha1": sha1h.Sum(nil), + prefixes.ChecksumPrefix + "md5": md5h.Sum(nil), + prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil), + } + + fixes := 0 + for attrName, sum := range checksums { + current, err := xattr.Get(path, attrName) + if err == nil && bytes.Equal(current, sum) { + continue + } + if err := xattr.Set(path, attrName, sum); err != nil { + logFailure("Failed to fix checksum '%s' for '%s': %v", attrName, path, err) + continue + } + fmt.Printf(" + Fixed checksum '%s' for '%s'\n", attrName, path) + c.restartRequired = true + fixes++ + } + return fixes +} + +func (c *consistencyChecker) checkNodes(spacePath string) { + rootID, err := xattr.Get(spacePath, prefixes.IDAttr) + if err != nil || len(rootID) == 0 { + logFailure("Space root '%s' missing '%s' attribute", spacePath, prefixes.IDAttr) + return + } + + fixes := c.walkNodes(spacePath, string(rootID)) + + if fixes > 0 { + fmt.Printf(" ✓ Fixed %d incorrect node attributes in %s\n", fixes, filepath.Base(spacePath)) + } +} + +// fixSpaceID updates the parentid attributes of all entries in a space to a new target ID, +// updates the space's own ID attributes, and removes obsolete IDs from the user index file. +func (c *consistencyChecker) fixSpaceID(spacePath string, obsoleteIDs []string, targetID string, entries []EntryInfo) { + // Set all parentid attributes to the proper space ID + err := setAllParentIDAttributes(entries, targetID) + if err != nil { + logFailure("an error occurred during file attribute update: %v", err) + return + } + + // Update space ID itself + fmt.Printf(" Updating directory '%s' with attribute '%s' -> %s\n", filepath.Base(spacePath), prefixes.IDAttr, targetID) + err = xattr.Set(spacePath, prefixes.IDAttr, []byte(targetID)) + if err != nil { + logFailure("Failed to set attribute on directory '%s': %v", spacePath, err) + return + } + err = xattr.Set(spacePath, prefixes.SpaceIDAttr, []byte(targetID)) + if err != nil { + logFailure("Failed to set attribute on directory '%s': %v", spacePath, err) + return + } + + // update the index + err = c.updateOwnerIndexFile(spacePath, obsoleteIDs) + if err != nil { + logFailure("Could not update the owner index file: %v", err) + } +} + +func (c *consistencyChecker) gatherAttributes(path string) ([]EntryInfo, map[string]struct{}, EntryInfo, error) { + dirEntries, err := os.ReadDir(path) + if err != nil { + return nil, nil, EntryInfo{}, fmt.Errorf("failed to read directory: %w", err) + } + + var allEntries []EntryInfo + uniqueIDs := make(map[string]struct{}) + var oldestEntry EntryInfo + oldestTime := time.Now().Add(100 * 365 * 24 * time.Hour) // Set to a future date to find the oldest entry + + for _, entry := range dirEntries { + fullPath := filepath.Join(path, entry.Name()) + if c.ignorer.IsIgnored(fullPath) { + continue + } + info, err := os.Stat(fullPath) + if err != nil { + fmt.Printf(" - Warning: could not stat %s: %v\n", entry.Name(), err) + continue + } + + parentID, err := xattr.Get(fullPath, prefixes.ParentidAttr) + if err != nil { + continue // Skip if attribute doesn't exist or can't be read + } + + entryInfo := EntryInfo{ + Path: fullPath, + ModTime: info.ModTime(), + ParentID: string(parentID), + } + + allEntries = append(allEntries, entryInfo) + uniqueIDs[string(parentID)] = struct{}{} + + if entryInfo.ModTime.Before(oldestTime) { + oldestTime = entryInfo.ModTime + oldestEntry = entryInfo + } + } + + return allEntries, uniqueIDs, oldestEntry, nil +} + +func setAllParentIDAttributes(entries []EntryInfo, targetID string) error { + fmt.Printf(" Setting all parent IDs to '%s':\n", targetID) + + for _, entry := range entries { + if entry.ParentID == targetID { + fmt.Printf(" - Skipping '%s' (already has target ID).\n", filepath.Base(entry.Path)) + continue + } + + fmt.Printf(" - Removing all attributes from '%s'. It will be re-assimilated\n", filepath.Base(entry.Path)) + filepath.WalkDir(entry.Path, func(path string, d os.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("error walking path '%s': %w", path, err) + } + + // Remove all attributes from the file. + if err := removeAttributes(path); err != nil { + fmt.Printf("failed to remove attributes from '%s': %v", path, err) + } + return nil + }) + } + return nil +} + +// updateOwnerIndexFile handles the logic of reading, modifying, and writing the MessagePack index file. +func (c *consistencyChecker) updateOwnerIndexFile(basePath string, obsoleteIDs []string) error { + fmt.Printf(" Rewriting index file '%s'\n", basePath) + + ownerID, err := xattr.Get(basePath, prefixes.OwnerIDAttr) + if err != nil { + return fmt.Errorf("could not get owner ID from oldest entry '%s' to find index: %w", basePath, err) + } + + indexPath := filepath.Join(c.cfg.Drivers.Posix.Root, "indexes", "by-user-id", string(ownerID)+".mpk") + indexPath = filepath.Clean(indexPath) + + // Read the MessagePack file + fileData, err := os.ReadFile(indexPath) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("index file does not exist, skipping update") + } + return fmt.Errorf("could not read index file: %w", err) + } + var indexMap map[string]string + if err := msgpack.Unmarshal(fileData, &indexMap); err != nil { + return fmt.Errorf("failed to parse MessagePack index file (is it corrupt?): %w", err) + } + + // Remove obsolete IDs from the map + itemsRemoved := 0 + for _, id := range obsoleteIDs { + if _, exists := indexMap[id]; exists { + fmt.Printf(" - Removing obsolete ID '%s' from index.\n", id) + delete(indexMap, id) + itemsRemoved++ + } else { + fmt.Printf(" - Obsolete ID '%s' not found in index\n", id) + } + } + + if itemsRemoved == 0 { + return nil + } + + // Write the data back to the file + updatedData, err := msgpack.Marshal(&indexMap) + if err != nil { + return fmt.Errorf("failed to marshal updated index map: %w", err) + } + if err := os.WriteFile(indexPath, updatedData, 0644); err != nil { + return fmt.Errorf("failed to write updated index file: %w", err) + } + + fmt.Printf(" ✓ Successfully removed %d item(s) and saved index file.\n", itemsRemoved) + return nil +} + +func removeAttributes(path string) error { + attrNames, err := xattr.List(path) + if err != nil { + return fmt.Errorf("failed to list attributes for '%s': %w", path, err) + } + + for _, attrName := range attrNames { + if err := xattr.Remove(path, attrName); err != nil { + return fmt.Errorf("failed to remove attribute '%s' from '%s': %w", attrName, path, err) + } + } + return nil +} diff --git a/opencloud/pkg/command/root.go b/opencloud/pkg/command/root.go index 834bc68e02..01ff6a77a7 100644 --- a/opencloud/pkg/command/root.go +++ b/opencloud/pkg/command/root.go @@ -6,11 +6,13 @@ import ( "os/signal" "syscall" + "github.com/rs/zerolog" + "github.com/spf13/cobra" + "github.com/opencloud-eu/opencloud/opencloud/pkg/register" "github.com/opencloud-eu/opencloud/pkg/clihelper" "github.com/opencloud-eu/opencloud/pkg/config" - - "github.com/spf13/cobra" + oclog "github.com/opencloud-eu/opencloud/pkg/log" ) // Execute is the entry point for the opencloud command. @@ -38,3 +40,11 @@ func Execute() error { ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP) return app.ExecuteContext(ctx) } + +func logger(name string) zerolog.Logger { + return oclog.NewLogger( + oclog.Name(name), + oclog.Level("info"), + oclog.Pretty(true), + oclog.Color(true)).Logger +} diff --git a/opencloud/pkg/command/shares.go b/opencloud/pkg/command/shares.go index 98cee2da4a..0cdfeea5d1 100644 --- a/opencloud/pkg/command/shares.go +++ b/opencloud/pkg/command/shares.go @@ -9,7 +9,6 @@ import ( "github.com/opencloud-eu/opencloud/pkg/config" "github.com/opencloud-eu/opencloud/pkg/config/configlog" "github.com/opencloud-eu/opencloud/pkg/config/parser" - oclog "github.com/opencloud-eu/opencloud/pkg/log" mregistry "github.com/opencloud-eu/opencloud/pkg/registry" sharing "github.com/opencloud-eu/opencloud/services/sharing/pkg/config" sharingparser "github.com/opencloud-eu/opencloud/services/sharing/pkg/config/parser" @@ -85,7 +84,7 @@ func cleanup(_ *cobra.Command, cfg *config.Config) error { return configlog.ReturnError(errors.New("cleanup is only implemented for the jsoncs3 share manager")) } - l := logger() + l := logger("migrate") zerolog.SetGlobalLevel(zerolog.InfoLevel) @@ -94,7 +93,7 @@ func cleanup(_ *cobra.Command, cfg *config.Config) error { if !ok { return configlog.ReturnError(errors.New("Unknown share manager type '" + driver + "'")) } - mgr, err := f(rcfg[driver].(map[string]any), l) + mgr, err := f(rcfg[driver].(map[string]any), &l) if err != nil { return configlog.ReturnError(err) } @@ -167,12 +166,3 @@ func revaShareConfig(cfg *sharing.Config) map[string]any { }, } } - -func logger() *zerolog.Logger { - log := oclog.NewLogger( - oclog.Name("migrate"), - oclog.Level("info"), - oclog.Pretty(true), - oclog.Color(true)).Logger - return &log -}