From 3bd188122ec43c49f33de94a8353870480451b68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 29 Jul 2026 11:21:55 +0200 Subject: [PATCH 01/10] Simplify output, remove spinner lib --- opencloud/pkg/command/posixfs.go | 52 ++++---------------------------- 1 file changed, 6 insertions(+), 46 deletions(-) diff --git a/opencloud/pkg/command/posixfs.go b/opencloud/pkg/command/posixfs.go index e38a400afb..00cd687bee 100644 --- a/opencloud/pkg/command/posixfs.go +++ b/opencloud/pkg/command/posixfs.go @@ -25,7 +25,6 @@ import ( "github.com/pkg/xattr" "github.com/rs/zerolog" "github.com/spf13/cobra" - "github.com/theckman/yacspin" "github.com/vmihailenco/msgpack/v5" ) @@ -39,7 +38,6 @@ const ( ) var ( - spinner *yacspin.Spinner restartRequired = false ignorer *ignore.Ignorer ) @@ -221,30 +219,11 @@ func checkPosixfsConsistency(cmd *cobra.Command, cfg *config.Config) error { return fmt.Errorf("error accessing '%s': %w", indexesPath, err) } - spinnerCfg := yacspin.Config{ - Frequency: 100 * time.Millisecond, - CharSet: yacspin.CharSets[11], - StopCharacter: "✓", - StopColors: []string{"fgGreen"}, - StopFailCharacter: "✗", - StopFailColors: []string{"fgRed"}, - } - - spinner, err = yacspin.New(spinnerCfg) - err = spinner.Start() - if err != nil { - return fmt.Errorf("error creating spinner: %w", err) - } - + fmt.Println("Checking personal spaces...") checkSpaces(filepath.Join(rootPath, "users")) - spinner.Suffix(" Personal spaces check ") - spinner.StopMessage("completed\n") - spinner.Stop() + fmt.Println("Checking project spaces...") checkSpaces(filepath.Join(rootPath, "projects")) - spinner.Suffix(" Project spaces check ") - spinner.StopMessage("completed") - spinner.Stop() if restartRequired { fmt.Println("\n\n ⚠️ Please restart your openCloud instance to apply changes.") @@ -255,8 +234,7 @@ func checkPosixfsConsistency(cmd *cobra.Command, cfg *config.Config) error { func checkSpaces(basePath string) { dirEntries, err := os.ReadDir(basePath) if err != nil { - spinner.Message(fmt.Sprintf("Error reading spaces directory '%s'\n", basePath)) - spinner.StopFail() + logFailure("Error reading spaces directory '%s': %v", basePath, err) return } @@ -269,9 +247,6 @@ func checkSpaces(basePath string) { } func checkSpace(spacePath string) { - spinner.Message("") - spinner.Suffix(fmt.Sprintf(" Checking space '%s'", spacePath)) - info, err := os.Stat(spacePath) if err != nil { logFailure("Error accessing path '%s': %v", spacePath, err) @@ -289,12 +264,10 @@ func checkSpace(spacePath string) { } checkSpaceID(spacePath) - checkNodeIDs(spacePath) + checkNodes(spacePath) } func checkSpaceID(spacePath string) { - spinner.Message(" - checking space ID uniqueness") - entries, uniqueIDs, oldestEntry, err := gatherAttributes(spacePath) if err != nil { logFailure("Failed to gather attributes: %v", err) @@ -306,7 +279,6 @@ func checkSpaceID(spacePath string) { } if len(uniqueIDs) > 1 { - spinner.Pause() fmt.Println("\n ⚠ Multiple space IDs found:") for id := range uniqueIDs { fmt.Printf(" - %s\n", id) @@ -325,7 +297,6 @@ func checkSpaceID(spacePath string) { input = strings.TrimSpace(strings.ToLower(input)) if input != "y" { - spinner.Unpause() logFailure("Operation cancelled by user.") return } @@ -338,7 +309,6 @@ func checkSpaceID(spacePath string) { } } fixSpaceID(spacePath, obsoleteIDs, targetID, entries) - spinner.Unpause() } } @@ -364,9 +334,7 @@ func walkNodes(dir string, parentID string) int { if err != nil { logFailure("Failed to fix parent ID for '%s': %v", fullPath, err) } else { - spinner.Pause() fmt.Printf(" + Fixed parent ID for '%s'", fullPath) - spinner.Unpause() fixes++ restartRequired = true } @@ -379,9 +347,7 @@ func walkNodes(dir string, parentID string) int { if err != nil { logFailure("Failed to fix name attribute for '%s': %v", fullPath, err) } else { - spinner.Pause() fmt.Printf(" + Fixed name attribute for '%s'", fullPath) - spinner.Unpause() fixes++ restartRequired = true } @@ -399,9 +365,7 @@ func walkNodes(dir string, parentID string) int { return fixes } -func checkNodeIDs(spacePath string) { - spinner.Message(" - checking nodes") - +func checkNodes(spacePath string) { rootID, err := xattr.Get(spacePath, idAttrName) if err != nil || len(rootID) == 0 { logFailure("Space root '%s' missing '%s' attribute", spacePath, idAttrName) @@ -411,9 +375,7 @@ func checkNodeIDs(spacePath string) { fixes := walkNodes(spacePath, string(rootID)) if fixes > 0 { - spinner.Pause() fmt.Printf("\n ✓ Fixed %d incorrect node attributes in %s\n", fixes, filepath.Base(spacePath)) - spinner.Unpause() } } @@ -584,7 +546,5 @@ func removeAttributes(path string) error { } func logFailure(message string, args ...any) { - spinner.StopFailMessage(fmt.Sprintf("\n"+message, args...)) - spinner.StopFail() - spinner.Start() + fmt.Fprintf(os.Stderr, message+"\n", args...) } From 40ced9997ecd482eb13b63997c15f2df4999400b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Wed, 29 Jul 2026 12:21:43 +0200 Subject: [PATCH 02/10] Also check blobsize and checksums (when --fix-checkums is set) --- opencloud/pkg/command/posixfs.go | 115 +++++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 27 deletions(-) diff --git a/opencloud/pkg/command/posixfs.go b/opencloud/pkg/command/posixfs.go index 00cd687bee..1150a8c473 100644 --- a/opencloud/pkg/command/posixfs.go +++ b/opencloud/pkg/command/posixfs.go @@ -2,9 +2,12 @@ package command import ( "bufio" + "bytes" + "context" "fmt" "os" "path/filepath" + "strconv" "strings" "time" @@ -21,6 +24,8 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/ignore" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" "github.com/pkg/xattr" "github.com/rs/zerolog" @@ -28,18 +33,10 @@ import ( "github.com/vmihailenco/msgpack/v5" ) -// Define the names of the extended attributes we are working with. -const ( - parentIDAttrName = "user.oc.parentid" - idAttrName = "user.oc.id" - nameAttrName = "user.oc.name" - spaceIDAttrName = "user.oc.space.id" - ownerIDAttrName = "user.oc.owner.id" -) - var ( - restartRequired = false - ignorer *ignore.Ignorer + restartRequired = false + recalculateChecksums = false + ignorer *ignore.Ignorer ) type IDCacher interface { @@ -196,6 +193,7 @@ func consistencyCmd(cfg *config.Config) *cobra.Command { } consCmd.Flags().StringP("root", "r", "", "Path to the root directory of the posixfs storage") _ = consCmd.MarkFlagRequired("root") + consCmd.Flags().Bool("fix-checksums", false, "Recalculate and fix the file checksums. This reads every file and can be slow on large storages.") return consCmd } @@ -203,6 +201,7 @@ func consistencyCmd(cfg *config.Config) *cobra.Command { // checkPosixfsConsistency checks the consistency of the posixfs storage. func checkPosixfsConsistency(cmd *cobra.Command, cfg *config.Config) error { rootPath, _ := cmd.Flags().GetString("root") + recalculateChecksums, _ = cmd.Flags().GetBool("fix-checksums") indexesPath := filepath.Join(rootPath, "indexes") opt, _ := options.New(map[string]interface{}{ @@ -257,9 +256,9 @@ func checkSpace(spacePath string) { return } - spaceID, err := xattr.Get(spacePath, spaceIDAttrName) + spaceID, err := xattr.Get(spacePath, prefixes.SpaceIDAttr) if err != nil || len(spaceID) == 0 { - logFailure("Error: The directory '%s' does not seem to be a space root, it's missing the '%s' attribute\n", spacePath, spaceIDAttrName) + logFailure("Error: The directory '%s' does not seem to be a space root, it's missing the '%s' attribute\n", spacePath, prefixes.SpaceIDAttr) return } @@ -328,9 +327,9 @@ func walkNodes(dir string, parentID string) int { } // Check if the parent ID attribute matches the expected parent ID, if not, fix it. - actualParentID, err := xattr.Get(fullPath, parentIDAttrName) + actualParentID, err := xattr.Get(fullPath, prefixes.ParentidAttr) if err != nil || string(actualParentID) != parentID { - err = xattr.Set(fullPath, parentIDAttrName, []byte(parentID)) + err = xattr.Set(fullPath, prefixes.ParentidAttr, []byte(parentID)) if err != nil { logFailure("Failed to fix parent ID for '%s': %v", fullPath, err) } else { @@ -341,9 +340,9 @@ func walkNodes(dir string, parentID string) int { } // Check that the name attribute matches the actual name of the file/directory, if not, fix it. - nameAttr, err := xattr.Get(fullPath, nameAttrName) + nameAttr, err := xattr.Get(fullPath, prefixes.NameAttr) if err != nil || string(nameAttr) != entry.Name() { - err = xattr.Set(fullPath, nameAttrName, []byte(entry.Name())) + err = xattr.Set(fullPath, prefixes.NameAttr, []byte(entry.Name())) if err != nil { logFailure("Failed to fix name attribute for '%s': %v", fullPath, err) } else { @@ -354,21 +353,83 @@ func walkNodes(dir string, parentID string) int { } if entry.IsDir() { - nodeID, err := xattr.Get(fullPath, idAttrName) + nodeID, err := xattr.Get(fullPath, prefixes.IDAttr) if err != nil || len(nodeID) == 0 { - logFailure("Directory '%s' missing '%s', skipping its children", fullPath, idAttrName) + logFailure("Directory '%s' missing '%s', skipping its children", fullPath, prefixes.IDAttr) continue } - walkNodes(fullPath, string(nodeID)) + fixes += walkNodes(fullPath, string(nodeID)) + } else { + fixes += checkBlobsize(fullPath) + if recalculateChecksums { + fixes += fixChecksums(fullPath) + } } } return fixes } +// checkBlobsize verifies that the stored blobsize attribute matches the actual +// file size and fixes it if it doesn't. It returns the number of fixes applied. +func checkBlobsize(path string) int { + info, err := os.Stat(path) + if err != nil { + logFailure("Error accessing file '%s': %v", path, err) + return 0 + } + + expectedSize := strconv.FormatInt(info.Size(), 10) + blobsize, err := xattr.Get(path, prefixes.BlobsizeAttr) + if err == nil && string(blobsize) == expectedSize { + return 0 + } + + if err := xattr.Set(path, prefixes.BlobsizeAttr, []byte(expectedSize)); err != nil { + logFailure("Failed to fix blobsize for '%s': %v", path, err) + return 0 + } + + fmt.Printf(" + Fixed blobsize for '%s'\n", path) + restartRequired = true + return 1 +} + +// fixChecksums recalculates the sha1, md5 and adler32 checksums of the file and +// updates the stored attributes if they differ. It returns the number of fixes applied. +func fixChecksums(path string) int { + sha1h, md5h, adler32h, err := node.CalculateChecksums(context.Background(), path) + if err != nil { + logFailure("Failed to calculate checksums for '%s': %v", path, err) + return 0 + } + + checksums := map[string][]byte{ + prefixes.ChecksumPrefix + "sha1": sha1h.Sum(nil), + prefixes.ChecksumPrefix + "md5": md5h.Sum(nil), + prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil), + } + + fixes := 0 + for attrName, sum := range checksums { + current, err := xattr.Get(path, attrName) + if err == nil && bytes.Equal(current, sum) { + continue + } + if err := xattr.Set(path, attrName, sum); err != nil { + logFailure("Failed to fix checksum '%s' for '%s': %v", attrName, path, err) + continue + } + fmt.Printf(" + Fixed checksum '%s' for '%s'\n", attrName, path) + restartRequired = true + fixes++ + } + return fixes +} + func checkNodes(spacePath string) { - rootID, err := xattr.Get(spacePath, idAttrName) + rootID, err := xattr.Get(spacePath, prefixes.IDAttr) if err != nil || len(rootID) == 0 { - logFailure("Space root '%s' missing '%s' attribute", spacePath, idAttrName) + logFailure("Space root '%s' missing '%s' attribute", spacePath, prefixes.IDAttr) return } @@ -388,13 +449,13 @@ func fixSpaceID(spacePath string, obsoleteIDs []string, targetID string, entries } // Update space ID itself - fmt.Printf(" Updating directory '%s' with attribute '%s' -> %s\n", filepath.Base(spacePath), idAttrName, targetID) - err = xattr.Set(spacePath, idAttrName, []byte(targetID)) + fmt.Printf(" Updating directory '%s' with attribute '%s' -> %s\n", filepath.Base(spacePath), prefixes.IDAttr, targetID) + err = xattr.Set(spacePath, prefixes.IDAttr, []byte(targetID)) if err != nil { logFailure("Failed to set attribute on directory '%s': %v", spacePath, err) return } - err = xattr.Set(spacePath, spaceIDAttrName, []byte(targetID)) + err = xattr.Set(spacePath, prefixes.SpaceIDAttr, []byte(targetID)) if err != nil { logFailure("Failed to set attribute on directory '%s': %v", spacePath, err) return @@ -429,7 +490,7 @@ func gatherAttributes(path string) ([]EntryInfo, map[string]struct{}, EntryInfo, continue } - parentID, err := xattr.Get(fullPath, parentIDAttrName) + parentID, err := xattr.Get(fullPath, prefixes.ParentidAttr) if err != nil { continue // Skip if attribute doesn't exist or can't be read } @@ -481,7 +542,7 @@ func setAllParentIDAttributes(entries []EntryInfo, targetID string) error { func updateOwnerIndexFile(basePath string, obsoleteIDs []string) error { fmt.Printf(" Rewriting index file '%s'\n", basePath) - ownerID, err := xattr.Get(basePath, ownerIDAttrName) + ownerID, err := xattr.Get(basePath, prefixes.OwnerIDAttr) if err != nil { return fmt.Errorf("could not get owner ID from oldest entry '%s' to find index: %w", basePath, err) } From 3d546693c63fa1c0473bdce26620afb8e531379c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Thu, 30 Jul 2026 08:46:35 +0200 Subject: [PATCH 03/10] Allow for checking specific spaces and files as well --- opencloud/pkg/command/posixfs.go | 222 +++++++++++++++++++++++-------- 1 file changed, 165 insertions(+), 57 deletions(-) diff --git a/opencloud/pkg/command/posixfs.go b/opencloud/pkg/command/posixfs.go index 1150a8c473..fa049d8731 100644 --- a/opencloud/pkg/command/posixfs.go +++ b/opencloud/pkg/command/posixfs.go @@ -17,6 +17,7 @@ import ( "github.com/opencloud-eu/opencloud/pkg/config/parser" oclog "github.com/opencloud-eu/opencloud/pkg/log" "github.com/opencloud-eu/opencloud/pkg/x/path/filepathx" + storageUsersConfig "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config" storageUsersParser "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config/parser" "github.com/opencloud-eu/opencloud/services/storage-users/pkg/event" "github.com/opencloud-eu/opencloud/services/storage-users/pkg/revaconfig" @@ -28,7 +29,6 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" "github.com/pkg/xattr" - "github.com/rs/zerolog" "github.com/spf13/cobra" "github.com/vmihailenco/msgpack/v5" ) @@ -183,46 +183,77 @@ func scanCmd(ocCfg *config.Config) *cobra.Command { } // consistencyCmd returns a command to check the consistency of the posixfs storage. -func consistencyCmd(cfg *config.Config) *cobra.Command { +func consistencyCmd(ocCfg *config.Config) *cobra.Command { consCmd := &cobra.Command{ - Use: "consistency", + Use: "consistency ", Short: "check the consistency of the posixfs storage", + Long: `check the consistency of the posixfs storage. + +The argument determines the scope of the check: + - a storage root: the whole storage (all personal and project spaces) is checked + - a space root: only that space is checked + - a file or folder: only that single entity is checked (and its children, if it is a folder)`, + Args: cobra.ExactArgs(1), + PreRunE: func(cmd *cobra.Command, args []string) error { + + if err := parser.ParseConfig(ocCfg, true); err != nil { + return configlog.ReturnError(err) + } + + // Parse storage users config + ocCfg.StorageUsers.Commons = ocCfg.Commons + + return configlog.ReturnFatal(storageUsersParser.ParseConfig(ocCfg.StorageUsers)) + }, RunE: func(cmd *cobra.Command, args []string) error { - return checkPosixfsConsistency(cmd, cfg) + cfg := ocCfg.StorageUsers + return checkPosixfsConsistency(cfg, cmd, args[0]) }, } - consCmd.Flags().StringP("root", "r", "", "Path to the root directory of the posixfs storage") - _ = consCmd.MarkFlagRequired("root") consCmd.Flags().Bool("fix-checksums", false, "Recalculate and fix the file checksums. This reads every file and can be slow on large storages.") return consCmd } -// checkPosixfsConsistency checks the consistency of the posixfs storage. -func checkPosixfsConsistency(cmd *cobra.Command, cfg *config.Config) error { - rootPath, _ := cmd.Flags().GetString("root") +// checkPosixfsConsistency checks the consistency of the posixfs storage. The +// given path determines the scope of the check: the whole storage, a single +// space or a single entity within a space. +func checkPosixfsConsistency(cfg *storageUsersConfig.Config, cmd *cobra.Command, path string) error { recalculateChecksums, _ = cmd.Flags().GetBool("fix-checksums") - indexesPath := filepath.Join(rootPath, "indexes") - opt, _ := options.New(map[string]interface{}{ - "root": rootPath, - }) - log := zerolog.Nop() - ignorer = ignore.NewIgnorer(opt, &log) - - _, err := os.Stat(indexesPath) - if err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("consistency check failed: '%s' is not a posixfs root", rootPath) - } - return fmt.Errorf("error accessing '%s': %w", indexesPath, err) + path = filepath.Clean(path) + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("error accessing '%s': %w", path, err) } - fmt.Println("Checking personal spaces...") - checkSpaces(filepath.Join(rootPath, "users")) + rootPath, err := findStorageRoot(path) + if err != nil { + return err + } - fmt.Println("Checking project spaces...") - checkSpaces(filepath.Join(rootPath, "projects")) + drivers := revaconfig.StorageProviderDrivers(cfg) + drivers["posix"] = revaconfig.Posix(cfg, false, false) + opts, err := options.New(drivers["posix"].(map[string]any)) + if err != nil { + return err + } + + ignorer = ignore.NewIgnorer(opts, nil) + + switch { + case path == rootPath: + fmt.Println("Checking personal spaces...") + checkSpaces(filepath.Join(path, "users")) + + fmt.Println("Checking project spaces...") + checkSpaces(filepath.Join(path, "projects")) + case isSpaceRoot(path): + fmt.Printf("Checking space '%s'...\n", path) + checkSpace(path) + default: + fmt.Printf("Checking '%s'...\n", path) + checkEntity(path) + } if restartRequired { fmt.Println("\n\n ⚠️ Please restart your openCloud instance to apply changes.") @@ -230,6 +261,38 @@ func checkPosixfsConsistency(cmd *cobra.Command, cfg *config.Config) error { return nil } +// findStorageRoot walks up the directory tree starting at path until it finds a +// directory that contains an "indexes" subdirectory which marks the root of a +// posixfs storage. A user folder inside a space might also be named "indexes", +// so to disambiguate we require that the "indexes" directory is an internal +// directory: the storage's own indexes directory is skipped during assimilation +// and therefore never receives a node ID attribute, whereas a regular user +// folder would have one. +func findStorageRoot(path string) (string, error) { + current := path + for { + indexesPath := filepath.Join(current, "indexes") + if info, err := os.Stat(indexesPath); err == nil && info.IsDir() { + if id, err := xattr.Get(indexesPath, prefixes.IDAttr); err != nil || len(id) == 0 { + return current, nil + } + } + + parent := filepath.Dir(current) + if parent == current { + return "", fmt.Errorf("'%s' does not appear to be inside a posixfs storage (no 'indexes' directory found)", path) + } + current = parent + } +} + +// isSpaceRoot reports whether the given path is a space root, which is +// identified by the presence of the space ID attribute. +func isSpaceRoot(path string) bool { + spaceID, err := xattr.Get(path, prefixes.SpaceIDAttr) + return err == nil && len(spaceID) > 0 +} + func checkSpaces(basePath string) { dirEntries, err := os.ReadDir(basePath) if err != nil { @@ -326,31 +389,7 @@ func walkNodes(dir string, parentID string) int { continue } - // Check if the parent ID attribute matches the expected parent ID, if not, fix it. - actualParentID, err := xattr.Get(fullPath, prefixes.ParentidAttr) - if err != nil || string(actualParentID) != parentID { - err = xattr.Set(fullPath, prefixes.ParentidAttr, []byte(parentID)) - if err != nil { - logFailure("Failed to fix parent ID for '%s': %v", fullPath, err) - } else { - fmt.Printf(" + Fixed parent ID for '%s'", fullPath) - fixes++ - restartRequired = true - } - } - - // Check that the name attribute matches the actual name of the file/directory, if not, fix it. - nameAttr, err := xattr.Get(fullPath, prefixes.NameAttr) - if err != nil || string(nameAttr) != entry.Name() { - err = xattr.Set(fullPath, prefixes.NameAttr, []byte(entry.Name())) - if err != nil { - logFailure("Failed to fix name attribute for '%s': %v", fullPath, err) - } else { - fmt.Printf(" + Fixed name attribute for '%s'", fullPath) - fixes++ - restartRequired = true - } - } + fixes += checkNodeAttributes(fullPath, entry.Name(), parentID, entry.IsDir()) if entry.IsDir() { nodeID, err := xattr.Get(fullPath, prefixes.IDAttr) @@ -359,16 +398,85 @@ func walkNodes(dir string, parentID string) int { continue } fixes += walkNodes(fullPath, string(nodeID)) - } else { - fixes += checkBlobsize(fullPath) - if recalculateChecksums { - fixes += fixChecksums(fullPath) - } } } return fixes } +// checkNodeAttributes checks and fixes the parent ID and name attributes of a +// single node. For files it additionally checks the blobsize and, when +// requested, the checksums. It returns the number of fixes applied. +func checkNodeAttributes(path, name, parentID string, isDir bool) int { + fixes := 0 + + // Check if the parent ID attribute matches the expected parent ID, if not, fix it. + actualParentID, err := xattr.Get(path, prefixes.ParentidAttr) + if err != nil || string(actualParentID) != parentID { + if err := xattr.Set(path, prefixes.ParentidAttr, []byte(parentID)); err != nil { + logFailure("Failed to fix parent ID for '%s': %v", path, err) + } else { + fmt.Printf(" + Fixed parent ID for '%s'\n", path) + fixes++ + restartRequired = true + } + } + + // Check that the name attribute matches the actual name of the file/directory, if not, fix it. + nameAttr, err := xattr.Get(path, prefixes.NameAttr) + if err != nil || string(nameAttr) != name { + if err := xattr.Set(path, prefixes.NameAttr, []byte(name)); err != nil { + logFailure("Failed to fix name attribute for '%s': %v", path, err) + } else { + fmt.Printf(" + Fixed name attribute for '%s'\n", path) + fixes++ + restartRequired = true + } + } + + if !isDir { + fixes += checkBlobsize(path) + if recalculateChecksums { + fixes += fixChecksums(path) + } + } + + return fixes +} + +// checkEntity checks a single file or folder within a space, including its own +// parent ID, name and (for files) blobsize/checksums. If the entity is a folder +// its children are checked recursively. +func checkEntity(path string) { + info, err := os.Stat(path) + if err != nil { + logFailure("Error accessing path '%s': %v", path, err) + return + } + + // The expected parent ID is the ID attribute of the containing directory. + parentDir := filepath.Dir(path) + parentID, err := xattr.Get(parentDir, prefixes.IDAttr) + if err != nil || len(parentID) == 0 { + logFailure("Parent directory '%s' is missing the '%s' attribute", parentDir, prefixes.IDAttr) + return + } + + fixes := checkNodeAttributes(path, info.Name(), string(parentID), info.IsDir()) + + if info.IsDir() { + nodeID, err := xattr.Get(path, prefixes.IDAttr) + if err != nil || len(nodeID) == 0 { + logFailure("Directory '%s' missing '%s' attribute", path, prefixes.IDAttr) + } else { + fixes += walkNodes(path, string(nodeID)) + } + } + + if fixes > 0 { + fmt.Printf(" ✓ Fixed %d incorrect node attributes for %s\n", fixes, filepath.Base(path)) + } +} + // checkBlobsize verifies that the stored blobsize attribute matches the actual // file size and fixes it if it doesn't. It returns the number of fixes applied. func checkBlobsize(path string) int { @@ -436,7 +544,7 @@ func checkNodes(spacePath string) { fixes := walkNodes(spacePath, string(rootID)) if fixes > 0 { - fmt.Printf("\n ✓ Fixed %d incorrect node attributes in %s\n", fixes, filepath.Base(spacePath)) + fmt.Printf(" ✓ Fixed %d incorrect node attributes in %s\n", fixes, filepath.Base(spacePath)) } } From 54cced7c567e087c18ab75f36de7c43ccc7c25d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 3 Aug 2026 08:58:19 +0200 Subject: [PATCH 04/10] Streamline logging in the commands --- opencloud/pkg/command/posixfs.go | 10 +++------- opencloud/pkg/command/root.go | 14 ++++++++++++-- opencloud/pkg/command/shares.go | 14 ++------------ 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/opencloud/pkg/command/posixfs.go b/opencloud/pkg/command/posixfs.go index fa049d8731..5b36d7fd93 100644 --- a/opencloud/pkg/command/posixfs.go +++ b/opencloud/pkg/command/posixfs.go @@ -15,7 +15,6 @@ import ( "github.com/opencloud-eu/opencloud/pkg/config" "github.com/opencloud-eu/opencloud/pkg/config/configlog" "github.com/opencloud-eu/opencloud/pkg/config/parser" - oclog "github.com/opencloud-eu/opencloud/pkg/log" "github.com/opencloud-eu/opencloud/pkg/x/path/filepathx" storageUsersConfig "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config" storageUsersParser "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config/parser" @@ -135,11 +134,7 @@ func scanCmd(ocCfg *config.Config) *cobra.Command { fmt.Fprintf(os.Stderr, "Failed to create event stream for posix driver: %v\n", err) os.Exit(1) } - log := oclog.NewLogger( - oclog.Name("posixfs scan"), - oclog.Level("error"), - oclog.Pretty(true), - oclog.Color(false)).Logger + log := logger("posixfs") if !defaultRoot { log = log.With().Str("basepath", root).Logger() @@ -219,6 +214,7 @@ The argument determines the scope of the check: // given path determines the scope of the check: the whole storage, a single // space or a single entity within a space. func checkPosixfsConsistency(cfg *storageUsersConfig.Config, cmd *cobra.Command, path string) error { + log := logger("posixfs") recalculateChecksums, _ = cmd.Flags().GetBool("fix-checksums") path = filepath.Clean(path) @@ -238,7 +234,7 @@ func checkPosixfsConsistency(cfg *storageUsersConfig.Config, cmd *cobra.Command, return err } - ignorer = ignore.NewIgnorer(opts, nil) + ignorer = ignore.NewIgnorer(opts, &log) switch { case path == rootPath: diff --git a/opencloud/pkg/command/root.go b/opencloud/pkg/command/root.go index 834bc68e02..01ff6a77a7 100644 --- a/opencloud/pkg/command/root.go +++ b/opencloud/pkg/command/root.go @@ -6,11 +6,13 @@ import ( "os/signal" "syscall" + "github.com/rs/zerolog" + "github.com/spf13/cobra" + "github.com/opencloud-eu/opencloud/opencloud/pkg/register" "github.com/opencloud-eu/opencloud/pkg/clihelper" "github.com/opencloud-eu/opencloud/pkg/config" - - "github.com/spf13/cobra" + oclog "github.com/opencloud-eu/opencloud/pkg/log" ) // Execute is the entry point for the opencloud command. @@ -38,3 +40,11 @@ func Execute() error { ctx, _ := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP) return app.ExecuteContext(ctx) } + +func logger(name string) zerolog.Logger { + return oclog.NewLogger( + oclog.Name(name), + oclog.Level("info"), + oclog.Pretty(true), + oclog.Color(true)).Logger +} diff --git a/opencloud/pkg/command/shares.go b/opencloud/pkg/command/shares.go index 98cee2da4a..0cdfeea5d1 100644 --- a/opencloud/pkg/command/shares.go +++ b/opencloud/pkg/command/shares.go @@ -9,7 +9,6 @@ import ( "github.com/opencloud-eu/opencloud/pkg/config" "github.com/opencloud-eu/opencloud/pkg/config/configlog" "github.com/opencloud-eu/opencloud/pkg/config/parser" - oclog "github.com/opencloud-eu/opencloud/pkg/log" mregistry "github.com/opencloud-eu/opencloud/pkg/registry" sharing "github.com/opencloud-eu/opencloud/services/sharing/pkg/config" sharingparser "github.com/opencloud-eu/opencloud/services/sharing/pkg/config/parser" @@ -85,7 +84,7 @@ func cleanup(_ *cobra.Command, cfg *config.Config) error { return configlog.ReturnError(errors.New("cleanup is only implemented for the jsoncs3 share manager")) } - l := logger() + l := logger("migrate") zerolog.SetGlobalLevel(zerolog.InfoLevel) @@ -94,7 +93,7 @@ func cleanup(_ *cobra.Command, cfg *config.Config) error { if !ok { return configlog.ReturnError(errors.New("Unknown share manager type '" + driver + "'")) } - mgr, err := f(rcfg[driver].(map[string]any), l) + mgr, err := f(rcfg[driver].(map[string]any), &l) if err != nil { return configlog.ReturnError(err) } @@ -167,12 +166,3 @@ func revaShareConfig(cfg *sharing.Config) map[string]any { }, } } - -func logger() *zerolog.Logger { - log := oclog.NewLogger( - oclog.Name("migrate"), - oclog.Level("info"), - oclog.Pretty(true), - oclog.Color(true)).Logger - return &log -} From 559b10e90a21c6928614f62c6ece99585dfe211a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 3 Aug 2026 09:49:10 +0200 Subject: [PATCH 05/10] Check if the given path is part of the storage --- opencloud/pkg/command/posixfs.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/opencloud/pkg/command/posixfs.go b/opencloud/pkg/command/posixfs.go index 5b36d7fd93..54e8e688c0 100644 --- a/opencloud/pkg/command/posixfs.go +++ b/opencloud/pkg/command/posixfs.go @@ -188,7 +188,7 @@ The argument determines the scope of the check: - a storage root: the whole storage (all personal and project spaces) is checked - a space root: only that space is checked - a file or folder: only that single entity is checked (and its children, if it is a folder)`, - Args: cobra.ExactArgs(1), + Args: cobra.MaximumNArgs(1), PreRunE: func(cmd *cobra.Command, args []string) error { if err := parser.ParseConfig(ocCfg, true); err != nil { @@ -202,7 +202,11 @@ The argument determines the scope of the check: }, RunE: func(cmd *cobra.Command, args []string) error { cfg := ocCfg.StorageUsers - return checkPosixfsConsistency(cfg, cmd, args[0]) + path := cfg.Drivers.Posix.Root + if len(args) > 0 { + path = args[0] + } + return checkPosixfsConsistency(cfg, cmd, path) }, } consCmd.Flags().Bool("fix-checksums", false, "Recalculate and fix the file checksums. This reads every file and can be slow on large storages.") @@ -235,6 +239,7 @@ func checkPosixfsConsistency(cfg *storageUsersConfig.Config, cmd *cobra.Command, } ignorer = ignore.NewIgnorer(opts, &log) + contained, _ := filepathx.IsSameOrContainedBy(rootPath, path) switch { case path == rootPath: @@ -246,9 +251,11 @@ func checkPosixfsConsistency(cfg *storageUsersConfig.Config, cmd *cobra.Command, case isSpaceRoot(path): fmt.Printf("Checking space '%s'...\n", path) checkSpace(path) - default: + case contained: fmt.Printf("Checking '%s'...\n", path) checkEntity(path) + default: + return fmt.Errorf("the provided path '%s' is neither a space root nor contained by the storage root '%s'", path, rootPath) } if restartRequired { From 20d16c5275f7af890311d3111fbef06630243bdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 3 Aug 2026 10:58:53 +0200 Subject: [PATCH 06/10] Allow for providing multiple paths to check --- opencloud/pkg/command/posixfs.go | 72 ++++++++++++++++---------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/opencloud/pkg/command/posixfs.go b/opencloud/pkg/command/posixfs.go index 54e8e688c0..37ce00abe8 100644 --- a/opencloud/pkg/command/posixfs.go +++ b/opencloud/pkg/command/posixfs.go @@ -180,15 +180,18 @@ func scanCmd(ocCfg *config.Config) *cobra.Command { // consistencyCmd returns a command to check the consistency of the posixfs storage. func consistencyCmd(ocCfg *config.Config) *cobra.Command { consCmd := &cobra.Command{ - Use: "consistency ", + Use: "consistency [path ...]", Short: "check the consistency of the posixfs storage", Long: `check the consistency of the posixfs storage. -The argument determines the scope of the check: +You can specify one or more paths to limit the scope of the check. +If no path is provided, the whole storage is checked. + +The provided arguments determines the scope of the check: - a storage root: the whole storage (all personal and project spaces) is checked - a space root: only that space is checked - a file or folder: only that single entity is checked (and its children, if it is a folder)`, - Args: cobra.MaximumNArgs(1), + Args: cobra.ArbitraryArgs, PreRunE: func(cmd *cobra.Command, args []string) error { if err := parser.ParseConfig(ocCfg, true); err != nil { @@ -202,11 +205,7 @@ The argument determines the scope of the check: }, RunE: func(cmd *cobra.Command, args []string) error { cfg := ocCfg.StorageUsers - path := cfg.Drivers.Posix.Root - if len(args) > 0 { - path = args[0] - } - return checkPosixfsConsistency(cfg, cmd, path) + return checkPosixfsConsistency(cfg, cmd, args) }, } consCmd.Flags().Bool("fix-checksums", false, "Recalculate and fix the file checksums. This reads every file and can be slow on large storages.") @@ -217,45 +216,48 @@ The argument determines the scope of the check: // checkPosixfsConsistency checks the consistency of the posixfs storage. The // given path determines the scope of the check: the whole storage, a single // space or a single entity within a space. -func checkPosixfsConsistency(cfg *storageUsersConfig.Config, cmd *cobra.Command, path string) error { +func checkPosixfsConsistency(cfg *storageUsersConfig.Config, cmd *cobra.Command, paths []string) error { + if len(paths) == 0 { + paths = []string{cfg.Drivers.Posix.Root} + } log := logger("posixfs") recalculateChecksums, _ = cmd.Flags().GetBool("fix-checksums") - path = filepath.Clean(path) - if _, err := os.Stat(path); err != nil { - return fmt.Errorf("error accessing '%s': %w", path, err) - } - - rootPath, err := findStorageRoot(path) - if err != nil { - return err - } - drivers := revaconfig.StorageProviderDrivers(cfg) drivers["posix"] = revaconfig.Posix(cfg, false, false) opts, err := options.New(drivers["posix"].(map[string]any)) if err != nil { return err } - ignorer = ignore.NewIgnorer(opts, &log) - contained, _ := filepathx.IsSameOrContainedBy(rootPath, path) - switch { - case path == rootPath: - fmt.Println("Checking personal spaces...") - checkSpaces(filepath.Join(path, "users")) + for _, path := range paths { + rootPath, err := findStorageRoot(path) + if err != nil { + return err + } + path = filepath.Clean(path) + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("error accessing '%s': %w", path, err) + } + contained, _ := filepathx.IsSameOrContainedBy(rootPath, path) - fmt.Println("Checking project spaces...") - checkSpaces(filepath.Join(path, "projects")) - case isSpaceRoot(path): - fmt.Printf("Checking space '%s'...\n", path) - checkSpace(path) - case contained: - fmt.Printf("Checking '%s'...\n", path) - checkEntity(path) - default: - return fmt.Errorf("the provided path '%s' is neither a space root nor contained by the storage root '%s'", path, rootPath) + switch { + case path == rootPath: + fmt.Println("Checking personal spaces...") + checkSpaces(filepath.Join(path, "users")) + + fmt.Println("Checking project spaces...") + checkSpaces(filepath.Join(path, "projects")) + case isSpaceRoot(path): + fmt.Printf("Checking space '%s'...\n", path) + checkSpace(path) + case contained: + fmt.Printf("Checking '%s'...\n", path) + checkEntity(path) + default: + return fmt.Errorf("the provided path '%s' is neither a space root nor contained by the storage root '%s'", path, rootPath) + } } if restartRequired { From b125f0681fcf4e9ba8a5a95696b918370e401df4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 3 Aug 2026 11:03:45 +0200 Subject: [PATCH 07/10] Fix typos --- opencloud/pkg/command/posixfs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opencloud/pkg/command/posixfs.go b/opencloud/pkg/command/posixfs.go index 37ce00abe8..94d90abc56 100644 --- a/opencloud/pkg/command/posixfs.go +++ b/opencloud/pkg/command/posixfs.go @@ -181,8 +181,8 @@ func scanCmd(ocCfg *config.Config) *cobra.Command { func consistencyCmd(ocCfg *config.Config) *cobra.Command { consCmd := &cobra.Command{ Use: "consistency [path ...]", - Short: "check the consistency of the posixfs storage", - Long: `check the consistency of the posixfs storage. + Short: "Check the consistency of the posixfs storage", + Long: `Check the consistency of the posixfs storage. You can specify one or more paths to limit the scope of the check. If no path is provided, the whole storage is checked. From c6fba8543f991e04510c94c901d0822e5c0155b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 3 Aug 2026 11:54:36 +0200 Subject: [PATCH 08/10] Improve wording --- opencloud/pkg/command/posixfs.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/opencloud/pkg/command/posixfs.go b/opencloud/pkg/command/posixfs.go index 94d90abc56..567d726a8f 100644 --- a/opencloud/pkg/command/posixfs.go +++ b/opencloud/pkg/command/posixfs.go @@ -190,7 +190,7 @@ If no path is provided, the whole storage is checked. The provided arguments determines the scope of the check: - a storage root: the whole storage (all personal and project spaces) is checked - a space root: only that space is checked - - a file or folder: only that single entity is checked (and its children, if it is a folder)`, + - a file or directory: only that single entity is checked (and its children, if it is a directory)`, Args: cobra.ArbitraryArgs, PreRunE: func(cmd *cobra.Command, args []string) error { @@ -268,11 +268,11 @@ func checkPosixfsConsistency(cfg *storageUsersConfig.Config, cmd *cobra.Command, // findStorageRoot walks up the directory tree starting at path until it finds a // directory that contains an "indexes" subdirectory which marks the root of a -// posixfs storage. A user folder inside a space might also be named "indexes", +// posixfs storage. A user directory inside a space might also be named "indexes", // so to disambiguate we require that the "indexes" directory is an internal // directory: the storage's own indexes directory is skipped during assimilation // and therefore never receives a node ID attribute, whereas a regular user -// folder would have one. +// directory would have one. func findStorageRoot(path string) (string, error) { current := path for { @@ -448,8 +448,8 @@ func checkNodeAttributes(path, name, parentID string, isDir bool) int { return fixes } -// checkEntity checks a single file or folder within a space, including its own -// parent ID, name and (for files) blobsize/checksums. If the entity is a folder +// checkEntity checks a single file or directory within a space, including its own +// parent ID, name and (for files) blobsize/checksums. If the entity is a directory // its children are checked recursively. func checkEntity(path string) { info, err := os.Stat(path) From deb3247b99bf30cdb8d28356a0759645e45b11c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 3 Aug 2026 12:17:19 +0200 Subject: [PATCH 09/10] Move consistency check logic into a checker struct This makes it possible to get rid of the globals without passing multiple state vars around. --- opencloud/pkg/command/posixfs.go | 514 +------------------ opencloud/pkg/command/posixfs_consistency.go | 492 ++++++++++++++++++ 2 files changed, 515 insertions(+), 491 deletions(-) create mode 100644 opencloud/pkg/command/posixfs_consistency.go diff --git a/opencloud/pkg/command/posixfs.go b/opencloud/pkg/command/posixfs.go index 567d726a8f..751e3db1be 100644 --- a/opencloud/pkg/command/posixfs.go +++ b/opencloud/pkg/command/posixfs.go @@ -1,14 +1,9 @@ package command import ( - "bufio" - "bytes" - "context" "fmt" "os" "path/filepath" - "strconv" - "strings" "time" "github.com/opencloud-eu/opencloud/opencloud/pkg/register" @@ -16,7 +11,6 @@ import ( "github.com/opencloud-eu/opencloud/pkg/config/configlog" "github.com/opencloud-eu/opencloud/pkg/config/parser" "github.com/opencloud-eu/opencloud/pkg/x/path/filepathx" - storageUsersConfig "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config" storageUsersParser "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config/parser" "github.com/opencloud-eu/opencloud/services/storage-users/pkg/event" "github.com/opencloud-eu/opencloud/services/storage-users/pkg/revaconfig" @@ -25,17 +19,9 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" "github.com/pkg/xattr" "github.com/spf13/cobra" - "github.com/vmihailenco/msgpack/v5" -) - -var ( - restartRequired = false - recalculateChecksums = false - ignorer *ignore.Ignorer ) type IDCacher interface { @@ -205,7 +191,27 @@ The provided arguments determines the scope of the check: }, RunE: func(cmd *cobra.Command, args []string) error { cfg := ocCfg.StorageUsers - return checkPosixfsConsistency(cfg, cmd, args) + if len(args) == 0 { + args = []string{cfg.Drivers.Posix.Root} + } + log := logger("posixfs") + recalculateChecksums, _ := cmd.Flags().GetBool("fix-checksums") + + drivers := revaconfig.StorageProviderDrivers(cfg) + drivers["posix"] = revaconfig.Posix(cfg, false, false) + opts, err := options.New(drivers["posix"].(map[string]any)) + if err != nil { + return err + } + ignorer := ignore.NewIgnorer(opts, &log) + + checker := &consistencyChecker{ + cfg: cfg, + ignorer: ignorer, + recalculateChecksums: recalculateChecksums, + } + + return checker.Check(args) }, } consCmd.Flags().Bool("fix-checksums", false, "Recalculate and fix the file checksums. This reads every file and can be slow on large storages.") @@ -213,57 +219,8 @@ The provided arguments determines the scope of the check: return consCmd } -// checkPosixfsConsistency checks the consistency of the posixfs storage. The -// given path determines the scope of the check: the whole storage, a single -// space or a single entity within a space. -func checkPosixfsConsistency(cfg *storageUsersConfig.Config, cmd *cobra.Command, paths []string) error { - if len(paths) == 0 { - paths = []string{cfg.Drivers.Posix.Root} - } - log := logger("posixfs") - recalculateChecksums, _ = cmd.Flags().GetBool("fix-checksums") - - drivers := revaconfig.StorageProviderDrivers(cfg) - drivers["posix"] = revaconfig.Posix(cfg, false, false) - opts, err := options.New(drivers["posix"].(map[string]any)) - if err != nil { - return err - } - ignorer = ignore.NewIgnorer(opts, &log) - - for _, path := range paths { - rootPath, err := findStorageRoot(path) - if err != nil { - return err - } - path = filepath.Clean(path) - if _, err := os.Stat(path); err != nil { - return fmt.Errorf("error accessing '%s': %w", path, err) - } - contained, _ := filepathx.IsSameOrContainedBy(rootPath, path) - - switch { - case path == rootPath: - fmt.Println("Checking personal spaces...") - checkSpaces(filepath.Join(path, "users")) - - fmt.Println("Checking project spaces...") - checkSpaces(filepath.Join(path, "projects")) - case isSpaceRoot(path): - fmt.Printf("Checking space '%s'...\n", path) - checkSpace(path) - case contained: - fmt.Printf("Checking '%s'...\n", path) - checkEntity(path) - default: - return fmt.Errorf("the provided path '%s' is neither a space root nor contained by the storage root '%s'", path, rootPath) - } - } - - if restartRequired { - fmt.Println("\n\n ⚠️ Please restart your openCloud instance to apply changes.") - } - return nil +func logFailure(message string, args ...any) { + fmt.Fprintf(os.Stderr, message+"\n", args...) } // findStorageRoot walks up the directory tree starting at path until it finds a @@ -297,428 +254,3 @@ func isSpaceRoot(path string) bool { spaceID, err := xattr.Get(path, prefixes.SpaceIDAttr) return err == nil && len(spaceID) > 0 } - -func checkSpaces(basePath string) { - dirEntries, err := os.ReadDir(basePath) - if err != nil { - logFailure("Error reading spaces directory '%s': %v", basePath, err) - return - } - - for _, entry := range dirEntries { - if entry.IsDir() { - fullPath := filepath.Join(basePath, entry.Name()) - checkSpace(fullPath) - } - } -} - -func checkSpace(spacePath string) { - info, err := os.Stat(spacePath) - if err != nil { - logFailure("Error accessing path '%s': %v", spacePath, err) - return - } - if !info.IsDir() { - logFailure("Error: The provided path '%s' is not a directory\n", spacePath) - return - } - - spaceID, err := xattr.Get(spacePath, prefixes.SpaceIDAttr) - if err != nil || len(spaceID) == 0 { - logFailure("Error: The directory '%s' does not seem to be a space root, it's missing the '%s' attribute\n", spacePath, prefixes.SpaceIDAttr) - return - } - - checkSpaceID(spacePath) - checkNodes(spacePath) -} - -func checkSpaceID(spacePath string) { - entries, uniqueIDs, oldestEntry, err := gatherAttributes(spacePath) - if err != nil { - logFailure("Failed to gather attributes: %v", err) - return - } - - if len(entries) == 0 { - return - } - - if len(uniqueIDs) > 1 { - fmt.Println("\n ⚠ Multiple space IDs found:") - for id := range uniqueIDs { - fmt.Printf(" - %s\n", id) - } - - fmt.Printf("\n ⏳ Oldest entry is '%s' (modified on %s).\n", - filepath.Base(oldestEntry.Path), oldestEntry.ModTime.Format(time.RFC1123)) - - targetID := oldestEntry.ParentID - fmt.Printf(" ✅ Proposed target Parent ID: %s\n", targetID) - - fmt.Printf("\n Do you want to unify all parent IDs to '%s'? This will modify %d entries, the directory, and the user index. (y/N): ", targetID, len(entries)) - - reader := bufio.NewReader(os.Stdin) - input, _ := reader.ReadString('\n') - input = strings.TrimSpace(strings.ToLower(input)) - - if input != "y" { - logFailure("Operation cancelled by user.") - return - } - restartRequired = true - - obsoleteIDs := []string{} - for id := range uniqueIDs { - if id != targetID { - obsoleteIDs = append(obsoleteIDs, id) - } - } - fixSpaceID(spacePath, obsoleteIDs, targetID, entries) - } -} - -func walkNodes(dir string, parentID string) int { - fixes := 0 - entries, err := os.ReadDir(dir) - if err != nil { - logFailure("Error reading directory '%s': %v", dir, err) - return 0 - } - - for _, entry := range entries { - fullPath := filepath.Join(dir, entry.Name()) - - if ignorer.IsIgnored(fullPath) { - continue - } - - fixes += checkNodeAttributes(fullPath, entry.Name(), parentID, entry.IsDir()) - - if entry.IsDir() { - nodeID, err := xattr.Get(fullPath, prefixes.IDAttr) - if err != nil || len(nodeID) == 0 { - logFailure("Directory '%s' missing '%s', skipping its children", fullPath, prefixes.IDAttr) - continue - } - fixes += walkNodes(fullPath, string(nodeID)) - } - } - return fixes -} - -// checkNodeAttributes checks and fixes the parent ID and name attributes of a -// single node. For files it additionally checks the blobsize and, when -// requested, the checksums. It returns the number of fixes applied. -func checkNodeAttributes(path, name, parentID string, isDir bool) int { - fixes := 0 - - // Check if the parent ID attribute matches the expected parent ID, if not, fix it. - actualParentID, err := xattr.Get(path, prefixes.ParentidAttr) - if err != nil || string(actualParentID) != parentID { - if err := xattr.Set(path, prefixes.ParentidAttr, []byte(parentID)); err != nil { - logFailure("Failed to fix parent ID for '%s': %v", path, err) - } else { - fmt.Printf(" + Fixed parent ID for '%s'\n", path) - fixes++ - restartRequired = true - } - } - - // Check that the name attribute matches the actual name of the file/directory, if not, fix it. - nameAttr, err := xattr.Get(path, prefixes.NameAttr) - if err != nil || string(nameAttr) != name { - if err := xattr.Set(path, prefixes.NameAttr, []byte(name)); err != nil { - logFailure("Failed to fix name attribute for '%s': %v", path, err) - } else { - fmt.Printf(" + Fixed name attribute for '%s'\n", path) - fixes++ - restartRequired = true - } - } - - if !isDir { - fixes += checkBlobsize(path) - if recalculateChecksums { - fixes += fixChecksums(path) - } - } - - return fixes -} - -// checkEntity checks a single file or directory within a space, including its own -// parent ID, name and (for files) blobsize/checksums. If the entity is a directory -// its children are checked recursively. -func checkEntity(path string) { - info, err := os.Stat(path) - if err != nil { - logFailure("Error accessing path '%s': %v", path, err) - return - } - - // The expected parent ID is the ID attribute of the containing directory. - parentDir := filepath.Dir(path) - parentID, err := xattr.Get(parentDir, prefixes.IDAttr) - if err != nil || len(parentID) == 0 { - logFailure("Parent directory '%s' is missing the '%s' attribute", parentDir, prefixes.IDAttr) - return - } - - fixes := checkNodeAttributes(path, info.Name(), string(parentID), info.IsDir()) - - if info.IsDir() { - nodeID, err := xattr.Get(path, prefixes.IDAttr) - if err != nil || len(nodeID) == 0 { - logFailure("Directory '%s' missing '%s' attribute", path, prefixes.IDAttr) - } else { - fixes += walkNodes(path, string(nodeID)) - } - } - - if fixes > 0 { - fmt.Printf(" ✓ Fixed %d incorrect node attributes for %s\n", fixes, filepath.Base(path)) - } -} - -// checkBlobsize verifies that the stored blobsize attribute matches the actual -// file size and fixes it if it doesn't. It returns the number of fixes applied. -func checkBlobsize(path string) int { - info, err := os.Stat(path) - if err != nil { - logFailure("Error accessing file '%s': %v", path, err) - return 0 - } - - expectedSize := strconv.FormatInt(info.Size(), 10) - blobsize, err := xattr.Get(path, prefixes.BlobsizeAttr) - if err == nil && string(blobsize) == expectedSize { - return 0 - } - - if err := xattr.Set(path, prefixes.BlobsizeAttr, []byte(expectedSize)); err != nil { - logFailure("Failed to fix blobsize for '%s': %v", path, err) - return 0 - } - - fmt.Printf(" + Fixed blobsize for '%s'\n", path) - restartRequired = true - return 1 -} - -// fixChecksums recalculates the sha1, md5 and adler32 checksums of the file and -// updates the stored attributes if they differ. It returns the number of fixes applied. -func fixChecksums(path string) int { - sha1h, md5h, adler32h, err := node.CalculateChecksums(context.Background(), path) - if err != nil { - logFailure("Failed to calculate checksums for '%s': %v", path, err) - return 0 - } - - checksums := map[string][]byte{ - prefixes.ChecksumPrefix + "sha1": sha1h.Sum(nil), - prefixes.ChecksumPrefix + "md5": md5h.Sum(nil), - prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil), - } - - fixes := 0 - for attrName, sum := range checksums { - current, err := xattr.Get(path, attrName) - if err == nil && bytes.Equal(current, sum) { - continue - } - if err := xattr.Set(path, attrName, sum); err != nil { - logFailure("Failed to fix checksum '%s' for '%s': %v", attrName, path, err) - continue - } - fmt.Printf(" + Fixed checksum '%s' for '%s'\n", attrName, path) - restartRequired = true - fixes++ - } - return fixes -} - -func checkNodes(spacePath string) { - rootID, err := xattr.Get(spacePath, prefixes.IDAttr) - if err != nil || len(rootID) == 0 { - logFailure("Space root '%s' missing '%s' attribute", spacePath, prefixes.IDAttr) - return - } - - fixes := walkNodes(spacePath, string(rootID)) - - if fixes > 0 { - fmt.Printf(" ✓ Fixed %d incorrect node attributes in %s\n", fixes, filepath.Base(spacePath)) - } -} - -func fixSpaceID(spacePath string, obsoleteIDs []string, targetID string, entries []EntryInfo) { - // Set all parentid attributes to the proper space ID - err := setAllParentIDAttributes(entries, targetID) - if err != nil { - logFailure("an error occurred during file attribute update: %v", err) - return - } - - // Update space ID itself - fmt.Printf(" Updating directory '%s' with attribute '%s' -> %s\n", filepath.Base(spacePath), prefixes.IDAttr, targetID) - err = xattr.Set(spacePath, prefixes.IDAttr, []byte(targetID)) - if err != nil { - logFailure("Failed to set attribute on directory '%s': %v", spacePath, err) - return - } - err = xattr.Set(spacePath, prefixes.SpaceIDAttr, []byte(targetID)) - if err != nil { - logFailure("Failed to set attribute on directory '%s': %v", spacePath, err) - return - } - - // update the index - err = updateOwnerIndexFile(spacePath, obsoleteIDs) - if err != nil { - logFailure("Could not update the owner index file: %v", err) - } -} - -func gatherAttributes(path string) ([]EntryInfo, map[string]struct{}, EntryInfo, error) { - dirEntries, err := os.ReadDir(path) - if err != nil { - return nil, nil, EntryInfo{}, fmt.Errorf("failed to read directory: %w", err) - } - - var allEntries []EntryInfo - uniqueIDs := make(map[string]struct{}) - var oldestEntry EntryInfo - oldestTime := time.Now().Add(100 * 365 * 24 * time.Hour) // Set to a future date to find the oldest entry - - for _, entry := range dirEntries { - fullPath := filepath.Join(path, entry.Name()) - if ignorer.IsIgnored(fullPath) { - continue - } - info, err := os.Stat(fullPath) - if err != nil { - fmt.Printf(" - Warning: could not stat %s: %v\n", entry.Name(), err) - continue - } - - parentID, err := xattr.Get(fullPath, prefixes.ParentidAttr) - if err != nil { - continue // Skip if attribute doesn't exist or can't be read - } - - entryInfo := EntryInfo{ - Path: fullPath, - ModTime: info.ModTime(), - ParentID: string(parentID), - } - - allEntries = append(allEntries, entryInfo) - uniqueIDs[string(parentID)] = struct{}{} - - if entryInfo.ModTime.Before(oldestTime) { - oldestTime = entryInfo.ModTime - oldestEntry = entryInfo - } - } - - return allEntries, uniqueIDs, oldestEntry, nil -} - -func setAllParentIDAttributes(entries []EntryInfo, targetID string) error { - fmt.Printf(" Setting all parent IDs to '%s':\n", targetID) - - for _, entry := range entries { - if entry.ParentID == targetID { - fmt.Printf(" - Skipping '%s' (already has target ID).\n", filepath.Base(entry.Path)) - continue - } - - fmt.Printf(" - Removing all attributes from '%s'. It will be re-assimilated\n", filepath.Base(entry.Path)) - filepath.WalkDir(entry.Path, func(path string, d os.DirEntry, err error) error { - if err != nil { - return fmt.Errorf("error walking path '%s': %w", path, err) - } - - // Remove all attributes from the file. - if err := removeAttributes(path); err != nil { - fmt.Printf("failed to remove attributes from '%s': %v", path, err) - } - return nil - }) - } - return nil -} - -// updateOwnerIndexFile handles the logic of reading, modifying, and writing the MessagePack index file. -func updateOwnerIndexFile(basePath string, obsoleteIDs []string) error { - fmt.Printf(" Rewriting index file '%s'\n", basePath) - - ownerID, err := xattr.Get(basePath, prefixes.OwnerIDAttr) - if err != nil { - return fmt.Errorf("could not get owner ID from oldest entry '%s' to find index: %w", basePath, err) - } - - indexPath := filepath.Join(basePath, "../../indexes/by-user-id", string(ownerID)+".mpk") - indexPath = filepath.Clean(indexPath) - - // Read the MessagePack file - fileData, err := os.ReadFile(indexPath) - if err != nil { - if os.IsNotExist(err) { - return fmt.Errorf("index file does not exist, skipping update") - } - return fmt.Errorf("could not read index file: %w", err) - } - var indexMap map[string]string - if err := msgpack.Unmarshal(fileData, &indexMap); err != nil { - return fmt.Errorf("failed to parse MessagePack index file (is it corrupt?): %w", err) - } - - // Remove obsolete IDs from the map - itemsRemoved := 0 - for _, id := range obsoleteIDs { - if _, exists := indexMap[id]; exists { - fmt.Printf(" - Removing obsolete ID '%s' from index.\n", id) - delete(indexMap, id) - itemsRemoved++ - } else { - fmt.Printf(" - Obsolete ID '%s' not found in index\n", id) - } - } - - if itemsRemoved == 0 { - return nil - } - - // Write the data back to the file - updatedData, err := msgpack.Marshal(&indexMap) - if err != nil { - return fmt.Errorf("failed to marshal updated index map: %w", err) - } - if err := os.WriteFile(indexPath, updatedData, 0644); err != nil { - return fmt.Errorf("failed to write updated index file: %w", err) - } - - fmt.Printf(" ✓ Successfully removed %d item(s) and saved index file.\n", itemsRemoved) - return nil -} - -func removeAttributes(path string) error { - attrNames, err := xattr.List(path) - if err != nil { - return fmt.Errorf("failed to list attributes for '%s': %w", path, err) - } - - for _, attrName := range attrNames { - if err := xattr.Remove(path, attrName); err != nil { - return fmt.Errorf("failed to remove attribute '%s' from '%s': %w", attrName, path, err) - } - } - return nil -} - -func logFailure(message string, args ...any) { - fmt.Fprintf(os.Stderr, message+"\n", args...) -} diff --git a/opencloud/pkg/command/posixfs_consistency.go b/opencloud/pkg/command/posixfs_consistency.go new file mode 100644 index 0000000000..c56677c65e --- /dev/null +++ b/opencloud/pkg/command/posixfs_consistency.go @@ -0,0 +1,492 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package command + +import ( + "bufio" + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/opencloud-eu/opencloud/pkg/x/path/filepathx" + storageUsersConfig "github.com/opencloud-eu/opencloud/services/storage-users/pkg/config" + "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/ignore" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" + "github.com/pkg/xattr" + "github.com/shamaton/msgpack/v2" +) + +type consistencyChecker struct { + cfg *storageUsersConfig.Config + ignorer *ignore.Ignorer + recalculateChecksums bool + + restartRequired bool +} + +// checkPosixfsConsistency checks the consistency of the posixfs storage. The +// given path determines the scope of the check: the whole storage, a single +// space or a single entity within a space. +func (c *consistencyChecker) Check(paths []string) error { + for _, path := range paths { + rootPath, err := findStorageRoot(path) + if err != nil { + return err + } + path = filepath.Clean(path) + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("error accessing '%s': %w", path, err) + } + contained, _ := filepathx.IsSameOrContainedBy(rootPath, path) + + switch { + case path == rootPath: + fmt.Println("Checking personal spaces...") + c.checkSpaces(filepath.Join(path, "users")) + + fmt.Println("Checking project spaces...") + c.checkSpaces(filepath.Join(path, "projects")) + case isSpaceRoot(path): + fmt.Printf("Checking space '%s'...\n", path) + c.checkSpace(path) + case contained: + fmt.Printf("Checking '%s'...\n", path) + c.checkEntity(path) + default: + return fmt.Errorf("the provided path '%s' is neither a space root nor contained by the storage root '%s'", path, rootPath) + } + } + + if c.restartRequired { + fmt.Println("\n\n ⚠️ Please restart your openCloud instance to apply changes.") + } + return nil +} + +func (c *consistencyChecker) checkSpaces(basePath string) { + dirEntries, err := os.ReadDir(basePath) + if err != nil { + logFailure("Error reading spaces directory '%s': %v", basePath, err) + return + } + + for _, entry := range dirEntries { + if entry.IsDir() { + fullPath := filepath.Join(basePath, entry.Name()) + c.checkSpace(fullPath) + } + } +} + +func (c *consistencyChecker) checkSpace(spacePath string) { + info, err := os.Stat(spacePath) + if err != nil { + logFailure("Error accessing path '%s': %v", spacePath, err) + return + } + if !info.IsDir() { + logFailure("Error: The provided path '%s' is not a directory\n", spacePath) + return + } + + spaceID, err := xattr.Get(spacePath, prefixes.SpaceIDAttr) + if err != nil || len(spaceID) == 0 { + logFailure("Error: The directory '%s' does not seem to be a space root, it's missing the '%s' attribute\n", spacePath, prefixes.SpaceIDAttr) + return + } + + c.checkSpaceID(spacePath) + c.checkNodes(spacePath) +} + +func (c *consistencyChecker) checkSpaceID(spacePath string) { + entries, uniqueIDs, oldestEntry, err := c.gatherAttributes(spacePath) + if err != nil { + logFailure("Failed to gather attributes: %v", err) + return + } + + if len(entries) == 0 { + return + } + + if len(uniqueIDs) > 1 { + fmt.Println("\n ⚠ Multiple space IDs found:") + for id := range uniqueIDs { + fmt.Printf(" - %s\n", id) + } + + fmt.Printf("\n ⏳ Oldest entry is '%s' (modified on %s).\n", + filepath.Base(oldestEntry.Path), oldestEntry.ModTime.Format(time.RFC1123)) + + targetID := oldestEntry.ParentID + fmt.Printf(" ✅ Proposed target Parent ID: %s\n", targetID) + + fmt.Printf("\n Do you want to unify all parent IDs to '%s'? This will modify %d entries, the directory, and the user index. (y/N): ", targetID, len(entries)) + + reader := bufio.NewReader(os.Stdin) + input, _ := reader.ReadString('\n') + input = strings.TrimSpace(strings.ToLower(input)) + + if input != "y" { + logFailure("Operation cancelled by user.") + return + } + c.restartRequired = true + + obsoleteIDs := []string{} + for id := range uniqueIDs { + if id != targetID { + obsoleteIDs = append(obsoleteIDs, id) + } + } + fixSpaceID(spacePath, obsoleteIDs, targetID, entries) + } +} + +func (c *consistencyChecker) walkNodes(dir string, parentID string) int { + fixes := 0 + entries, err := os.ReadDir(dir) + if err != nil { + logFailure("Error reading directory '%s': %v", dir, err) + return 0 + } + + for _, entry := range entries { + fullPath := filepath.Join(dir, entry.Name()) + + if c.ignorer.IsIgnored(fullPath) { + continue + } + + fixes += c.checkNodeAttributes(fullPath, entry.Name(), parentID, entry.IsDir()) + + if entry.IsDir() { + nodeID, err := xattr.Get(fullPath, prefixes.IDAttr) + if err != nil || len(nodeID) == 0 { + logFailure("Directory '%s' missing '%s', skipping its children", fullPath, prefixes.IDAttr) + continue + } + fixes += c.walkNodes(fullPath, string(nodeID)) + } + } + return fixes +} + +// checkNodeAttributes checks and fixes the parent ID and name attributes of a +// single node. For files it additionally checks the blobsize and, when +// requested, the checksums. It returns the number of fixes applied. +func (c *consistencyChecker) checkNodeAttributes(path, name, parentID string, isDir bool) int { + fixes := 0 + + // Check if the parent ID attribute matches the expected parent ID, if not, fix it. + actualParentID, err := xattr.Get(path, prefixes.ParentidAttr) + if err != nil || string(actualParentID) != parentID { + if err := xattr.Set(path, prefixes.ParentidAttr, []byte(parentID)); err != nil { + logFailure("Failed to fix parent ID for '%s': %v", path, err) + } else { + fmt.Printf(" + Fixed parent ID for '%s'\n", path) + fixes++ + c.restartRequired = true + } + } + + // Check that the name attribute matches the actual name of the file/directory, if not, fix it. + nameAttr, err := xattr.Get(path, prefixes.NameAttr) + if err != nil || string(nameAttr) != name { + if err := xattr.Set(path, prefixes.NameAttr, []byte(name)); err != nil { + logFailure("Failed to fix name attribute for '%s': %v", path, err) + } else { + fmt.Printf(" + Fixed name attribute for '%s'\n", path) + fixes++ + c.restartRequired = true + } + } + + if !isDir { + fixes += c.checkBlobsize(path) + if c.recalculateChecksums { + fixes += c.fixChecksums(path) + } + } + + return fixes +} + +// checkEntity checks a single file or directory within a space, including its own +// parent ID, name and (for files) blobsize/checksums. If the entity is a directory +// its children are checked recursively. +func (c *consistencyChecker) checkEntity(path string) { + info, err := os.Stat(path) + if err != nil { + logFailure("Error accessing path '%s': %v", path, err) + return + } + + // The expected parent ID is the ID attribute of the containing directory. + parentDir := filepath.Dir(path) + parentID, err := xattr.Get(parentDir, prefixes.IDAttr) + if err != nil || len(parentID) == 0 { + logFailure("Parent directory '%s' is missing the '%s' attribute", parentDir, prefixes.IDAttr) + return + } + + fixes := c.checkNodeAttributes(path, info.Name(), string(parentID), info.IsDir()) + + if info.IsDir() { + nodeID, err := xattr.Get(path, prefixes.IDAttr) + if err != nil || len(nodeID) == 0 { + logFailure("Directory '%s' missing '%s' attribute", path, prefixes.IDAttr) + } else { + fixes += c.walkNodes(path, string(nodeID)) + } + } + + if fixes > 0 { + fmt.Printf(" ✓ Fixed %d incorrect node attributes for %s\n", fixes, filepath.Base(path)) + } +} + +// checkBlobsize verifies that the stored blobsize attribute matches the actual +// file size and fixes it if it doesn't. It returns the number of fixes applied. +func (c *consistencyChecker) checkBlobsize(path string) int { + info, err := os.Stat(path) + if err != nil { + logFailure("Error accessing file '%s': %v", path, err) + return 0 + } + + expectedSize := strconv.FormatInt(info.Size(), 10) + blobsize, err := xattr.Get(path, prefixes.BlobsizeAttr) + if err == nil && string(blobsize) == expectedSize { + return 0 + } + + if err := xattr.Set(path, prefixes.BlobsizeAttr, []byte(expectedSize)); err != nil { + logFailure("Failed to fix blobsize for '%s': %v", path, err) + return 0 + } + + fmt.Printf(" + Fixed blobsize for '%s'\n", path) + c.restartRequired = true + return 1 +} + +// fixChecksums recalculates the sha1, md5 and adler32 checksums of the file and +// updates the stored attributes if they differ. It returns the number of fixes applied. +func (c *consistencyChecker) fixChecksums(path string) int { + sha1h, md5h, adler32h, err := node.CalculateChecksums(context.Background(), path) + if err != nil { + logFailure("Failed to calculate checksums for '%s': %v", path, err) + return 0 + } + + checksums := map[string][]byte{ + prefixes.ChecksumPrefix + "sha1": sha1h.Sum(nil), + prefixes.ChecksumPrefix + "md5": md5h.Sum(nil), + prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil), + } + + fixes := 0 + for attrName, sum := range checksums { + current, err := xattr.Get(path, attrName) + if err == nil && bytes.Equal(current, sum) { + continue + } + if err := xattr.Set(path, attrName, sum); err != nil { + logFailure("Failed to fix checksum '%s' for '%s': %v", attrName, path, err) + continue + } + fmt.Printf(" + Fixed checksum '%s' for '%s'\n", attrName, path) + c.restartRequired = true + fixes++ + } + return fixes +} + +func (c *consistencyChecker) checkNodes(spacePath string) { + rootID, err := xattr.Get(spacePath, prefixes.IDAttr) + if err != nil || len(rootID) == 0 { + logFailure("Space root '%s' missing '%s' attribute", spacePath, prefixes.IDAttr) + return + } + + fixes := c.walkNodes(spacePath, string(rootID)) + + if fixes > 0 { + fmt.Printf(" ✓ Fixed %d incorrect node attributes in %s\n", fixes, filepath.Base(spacePath)) + } +} + +func fixSpaceID(spacePath string, obsoleteIDs []string, targetID string, entries []EntryInfo) { + // Set all parentid attributes to the proper space ID + err := setAllParentIDAttributes(entries, targetID) + if err != nil { + logFailure("an error occurred during file attribute update: %v", err) + return + } + + // Update space ID itself + fmt.Printf(" Updating directory '%s' with attribute '%s' -> %s\n", filepath.Base(spacePath), prefixes.IDAttr, targetID) + err = xattr.Set(spacePath, prefixes.IDAttr, []byte(targetID)) + if err != nil { + logFailure("Failed to set attribute on directory '%s': %v", spacePath, err) + return + } + err = xattr.Set(spacePath, prefixes.SpaceIDAttr, []byte(targetID)) + if err != nil { + logFailure("Failed to set attribute on directory '%s': %v", spacePath, err) + return + } + + // update the index + err = updateOwnerIndexFile(spacePath, obsoleteIDs) + if err != nil { + logFailure("Could not update the owner index file: %v", err) + } +} + +func (c *consistencyChecker) gatherAttributes(path string) ([]EntryInfo, map[string]struct{}, EntryInfo, error) { + dirEntries, err := os.ReadDir(path) + if err != nil { + return nil, nil, EntryInfo{}, fmt.Errorf("failed to read directory: %w", err) + } + + var allEntries []EntryInfo + uniqueIDs := make(map[string]struct{}) + var oldestEntry EntryInfo + oldestTime := time.Now().Add(100 * 365 * 24 * time.Hour) // Set to a future date to find the oldest entry + + for _, entry := range dirEntries { + fullPath := filepath.Join(path, entry.Name()) + if c.ignorer.IsIgnored(fullPath) { + continue + } + info, err := os.Stat(fullPath) + if err != nil { + fmt.Printf(" - Warning: could not stat %s: %v\n", entry.Name(), err) + continue + } + + parentID, err := xattr.Get(fullPath, prefixes.ParentidAttr) + if err != nil { + continue // Skip if attribute doesn't exist or can't be read + } + + entryInfo := EntryInfo{ + Path: fullPath, + ModTime: info.ModTime(), + ParentID: string(parentID), + } + + allEntries = append(allEntries, entryInfo) + uniqueIDs[string(parentID)] = struct{}{} + + if entryInfo.ModTime.Before(oldestTime) { + oldestTime = entryInfo.ModTime + oldestEntry = entryInfo + } + } + + return allEntries, uniqueIDs, oldestEntry, nil +} + +func setAllParentIDAttributes(entries []EntryInfo, targetID string) error { + fmt.Printf(" Setting all parent IDs to '%s':\n", targetID) + + for _, entry := range entries { + if entry.ParentID == targetID { + fmt.Printf(" - Skipping '%s' (already has target ID).\n", filepath.Base(entry.Path)) + continue + } + + fmt.Printf(" - Removing all attributes from '%s'. It will be re-assimilated\n", filepath.Base(entry.Path)) + filepath.WalkDir(entry.Path, func(path string, d os.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("error walking path '%s': %w", path, err) + } + + // Remove all attributes from the file. + if err := removeAttributes(path); err != nil { + fmt.Printf("failed to remove attributes from '%s': %v", path, err) + } + return nil + }) + } + return nil +} + +// updateOwnerIndexFile handles the logic of reading, modifying, and writing the MessagePack index file. +func updateOwnerIndexFile(basePath string, obsoleteIDs []string) error { + fmt.Printf(" Rewriting index file '%s'\n", basePath) + + ownerID, err := xattr.Get(basePath, prefixes.OwnerIDAttr) + if err != nil { + return fmt.Errorf("could not get owner ID from oldest entry '%s' to find index: %w", basePath, err) + } + + indexPath := filepath.Join(basePath, "../../indexes/by-user-id", string(ownerID)+".mpk") + indexPath = filepath.Clean(indexPath) + + // Read the MessagePack file + fileData, err := os.ReadFile(indexPath) + if err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("index file does not exist, skipping update") + } + return fmt.Errorf("could not read index file: %w", err) + } + var indexMap map[string]string + if err := msgpack.Unmarshal(fileData, &indexMap); err != nil { + return fmt.Errorf("failed to parse MessagePack index file (is it corrupt?): %w", err) + } + + // Remove obsolete IDs from the map + itemsRemoved := 0 + for _, id := range obsoleteIDs { + if _, exists := indexMap[id]; exists { + fmt.Printf(" - Removing obsolete ID '%s' from index.\n", id) + delete(indexMap, id) + itemsRemoved++ + } else { + fmt.Printf(" - Obsolete ID '%s' not found in index\n", id) + } + } + + if itemsRemoved == 0 { + return nil + } + + // Write the data back to the file + updatedData, err := msgpack.Marshal(&indexMap) + if err != nil { + return fmt.Errorf("failed to marshal updated index map: %w", err) + } + if err := os.WriteFile(indexPath, updatedData, 0644); err != nil { + return fmt.Errorf("failed to write updated index file: %w", err) + } + + fmt.Printf(" ✓ Successfully removed %d item(s) and saved index file.\n", itemsRemoved) + return nil +} + +func removeAttributes(path string) error { + attrNames, err := xattr.List(path) + if err != nil { + return fmt.Errorf("failed to list attributes for '%s': %w", path, err) + } + + for _, attrName := range attrNames { + if err := xattr.Remove(path, attrName); err != nil { + return fmt.Errorf("failed to remove attribute '%s' from '%s': %w", attrName, path, err) + } + } + return nil +} From 97ad525d5becb043af8a137ebafba84b07bc3879 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Duffeck?= Date: Mon, 3 Aug 2026 13:24:23 +0200 Subject: [PATCH 10/10] Infer the indexes path from the config instead of calculating it --- opencloud/pkg/command/posixfs_consistency.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/opencloud/pkg/command/posixfs_consistency.go b/opencloud/pkg/command/posixfs_consistency.go index c56677c65e..f822143bbc 100644 --- a/opencloud/pkg/command/posixfs_consistency.go +++ b/opencloud/pkg/command/posixfs_consistency.go @@ -147,7 +147,7 @@ func (c *consistencyChecker) checkSpaceID(spacePath string) { obsoleteIDs = append(obsoleteIDs, id) } } - fixSpaceID(spacePath, obsoleteIDs, targetID, entries) + c.fixSpaceID(spacePath, obsoleteIDs, targetID, entries) } } @@ -325,7 +325,9 @@ func (c *consistencyChecker) checkNodes(spacePath string) { } } -func fixSpaceID(spacePath string, obsoleteIDs []string, targetID string, entries []EntryInfo) { +// fixSpaceID updates the parentid attributes of all entries in a space to a new target ID, +// updates the space's own ID attributes, and removes obsolete IDs from the user index file. +func (c *consistencyChecker) fixSpaceID(spacePath string, obsoleteIDs []string, targetID string, entries []EntryInfo) { // Set all parentid attributes to the proper space ID err := setAllParentIDAttributes(entries, targetID) if err != nil { @@ -347,7 +349,7 @@ func fixSpaceID(spacePath string, obsoleteIDs []string, targetID string, entries } // update the index - err = updateOwnerIndexFile(spacePath, obsoleteIDs) + err = c.updateOwnerIndexFile(spacePath, obsoleteIDs) if err != nil { logFailure("Could not update the owner index file: %v", err) } @@ -424,7 +426,7 @@ func setAllParentIDAttributes(entries []EntryInfo, targetID string) error { } // updateOwnerIndexFile handles the logic of reading, modifying, and writing the MessagePack index file. -func updateOwnerIndexFile(basePath string, obsoleteIDs []string) error { +func (c *consistencyChecker) updateOwnerIndexFile(basePath string, obsoleteIDs []string) error { fmt.Printf(" Rewriting index file '%s'\n", basePath) ownerID, err := xattr.Get(basePath, prefixes.OwnerIDAttr) @@ -432,7 +434,7 @@ func updateOwnerIndexFile(basePath string, obsoleteIDs []string) error { return fmt.Errorf("could not get owner ID from oldest entry '%s' to find index: %w", basePath, err) } - indexPath := filepath.Join(basePath, "../../indexes/by-user-id", string(ownerID)+".mpk") + indexPath := filepath.Join(c.cfg.Drivers.Posix.Root, "indexes", "by-user-id", string(ownerID)+".mpk") indexPath = filepath.Clean(indexPath) // Read the MessagePack file