package scanner import ( "context" "io/fs" "maps" "path" "slices" "sort" "strings" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/utils" ) // walkDirTree recursively walks the directory tree starting from the given targetFolders. // If no targetFolders are provided, it starts from the root folder ("."). // It returns a channel of folderEntry pointers representing each folder found. func walkDirTree(ctx context.Context, job *scanJob, targetFolders ...string) (<-chan *folderEntry, error) { results := make(chan *folderEntry) folders := targetFolders if len(targetFolders) == 0 { // No specific folders provided, scan the root folder folders = []string{"."} } go func() { defer close(results) for _, folderPath := range folders { if utils.IsCtxDone(ctx) { return } // Check if target folder exists before walking it // If it doesn't exist (e.g., deleted between watcher detection and scan execution), // skip it so it remains in job.lastUpdates and gets handled in following steps _, err := fs.Stat(job.fs, folderPath) if err != nil { log.Warn(ctx, "Scanner: Target folder does not exist.", "path", folderPath, err) continue } // Create checker and push patterns from root to this folder checker := newIgnoreChecker(job.fs) err = checker.PushAllParents(ctx, folderPath) if err != nil { log.Error(ctx, "Scanner: Error pushing ignore patterns for target folder", "path", folderPath, err) continue } // Recursively walk this folder and all its children err = walkFolder(ctx, job, folderPath, checker, results) if err != nil { log.Error(ctx, "Scanner: Error walking target folder", "path", folderPath, err) continue } } log.Debug(ctx, "Scanner: Finished reading target folders", "lib", job.lib.Name, "path", job.lib.Path, "numFolders", job.numFolders.Load()) }() return results, nil } func walkFolder(ctx context.Context, job *scanJob, currentFolder string, checker *IgnoreChecker, results chan<- *folderEntry) error { // Push patterns for this folder onto the stack _ = checker.Push(ctx, currentFolder) defer checker.Pop() // Pop patterns when leaving this folder folder, children, err := loadDir(ctx, job, currentFolder, checker) if err != nil { log.Warn(ctx, "Scanner: Error loading dir. Skipping", "path", currentFolder, err) return nil } for _, c := range children { err := walkFolder(ctx, job, c, checker, results) if err != nil { return err } } dir := path.Clean(currentFolder) log.Trace(ctx, "Scanner: Found directory", " path", dir, "audioFiles", maps.Keys(folder.audioFiles), "images", maps.Keys(folder.imageFiles), "playlists", folder.numPlaylists, "imagesUpdatedAt", folder.imagesUpdatedAt, "updTime", folder.updTime, "modTime", folder.modTime, "numChildren", len(children)) folder.path = dir folder.elapsed.Start() results <- folder return nil } func loadDir(ctx context.Context, job *scanJob, dirPath string, checker *IgnoreChecker) (folder *folderEntry, children []string, err error) { // Check if directory exists before creating the folder entry // This is important to avoid removing the folder from lastUpdates if it doesn't exist dirInfo, err := fs.Stat(job.fs, dirPath) if err != nil { log.Warn(ctx, "Scanner: Error stating dir", "path", dirPath, err) return nil, nil, err } // Now that we know the folder exists, create the entry (which removes it from lastUpdates) folder = job.createFolderEntry(dirPath) folder.modTime = dirInfo.ModTime() dir, err := job.fs.Open(dirPath) if err != nil { log.Warn(ctx, "Scanner: Error in Opening directory", "path", dirPath, err) return folder, children, err } defer dir.Close() dirFile, ok := dir.(fs.ReadDirFile) if !ok { log.Error(ctx, "Not a directory", "path", dirPath) return folder, children, err } entries := fullReadDir(ctx, dirFile) children = make([]string, 0, len(entries)) for _, entry := range entries { entryPath := path.Join(dirPath, entry.Name()) if checker.ShouldIgnore(ctx, entryPath) { log.Trace(ctx, "Scanner: Ignoring entry", "path", entryPath) continue } if isEntryIgnored(entry.Name()) { continue } if ctx.Err() != nil { return folder, children, ctx.Err() } isDir, err := isDirOrSymlinkToDir(job.fs, dirPath, entry) // Skip invalid symlinks if err != nil { log.Warn(ctx, "Scanner: Invalid symlink", "dir", entryPath, err) continue } if isDir && !isDirIgnored(entry.Name()) && isDirReadable(ctx, job.fs, entryPath) { children = append(children, entryPath) folder.numSubFolders++ } else { fileInfo, err := entry.Info() if err != nil { log.Warn(ctx, "Scanner: Error getting fileInfo", "name", entry.Name(), err) return folder, children, err } if fileInfo.ModTime().After(folder.modTime) { folder.modTime = fileInfo.ModTime() } switch { case model.IsAudioFile(entry.Name()): folder.audioFiles[entry.Name()] = entry case model.IsValidPlaylist(entry.Name()): folder.numPlaylists++ case model.IsImageFile(entry.Name()): folder.imageFiles[entry.Name()] = entry folder.imagesUpdatedAt = utils.TimeNewest(folder.imagesUpdatedAt, fileInfo.ModTime(), folder.modTime) } } } return folder, children, nil } // fullReadDir reads all files in the folder, skipping the ones with errors. // It also detects when it is "stuck" with an error in the same directory over and over. // In this case, it stops and returns whatever it was able to read until it got stuck. // See discussion here: https://github.com/navidrome/navidrome/issues/1164#issuecomment-881922850 func fullReadDir(ctx context.Context, dir fs.ReadDirFile) []fs.DirEntry { var allEntries []fs.DirEntry var prevErrStr = "" for { if ctx.Err() != nil { return nil } entries, err := dir.ReadDir(-1) allEntries = append(allEntries, entries...) if err == nil { break } log.Warn(ctx, "Skipping DirEntry", err) if prevErrStr == err.Error() { log.Error(ctx, "Scanner: Duplicate DirEntry failure, bailing", err) break } prevErrStr = err.Error() } sort.Slice(allEntries, func(i, j int) bool { return allEntries[i].Name() < allEntries[j].Name() }) return allEntries } // isDirOrSymlinkToDir returns true if and only if the dirEnt represents a file // system directory, or a symbolic link to a directory. Note that if the dirEnt // is not a directory but is a symbolic link, this method will resolve by // sending a request to the operating system to follow the symbolic link. // originally copied from github.com/karrick/godirwalk, modified to use dirEntry for // efficiency for go 1.16 and beyond func isDirOrSymlinkToDir(fsys fs.FS, baseDir string, dirEnt fs.DirEntry) (bool, error) { if dirEnt.IsDir() { return true, nil } if dirEnt.Type()&fs.ModeSymlink == 0 { return false, nil } // If symlinks are disabled, return false for symlinks if !conf.Server.Scanner.FollowSymlinks { return false, nil } // Does this symlink point to a directory? fileInfo, err := fs.Stat(fsys, path.Join(baseDir, dirEnt.Name())) if err != nil { return false, err } return fileInfo.IsDir(), nil } // isDirReadable returns true if the directory represented by dirEnt is readable func isDirReadable(ctx context.Context, fsys fs.FS, dirPath string) bool { dir, err := fsys.Open(dirPath) if err != nil { log.Warn("Scanner: Skipping unreadable directory", "path", dirPath, err) return false } err = dir.Close() if err != nil { log.Warn(ctx, "Scanner: Error closing directory", "path", dirPath, err) } return true } // List of special directories to ignore var ignoredDirs = []string{ "$RECYCLE.BIN", "#snapshot", "@Recycle", "@Recently-Snapshot", ".streams", "lost+found", } // isDirIgnored returns true if the directory represented by dirEnt should be ignored func isDirIgnored(name string) bool { // allows Album folders for albums which eg start with ellipses if strings.HasPrefix(name, ".") && !strings.HasPrefix(name, "..") { return true } if slices.ContainsFunc(ignoredDirs, func(s string) bool { return strings.EqualFold(s, name) }) { return true } return false } func isEntryIgnored(name string) bool { return strings.HasPrefix(name, ".") && !strings.HasPrefix(name, "..") }