mirror of
https://github.com/navidrome/navidrome.git
synced 2025-12-23 15:08:04 -05:00
* feat: Add selective folder scanning capability Implement targeted scanning of specific library/folder pairs without full recursion. This enables efficient rescanning of individual folders when changes are detected, significantly reducing scan time for large libraries. Key changes: - Add ScanTarget struct and ScanFolders API to Scanner interface - Implement CLI flag --targets for specifying libraryID:folderPath pairs - Add FolderRepository.GetByPaths() for batch folder info retrieval - Create loadSpecificFolders() for non-recursive directory loading - Scope GC operations to affected libraries only (with TODO for full impl) - Add comprehensive tests for selective scanning behavior The selective scan: - Only processes specified folders (no subdirectory recursion) - Maintains library isolation - Runs full maintenance pipeline scoped to affected libraries - Supports both full and quick scan modes Examples: navidrome scan --targets "1:Music/Rock,1:Music/Jazz" navidrome scan --full --targets "2:Classical" * feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval Signed-off-by: Deluan <deluan@navidrome.org> * test: update parseTargets test to handle folder names with spaces Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): remove unused LibraryPath struct and update GC logging message Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): enhance external scanner to support target-specific scanning Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify scanner methods Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement folder scanning notifications with deduplication Signed-off-by: Deluan <deluan@navidrome.org> * refactor(watcher): add resolveFolderPath function for testability Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement path ignoring based on .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): implement IgnoreChecker for managing .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(ignore_checker): rename scanner to lineScanner for clarity Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance ScanTarget struct with String method for better target representation Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): validate library ID to prevent negative values Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify GC method by removing library ID parameter Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): update folder scanning to include all descendants of specified folders Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): allow selective scan in the /startScan endpoint Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update CallScan to handle specific library/folder pairs Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline scanning logic by removing scanAll method Signed-off-by: Deluan <deluan@navidrome.org> * test: enhance mockScanner for thread safety and improve test reliability Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move scanner.ScanTarget to model.ScanTarget Signed-off-by: Deluan <deluan@navidrome.org> * refactor: move scanner types to model,implement MockScanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update scanner interface and implementations to use model.Scanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder_repository): normalize target path handling by using filepath.Clean Signed-off-by: Deluan <deluan@navidrome.org> * test(folder_repository): add comprehensive tests for folder retrieval and child exclusion Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify selective scan logic using slice.Filter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move initialization logic from phase_1 to the scanner itself Signed-off-by: Deluan <deluan@navidrome.org> * refactor(tests): rename selective scan test file to scanner_selective_test.go Signed-off-by: Deluan <deluan@navidrome.org> * feat(configuration): add DevSelectiveWatcher configuration option Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): enhance .ndignore handling for folder deletions and file changes Signed-off-by: Deluan <deluan@navidrome.org> * docs(scanner): comments Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance walkDirTree to support target folder scanning Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner, watcher): handle errors when pushing ignore patterns for folders Signed-off-by: Deluan <deluan@navidrome.org> * Update scanner/phase_1_folders.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): add tests for ScanBegin and ScanEnd functionality Signed-off-by: Deluan <deluan@navidrome.org> * fix(library): update PRAGMA optimize to check table sizes without ANALYZE Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): refactor tests Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add selective scan options and update translations Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add quick and full scan options for individual libraries Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add Scan buttonsto the LibraryList Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): update scanning parameters from 'path' to 'target' for selective scans. * refactor(scan): move ParseTargets function to model package * test(scan): suppress unused return value from SetUserLibraries in tests * feat(gc): enhance garbage collection to support selective library purging Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): prevent race condition when scanning deleted folders When the watcher detects changes in a folder that gets deleted before the scanner runs (due to the 10-second delay), the scanner was prematurely removing these folders from the tracking map, preventing them from being marked as missing. The issue occurred because `newFolderEntry` was calling `popLastUpdate` before verifying the folder actually exists on the filesystem. Changes: - Move fs.Stat check before newFolderEntry creation in loadDir to ensure deleted folders remain in lastUpdates for finalize() to handle - Add early existence check in walkDirTree to skip non-existent target folders with a warning log - Add unit test verifying non-existent folders aren't removed from lastUpdates prematurely - Add integration test for deleted folder scenario with ScanFolders Fixes the issue where deleting entire folders (e.g., /music/AC_DC) wouldn't mark tracks as missing when using selective folder scanning. * refactor(scan): streamline folder entry creation and update handling Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): add '@Recycle' (QNAP) to ignored directories list Signed-off-by: Deluan <deluan@navidrome.org> * fix(log): improve thread safety in logging level management * test(scan): move unit tests for ParseTargets function Signed-off-by: Deluan <deluan@navidrome.org> * review Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: deluan <deluan.quintao@mechanical-orchard.com>
314 lines
10 KiB
Go
314 lines
10 KiB
Go
package scanner
|
|
|
|
import (
|
|
"context"
|
|
"testing/fstest"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
)
|
|
|
|
var _ = Describe("IgnoreChecker", func() {
|
|
Describe("loadPatternsFromFolder", func() {
|
|
var ic *IgnoreChecker
|
|
var ctx context.Context
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
})
|
|
|
|
Context("when .ndignore file does not exist", func() {
|
|
It("should return empty patterns", func() {
|
|
fsys := fstest.MapFS{}
|
|
ic = newIgnoreChecker(fsys)
|
|
patterns := ic.loadPatternsFromFolder(ctx, ".")
|
|
Expect(patterns).To(BeEmpty())
|
|
})
|
|
})
|
|
|
|
Context("when .ndignore file is empty", func() {
|
|
It("should return wildcard to ignore everything", func() {
|
|
fsys := fstest.MapFS{
|
|
".ndignore": &fstest.MapFile{Data: []byte("")},
|
|
}
|
|
ic = newIgnoreChecker(fsys)
|
|
patterns := ic.loadPatternsFromFolder(ctx, ".")
|
|
Expect(patterns).To(Equal([]string{"**/*"}))
|
|
})
|
|
})
|
|
|
|
DescribeTable("parsing .ndignore content",
|
|
func(content string, expectedPatterns []string) {
|
|
fsys := fstest.MapFS{
|
|
".ndignore": &fstest.MapFile{Data: []byte(content)},
|
|
}
|
|
ic = newIgnoreChecker(fsys)
|
|
patterns := ic.loadPatternsFromFolder(ctx, ".")
|
|
Expect(patterns).To(Equal(expectedPatterns))
|
|
},
|
|
Entry("single pattern", "*.txt", []string{"*.txt"}),
|
|
Entry("multiple patterns", "*.txt\n*.log", []string{"*.txt", "*.log"}),
|
|
Entry("with comments", "# comment\n*.txt\n# another\n*.log", []string{"*.txt", "*.log"}),
|
|
Entry("with empty lines", "*.txt\n\n*.log\n\n", []string{"*.txt", "*.log"}),
|
|
Entry("mixed content", "# header\n\n*.txt\n# middle\n*.log\n\n", []string{"*.txt", "*.log"}),
|
|
Entry("only comments and empty lines", "# comment\n\n# another\n", []string{"**/*"}),
|
|
Entry("trailing newline", "*.txt\n*.log\n", []string{"*.txt", "*.log"}),
|
|
Entry("directory pattern", "temp/", []string{"temp/"}),
|
|
Entry("wildcard pattern", "**/*.mp3", []string{"**/*.mp3"}),
|
|
Entry("multiple wildcards", "**/*.mp3\n**/*.flac\n*.log", []string{"**/*.mp3", "**/*.flac", "*.log"}),
|
|
Entry("negation pattern", "!important.txt", []string{"!important.txt"}),
|
|
Entry("comment with hash not at start is pattern", "not#comment", []string{"not#comment"}),
|
|
Entry("whitespace-only lines skipped", "*.txt\n \n*.log\n\t\n", []string{"*.txt", "*.log"}),
|
|
Entry("patterns with whitespace trimmed", " *.txt \n\t*.log\t", []string{"*.txt", "*.log"}),
|
|
)
|
|
})
|
|
|
|
Describe("Push and Pop", func() {
|
|
var ic *IgnoreChecker
|
|
var fsys fstest.MapFS
|
|
var ctx context.Context
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
fsys = fstest.MapFS{
|
|
".ndignore": &fstest.MapFile{Data: []byte("*.txt")},
|
|
"folder1/.ndignore": &fstest.MapFile{Data: []byte("*.mp3")},
|
|
"folder2/.ndignore": &fstest.MapFile{Data: []byte("*.flac")},
|
|
}
|
|
ic = newIgnoreChecker(fsys)
|
|
})
|
|
|
|
Context("Push", func() {
|
|
It("should add patterns to stack", func() {
|
|
err := ic.Push(ctx, ".")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(len(ic.patternStack)).To(Equal(1))
|
|
Expect(ic.currentPatterns).To(ContainElement("*.txt"))
|
|
})
|
|
|
|
It("should compile matcher after push", func() {
|
|
err := ic.Push(ctx, ".")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(ic.matcher).ToNot(BeNil())
|
|
})
|
|
|
|
It("should accumulate patterns from multiple levels", func() {
|
|
err := ic.Push(ctx, ".")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
err = ic.Push(ctx, "folder1")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(len(ic.patternStack)).To(Equal(2))
|
|
Expect(ic.currentPatterns).To(ConsistOf("*.txt", "*.mp3"))
|
|
})
|
|
|
|
It("should handle push when no .ndignore exists", func() {
|
|
err := ic.Push(ctx, "nonexistent")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(len(ic.patternStack)).To(Equal(1))
|
|
Expect(ic.currentPatterns).To(BeEmpty())
|
|
})
|
|
})
|
|
|
|
Context("Pop", func() {
|
|
It("should remove most recent patterns", func() {
|
|
err := ic.Push(ctx, ".")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
err = ic.Push(ctx, "folder1")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
ic.Pop()
|
|
Expect(len(ic.patternStack)).To(Equal(1))
|
|
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
|
|
})
|
|
|
|
It("should handle Pop on empty stack gracefully", func() {
|
|
Expect(func() { ic.Pop() }).ToNot(Panic())
|
|
Expect(ic.patternStack).To(BeEmpty())
|
|
})
|
|
|
|
It("should set matcher to nil when all patterns popped", func() {
|
|
err := ic.Push(ctx, ".")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(ic.matcher).ToNot(BeNil())
|
|
ic.Pop()
|
|
Expect(ic.matcher).To(BeNil())
|
|
})
|
|
|
|
It("should update matcher after pop", func() {
|
|
err := ic.Push(ctx, ".")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
err = ic.Push(ctx, "folder1")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
matcher1 := ic.matcher
|
|
ic.Pop()
|
|
matcher2 := ic.matcher
|
|
Expect(matcher1).ToNot(Equal(matcher2))
|
|
})
|
|
})
|
|
|
|
Context("multiple Push/Pop cycles", func() {
|
|
It("should maintain correct state through cycles", func() {
|
|
err := ic.Push(ctx, ".")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
|
|
|
|
err = ic.Push(ctx, "folder1")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(ic.currentPatterns).To(ConsistOf("*.txt", "*.mp3"))
|
|
|
|
ic.Pop()
|
|
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
|
|
|
|
err = ic.Push(ctx, "folder2")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(ic.currentPatterns).To(ConsistOf("*.txt", "*.flac"))
|
|
|
|
ic.Pop()
|
|
Expect(ic.currentPatterns).To(Equal([]string{"*.txt"}))
|
|
|
|
ic.Pop()
|
|
Expect(ic.currentPatterns).To(BeEmpty())
|
|
})
|
|
})
|
|
})
|
|
|
|
Describe("PushAllParents", func() {
|
|
var ic *IgnoreChecker
|
|
var ctx context.Context
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
fsys := fstest.MapFS{
|
|
".ndignore": &fstest.MapFile{Data: []byte("root.txt")},
|
|
"folder1/.ndignore": &fstest.MapFile{Data: []byte("level1.txt")},
|
|
"folder1/folder2/.ndignore": &fstest.MapFile{Data: []byte("level2.txt")},
|
|
"folder1/folder2/folder3/.ndignore": &fstest.MapFile{Data: []byte("level3.txt")},
|
|
}
|
|
ic = newIgnoreChecker(fsys)
|
|
})
|
|
|
|
DescribeTable("loading parent patterns",
|
|
func(targetPath string, expectedStackDepth int, expectedPatterns []string) {
|
|
err := ic.PushAllParents(ctx, targetPath)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(len(ic.patternStack)).To(Equal(expectedStackDepth))
|
|
Expect(ic.currentPatterns).To(ConsistOf(expectedPatterns))
|
|
},
|
|
Entry("root path", ".", 1, []string{"root.txt"}),
|
|
Entry("empty path", "", 1, []string{"root.txt"}),
|
|
Entry("single level", "folder1", 2, []string{"root.txt", "level1.txt"}),
|
|
Entry("two levels", "folder1/folder2", 3, []string{"root.txt", "level1.txt", "level2.txt"}),
|
|
Entry("three levels", "folder1/folder2/folder3", 4, []string{"root.txt", "level1.txt", "level2.txt", "level3.txt"}),
|
|
)
|
|
|
|
It("should only compile patterns once at the end", func() {
|
|
// This is more of a behavioral test - we verify the matcher is not nil after PushAllParents
|
|
err := ic.PushAllParents(ctx, "folder1/folder2")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(ic.matcher).ToNot(BeNil())
|
|
})
|
|
|
|
It("should handle paths with dot", func() {
|
|
err := ic.PushAllParents(ctx, "./folder1")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(len(ic.patternStack)).To(Equal(2))
|
|
})
|
|
|
|
Context("when some parent folders have no .ndignore", func() {
|
|
BeforeEach(func() {
|
|
fsys := fstest.MapFS{
|
|
".ndignore": &fstest.MapFile{Data: []byte("root.txt")},
|
|
"folder1/folder2/.ndignore": &fstest.MapFile{Data: []byte("level2.txt")},
|
|
}
|
|
ic = newIgnoreChecker(fsys)
|
|
})
|
|
|
|
It("should still push all parent levels", func() {
|
|
err := ic.PushAllParents(ctx, "folder1/folder2")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(len(ic.patternStack)).To(Equal(3)) // root, folder1 (empty), folder2
|
|
Expect(ic.currentPatterns).To(ConsistOf("root.txt", "level2.txt"))
|
|
})
|
|
})
|
|
})
|
|
|
|
Describe("ShouldIgnore", func() {
|
|
var ic *IgnoreChecker
|
|
var ctx context.Context
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
})
|
|
|
|
Context("with no patterns loaded", func() {
|
|
It("should not ignore any path", func() {
|
|
fsys := fstest.MapFS{}
|
|
ic = newIgnoreChecker(fsys)
|
|
Expect(ic.ShouldIgnore(ctx, "anything.txt")).To(BeFalse())
|
|
Expect(ic.ShouldIgnore(ctx, "folder/file.mp3")).To(BeFalse())
|
|
})
|
|
})
|
|
|
|
Context("special paths", func() {
|
|
BeforeEach(func() {
|
|
fsys := fstest.MapFS{
|
|
".ndignore": &fstest.MapFile{Data: []byte("**/*")},
|
|
}
|
|
ic = newIgnoreChecker(fsys)
|
|
err := ic.Push(ctx, ".")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
})
|
|
|
|
It("should never ignore root or empty paths", func() {
|
|
Expect(ic.ShouldIgnore(ctx, "")).To(BeFalse())
|
|
Expect(ic.ShouldIgnore(ctx, ".")).To(BeFalse())
|
|
})
|
|
|
|
It("should ignore all other paths with wildcard", func() {
|
|
Expect(ic.ShouldIgnore(ctx, "file.txt")).To(BeTrue())
|
|
Expect(ic.ShouldIgnore(ctx, "folder/file.mp3")).To(BeTrue())
|
|
})
|
|
})
|
|
|
|
DescribeTable("pattern matching",
|
|
func(pattern string, path string, shouldMatch bool) {
|
|
fsys := fstest.MapFS{
|
|
".ndignore": &fstest.MapFile{Data: []byte(pattern)},
|
|
}
|
|
ic = newIgnoreChecker(fsys)
|
|
err := ic.Push(ctx, ".")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(ic.ShouldIgnore(ctx, path)).To(Equal(shouldMatch))
|
|
},
|
|
Entry("glob match", "*.txt", "file.txt", true),
|
|
Entry("glob no match", "*.txt", "file.mp3", false),
|
|
Entry("directory pattern match", "tmp/", "tmp/file.txt", true),
|
|
Entry("directory pattern no match", "tmp/", "temporary/file.txt", false),
|
|
Entry("nested glob match", "**/*.log", "deep/nested/file.log", true),
|
|
Entry("nested glob no match", "**/*.log", "deep/nested/file.txt", false),
|
|
Entry("specific file match", "ignore.me", "ignore.me", true),
|
|
Entry("specific file no match", "ignore.me", "keep.me", false),
|
|
Entry("wildcard all", "**/*", "any/path/file.txt", true),
|
|
Entry("nested specific match", "temp/*", "temp/cache.db", true),
|
|
Entry("nested specific no match", "temp/*", "temporary/cache.db", false),
|
|
)
|
|
|
|
Context("with multiple patterns", func() {
|
|
BeforeEach(func() {
|
|
fsys := fstest.MapFS{
|
|
".ndignore": &fstest.MapFile{Data: []byte("*.txt\n*.log\ntemp/")},
|
|
}
|
|
ic = newIgnoreChecker(fsys)
|
|
err := ic.Push(ctx, ".")
|
|
Expect(err).ToNot(HaveOccurred())
|
|
})
|
|
|
|
It("should match any of the patterns", func() {
|
|
Expect(ic.ShouldIgnore(ctx, "file.txt")).To(BeTrue())
|
|
Expect(ic.ShouldIgnore(ctx, "debug.log")).To(BeTrue())
|
|
Expect(ic.ShouldIgnore(ctx, "temp/cache")).To(BeTrue())
|
|
Expect(ic.ShouldIgnore(ctx, "music.mp3")).To(BeFalse())
|
|
})
|
|
})
|
|
})
|
|
})
|