mirror of
https://github.com/navidrome/navidrome.git
synced 2025-12-23 23:18:05 -05:00
* feat: Add selective folder scanning capability Implement targeted scanning of specific library/folder pairs without full recursion. This enables efficient rescanning of individual folders when changes are detected, significantly reducing scan time for large libraries. Key changes: - Add ScanTarget struct and ScanFolders API to Scanner interface - Implement CLI flag --targets for specifying libraryID:folderPath pairs - Add FolderRepository.GetByPaths() for batch folder info retrieval - Create loadSpecificFolders() for non-recursive directory loading - Scope GC operations to affected libraries only (with TODO for full impl) - Add comprehensive tests for selective scanning behavior The selective scan: - Only processes specified folders (no subdirectory recursion) - Maintains library isolation - Runs full maintenance pipeline scoped to affected libraries - Supports both full and quick scan modes Examples: navidrome scan --targets "1:Music/Rock,1:Music/Jazz" navidrome scan --full --targets "2:Classical" * feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval Signed-off-by: Deluan <deluan@navidrome.org> * test: update parseTargets test to handle folder names with spaces Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): remove unused LibraryPath struct and update GC logging message Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): enhance external scanner to support target-specific scanning Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify scanner methods Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement folder scanning notifications with deduplication Signed-off-by: Deluan <deluan@navidrome.org> * refactor(watcher): add resolveFolderPath function for testability Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement path ignoring based on .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): implement IgnoreChecker for managing .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(ignore_checker): rename scanner to lineScanner for clarity Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance ScanTarget struct with String method for better target representation Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): validate library ID to prevent negative values Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify GC method by removing library ID parameter Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): update folder scanning to include all descendants of specified folders Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): allow selective scan in the /startScan endpoint Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update CallScan to handle specific library/folder pairs Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline scanning logic by removing scanAll method Signed-off-by: Deluan <deluan@navidrome.org> * test: enhance mockScanner for thread safety and improve test reliability Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move scanner.ScanTarget to model.ScanTarget Signed-off-by: Deluan <deluan@navidrome.org> * refactor: move scanner types to model,implement MockScanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update scanner interface and implementations to use model.Scanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder_repository): normalize target path handling by using filepath.Clean Signed-off-by: Deluan <deluan@navidrome.org> * test(folder_repository): add comprehensive tests for folder retrieval and child exclusion Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify selective scan logic using slice.Filter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move initialization logic from phase_1 to the scanner itself Signed-off-by: Deluan <deluan@navidrome.org> * refactor(tests): rename selective scan test file to scanner_selective_test.go Signed-off-by: Deluan <deluan@navidrome.org> * feat(configuration): add DevSelectiveWatcher configuration option Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): enhance .ndignore handling for folder deletions and file changes Signed-off-by: Deluan <deluan@navidrome.org> * docs(scanner): comments Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance walkDirTree to support target folder scanning Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner, watcher): handle errors when pushing ignore patterns for folders Signed-off-by: Deluan <deluan@navidrome.org> * Update scanner/phase_1_folders.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): add tests for ScanBegin and ScanEnd functionality Signed-off-by: Deluan <deluan@navidrome.org> * fix(library): update PRAGMA optimize to check table sizes without ANALYZE Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): refactor tests Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add selective scan options and update translations Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add quick and full scan options for individual libraries Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add Scan buttonsto the LibraryList Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): update scanning parameters from 'path' to 'target' for selective scans. * refactor(scan): move ParseTargets function to model package * test(scan): suppress unused return value from SetUserLibraries in tests * feat(gc): enhance garbage collection to support selective library purging Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): prevent race condition when scanning deleted folders When the watcher detects changes in a folder that gets deleted before the scanner runs (due to the 10-second delay), the scanner was prematurely removing these folders from the tracking map, preventing them from being marked as missing. The issue occurred because `newFolderEntry` was calling `popLastUpdate` before verifying the folder actually exists on the filesystem. Changes: - Move fs.Stat check before newFolderEntry creation in loadDir to ensure deleted folders remain in lastUpdates for finalize() to handle - Add early existence check in walkDirTree to skip non-existent target folders with a warning log - Add unit test verifying non-existent folders aren't removed from lastUpdates prematurely - Add integration test for deleted folder scenario with ScanFolders Fixes the issue where deleting entire folders (e.g., /music/AC_DC) wouldn't mark tracks as missing when using selective folder scanning. * refactor(scan): streamline folder entry creation and update handling Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): add '@Recycle' (QNAP) to ignored directories list Signed-off-by: Deluan <deluan@navidrome.org> * fix(log): improve thread safety in logging level management * test(scan): move unit tests for ParseTargets function Signed-off-by: Deluan <deluan@navidrome.org> * review Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: deluan <deluan.quintao@mechanical-orchard.com>
365 lines
10 KiB
Go
365 lines
10 KiB
Go
package core
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sync"
|
|
|
|
"github.com/navidrome/navidrome/model"
|
|
"github.com/navidrome/navidrome/model/request"
|
|
"github.com/navidrome/navidrome/tests"
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
"github.com/sirupsen/logrus"
|
|
)
|
|
|
|
var _ = Describe("Maintenance", func() {
|
|
var ds *tests.MockDataStore
|
|
var mfRepo *extendedMediaFileRepo
|
|
var service Maintenance
|
|
var ctx context.Context
|
|
|
|
BeforeEach(func() {
|
|
ctx = context.Background()
|
|
ctx = request.WithUser(ctx, model.User{ID: "user1", IsAdmin: true})
|
|
|
|
ds = createTestDataStore()
|
|
mfRepo = ds.MockedMediaFile.(*extendedMediaFileRepo)
|
|
service = NewMaintenance(ds)
|
|
})
|
|
|
|
Describe("DeleteMissingFiles", func() {
|
|
Context("with specific IDs", func() {
|
|
It("deletes specific missing files and runs GC", func() {
|
|
// Setup: mock missing files with album IDs
|
|
mfRepo.SetData(model.MediaFiles{
|
|
{ID: "mf1", AlbumID: "album1", Missing: true},
|
|
{ID: "mf2", AlbumID: "album2", Missing: true},
|
|
})
|
|
|
|
err := service.DeleteMissingFiles(ctx, []string{"mf1", "mf2"})
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(mfRepo.deleteMissingCalled).To(BeTrue())
|
|
Expect(mfRepo.deletedIDs).To(Equal([]string{"mf1", "mf2"}))
|
|
Expect(ds.GCCalled).To(BeTrue(), "GC should be called after deletion")
|
|
})
|
|
|
|
It("triggers artist stats refresh and album refresh after deletion", func() {
|
|
artistRepo := ds.MockedArtist.(*extendedArtistRepo)
|
|
// Setup: mock missing files with albums
|
|
albumRepo := ds.MockedAlbum.(*extendedAlbumRepo)
|
|
albumRepo.SetData(model.Albums{
|
|
{ID: "album1", Name: "Test Album", SongCount: 5},
|
|
})
|
|
mfRepo.SetData(model.MediaFiles{
|
|
{ID: "mf1", AlbumID: "album1", Missing: true},
|
|
{ID: "mf2", AlbumID: "album1", Missing: false, Size: 1000, Duration: 180},
|
|
{ID: "mf3", AlbumID: "album1", Missing: false, Size: 2000, Duration: 200},
|
|
})
|
|
|
|
err := service.DeleteMissingFiles(ctx, []string{"mf1"})
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
// Wait for background goroutines to complete
|
|
service.(*maintenanceService).wait()
|
|
|
|
// RefreshStats should be called
|
|
Expect(artistRepo.IsRefreshStatsCalled()).To(BeTrue(), "Artist stats should be refreshed")
|
|
|
|
// Album should be updated with new calculated values
|
|
Expect(albumRepo.GetPutCallCount()).To(BeNumerically(">", 0), "Album.Put() should be called to refresh album data")
|
|
})
|
|
|
|
It("returns error if deletion fails", func() {
|
|
mfRepo.deleteMissingError = errors.New("delete failed")
|
|
|
|
err := service.DeleteMissingFiles(ctx, []string{"mf1"})
|
|
|
|
Expect(err).To(HaveOccurred())
|
|
Expect(err.Error()).To(ContainSubstring("delete failed"))
|
|
})
|
|
|
|
It("continues even if album tracking fails", func() {
|
|
mfRepo.SetError(true)
|
|
|
|
err := service.DeleteMissingFiles(ctx, []string{"mf1"})
|
|
|
|
// Should not fail, just log warning
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(mfRepo.deleteMissingCalled).To(BeTrue())
|
|
})
|
|
|
|
It("returns error if GC fails", func() {
|
|
mfRepo.SetData(model.MediaFiles{
|
|
{ID: "mf1", AlbumID: "album1", Missing: true},
|
|
})
|
|
|
|
// Set GC to return error
|
|
ds.GCError = errors.New("gc failed")
|
|
|
|
err := service.DeleteMissingFiles(ctx, []string{"mf1"})
|
|
|
|
Expect(err).To(HaveOccurred())
|
|
Expect(err.Error()).To(ContainSubstring("gc failed"))
|
|
})
|
|
})
|
|
|
|
Context("album ID extraction", func() {
|
|
It("extracts unique album IDs from missing files", func() {
|
|
mfRepo.SetData(model.MediaFiles{
|
|
{ID: "mf1", AlbumID: "album1", Missing: true},
|
|
{ID: "mf2", AlbumID: "album1", Missing: true},
|
|
{ID: "mf3", AlbumID: "album2", Missing: true},
|
|
})
|
|
|
|
err := service.DeleteMissingFiles(ctx, []string{"mf1", "mf2", "mf3"})
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
|
})
|
|
|
|
It("skips files without album IDs", func() {
|
|
mfRepo.SetData(model.MediaFiles{
|
|
{ID: "mf1", AlbumID: "", Missing: true},
|
|
{ID: "mf2", AlbumID: "album1", Missing: true},
|
|
})
|
|
|
|
err := service.DeleteMissingFiles(ctx, []string{"mf1", "mf2"})
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
|
})
|
|
})
|
|
})
|
|
|
|
Describe("DeleteAllMissingFiles", func() {
|
|
It("deletes all missing files and runs GC", func() {
|
|
mfRepo.SetData(model.MediaFiles{
|
|
{ID: "mf1", AlbumID: "album1", Missing: true},
|
|
{ID: "mf2", AlbumID: "album2", Missing: true},
|
|
{ID: "mf3", AlbumID: "album3", Missing: true},
|
|
})
|
|
|
|
err := service.DeleteAllMissingFiles(ctx)
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(ds.GCCalled).To(BeTrue(), "GC should be called after deletion")
|
|
})
|
|
|
|
It("returns error if deletion fails", func() {
|
|
mfRepo.SetError(true)
|
|
|
|
err := service.DeleteAllMissingFiles(ctx)
|
|
|
|
Expect(err).To(HaveOccurred())
|
|
})
|
|
|
|
It("handles empty result gracefully", func() {
|
|
mfRepo.SetData(model.MediaFiles{})
|
|
|
|
err := service.DeleteAllMissingFiles(ctx)
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
|
})
|
|
})
|
|
|
|
Describe("Album refresh logic", func() {
|
|
var albumRepo *extendedAlbumRepo
|
|
|
|
BeforeEach(func() {
|
|
albumRepo = ds.MockedAlbum.(*extendedAlbumRepo)
|
|
})
|
|
|
|
Context("when album has no tracks after deletion", func() {
|
|
It("skips the album without updating it", func() {
|
|
// Setup album with no remaining tracks
|
|
albumRepo.SetData(model.Albums{
|
|
{ID: "album1", Name: "Empty Album", SongCount: 1},
|
|
})
|
|
mfRepo.SetData(model.MediaFiles{
|
|
{ID: "mf1", AlbumID: "album1", Missing: true},
|
|
})
|
|
|
|
err := service.DeleteMissingFiles(ctx, []string{"mf1"})
|
|
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
// Wait for background goroutines to complete
|
|
service.(*maintenanceService).wait()
|
|
|
|
// Album should NOT be updated because it has no tracks left
|
|
Expect(albumRepo.GetPutCallCount()).To(Equal(0), "Album with no tracks should not be updated")
|
|
})
|
|
})
|
|
|
|
Context("when Put fails for one album", func() {
|
|
It("continues processing other albums", func() {
|
|
albumRepo.SetData(model.Albums{
|
|
{ID: "album1", Name: "Album 1"},
|
|
{ID: "album2", Name: "Album 2"},
|
|
})
|
|
mfRepo.SetData(model.MediaFiles{
|
|
{ID: "mf1", AlbumID: "album1", Missing: true},
|
|
{ID: "mf2", AlbumID: "album1", Missing: false, Size: 1000, Duration: 180},
|
|
{ID: "mf3", AlbumID: "album2", Missing: true},
|
|
{ID: "mf4", AlbumID: "album2", Missing: false, Size: 2000, Duration: 200},
|
|
})
|
|
|
|
// Make Put fail on first call but succeed on subsequent calls
|
|
albumRepo.putError = errors.New("put failed")
|
|
albumRepo.failOnce = true
|
|
|
|
err := service.DeleteMissingFiles(ctx, []string{"mf1", "mf3"})
|
|
|
|
// Should not fail even if one album's Put fails
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
// Wait for background goroutines to complete
|
|
service.(*maintenanceService).wait()
|
|
|
|
// Put should have been called multiple times
|
|
Expect(albumRepo.GetPutCallCount()).To(BeNumerically(">", 0), "Put should be attempted")
|
|
})
|
|
})
|
|
|
|
Context("when media file loading fails", func() {
|
|
It("logs warning but continues when tracking affected albums fails", func() {
|
|
// Set up log capturing
|
|
hook, cleanup := tests.LogHook()
|
|
defer cleanup()
|
|
|
|
albumRepo.SetData(model.Albums{
|
|
{ID: "album1", Name: "Album 1"},
|
|
})
|
|
mfRepo.SetData(model.MediaFiles{
|
|
{ID: "mf1", AlbumID: "album1", Missing: true},
|
|
})
|
|
// Make GetAll fail when loading media files
|
|
mfRepo.SetError(true)
|
|
|
|
err := service.DeleteMissingFiles(ctx, []string{"mf1"})
|
|
|
|
// Deletion should succeed despite the tracking error
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(mfRepo.deleteMissingCalled).To(BeTrue())
|
|
|
|
// Verify the warning was logged
|
|
Expect(hook.LastEntry()).ToNot(BeNil())
|
|
Expect(hook.LastEntry().Level).To(Equal(logrus.WarnLevel))
|
|
Expect(hook.LastEntry().Message).To(Equal("Error tracking affected albums for refresh"))
|
|
})
|
|
})
|
|
})
|
|
})
|
|
|
|
// Test helper to create a mock DataStore with controllable behavior
|
|
func createTestDataStore() *tests.MockDataStore {
|
|
ds := &tests.MockDataStore{}
|
|
|
|
// Create extended album repo with Put tracking
|
|
albumRepo := &extendedAlbumRepo{
|
|
MockAlbumRepo: tests.CreateMockAlbumRepo(),
|
|
}
|
|
ds.MockedAlbum = albumRepo
|
|
|
|
// Create extended artist repo with RefreshStats tracking
|
|
artistRepo := &extendedArtistRepo{
|
|
MockArtistRepo: tests.CreateMockArtistRepo(),
|
|
}
|
|
ds.MockedArtist = artistRepo
|
|
|
|
// Create extended media file repo with DeleteMissing support
|
|
mfRepo := &extendedMediaFileRepo{
|
|
MockMediaFileRepo: tests.CreateMockMediaFileRepo(),
|
|
}
|
|
ds.MockedMediaFile = mfRepo
|
|
|
|
return ds
|
|
}
|
|
|
|
// Extension of MockMediaFileRepo to add DeleteMissing method
|
|
type extendedMediaFileRepo struct {
|
|
*tests.MockMediaFileRepo
|
|
deleteMissingCalled bool
|
|
deletedIDs []string
|
|
deleteMissingError error
|
|
}
|
|
|
|
func (m *extendedMediaFileRepo) DeleteMissing(ids []string) error {
|
|
m.deleteMissingCalled = true
|
|
m.deletedIDs = ids
|
|
if m.deleteMissingError != nil {
|
|
return m.deleteMissingError
|
|
}
|
|
// Actually delete from the mock data
|
|
for _, id := range ids {
|
|
delete(m.Data, id)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Extension of MockAlbumRepo to track Put calls
|
|
type extendedAlbumRepo struct {
|
|
*tests.MockAlbumRepo
|
|
mu sync.RWMutex
|
|
putCallCount int
|
|
lastPutData *model.Album
|
|
putError error
|
|
failOnce bool
|
|
}
|
|
|
|
func (m *extendedAlbumRepo) Put(album *model.Album) error {
|
|
m.mu.Lock()
|
|
m.putCallCount++
|
|
m.lastPutData = album
|
|
|
|
// Handle failOnce behavior
|
|
var err error
|
|
if m.putError != nil {
|
|
if m.failOnce {
|
|
err = m.putError
|
|
m.putError = nil // Clear error after first failure
|
|
m.mu.Unlock()
|
|
return err
|
|
}
|
|
err = m.putError
|
|
m.mu.Unlock()
|
|
return err
|
|
}
|
|
m.mu.Unlock()
|
|
|
|
return m.MockAlbumRepo.Put(album)
|
|
}
|
|
|
|
func (m *extendedAlbumRepo) GetPutCallCount() int {
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
return m.putCallCount
|
|
}
|
|
|
|
// Extension of MockArtistRepo to track RefreshStats calls
|
|
type extendedArtistRepo struct {
|
|
*tests.MockArtistRepo
|
|
mu sync.RWMutex
|
|
refreshStatsCalled bool
|
|
refreshStatsError error
|
|
}
|
|
|
|
func (m *extendedArtistRepo) RefreshStats(allArtists bool) (int64, error) {
|
|
m.mu.Lock()
|
|
m.refreshStatsCalled = true
|
|
err := m.refreshStatsError
|
|
m.mu.Unlock()
|
|
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return m.MockArtistRepo.RefreshStats(allArtists)
|
|
}
|
|
|
|
func (m *extendedArtistRepo) IsRefreshStatsCalled() bool {
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
return m.refreshStatsCalled
|
|
}
|