mirror of
https://github.com/navidrome/navidrome.git
synced 2025-12-23 15:08:04 -05:00
* feat: Add selective folder scanning capability Implement targeted scanning of specific library/folder pairs without full recursion. This enables efficient rescanning of individual folders when changes are detected, significantly reducing scan time for large libraries. Key changes: - Add ScanTarget struct and ScanFolders API to Scanner interface - Implement CLI flag --targets for specifying libraryID:folderPath pairs - Add FolderRepository.GetByPaths() for batch folder info retrieval - Create loadSpecificFolders() for non-recursive directory loading - Scope GC operations to affected libraries only (with TODO for full impl) - Add comprehensive tests for selective scanning behavior The selective scan: - Only processes specified folders (no subdirectory recursion) - Maintains library isolation - Runs full maintenance pipeline scoped to affected libraries - Supports both full and quick scan modes Examples: navidrome scan --targets "1:Music/Rock,1:Music/Jazz" navidrome scan --full --targets "2:Classical" * feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval Signed-off-by: Deluan <deluan@navidrome.org> * test: update parseTargets test to handle folder names with spaces Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): remove unused LibraryPath struct and update GC logging message Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): enhance external scanner to support target-specific scanning Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify scanner methods Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement folder scanning notifications with deduplication Signed-off-by: Deluan <deluan@navidrome.org> * refactor(watcher): add resolveFolderPath function for testability Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement path ignoring based on .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): implement IgnoreChecker for managing .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(ignore_checker): rename scanner to lineScanner for clarity Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance ScanTarget struct with String method for better target representation Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): validate library ID to prevent negative values Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify GC method by removing library ID parameter Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): update folder scanning to include all descendants of specified folders Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): allow selective scan in the /startScan endpoint Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update CallScan to handle specific library/folder pairs Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline scanning logic by removing scanAll method Signed-off-by: Deluan <deluan@navidrome.org> * test: enhance mockScanner for thread safety and improve test reliability Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move scanner.ScanTarget to model.ScanTarget Signed-off-by: Deluan <deluan@navidrome.org> * refactor: move scanner types to model,implement MockScanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update scanner interface and implementations to use model.Scanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder_repository): normalize target path handling by using filepath.Clean Signed-off-by: Deluan <deluan@navidrome.org> * test(folder_repository): add comprehensive tests for folder retrieval and child exclusion Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify selective scan logic using slice.Filter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move initialization logic from phase_1 to the scanner itself Signed-off-by: Deluan <deluan@navidrome.org> * refactor(tests): rename selective scan test file to scanner_selective_test.go Signed-off-by: Deluan <deluan@navidrome.org> * feat(configuration): add DevSelectiveWatcher configuration option Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): enhance .ndignore handling for folder deletions and file changes Signed-off-by: Deluan <deluan@navidrome.org> * docs(scanner): comments Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance walkDirTree to support target folder scanning Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner, watcher): handle errors when pushing ignore patterns for folders Signed-off-by: Deluan <deluan@navidrome.org> * Update scanner/phase_1_folders.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): add tests for ScanBegin and ScanEnd functionality Signed-off-by: Deluan <deluan@navidrome.org> * fix(library): update PRAGMA optimize to check table sizes without ANALYZE Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): refactor tests Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add selective scan options and update translations Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add quick and full scan options for individual libraries Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add Scan buttonsto the LibraryList Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): update scanning parameters from 'path' to 'target' for selective scans. * refactor(scan): move ParseTargets function to model package * test(scan): suppress unused return value from SetUserLibraries in tests * feat(gc): enhance garbage collection to support selective library purging Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): prevent race condition when scanning deleted folders When the watcher detects changes in a folder that gets deleted before the scanner runs (due to the 10-second delay), the scanner was prematurely removing these folders from the tracking map, preventing them from being marked as missing. The issue occurred because `newFolderEntry` was calling `popLastUpdate` before verifying the folder actually exists on the filesystem. Changes: - Move fs.Stat check before newFolderEntry creation in loadDir to ensure deleted folders remain in lastUpdates for finalize() to handle - Add early existence check in walkDirTree to skip non-existent target folders with a warning log - Add unit test verifying non-existent folders aren't removed from lastUpdates prematurely - Add integration test for deleted folder scenario with ScanFolders Fixes the issue where deleting entire folders (e.g., /music/AC_DC) wouldn't mark tracks as missing when using selective folder scanning. * refactor(scan): streamline folder entry creation and update handling Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): add '@Recycle' (QNAP) to ignored directories list Signed-off-by: Deluan <deluan@navidrome.org> * fix(log): improve thread safety in logging level management * test(scan): move unit tests for ParseTargets function Signed-off-by: Deluan <deluan@navidrome.org> * review Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: deluan <deluan.quintao@mechanical-orchard.com>
204 lines
7.1 KiB
Go
204 lines
7.1 KiB
Go
package persistence
|
|
|
|
import (
|
|
"context"
|
|
|
|
"github.com/navidrome/navidrome/log"
|
|
"github.com/navidrome/navidrome/model"
|
|
"github.com/navidrome/navidrome/model/request"
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
"github.com/pocketbase/dbx"
|
|
)
|
|
|
|
var _ = Describe("LibraryRepository", func() {
|
|
var repo model.LibraryRepository
|
|
var ctx context.Context
|
|
var conn *dbx.DB
|
|
|
|
BeforeEach(func() {
|
|
ctx = request.WithUser(log.NewContext(context.TODO()), model.User{ID: "userid"})
|
|
conn = GetDBXBuilder()
|
|
repo = NewLibraryRepository(ctx, conn)
|
|
})
|
|
|
|
AfterEach(func() {
|
|
// Clean up test libraries (keep ID 1 which is the default library)
|
|
_, _ = conn.NewQuery("DELETE FROM library WHERE id > 1").Execute()
|
|
})
|
|
|
|
Describe("Put", func() {
|
|
Context("when ID is 0", func() {
|
|
It("inserts a new library with autoassigned ID", func() {
|
|
lib := &model.Library{
|
|
ID: 0,
|
|
Name: "Test Library",
|
|
Path: "/music/test",
|
|
}
|
|
|
|
err := repo.Put(lib)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(lib.ID).To(BeNumerically(">", 0))
|
|
Expect(lib.CreatedAt).ToNot(BeZero())
|
|
Expect(lib.UpdatedAt).ToNot(BeZero())
|
|
|
|
// Verify it was inserted
|
|
savedLib, err := repo.Get(lib.ID)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(savedLib.Name).To(Equal("Test Library"))
|
|
Expect(savedLib.Path).To(Equal("/music/test"))
|
|
})
|
|
})
|
|
|
|
Context("when ID is non-zero and record exists", func() {
|
|
It("updates the existing record", func() {
|
|
// First create a library
|
|
lib := &model.Library{
|
|
ID: 0,
|
|
Name: "Original Library",
|
|
Path: "/music/original",
|
|
}
|
|
err := repo.Put(lib)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
originalID := lib.ID
|
|
originalCreatedAt := lib.CreatedAt
|
|
|
|
// Now update it
|
|
lib.Name = "Updated Library"
|
|
lib.Path = "/music/updated"
|
|
err = repo.Put(lib)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
// Verify it was updated, not inserted
|
|
Expect(lib.ID).To(Equal(originalID))
|
|
Expect(lib.CreatedAt).To(Equal(originalCreatedAt))
|
|
Expect(lib.UpdatedAt).To(BeTemporally(">", originalCreatedAt))
|
|
|
|
// Verify the changes were saved
|
|
savedLib, err := repo.Get(lib.ID)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(savedLib.Name).To(Equal("Updated Library"))
|
|
Expect(savedLib.Path).To(Equal("/music/updated"))
|
|
})
|
|
})
|
|
|
|
Context("when ID is non-zero but record doesn't exist", func() {
|
|
It("inserts a new record with the specified ID", func() {
|
|
lib := &model.Library{
|
|
ID: 999,
|
|
Name: "New Library with ID",
|
|
Path: "/music/new",
|
|
}
|
|
|
|
// Ensure the record doesn't exist
|
|
_, err := repo.Get(999)
|
|
Expect(err).To(HaveOccurred())
|
|
|
|
// Put should insert it
|
|
err = repo.Put(lib)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(lib.ID).To(Equal(999))
|
|
Expect(lib.CreatedAt).ToNot(BeZero())
|
|
Expect(lib.UpdatedAt).ToNot(BeZero())
|
|
|
|
// Verify it was inserted with the correct ID
|
|
savedLib, err := repo.Get(999)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(savedLib.ID).To(Equal(999))
|
|
Expect(savedLib.Name).To(Equal("New Library with ID"))
|
|
Expect(savedLib.Path).To(Equal("/music/new"))
|
|
})
|
|
})
|
|
})
|
|
|
|
It("refreshes stats", func() {
|
|
libBefore, err := repo.Get(1)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(repo.RefreshStats(1)).To(Succeed())
|
|
libAfter, err := repo.Get(1)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(libAfter.UpdatedAt).To(BeTemporally(">", libBefore.UpdatedAt))
|
|
|
|
var songsRes, albumsRes, artistsRes, foldersRes, filesRes, missingRes struct{ Count int64 }
|
|
var sizeRes struct{ Sum int64 }
|
|
var durationRes struct{ Sum float64 }
|
|
|
|
Expect(conn.NewQuery("select count(*) as count from media_file where library_id = {:id} and missing = 0").Bind(dbx.Params{"id": 1}).One(&songsRes)).To(Succeed())
|
|
Expect(conn.NewQuery("select count(*) as count from album where library_id = {:id} and missing = 0").Bind(dbx.Params{"id": 1}).One(&albumsRes)).To(Succeed())
|
|
Expect(conn.NewQuery("select count(*) as count from library_artist la join artist a on la.artist_id = a.id where la.library_id = {:id} and a.missing = 0").Bind(dbx.Params{"id": 1}).One(&artistsRes)).To(Succeed())
|
|
Expect(conn.NewQuery("select count(*) as count from folder where library_id = {:id} and missing = 0").Bind(dbx.Params{"id": 1}).One(&foldersRes)).To(Succeed())
|
|
Expect(conn.NewQuery("select ifnull(sum(num_audio_files + num_playlists + json_array_length(image_files)),0) as count from folder where library_id = {:id} and missing = 0").Bind(dbx.Params{"id": 1}).One(&filesRes)).To(Succeed())
|
|
Expect(conn.NewQuery("select count(*) as count from media_file where library_id = {:id} and missing = 1").Bind(dbx.Params{"id": 1}).One(&missingRes)).To(Succeed())
|
|
Expect(conn.NewQuery("select ifnull(sum(size),0) as sum from album where library_id = {:id} and missing = 0").Bind(dbx.Params{"id": 1}).One(&sizeRes)).To(Succeed())
|
|
Expect(conn.NewQuery("select ifnull(sum(duration),0) as sum from album where library_id = {:id} and missing = 0").Bind(dbx.Params{"id": 1}).One(&durationRes)).To(Succeed())
|
|
|
|
Expect(libAfter.TotalSongs).To(Equal(int(songsRes.Count)))
|
|
Expect(libAfter.TotalAlbums).To(Equal(int(albumsRes.Count)))
|
|
Expect(libAfter.TotalArtists).To(Equal(int(artistsRes.Count)))
|
|
Expect(libAfter.TotalFolders).To(Equal(int(foldersRes.Count)))
|
|
Expect(libAfter.TotalFiles).To(Equal(int(filesRes.Count)))
|
|
Expect(libAfter.TotalMissingFiles).To(Equal(int(missingRes.Count)))
|
|
Expect(libAfter.TotalSize).To(Equal(sizeRes.Sum))
|
|
Expect(libAfter.TotalDuration).To(Equal(durationRes.Sum))
|
|
})
|
|
|
|
Describe("ScanBegin and ScanEnd", func() {
|
|
var lib *model.Library
|
|
|
|
BeforeEach(func() {
|
|
lib = &model.Library{
|
|
ID: 0,
|
|
Name: "Test Scan Library",
|
|
Path: "/music/test-scan",
|
|
}
|
|
err := repo.Put(lib)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
})
|
|
|
|
DescribeTable("ScanBegin",
|
|
func(fullScan bool, expectedFullScanInProgress bool) {
|
|
err := repo.ScanBegin(lib.ID, fullScan)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
updatedLib, err := repo.Get(lib.ID)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(updatedLib.LastScanStartedAt).ToNot(BeZero())
|
|
Expect(updatedLib.FullScanInProgress).To(Equal(expectedFullScanInProgress))
|
|
},
|
|
Entry("sets FullScanInProgress to true for full scan", true, true),
|
|
Entry("sets FullScanInProgress to false for quick scan", false, false),
|
|
)
|
|
|
|
Context("ScanEnd", func() {
|
|
BeforeEach(func() {
|
|
err := repo.ScanBegin(lib.ID, true)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
})
|
|
|
|
It("sets LastScanAt and clears FullScanInProgress and LastScanStartedAt", func() {
|
|
err := repo.ScanEnd(lib.ID)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
updatedLib, err := repo.Get(lib.ID)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(updatedLib.LastScanAt).ToNot(BeZero())
|
|
Expect(updatedLib.FullScanInProgress).To(BeFalse())
|
|
Expect(updatedLib.LastScanStartedAt).To(BeZero())
|
|
})
|
|
|
|
It("sets LastScanAt to be after LastScanStartedAt", func() {
|
|
libBefore, err := repo.Get(lib.ID)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
err = repo.ScanEnd(lib.ID)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
|
|
libAfter, err := repo.Get(lib.ID)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(libAfter.LastScanAt).To(BeTemporally(">=", libBefore.LastScanStartedAt))
|
|
})
|
|
})
|
|
})
|
|
})
|