diff --git a/conf/configuration.go b/conf/configuration.go index eebf1c004..8561f343f 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -134,6 +134,7 @@ type scannerOptions struct { GenreSeparators string // Deprecated: Use Tags.genre.Split instead GroupAlbumReleases bool // Deprecated: Use PID.Album instead FollowSymlinks bool // Whether to follow symlinks when scanning directories + PurgeMissing string // Values: "never", "always", "full" } type subsonicOptions struct { @@ -277,6 +278,7 @@ func Load(noConfigDump bool) { validateScanSchedule, validateBackupSchedule, validatePlaylistsPath, + validatePurgeMissingOption, ) if err != nil { os.Exit(1) @@ -382,6 +384,24 @@ func validatePlaylistsPath() error { return nil } +func validatePurgeMissingOption() error { + allowedValues := []string{consts.PurgeMissingNever, consts.PurgeMissingAlways, consts.PurgeMissingFull} + valid := false + for _, v := range allowedValues { + if v == Server.Scanner.PurgeMissing { + valid = true + break + } + } + if !valid { + err := fmt.Errorf("Invalid Scanner.PurgeMissing value: '%s'. Must be one of: %v", Server.Scanner.PurgeMissing, allowedValues) + log.Error(err.Error()) + Server.Scanner.PurgeMissing = consts.PurgeMissingNever + return err + } + return nil +} + func validateScanSchedule() error { if Server.Scanner.Schedule == "0" || Server.Scanner.Schedule == "" { Server.Scanner.Schedule = "" @@ -421,7 +441,7 @@ func AddHook(hook func()) { hooks = append(hooks, hook) } -func init() { +func setViperDefaults() { viper.SetDefault("musicfolder", filepath.Join(".", "music")) viper.SetDefault("cachefolder", "") viper.SetDefault("datafolder", ".") @@ -458,7 +478,6 @@ func init() { viper.SetDefault("indexgroups", "A B C D E F G H I J K L M N O P Q R S T U V W X-Z(XYZ) [Unknown]([)") viper.SetDefault("ffmpegpath", "") viper.SetDefault("mpvcmdtemplate", "mpv --audio-device=%d --no-audio-display --pause %f --input-ipc-server=%s") - viper.SetDefault("coverartpriority", "cover.*, folder.*, front.*, embedded, external") viper.SetDefault("coverjpegquality", 75) viper.SetDefault("artistartpriority", "artist.*, album/artist.*, external") @@ -481,19 +500,15 @@ func init() { viper.SetDefault("authrequestlimit", 5) viper.SetDefault("authwindowlength", 20*time.Second) viper.SetDefault("passwordencryptionkey", "") - viper.SetDefault("reverseproxyuserheader", "Remote-User") viper.SetDefault("reverseproxywhitelist", "") - viper.SetDefault("prometheus.enabled", false) viper.SetDefault("prometheus.metricspath", consts.PrometheusDefaultPath) viper.SetDefault("prometheus.password", "") - viper.SetDefault("jukebox.enabled", false) viper.SetDefault("jukebox.devices", []AudioDeviceDefinition{}) viper.SetDefault("jukebox.default", "") viper.SetDefault("jukebox.adminonly", true) - viper.SetDefault("scanner.enabled", true) viper.SetDefault("scanner.schedule", "0") viper.SetDefault("scanner.extractor", consts.DefaultScannerExtractor) @@ -503,12 +518,11 @@ func init() { viper.SetDefault("scanner.genreseparators", "") viper.SetDefault("scanner.groupalbumreleases", false) viper.SetDefault("scanner.followsymlinks", true) - + viper.SetDefault("scanner.purgemissing", "never") viper.SetDefault("subsonic.appendsubtitle", true) viper.SetDefault("subsonic.artistparticipations", false) viper.SetDefault("subsonic.defaultreportrealpath", false) viper.SetDefault("subsonic.legacyclients", "DSub") - viper.SetDefault("agents", "lastfm,spotify") viper.SetDefault("lastfm.enabled", true) viper.SetDefault("lastfm.language", "en") @@ -518,24 +532,17 @@ func init() { viper.SetDefault("spotify.secret", "") viper.SetDefault("listenbrainz.enabled", true) viper.SetDefault("listenbrainz.baseurl", "https://api.listenbrainz.org/1/") - viper.SetDefault("httpsecurityheaders.customframeoptionsvalue", "DENY") - viper.SetDefault("backup.path", "") viper.SetDefault("backup.schedule", "") viper.SetDefault("backup.count", 0) - viper.SetDefault("pid.track", consts.DefaultTrackPID) viper.SetDefault("pid.album", consts.DefaultAlbumPID) - viper.SetDefault("inspect.enabled", true) viper.SetDefault("inspect.maxrequests", 1) viper.SetDefault("inspect.backloglimit", consts.RequestThrottleBacklogLimit) viper.SetDefault("inspect.backlogtimeout", consts.RequestThrottleBacklogTimeout) - viper.SetDefault("lyricspriority", ".lrc,.txt,embedded") - - // DevFlags. These are used to enable/disable debugging and incomplete features viper.SetDefault("devlogsourceline", false) viper.SetDefault("devenableprofiler", false) viper.SetDefault("devautocreateadminpassword", "") @@ -556,6 +563,10 @@ func init() { viper.SetDefault("devenableplayerinsights", true) } +func init() { + setViperDefaults() +} + func InitConfig(cfgFile string) { codecRegistry := viper.NewCodecRegistry() _ = codecRegistry.RegisterCodec("ini", ini.Codec{}) diff --git a/conf/configuration_test.go b/conf/configuration_test.go index f57764709..5b54e4975 100644 --- a/conf/configuration_test.go +++ b/conf/configuration_test.go @@ -5,7 +5,7 @@ import ( "path/filepath" "testing" - . "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/conf" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/spf13/viper" @@ -20,9 +20,10 @@ var _ = Describe("Configuration", func() { BeforeEach(func() { // Reset viper configuration viper.Reset() + conf.SetViperDefaults() viper.SetDefault("datafolder", GinkgoT().TempDir()) viper.SetDefault("loglevel", "error") - ResetConf() + conf.ResetConf() }) DescribeTable("should load configuration from", @@ -30,17 +31,17 @@ var _ = Describe("Configuration", func() { filename := filepath.Join("testdata", "cfg."+format) // Initialize config with the test file - InitConfig(filename) + conf.InitConfig(filename) // Load the configuration (with noConfigDump=true) - Load(true) + conf.Load(true) // Execute the format-specific assertions - Expect(Server.MusicFolder).To(Equal(fmt.Sprintf("/%s/music", format))) - Expect(Server.UIWelcomeMessage).To(Equal("Welcome " + format)) - Expect(Server.Tags["custom"].Aliases).To(Equal([]string{format, "test"})) + Expect(conf.Server.MusicFolder).To(Equal(fmt.Sprintf("/%s/music", format))) + Expect(conf.Server.UIWelcomeMessage).To(Equal("Welcome " + format)) + Expect(conf.Server.Tags["custom"].Aliases).To(Equal([]string{format, "test"})) // The config file used should be the one we created - Expect(Server.ConfigFile).To(Equal(filename)) + Expect(conf.Server.ConfigFile).To(Equal(filename)) }, Entry("TOML format", "toml"), Entry("YAML format", "yaml"), diff --git a/conf/export_test.go b/conf/export_test.go index 0bd7819eb..1b6daf036 100644 --- a/conf/export_test.go +++ b/conf/export_test.go @@ -3,3 +3,5 @@ package conf func ResetConf() { Server = &configOptions{} } + +var SetViperDefaults = setViperDefaults diff --git a/consts/consts.go b/consts/consts.go index 2dbb46d07..fbb2c9429 100644 --- a/consts/consts.go +++ b/consts/consts.go @@ -115,6 +115,12 @@ const ( InsightsInitialDelay = 30 * time.Minute ) +const ( + PurgeMissingNever = "never" + PurgeMissingAlways = "always" + PurgeMissingFull = "full" +) + var ( DefaultDownsamplingFormat = "opus" DefaultTranscodings = []struct { diff --git a/model/mediafile.go b/model/mediafile.go index 354c5ea47..cdb001c85 100644 --- a/model/mediafile.go +++ b/model/mediafile.go @@ -342,6 +342,7 @@ type MediaFileRepository interface { GetCursor(options ...QueryOptions) (MediaFileCursor, error) Delete(id string) error DeleteMissing(ids []string) error + DeleteAllMissing() (int64, error) FindByPaths(paths []string) (MediaFiles, error) // The following methods are used exclusively by the scanner: diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go index e236b1b29..b0ed637c1 100644 --- a/persistence/mediafile_repository.go +++ b/persistence/mediafile_repository.go @@ -192,6 +192,15 @@ func (r *mediaFileRepository) Delete(id string) error { return r.delete(Eq{"id": id}) } +func (r *mediaFileRepository) DeleteAllMissing() (int64, error) { + user := loggedUser(r.ctx) + if !user.IsAdmin { + return 0, rest.ErrPermissionDenied + } + del := Delete(r.tableName).Where(Eq{"missing": true}) + return r.executeSQL(del) +} + func (r *mediaFileRepository) DeleteMissing(ids []string) error { user := loggedUser(r.ctx) if !user.IsAdmin { diff --git a/scanner/phase_2_missing_tracks.go b/scanner/phase_2_missing_tracks.go index 352f92c34..6f56f6a52 100644 --- a/scanner/phase_2_missing_tracks.go +++ b/scanner/phase_2_missing_tracks.go @@ -6,6 +6,8 @@ import ( "sync/atomic" ppl "github.com/google/go-pipeline/pkg/pipeline" + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" ) @@ -182,7 +184,35 @@ func (p *phaseMissingTracks) finalize(err error) error { if matched > 0 { log.Info(p.ctx, "Scanner: Found moved files", "total", matched, err) } + if err != nil { + return err + } + + // Check if we should purge missing items + if conf.Server.Scanner.PurgeMissing == consts.PurgeMissingAlways || (conf.Server.Scanner.PurgeMissing == consts.PurgeMissingFull && p.state.fullScan) { + if err = p.purgeMissing(); err != nil { + log.Error(p.ctx, "Scanner: Error purging missing items", err) + } + } + return err } +func (p *phaseMissingTracks) purgeMissing() error { + deletedCount, err := p.ds.MediaFile(p.ctx).DeleteAllMissing() + if err != nil { + return fmt.Errorf("error deleting missing files: %w", err) + } + + if deletedCount > 0 { + log.Info(p.ctx, "Scanner: Purged missing items from the database", "mediaFiles", deletedCount) + // Set changesDetected to true so that garbage collection will run at the end of the scan process + p.state.changesDetected.Store(true) + } else { + log.Debug(p.ctx, "Scanner: No missing items to purge") + } + + return nil +} + var _ phase[*missingTracks] = (*phaseMissingTracks)(nil) diff --git a/scanner/phase_2_missing_tracks_test.go b/scanner/phase_2_missing_tracks_test.go index 2cd686604..5dd6cc679 100644 --- a/scanner/phase_2_missing_tracks_test.go +++ b/scanner/phase_2_missing_tracks_test.go @@ -4,6 +4,8 @@ import ( "context" "time" + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/tests" . "github.com/onsi/ginkgo/v2" @@ -222,4 +224,66 @@ var _ = Describe("phaseMissingTracks", func() { Expect(state.changesDetected.Load()).To(BeFalse()) }) }) + + Describe("finalize", func() { + It("should return nil if no error", func() { + err := phase.finalize(nil) + Expect(err).To(BeNil()) + Expect(state.changesDetected.Load()).To(BeFalse()) + }) + + It("should return the error if provided", func() { + err := phase.finalize(context.DeadlineExceeded) + Expect(err).To(Equal(context.DeadlineExceeded)) + Expect(state.changesDetected.Load()).To(BeFalse()) + }) + + When("PurgeMissing is 'always'", func() { + BeforeEach(func() { + conf.Server.Scanner.PurgeMissing = consts.PurgeMissingAlways + mr.CountAllValue = 3 + mr.DeleteAllMissingValue = 3 + }) + It("should purge missing files", func() { + Expect(state.changesDetected.Load()).To(BeFalse()) + err := phase.finalize(nil) + Expect(err).To(BeNil()) + Expect(state.changesDetected.Load()).To(BeTrue()) + }) + }) + + When("PurgeMissing is 'full'", func() { + BeforeEach(func() { + conf.Server.Scanner.PurgeMissing = consts.PurgeMissingFull + mr.CountAllValue = 2 + mr.DeleteAllMissingValue = 2 + }) + It("should not purge missing files if not a full scan", func() { + state.fullScan = false + err := phase.finalize(nil) + Expect(err).To(BeNil()) + Expect(state.changesDetected.Load()).To(BeFalse()) + }) + It("should purge missing files if full scan", func() { + Expect(state.changesDetected.Load()).To(BeFalse()) + state.fullScan = true + err := phase.finalize(nil) + Expect(err).To(BeNil()) + Expect(state.changesDetected.Load()).To(BeTrue()) + }) + }) + + When("PurgeMissing is 'never'", func() { + BeforeEach(func() { + conf.Server.Scanner.PurgeMissing = consts.PurgeMissingNever + mr.CountAllValue = 1 + mr.DeleteAllMissingValue = 1 + }) + It("should not purge missing files", func() { + err := phase.finalize(nil) + Expect(err).To(BeNil()) + Expect(state.changesDetected.Load()).To(BeFalse()) + }) + }) + }) }) diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 33c78fe7d..7c74e3f81 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -10,6 +10,7 @@ import ( "github.com/google/uuid" "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/conf/configtest" + "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/core" "github.com/navidrome/navidrome/core/artwork" "github.com/navidrome/navidrome/core/metrics" @@ -17,6 +18,7 @@ import ( "github.com/navidrome/navidrome/db" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/model/request" "github.com/navidrome/navidrome/persistence" "github.com/navidrome/navidrome/scanner" "github.com/navidrome/navidrome/server/events" @@ -47,6 +49,7 @@ var _ = Describe("Scanner", Ordered, func() { } BeforeAll(func() { + ctx = request.WithUser(GinkgoT().Context(), model.User{ID: "123", IsAdmin: true}) tmpDir := GinkgoT().TempDir() conf.Server.DbPath = filepath.Join(tmpDir, "test-scanner.db?_journal_mode=WAL") log.Warn("Using DB at " + conf.Server.DbPath) @@ -54,7 +57,6 @@ var _ = Describe("Scanner", Ordered, func() { }) BeforeEach(func() { - ctx = context.Background() db.Init(ctx) DeferCleanup(func() { Expect(tests.ClearDB()).To(Succeed()) @@ -501,6 +503,113 @@ var _ = Describe("Scanner", Ordered, func() { Expect(aa[0].MbzArtistID).To(Equal(beatlesMBID)) Expect(aa[0].SortArtistName).To(Equal("Beatles, The")) }) + + Context("When PurgeMissing is configured", func() { + When("PurgeMissing is set to 'never'", func() { + BeforeEach(func() { + DeferCleanup(configtest.SetupConfig()) + conf.Server.Scanner.PurgeMissing = consts.PurgeMissingNever + }) + + It("should mark files as missing but not delete them", func() { + By("Running initial scan") + Expect(runScanner(ctx, true)).To(Succeed()) + + By("Removing a file") + fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3") + + By("Running another scan") + Expect(runScanner(ctx, true)).To(Succeed()) + + By("Checking files are marked as missing but not deleted") + count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{ + Filters: squirrel.Eq{"missing": true}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(int64(1))) + + mf, err := findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3") + Expect(err).ToNot(HaveOccurred()) + Expect(mf.Missing).To(BeTrue()) + }) + }) + + When("PurgeMissing is set to 'always'", func() { + BeforeEach(func() { + conf.Server.Scanner.PurgeMissing = consts.PurgeMissingAlways + }) + + It("should purge missing files on any scan", func() { + By("Running initial scan") + Expect(runScanner(ctx, false)).To(Succeed()) + + By("Removing a file") + fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3") + + By("Running an incremental scan") + Expect(runScanner(ctx, false)).To(Succeed()) + + By("Checking missing files are deleted") + count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{ + Filters: squirrel.Eq{"missing": true}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(BeZero()) + + _, err = findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3") + Expect(err).To(MatchError(model.ErrNotFound)) + }) + }) + + When("PurgeMissing is set to 'full'", func() { + BeforeEach(func() { + conf.Server.Scanner.PurgeMissing = consts.PurgeMissingFull + }) + + It("should not purge missing files on incremental scans", func() { + By("Running initial scan") + Expect(runScanner(ctx, true)).To(Succeed()) + + By("Removing a file") + fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3") + + By("Running an incremental scan") + Expect(runScanner(ctx, false)).To(Succeed()) + + By("Checking files are marked as missing but not deleted") + count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{ + Filters: squirrel.Eq{"missing": true}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(int64(1))) + + mf, err := findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3") + Expect(err).ToNot(HaveOccurred()) + Expect(mf.Missing).To(BeTrue()) + }) + + It("should purge missing files only on full scans", func() { + By("Running initial scan") + Expect(runScanner(ctx, true)).To(Succeed()) + + By("Removing a file") + fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3") + + By("Running a full scan") + Expect(runScanner(ctx, true)).To(Succeed()) + + By("Checking missing files are deleted") + count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{ + Filters: squirrel.Eq{"missing": true}, + }) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(BeZero()) + + _, err = findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3") + Expect(err).To(MatchError(model.ErrNotFound)) + }) + }) + }) }) }) diff --git a/tests/mock_mediafile_repo.go b/tests/mock_mediafile_repo.go index 01d82e03b..85adb8a25 100644 --- a/tests/mock_mediafile_repo.go +++ b/tests/mock_mediafile_repo.go @@ -22,6 +22,10 @@ type MockMediaFileRepo struct { model.MediaFileRepository Data map[string]*model.MediaFile Err bool + // Add fields and methods for controlling CountAll and DeleteAllMissing in tests + CountAllValue int64 + CountAllOptions model.QueryOptions + DeleteAllMissingValue int64 } func (m *MockMediaFileRepo) SetError(err bool) { @@ -161,4 +165,35 @@ func (m *MockMediaFileRepo) GetMissingAndMatching(libId int) (model.MediaFileCur }, nil } +func (m *MockMediaFileRepo) CountAll(opts ...model.QueryOptions) (int64, error) { + if m.Err { + return 0, errors.New("error") + } + if m.CountAllValue != 0 { + if len(opts) > 0 { + m.CountAllOptions = opts[0] + } + return m.CountAllValue, nil + } + return int64(len(m.Data)), nil +} + +func (m *MockMediaFileRepo) DeleteAllMissing() (int64, error) { + if m.Err { + return 0, errors.New("error") + } + if m.DeleteAllMissingValue != 0 { + return m.DeleteAllMissingValue, nil + } + // Remove all missing files from Data + var count int64 + for id, mf := range m.Data { + if mf.Missing { + delete(m.Data, id) + count++ + } + } + return count, nil +} + var _ model.MediaFileRepository = (*MockMediaFileRepo)(nil)