feat(scanner): add Scanner.PurgeMissing configuration option (#4107)

* Initial plan for issue

* Add Scanner.PurgeMissing configuration option

Co-authored-by: deluan <331353+deluan@users.noreply.github.com>

* Remove GC call from phaseMissingTracks.purgeMissing method

Co-authored-by: deluan <331353+deluan@users.noreply.github.com>

* Address PR comments for Scanner.PurgeMissing feature

Co-authored-by: deluan <331353+deluan@users.noreply.github.com>

* Address PR comments and add DeleteAllMissing method

Co-authored-by: deluan <331353+deluan@users.noreply.github.com>

* refactor(scanner): simplify purgeMissing logic and improve error handling

Signed-off-by: Deluan <deluan@navidrome.org>

* fix configuration test

Signed-off-by: Deluan <deluan@navidrome.org>

---------

Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: deluan <331353+deluan@users.noreply.github.com>
Co-authored-by: Deluan <deluan@navidrome.org>

conf/configuration.go

@@ -134,6 +134,7 @@ type scannerOptions struct {
 	GenreSeparators    string // Deprecated: Use Tags.genre.Split instead
 	GroupAlbumReleases bool   // Deprecated: Use PID.Album instead
 	FollowSymlinks     bool   // Whether to follow symlinks when scanning directories
+	PurgeMissing       string // Values: "never", "always", "full"
 }
 
 type subsonicOptions struct {
@@ -277,6 +278,7 @@ func Load(noConfigDump bool) {
 		validateScanSchedule,
 		validateBackupSchedule,
 		validatePlaylistsPath,
+		validatePurgeMissingOption,
 	)
 	if err != nil {
 		os.Exit(1)
@@ -382,6 +384,24 @@ func validatePlaylistsPath() error {
 	return nil
 }
 
+func validatePurgeMissingOption() error {
+	allowedValues := []string{consts.PurgeMissingNever, consts.PurgeMissingAlways, consts.PurgeMissingFull}
+	valid := false
+	for _, v := range allowedValues {
+		if v == Server.Scanner.PurgeMissing {
+			valid = true
+			break
+		}
+	}
+	if !valid {
+		err := fmt.Errorf("Invalid Scanner.PurgeMissing value: '%s'. Must be one of: %v", Server.Scanner.PurgeMissing, allowedValues)
+		log.Error(err.Error())
+		Server.Scanner.PurgeMissing = consts.PurgeMissingNever
+		return err
+	}
+	return nil
+}
+
 func validateScanSchedule() error {
 	if Server.Scanner.Schedule == "0" || Server.Scanner.Schedule == "" {
 		Server.Scanner.Schedule = ""
@@ -421,7 +441,7 @@ func AddHook(hook func()) {
 	hooks = append(hooks, hook)
 }
 
-func init() {
+func setViperDefaults() {
 	viper.SetDefault("musicfolder", filepath.Join(".", "music"))
 	viper.SetDefault("cachefolder", "")
 	viper.SetDefault("datafolder", ".")
@@ -458,7 +478,6 @@ func init() {
 	viper.SetDefault("indexgroups", "A B C D E F G H I J K L M N O P Q R S T U V W X-Z(XYZ) [Unknown]([)")
 	viper.SetDefault("ffmpegpath", "")
 	viper.SetDefault("mpvcmdtemplate", "mpv --audio-device=%d --no-audio-display --pause %f --input-ipc-server=%s")
-
 	viper.SetDefault("coverartpriority", "cover.*, folder.*, front.*, embedded, external")
 	viper.SetDefault("coverjpegquality", 75)
 	viper.SetDefault("artistartpriority", "artist.*, album/artist.*, external")
@@ -481,19 +500,15 @@ func init() {
 	viper.SetDefault("authrequestlimit", 5)
 	viper.SetDefault("authwindowlength", 20*time.Second)
 	viper.SetDefault("passwordencryptionkey", "")
-
 	viper.SetDefault("reverseproxyuserheader", "Remote-User")
 	viper.SetDefault("reverseproxywhitelist", "")
-
 	viper.SetDefault("prometheus.enabled", false)
 	viper.SetDefault("prometheus.metricspath", consts.PrometheusDefaultPath)
 	viper.SetDefault("prometheus.password", "")
-
 	viper.SetDefault("jukebox.enabled", false)
 	viper.SetDefault("jukebox.devices", []AudioDeviceDefinition{})
 	viper.SetDefault("jukebox.default", "")
 	viper.SetDefault("jukebox.adminonly", true)
-
 	viper.SetDefault("scanner.enabled", true)
 	viper.SetDefault("scanner.schedule", "0")
 	viper.SetDefault("scanner.extractor", consts.DefaultScannerExtractor)
@@ -503,12 +518,11 @@ func init() {
 	viper.SetDefault("scanner.genreseparators", "")
 	viper.SetDefault("scanner.groupalbumreleases", false)
 	viper.SetDefault("scanner.followsymlinks", true)
-
+	viper.SetDefault("scanner.purgemissing", "never")
 	viper.SetDefault("subsonic.appendsubtitle", true)
 	viper.SetDefault("subsonic.artistparticipations", false)
 	viper.SetDefault("subsonic.defaultreportrealpath", false)
 	viper.SetDefault("subsonic.legacyclients", "DSub")
-
 	viper.SetDefault("agents", "lastfm,spotify")
 	viper.SetDefault("lastfm.enabled", true)
 	viper.SetDefault("lastfm.language", "en")
@@ -518,24 +532,17 @@ func init() {
 	viper.SetDefault("spotify.secret", "")
 	viper.SetDefault("listenbrainz.enabled", true)
 	viper.SetDefault("listenbrainz.baseurl", "https://api.listenbrainz.org/1/")
-
 	viper.SetDefault("httpsecurityheaders.customframeoptionsvalue", "DENY")
-
 	viper.SetDefault("backup.path", "")
 	viper.SetDefault("backup.schedule", "")
 	viper.SetDefault("backup.count", 0)
-
 	viper.SetDefault("pid.track", consts.DefaultTrackPID)
 	viper.SetDefault("pid.album", consts.DefaultAlbumPID)
-
 	viper.SetDefault("inspect.enabled", true)
 	viper.SetDefault("inspect.maxrequests", 1)
 	viper.SetDefault("inspect.backloglimit", consts.RequestThrottleBacklogLimit)
 	viper.SetDefault("inspect.backlogtimeout", consts.RequestThrottleBacklogTimeout)
-
 	viper.SetDefault("lyricspriority", ".lrc,.txt,embedded")
-
-	// DevFlags. These are used to enable/disable debugging and incomplete features
 	viper.SetDefault("devlogsourceline", false)
 	viper.SetDefault("devenableprofiler", false)
 	viper.SetDefault("devautocreateadminpassword", "")
@@ -556,6 +563,10 @@ func init() {
 	viper.SetDefault("devenableplayerinsights", true)
 }
 
+func init() {
+	setViperDefaults()
+}
+
 func InitConfig(cfgFile string) {
 	codecRegistry := viper.NewCodecRegistry()
 	_ = codecRegistry.RegisterCodec("ini", ini.Codec{})

conf/configuration_test.go

@@ -5,7 +5,7 @@ import (
 	"path/filepath"
 	"testing"
 
-	. "github.com/navidrome/navidrome/conf"
+	"github.com/navidrome/navidrome/conf"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/spf13/viper"
@@ -20,9 +20,10 @@ var _ = Describe("Configuration", func() {
 	BeforeEach(func() {
 		// Reset viper configuration
 		viper.Reset()
+		conf.SetViperDefaults()
 		viper.SetDefault("datafolder", GinkgoT().TempDir())
 		viper.SetDefault("loglevel", "error")
-		ResetConf()
+		conf.ResetConf()
 	})
 
 	DescribeTable("should load configuration from",
@@ -30,17 +31,17 @@ var _ = Describe("Configuration", func() {
 			filename := filepath.Join("testdata", "cfg."+format)
 
 			// Initialize config with the test file
-			InitConfig(filename)
+			conf.InitConfig(filename)
 			// Load the configuration (with noConfigDump=true)
-			Load(true)
+			conf.Load(true)
 
 			// Execute the format-specific assertions
-			Expect(Server.MusicFolder).To(Equal(fmt.Sprintf("/%s/music", format)))
-			Expect(Server.UIWelcomeMessage).To(Equal("Welcome " + format))
-			Expect(Server.Tags["custom"].Aliases).To(Equal([]string{format, "test"}))
+			Expect(conf.Server.MusicFolder).To(Equal(fmt.Sprintf("/%s/music", format)))
+			Expect(conf.Server.UIWelcomeMessage).To(Equal("Welcome " + format))
+			Expect(conf.Server.Tags["custom"].Aliases).To(Equal([]string{format, "test"}))
 
 			// The config file used should be the one we created
-			Expect(Server.ConfigFile).To(Equal(filename))
+			Expect(conf.Server.ConfigFile).To(Equal(filename))
 		},
 		Entry("TOML format", "toml"),
 		Entry("YAML format", "yaml"),

conf/export_test.go

@@ -3,3 +3,5 @@ package conf
 func ResetConf() {
 	Server = &configOptions{}
 }
+
+var SetViperDefaults = setViperDefaults

consts/consts.go

@@ -115,6 +115,12 @@ const (
 	InsightsInitialDelay   = 30 * time.Minute
 )
 
+const (
+	PurgeMissingNever  = "never"
+	PurgeMissingAlways = "always"
+	PurgeMissingFull   = "full"
+)
+
 var (
 	DefaultDownsamplingFormat = "opus"
 	DefaultTranscodings       = []struct {

model/mediafile.go

@@ -342,6 +342,7 @@ type MediaFileRepository interface {
 	GetCursor(options ...QueryOptions) (MediaFileCursor, error)
 	Delete(id string) error
 	DeleteMissing(ids []string) error
+	DeleteAllMissing() (int64, error)
 	FindByPaths(paths []string) (MediaFiles, error)
 
 	// The following methods are used exclusively by the scanner:

persistence/mediafile_repository.go

@@ -192,6 +192,15 @@ func (r *mediaFileRepository) Delete(id string) error {
 	return r.delete(Eq{"id": id})
 }
 
+func (r *mediaFileRepository) DeleteAllMissing() (int64, error) {
+	user := loggedUser(r.ctx)
+	if !user.IsAdmin {
+		return 0, rest.ErrPermissionDenied
+	}
+	del := Delete(r.tableName).Where(Eq{"missing": true})
+	return r.executeSQL(del)
+}
+
 func (r *mediaFileRepository) DeleteMissing(ids []string) error {
 	user := loggedUser(r.ctx)
 	if !user.IsAdmin {

scanner/phase_2_missing_tracks.go

@@ -6,6 +6,8 @@ import (
 	"sync/atomic"
 
 	ppl "github.com/google/go-pipeline/pkg/pipeline"
+	"github.com/navidrome/navidrome/conf"
+	"github.com/navidrome/navidrome/consts"
 	"github.com/navidrome/navidrome/log"
 	"github.com/navidrome/navidrome/model"
 )
@@ -182,7 +184,35 @@ func (p *phaseMissingTracks) finalize(err error) error {
 	if matched > 0 {
 		log.Info(p.ctx, "Scanner: Found moved files", "total", matched, err)
 	}
+	if err != nil {
+		return err
+	}
+
+	// Check if we should purge missing items
+	if conf.Server.Scanner.PurgeMissing == consts.PurgeMissingAlways || (conf.Server.Scanner.PurgeMissing == consts.PurgeMissingFull && p.state.fullScan) {
+		if err = p.purgeMissing(); err != nil {
+			log.Error(p.ctx, "Scanner: Error purging missing items", err)
+		}
+	}
+
 	return err
 }
 
+func (p *phaseMissingTracks) purgeMissing() error {
+	deletedCount, err := p.ds.MediaFile(p.ctx).DeleteAllMissing()
+	if err != nil {
+		return fmt.Errorf("error deleting missing files: %w", err)
+	}
+
+	if deletedCount > 0 {
+		log.Info(p.ctx, "Scanner: Purged missing items from the database", "mediaFiles", deletedCount)
+		// Set changesDetected to true so that garbage collection will run at the end of the scan process
+		p.state.changesDetected.Store(true)
+	} else {
+		log.Debug(p.ctx, "Scanner: No missing items to purge")
+	}
+
+	return nil
+}
+
 var _ phase[*missingTracks] = (*phaseMissingTracks)(nil)

scanner/phase_2_missing_tracks_test.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"time"
 
+	"github.com/navidrome/navidrome/conf"
+	"github.com/navidrome/navidrome/consts"
 	"github.com/navidrome/navidrome/model"
 	"github.com/navidrome/navidrome/tests"
 	. "github.com/onsi/ginkgo/v2"
@@ -222,4 +224,66 @@ var _ = Describe("phaseMissingTracks", func() {
 			Expect(state.changesDetected.Load()).To(BeFalse())
 		})
 	})
+
+	Describe("finalize", func() {
+		It("should return nil if no error", func() {
+			err := phase.finalize(nil)
+			Expect(err).To(BeNil())
+			Expect(state.changesDetected.Load()).To(BeFalse())
+		})
+
+		It("should return the error if provided", func() {
+			err := phase.finalize(context.DeadlineExceeded)
+			Expect(err).To(Equal(context.DeadlineExceeded))
+			Expect(state.changesDetected.Load()).To(BeFalse())
+		})
+
+		When("PurgeMissing is 'always'", func() {
+			BeforeEach(func() {
+				conf.Server.Scanner.PurgeMissing = consts.PurgeMissingAlways
+				mr.CountAllValue = 3
+				mr.DeleteAllMissingValue = 3
+			})
+			It("should purge missing files", func() {
+				Expect(state.changesDetected.Load()).To(BeFalse())
+				err := phase.finalize(nil)
+				Expect(err).To(BeNil())
+				Expect(state.changesDetected.Load()).To(BeTrue())
+			})
+		})
+
+		When("PurgeMissing is 'full'", func() {
+			BeforeEach(func() {
+				conf.Server.Scanner.PurgeMissing = consts.PurgeMissingFull
+				mr.CountAllValue = 2
+				mr.DeleteAllMissingValue = 2
+			})
+			It("should not purge missing files if not a full scan", func() {
+				state.fullScan = false
+				err := phase.finalize(nil)
+				Expect(err).To(BeNil())
+				Expect(state.changesDetected.Load()).To(BeFalse())
+			})
+			It("should purge missing files if full scan", func() {
+				Expect(state.changesDetected.Load()).To(BeFalse())
+				state.fullScan = true
+				err := phase.finalize(nil)
+				Expect(err).To(BeNil())
+				Expect(state.changesDetected.Load()).To(BeTrue())
+			})
+		})
+
+		When("PurgeMissing is 'never'", func() {
+			BeforeEach(func() {
+				conf.Server.Scanner.PurgeMissing = consts.PurgeMissingNever
+				mr.CountAllValue = 1
+				mr.DeleteAllMissingValue = 1
+			})
+			It("should not purge missing files", func() {
+				err := phase.finalize(nil)
+				Expect(err).To(BeNil())
+				Expect(state.changesDetected.Load()).To(BeFalse())
+			})
+		})
+	})
 })

scanner/scanner_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/navidrome/navidrome/conf"
 	"github.com/navidrome/navidrome/conf/configtest"
+	"github.com/navidrome/navidrome/consts"
 	"github.com/navidrome/navidrome/core"
 	"github.com/navidrome/navidrome/core/artwork"
 	"github.com/navidrome/navidrome/core/metrics"
@@ -17,6 +18,7 @@ import (
 	"github.com/navidrome/navidrome/db"
 	"github.com/navidrome/navidrome/log"
 	"github.com/navidrome/navidrome/model"
+	"github.com/navidrome/navidrome/model/request"
 	"github.com/navidrome/navidrome/persistence"
 	"github.com/navidrome/navidrome/scanner"
 	"github.com/navidrome/navidrome/server/events"
@@ -47,6 +49,7 @@ var _ = Describe("Scanner", Ordered, func() {
 	}
 
 	BeforeAll(func() {
+		ctx = request.WithUser(GinkgoT().Context(), model.User{ID: "123", IsAdmin: true})
 		tmpDir := GinkgoT().TempDir()
 		conf.Server.DbPath = filepath.Join(tmpDir, "test-scanner.db?_journal_mode=WAL")
 		log.Warn("Using DB at " + conf.Server.DbPath)
@@ -54,7 +57,6 @@ var _ = Describe("Scanner", Ordered, func() {
 	})
 
 	BeforeEach(func() {
-		ctx = context.Background()
 		db.Init(ctx)
 		DeferCleanup(func() {
 			Expect(tests.ClearDB()).To(Succeed())
@@ -501,6 +503,113 @@ var _ = Describe("Scanner", Ordered, func() {
 			Expect(aa[0].MbzArtistID).To(Equal(beatlesMBID))
 			Expect(aa[0].SortArtistName).To(Equal("Beatles, The"))
 		})
+
+		Context("When PurgeMissing is configured", func() {
+			When("PurgeMissing is set to 'never'", func() {
+				BeforeEach(func() {
+					DeferCleanup(configtest.SetupConfig())
+					conf.Server.Scanner.PurgeMissing = consts.PurgeMissingNever
+				})
+
+				It("should mark files as missing but not delete them", func() {
+					By("Running initial scan")
+					Expect(runScanner(ctx, true)).To(Succeed())
+
+					By("Removing a file")
+					fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+
+					By("Running another scan")
+					Expect(runScanner(ctx, true)).To(Succeed())
+
+					By("Checking files are marked as missing but not deleted")
+					count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{
+						Filters: squirrel.Eq{"missing": true},
+					})
+					Expect(err).ToNot(HaveOccurred())
+					Expect(count).To(Equal(int64(1)))
+
+					mf, err := findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+					Expect(err).ToNot(HaveOccurred())
+					Expect(mf.Missing).To(BeTrue())
+				})
+			})
+
+			When("PurgeMissing is set to 'always'", func() {
+				BeforeEach(func() {
+					conf.Server.Scanner.PurgeMissing = consts.PurgeMissingAlways
+				})
+
+				It("should purge missing files on any scan", func() {
+					By("Running initial scan")
+					Expect(runScanner(ctx, false)).To(Succeed())
+
+					By("Removing a file")
+					fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+
+					By("Running an incremental scan")
+					Expect(runScanner(ctx, false)).To(Succeed())
+
+					By("Checking missing files are deleted")
+					count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{
+						Filters: squirrel.Eq{"missing": true},
+					})
+					Expect(err).ToNot(HaveOccurred())
+					Expect(count).To(BeZero())
+
+					_, err = findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+					Expect(err).To(MatchError(model.ErrNotFound))
+				})
+			})
+
+			When("PurgeMissing is set to 'full'", func() {
+				BeforeEach(func() {
+					conf.Server.Scanner.PurgeMissing = consts.PurgeMissingFull
+				})
+
+				It("should not purge missing files on incremental scans", func() {
+					By("Running initial scan")
+					Expect(runScanner(ctx, true)).To(Succeed())
+
+					By("Removing a file")
+					fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+
+					By("Running an incremental scan")
+					Expect(runScanner(ctx, false)).To(Succeed())
+
+					By("Checking files are marked as missing but not deleted")
+					count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{
+						Filters: squirrel.Eq{"missing": true},
+					})
+					Expect(err).ToNot(HaveOccurred())
+					Expect(count).To(Equal(int64(1)))
+
+					mf, err := findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+					Expect(err).ToNot(HaveOccurred())
+					Expect(mf.Missing).To(BeTrue())
+				})
+
+				It("should purge missing files only on full scans", func() {
+					By("Running initial scan")
+					Expect(runScanner(ctx, true)).To(Succeed())
+
+					By("Removing a file")
+					fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+
+					By("Running a full scan")
+					Expect(runScanner(ctx, true)).To(Succeed())
+
+					By("Checking missing files are deleted")
+					count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{
+						Filters: squirrel.Eq{"missing": true},
+					})
+					Expect(err).ToNot(HaveOccurred())
+					Expect(count).To(BeZero())
+
+					_, err = findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+					Expect(err).To(MatchError(model.ErrNotFound))
+				})
+			})
+		})
 	})
 })
 

tests/mock_mediafile_repo.go

@@ -22,6 +22,10 @@ type MockMediaFileRepo struct {
 	model.MediaFileRepository
 	Data map[string]*model.MediaFile
 	Err  bool
+	// Add fields and methods for controlling CountAll and DeleteAllMissing in tests
+	CountAllValue         int64
+	CountAllOptions       model.QueryOptions
+	DeleteAllMissingValue int64
 }
 
 func (m *MockMediaFileRepo) SetError(err bool) {
@@ -161,4 +165,35 @@ func (m *MockMediaFileRepo) GetMissingAndMatching(libId int) (model.MediaFileCur
 	}, nil
 }
 
+func (m *MockMediaFileRepo) CountAll(opts ...model.QueryOptions) (int64, error) {
+	if m.Err {
+		return 0, errors.New("error")
+	}
+	if m.CountAllValue != 0 {
+		if len(opts) > 0 {
+			m.CountAllOptions = opts[0]
+		}
+		return m.CountAllValue, nil
+	}
+	return int64(len(m.Data)), nil
+}
+
+func (m *MockMediaFileRepo) DeleteAllMissing() (int64, error) {
+	if m.Err {
+		return 0, errors.New("error")
+	}
+	if m.DeleteAllMissingValue != 0 {
+		return m.DeleteAllMissingValue, nil
+	}
+	// Remove all missing files from Data
+	var count int64
+	for id, mf := range m.Data {
+		if mf.Missing {
+			delete(m.Data, id)
+			count++
+		}
+	}
+	return count, nil
+}
+
 var _ model.MediaFileRepository = (*MockMediaFileRepo)(nil)