Files
navidrome/log/log.go
Deluan Quintão 28d5299ffc feat(scanner): implement selective folder scanning and file system watcher improvements (#4674)
* feat: Add selective folder scanning capability

Implement targeted scanning of specific library/folder pairs without
full recursion. This enables efficient rescanning of individual folders
when changes are detected, significantly reducing scan time for large
libraries.

Key changes:
- Add ScanTarget struct and ScanFolders API to Scanner interface
- Implement CLI flag --targets for specifying libraryID:folderPath pairs
- Add FolderRepository.GetByPaths() for batch folder info retrieval
- Create loadSpecificFolders() for non-recursive directory loading
- Scope GC operations to affected libraries only (with TODO for full impl)
- Add comprehensive tests for selective scanning behavior

The selective scan:
- Only processes specified folders (no subdirectory recursion)
- Maintains library isolation
- Runs full maintenance pipeline scoped to affected libraries
- Supports both full and quick scan modes

Examples:
  navidrome scan --targets "1:Music/Rock,1:Music/Jazz"
  navidrome scan --full --targets "2:Classical"

* feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval

Signed-off-by: Deluan <deluan@navidrome.org>

* test: update parseTargets test to handle folder names with spaces

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(folder): remove unused LibraryPath struct and update GC logging message

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(folder): enhance external scanner to support target-specific scanning

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): simplify scanner methods

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(watcher): implement folder scanning notifications with deduplication

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(watcher): add resolveFolderPath function for testability

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(watcher): implement path ignoring based on .ndignore patterns

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): implement IgnoreChecker for managing .ndignore patterns

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(ignore_checker): rename scanner to lineScanner for clarity

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): enhance ScanTarget struct with String method for better target representation

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner): validate library ID to prevent negative values

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): simplify GC method by removing library ID parameter

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(scanner): update folder scanning to include all descendants of specified folders

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(subsonic): allow selective scan in the /startScan endpoint

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): update CallScan to handle specific library/folder pairs

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): streamline scanning logic by removing scanAll method

Signed-off-by: Deluan <deluan@navidrome.org>

* test: enhance mockScanner for thread safety and improve test reliability

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): move scanner.ScanTarget to model.ScanTarget

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor: move scanner types to model,implement MockScanner

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): update scanner interface and implementations to use model.Scanner

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(folder_repository): normalize target path handling by using filepath.Clean

Signed-off-by: Deluan <deluan@navidrome.org>

* test(folder_repository): add comprehensive tests for folder retrieval and child exclusion

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): simplify selective scan logic using slice.Filter

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): move initialization logic from phase_1 to the scanner itself

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(tests): rename selective scan test file to scanner_selective_test.go

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(configuration): add DevSelectiveWatcher configuration option

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(watcher): enhance .ndignore handling for folder deletions and file changes

Signed-off-by: Deluan <deluan@navidrome.org>

* docs(scanner): comments

Signed-off-by: Deluan <deluan@navidrome.org>

* refactor(scanner): enhance walkDirTree to support target folder scanning

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner, watcher): handle errors when pushing ignore patterns for folders

Signed-off-by: Deluan <deluan@navidrome.org>

* Update scanner/phase_1_folders.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets

Signed-off-by: Deluan <deluan@navidrome.org>

* test(scanner): add tests for ScanBegin and ScanEnd functionality

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(library): update PRAGMA optimize to check table sizes without ANALYZE

Signed-off-by: Deluan <deluan@navidrome.org>

* test(scanner): refactor tests

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(ui): add selective scan options and update translations

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(ui): add quick and full scan options for individual libraries

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(ui): add Scan buttonsto the LibraryList

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(scan): update scanning parameters from 'path' to 'target' for selective scans.

* refactor(scan): move ParseTargets function to model package

* test(scan): suppress unused return value from SetUserLibraries in tests

* feat(gc): enhance garbage collection to support selective library purging

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(scanner): prevent race condition when scanning deleted folders

When the watcher detects changes in a folder that gets deleted before
the scanner runs (due to the 10-second delay), the scanner was
prematurely removing these folders from the tracking map, preventing
them from being marked as missing.

The issue occurred because `newFolderEntry` was calling `popLastUpdate`
before verifying the folder actually exists on the filesystem.

Changes:
- Move fs.Stat check before newFolderEntry creation in loadDir to
  ensure deleted folders remain in lastUpdates for finalize() to handle
- Add early existence check in walkDirTree to skip non-existent target
  folders with a warning log
- Add unit test verifying non-existent folders aren't removed from
  lastUpdates prematurely
- Add integration test for deleted folder scenario with ScanFolders

Fixes the issue where deleting entire folders (e.g., /music/AC_DC)
wouldn't mark tracks as missing when using selective folder scanning.

* refactor(scan): streamline folder entry creation and update handling

Signed-off-by: Deluan <deluan@navidrome.org>

* feat(scan): add '@Recycle' (QNAP) to ignored directories list

Signed-off-by: Deluan <deluan@navidrome.org>

* fix(log): improve thread safety in logging level management

* test(scan): move unit tests for ParseTargets function

Signed-off-by: Deluan <deluan@navidrome.org>

* review

Signed-off-by: Deluan <deluan@navidrome.org>

---------

Signed-off-by: Deluan <deluan@navidrome.org>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: deluan <deluan.quintao@mechanical-orchard.com>
2025-11-14 22:15:43 -05:00

352 lines
7.2 KiB
Go

package log
import (
"context"
"errors"
"fmt"
"io"
"iter"
"net/http"
"os"
"runtime"
"sort"
"strings"
"sync"
"time"
"github.com/sirupsen/logrus"
)
type Level uint32
type LevelFunc = func(ctx interface{}, msg interface{}, keyValuePairs ...interface{})
var redacted = &Hook{
AcceptedLevels: logrus.AllLevels,
RedactionList: []string{
// Keys from the config
"(ApiKey:\")[\\w]*",
"(Secret:\")[\\w]*",
"(Spotify.*ID:\")[\\w]*",
"(PasswordEncryptionKey:[\\s]*\")[^\"]*",
"(ReverseProxyUserHeader:[\\s]*\")[^\"]*",
"(ReverseProxyWhitelist:[\\s]*\")[^\"]*",
"(MetricsPath:[\\s]*\")[^\"]*",
"(DevAutoCreateAdminPassword:[\\s]*\")[^\"]*",
"(DevAutoLoginUsername:[\\s]*\")[^\"]*",
// UI appConfig
"(subsonicToken:)[\\w]+(\\s)",
"(subsonicSalt:)[\\w]+(\\s)",
"(token:)[^\\s]+",
// Subsonic query params
"([^\\w]t=)[\\w]+",
"([^\\w]s=)[^&]+",
"([^\\w]p=)[^&]+",
"([^\\w]jwt=)[^&]+",
// External services query params
"([^\\w]api_key=)[\\w]+",
},
}
const (
LevelFatal = Level(logrus.FatalLevel)
LevelError = Level(logrus.ErrorLevel)
LevelWarn = Level(logrus.WarnLevel)
LevelInfo = Level(logrus.InfoLevel)
LevelDebug = Level(logrus.DebugLevel)
LevelTrace = Level(logrus.TraceLevel)
)
type contextKey string
const loggerCtxKey = contextKey("logger")
type levelPath struct {
path string
level Level
}
var (
currentLevel Level
loggerMu sync.RWMutex
defaultLogger = logrus.New()
logSourceLine = false
rootPath string
logLevels []levelPath
)
// SetLevel sets the global log level used by the simple logger.
func SetLevel(l Level) {
loggerMu.Lock()
currentLevel = l
defaultLogger.Level = logrus.TraceLevel
loggerMu.Unlock()
logrus.SetLevel(logrus.Level(l))
}
func SetLevelString(l string) {
level := levelFromString(l)
SetLevel(level)
}
func levelFromString(l string) Level {
envLevel := strings.ToLower(l)
var level Level
switch envLevel {
case "fatal":
level = LevelFatal
case "error":
level = LevelError
case "warn":
level = LevelWarn
case "debug":
level = LevelDebug
case "trace":
level = LevelTrace
default:
level = LevelInfo
}
return level
}
// SetLogLevels sets the log levels for specific paths in the codebase.
func SetLogLevels(levels map[string]string) {
loggerMu.Lock()
defer loggerMu.Unlock()
logLevels = nil
for k, v := range levels {
logLevels = append(logLevels, levelPath{path: k, level: levelFromString(v)})
}
sort.Slice(logLevels, func(i, j int) bool {
return logLevels[i].path > logLevels[j].path
})
}
func SetLogSourceLine(enabled bool) {
logSourceLine = enabled
}
func SetRedacting(enabled bool) {
if enabled {
loggerMu.Lock()
defer loggerMu.Unlock()
defaultLogger.AddHook(redacted)
}
}
func SetOutput(w io.Writer) {
if runtime.GOOS == "windows" {
w = CRLFWriter(w)
}
loggerMu.Lock()
defer loggerMu.Unlock()
defaultLogger.SetOutput(w)
}
// Redact applies redaction to a single string
func Redact(msg string) string {
r, _ := redacted.redact(msg)
return r
}
func NewContext(ctx context.Context, keyValuePairs ...interface{}) context.Context {
if ctx == nil {
ctx = context.Background()
}
logger, ok := ctx.Value(loggerCtxKey).(*logrus.Entry)
if !ok {
logger = createNewLogger()
}
logger = addFields(logger, keyValuePairs)
ctx = context.WithValue(ctx, loggerCtxKey, logger)
return ctx
}
func SetDefaultLogger(l *logrus.Logger) {
loggerMu.Lock()
defer loggerMu.Unlock()
defaultLogger = l
}
func CurrentLevel() Level {
loggerMu.RLock()
defer loggerMu.RUnlock()
return currentLevel
}
// IsGreaterOrEqualTo returns true if the caller's current log level is equal or greater than the provided level.
func IsGreaterOrEqualTo(level Level) bool {
return shouldLog(level, 2)
}
func Fatal(args ...interface{}) {
log(LevelFatal, args...)
os.Exit(1)
}
func Error(args ...interface{}) {
log(LevelError, args...)
}
func Warn(args ...interface{}) {
log(LevelWarn, args...)
}
func Info(args ...interface{}) {
log(LevelInfo, args...)
}
func Debug(args ...interface{}) {
log(LevelDebug, args...)
}
func Trace(args ...interface{}) {
log(LevelTrace, args...)
}
func log(level Level, args ...interface{}) {
if !shouldLog(level, 3) {
return
}
logger, msg := parseArgs(args)
logger.Log(logrus.Level(level), msg)
}
func Writer() io.Writer {
loggerMu.RLock()
defer loggerMu.RUnlock()
return defaultLogger.Writer()
}
func shouldLog(requiredLevel Level, skip int) bool {
loggerMu.RLock()
level := currentLevel
levels := logLevels
loggerMu.RUnlock()
if level >= requiredLevel {
return true
}
if len(levels) == 0 {
return false
}
_, file, _, ok := runtime.Caller(skip)
if !ok {
return false
}
file = strings.TrimPrefix(file, rootPath)
for _, lp := range levels {
if strings.HasPrefix(file, lp.path) {
return lp.level >= requiredLevel
}
}
return false
}
func parseArgs(args []interface{}) (*logrus.Entry, string) {
var l *logrus.Entry
var err error
if args[0] == nil {
l = createNewLogger()
args = args[1:]
} else {
l, err = extractLogger(args[0])
if err != nil {
l = createNewLogger()
} else {
args = args[1:]
}
}
if len(args) > 1 {
kvPairs := args[1:]
l = addFields(l, kvPairs)
}
if logSourceLine {
_, file, line, ok := runtime.Caller(3)
if !ok {
file = "???"
line = 0
}
//_, filename := path.Split(file)
//l = l.WithField("filename", filename).WithField("line", line)
l = l.WithField(" source", fmt.Sprintf("file://%s:%d", file, line))
}
switch msg := args[0].(type) {
case error:
return l, msg.Error()
case string:
return l, msg
}
return l, ""
}
func addFields(logger *logrus.Entry, keyValuePairs []interface{}) *logrus.Entry {
for i := 0; i < len(keyValuePairs); i += 2 {
switch name := keyValuePairs[i].(type) {
case error:
logger = logger.WithField("error", name.Error())
case string:
if i+1 >= len(keyValuePairs) {
logger = logger.WithField(name, "!!!!Invalid number of arguments in log call!!!!")
} else {
switch v := keyValuePairs[i+1].(type) {
case time.Duration:
logger = logger.WithField(name, ShortDur(v))
case fmt.Stringer:
logger = logger.WithField(name, StringerValue(v))
case iter.Seq[string]:
logger = logger.WithField(name, formatSeq(v))
case []string:
logger = logger.WithField(name, formatSlice(v))
default:
logger = logger.WithField(name, v)
}
}
}
}
return logger
}
func extractLogger(ctx interface{}) (*logrus.Entry, error) {
switch ctx := ctx.(type) {
case *logrus.Entry:
return ctx, nil
case context.Context:
logger := ctx.Value(loggerCtxKey)
if logger != nil {
return logger.(*logrus.Entry), nil
}
return extractLogger(NewContext(ctx))
case *http.Request:
return extractLogger(ctx.Context())
}
return nil, errors.New("no logger found")
}
func createNewLogger() *logrus.Entry {
//logrus.SetFormatter(&logrus.TextFormatter{ForceColors: true, DisableTimestamp: false, FullTimestamp: true})
//l.Formatter = &logrus.TextFormatter{ForceColors: true, DisableTimestamp: false, FullTimestamp: true}
loggerMu.RLock()
defer loggerMu.RUnlock()
logger := logrus.NewEntry(defaultLogger)
return logger
}
func init() {
defaultLogger.Level = logrus.TraceLevel
_, file, _, ok := runtime.Caller(0)
if !ok {
return
}
rootPath = strings.TrimSuffix(file, "log/log.go")
}