mirror of
https://github.com/navidrome/navidrome.git
synced 2025-12-23 23:18:05 -05:00
* feat: Add selective folder scanning capability Implement targeted scanning of specific library/folder pairs without full recursion. This enables efficient rescanning of individual folders when changes are detected, significantly reducing scan time for large libraries. Key changes: - Add ScanTarget struct and ScanFolders API to Scanner interface - Implement CLI flag --targets for specifying libraryID:folderPath pairs - Add FolderRepository.GetByPaths() for batch folder info retrieval - Create loadSpecificFolders() for non-recursive directory loading - Scope GC operations to affected libraries only (with TODO for full impl) - Add comprehensive tests for selective scanning behavior The selective scan: - Only processes specified folders (no subdirectory recursion) - Maintains library isolation - Runs full maintenance pipeline scoped to affected libraries - Supports both full and quick scan modes Examples: navidrome scan --targets "1:Music/Rock,1:Music/Jazz" navidrome scan --full --targets "2:Classical" * feat(folder): replace GetByPaths with GetFolderUpdateInfo for improved folder updates retrieval Signed-off-by: Deluan <deluan@navidrome.org> * test: update parseTargets test to handle folder names with spaces Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): remove unused LibraryPath struct and update GC logging message Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder): enhance external scanner to support target-specific scanning Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify scanner methods Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement folder scanning notifications with deduplication Signed-off-by: Deluan <deluan@navidrome.org> * refactor(watcher): add resolveFolderPath function for testability Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): implement path ignoring based on .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): implement IgnoreChecker for managing .ndignore patterns Signed-off-by: Deluan <deluan@navidrome.org> * refactor(ignore_checker): rename scanner to lineScanner for clarity Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance ScanTarget struct with String method for better target representation Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): validate library ID to prevent negative values Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify GC method by removing library ID parameter Signed-off-by: Deluan <deluan@navidrome.org> * feat(scanner): update folder scanning to include all descendants of specified folders Signed-off-by: Deluan <deluan@navidrome.org> * feat(subsonic): allow selective scan in the /startScan endpoint Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update CallScan to handle specific library/folder pairs Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline scanning logic by removing scanAll method Signed-off-by: Deluan <deluan@navidrome.org> * test: enhance mockScanner for thread safety and improve test reliability Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move scanner.ScanTarget to model.ScanTarget Signed-off-by: Deluan <deluan@navidrome.org> * refactor: move scanner types to model,implement MockScanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): update scanner interface and implementations to use model.Scanner Signed-off-by: Deluan <deluan@navidrome.org> * refactor(folder_repository): normalize target path handling by using filepath.Clean Signed-off-by: Deluan <deluan@navidrome.org> * test(folder_repository): add comprehensive tests for folder retrieval and child exclusion Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): simplify selective scan logic using slice.Filter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): streamline phase folder and album creation by removing unnecessary library parameter Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): move initialization logic from phase_1 to the scanner itself Signed-off-by: Deluan <deluan@navidrome.org> * refactor(tests): rename selective scan test file to scanner_selective_test.go Signed-off-by: Deluan <deluan@navidrome.org> * feat(configuration): add DevSelectiveWatcher configuration option Signed-off-by: Deluan <deluan@navidrome.org> * feat(watcher): enhance .ndignore handling for folder deletions and file changes Signed-off-by: Deluan <deluan@navidrome.org> * docs(scanner): comments Signed-off-by: Deluan <deluan@navidrome.org> * refactor(scanner): enhance walkDirTree to support target folder scanning Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner, watcher): handle errors when pushing ignore patterns for folders Signed-off-by: Deluan <deluan@navidrome.org> * Update scanner/phase_1_folders.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * refactor(scanner): replace parseTargets function with direct call to scanner.ParseTargets Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): add tests for ScanBegin and ScanEnd functionality Signed-off-by: Deluan <deluan@navidrome.org> * fix(library): update PRAGMA optimize to check table sizes without ANALYZE Signed-off-by: Deluan <deluan@navidrome.org> * test(scanner): refactor tests Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add selective scan options and update translations Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add quick and full scan options for individual libraries Signed-off-by: Deluan <deluan@navidrome.org> * feat(ui): add Scan buttonsto the LibraryList Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): update scanning parameters from 'path' to 'target' for selective scans. * refactor(scan): move ParseTargets function to model package * test(scan): suppress unused return value from SetUserLibraries in tests * feat(gc): enhance garbage collection to support selective library purging Signed-off-by: Deluan <deluan@navidrome.org> * fix(scanner): prevent race condition when scanning deleted folders When the watcher detects changes in a folder that gets deleted before the scanner runs (due to the 10-second delay), the scanner was prematurely removing these folders from the tracking map, preventing them from being marked as missing. The issue occurred because `newFolderEntry` was calling `popLastUpdate` before verifying the folder actually exists on the filesystem. Changes: - Move fs.Stat check before newFolderEntry creation in loadDir to ensure deleted folders remain in lastUpdates for finalize() to handle - Add early existence check in walkDirTree to skip non-existent target folders with a warning log - Add unit test verifying non-existent folders aren't removed from lastUpdates prematurely - Add integration test for deleted folder scenario with ScanFolders Fixes the issue where deleting entire folders (e.g., /music/AC_DC) wouldn't mark tracks as missing when using selective folder scanning. * refactor(scan): streamline folder entry creation and update handling Signed-off-by: Deluan <deluan@navidrome.org> * feat(scan): add '@Recycle' (QNAP) to ignored directories list Signed-off-by: Deluan <deluan@navidrome.org> * fix(log): improve thread safety in logging level management * test(scan): move unit tests for ParseTargets function Signed-off-by: Deluan <deluan@navidrome.org> * review Signed-off-by: Deluan <deluan@navidrome.org> --------- Signed-off-by: Deluan <deluan@navidrome.org> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: deluan <deluan.quintao@mechanical-orchard.com>
352 lines
7.2 KiB
Go
352 lines
7.2 KiB
Go
package log
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"iter"
|
|
"net/http"
|
|
"os"
|
|
"runtime"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/sirupsen/logrus"
|
|
)
|
|
|
|
type Level uint32
|
|
|
|
type LevelFunc = func(ctx interface{}, msg interface{}, keyValuePairs ...interface{})
|
|
|
|
var redacted = &Hook{
|
|
AcceptedLevels: logrus.AllLevels,
|
|
RedactionList: []string{
|
|
// Keys from the config
|
|
"(ApiKey:\")[\\w]*",
|
|
"(Secret:\")[\\w]*",
|
|
"(Spotify.*ID:\")[\\w]*",
|
|
"(PasswordEncryptionKey:[\\s]*\")[^\"]*",
|
|
"(ReverseProxyUserHeader:[\\s]*\")[^\"]*",
|
|
"(ReverseProxyWhitelist:[\\s]*\")[^\"]*",
|
|
"(MetricsPath:[\\s]*\")[^\"]*",
|
|
"(DevAutoCreateAdminPassword:[\\s]*\")[^\"]*",
|
|
"(DevAutoLoginUsername:[\\s]*\")[^\"]*",
|
|
|
|
// UI appConfig
|
|
"(subsonicToken:)[\\w]+(\\s)",
|
|
"(subsonicSalt:)[\\w]+(\\s)",
|
|
"(token:)[^\\s]+",
|
|
|
|
// Subsonic query params
|
|
"([^\\w]t=)[\\w]+",
|
|
"([^\\w]s=)[^&]+",
|
|
"([^\\w]p=)[^&]+",
|
|
"([^\\w]jwt=)[^&]+",
|
|
|
|
// External services query params
|
|
"([^\\w]api_key=)[\\w]+",
|
|
},
|
|
}
|
|
|
|
const (
|
|
LevelFatal = Level(logrus.FatalLevel)
|
|
LevelError = Level(logrus.ErrorLevel)
|
|
LevelWarn = Level(logrus.WarnLevel)
|
|
LevelInfo = Level(logrus.InfoLevel)
|
|
LevelDebug = Level(logrus.DebugLevel)
|
|
LevelTrace = Level(logrus.TraceLevel)
|
|
)
|
|
|
|
type contextKey string
|
|
|
|
const loggerCtxKey = contextKey("logger")
|
|
|
|
type levelPath struct {
|
|
path string
|
|
level Level
|
|
}
|
|
|
|
var (
|
|
currentLevel Level
|
|
loggerMu sync.RWMutex
|
|
defaultLogger = logrus.New()
|
|
logSourceLine = false
|
|
rootPath string
|
|
logLevels []levelPath
|
|
)
|
|
|
|
// SetLevel sets the global log level used by the simple logger.
|
|
func SetLevel(l Level) {
|
|
loggerMu.Lock()
|
|
currentLevel = l
|
|
defaultLogger.Level = logrus.TraceLevel
|
|
loggerMu.Unlock()
|
|
logrus.SetLevel(logrus.Level(l))
|
|
}
|
|
|
|
func SetLevelString(l string) {
|
|
level := levelFromString(l)
|
|
SetLevel(level)
|
|
}
|
|
|
|
func levelFromString(l string) Level {
|
|
envLevel := strings.ToLower(l)
|
|
var level Level
|
|
switch envLevel {
|
|
case "fatal":
|
|
level = LevelFatal
|
|
case "error":
|
|
level = LevelError
|
|
case "warn":
|
|
level = LevelWarn
|
|
case "debug":
|
|
level = LevelDebug
|
|
case "trace":
|
|
level = LevelTrace
|
|
default:
|
|
level = LevelInfo
|
|
}
|
|
return level
|
|
}
|
|
|
|
// SetLogLevels sets the log levels for specific paths in the codebase.
|
|
func SetLogLevels(levels map[string]string) {
|
|
loggerMu.Lock()
|
|
defer loggerMu.Unlock()
|
|
logLevels = nil
|
|
for k, v := range levels {
|
|
logLevels = append(logLevels, levelPath{path: k, level: levelFromString(v)})
|
|
}
|
|
sort.Slice(logLevels, func(i, j int) bool {
|
|
return logLevels[i].path > logLevels[j].path
|
|
})
|
|
}
|
|
|
|
func SetLogSourceLine(enabled bool) {
|
|
logSourceLine = enabled
|
|
}
|
|
|
|
func SetRedacting(enabled bool) {
|
|
if enabled {
|
|
loggerMu.Lock()
|
|
defer loggerMu.Unlock()
|
|
defaultLogger.AddHook(redacted)
|
|
}
|
|
}
|
|
|
|
func SetOutput(w io.Writer) {
|
|
if runtime.GOOS == "windows" {
|
|
w = CRLFWriter(w)
|
|
}
|
|
loggerMu.Lock()
|
|
defer loggerMu.Unlock()
|
|
defaultLogger.SetOutput(w)
|
|
}
|
|
|
|
// Redact applies redaction to a single string
|
|
func Redact(msg string) string {
|
|
r, _ := redacted.redact(msg)
|
|
return r
|
|
}
|
|
|
|
func NewContext(ctx context.Context, keyValuePairs ...interface{}) context.Context {
|
|
if ctx == nil {
|
|
ctx = context.Background()
|
|
}
|
|
|
|
logger, ok := ctx.Value(loggerCtxKey).(*logrus.Entry)
|
|
if !ok {
|
|
logger = createNewLogger()
|
|
}
|
|
logger = addFields(logger, keyValuePairs)
|
|
ctx = context.WithValue(ctx, loggerCtxKey, logger)
|
|
|
|
return ctx
|
|
}
|
|
|
|
func SetDefaultLogger(l *logrus.Logger) {
|
|
loggerMu.Lock()
|
|
defer loggerMu.Unlock()
|
|
defaultLogger = l
|
|
}
|
|
|
|
func CurrentLevel() Level {
|
|
loggerMu.RLock()
|
|
defer loggerMu.RUnlock()
|
|
return currentLevel
|
|
}
|
|
|
|
// IsGreaterOrEqualTo returns true if the caller's current log level is equal or greater than the provided level.
|
|
func IsGreaterOrEqualTo(level Level) bool {
|
|
return shouldLog(level, 2)
|
|
}
|
|
|
|
func Fatal(args ...interface{}) {
|
|
log(LevelFatal, args...)
|
|
os.Exit(1)
|
|
}
|
|
|
|
func Error(args ...interface{}) {
|
|
log(LevelError, args...)
|
|
}
|
|
|
|
func Warn(args ...interface{}) {
|
|
log(LevelWarn, args...)
|
|
}
|
|
|
|
func Info(args ...interface{}) {
|
|
log(LevelInfo, args...)
|
|
}
|
|
|
|
func Debug(args ...interface{}) {
|
|
log(LevelDebug, args...)
|
|
}
|
|
|
|
func Trace(args ...interface{}) {
|
|
log(LevelTrace, args...)
|
|
}
|
|
|
|
func log(level Level, args ...interface{}) {
|
|
if !shouldLog(level, 3) {
|
|
return
|
|
}
|
|
logger, msg := parseArgs(args)
|
|
logger.Log(logrus.Level(level), msg)
|
|
}
|
|
|
|
func Writer() io.Writer {
|
|
loggerMu.RLock()
|
|
defer loggerMu.RUnlock()
|
|
return defaultLogger.Writer()
|
|
}
|
|
|
|
func shouldLog(requiredLevel Level, skip int) bool {
|
|
loggerMu.RLock()
|
|
level := currentLevel
|
|
levels := logLevels
|
|
loggerMu.RUnlock()
|
|
|
|
if level >= requiredLevel {
|
|
return true
|
|
}
|
|
if len(levels) == 0 {
|
|
return false
|
|
}
|
|
|
|
_, file, _, ok := runtime.Caller(skip)
|
|
if !ok {
|
|
return false
|
|
}
|
|
|
|
file = strings.TrimPrefix(file, rootPath)
|
|
for _, lp := range levels {
|
|
if strings.HasPrefix(file, lp.path) {
|
|
return lp.level >= requiredLevel
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func parseArgs(args []interface{}) (*logrus.Entry, string) {
|
|
var l *logrus.Entry
|
|
var err error
|
|
if args[0] == nil {
|
|
l = createNewLogger()
|
|
args = args[1:]
|
|
} else {
|
|
l, err = extractLogger(args[0])
|
|
if err != nil {
|
|
l = createNewLogger()
|
|
} else {
|
|
args = args[1:]
|
|
}
|
|
}
|
|
if len(args) > 1 {
|
|
kvPairs := args[1:]
|
|
l = addFields(l, kvPairs)
|
|
}
|
|
if logSourceLine {
|
|
_, file, line, ok := runtime.Caller(3)
|
|
if !ok {
|
|
file = "???"
|
|
line = 0
|
|
}
|
|
//_, filename := path.Split(file)
|
|
//l = l.WithField("filename", filename).WithField("line", line)
|
|
l = l.WithField(" source", fmt.Sprintf("file://%s:%d", file, line))
|
|
}
|
|
|
|
switch msg := args[0].(type) {
|
|
case error:
|
|
return l, msg.Error()
|
|
case string:
|
|
return l, msg
|
|
}
|
|
|
|
return l, ""
|
|
}
|
|
|
|
func addFields(logger *logrus.Entry, keyValuePairs []interface{}) *logrus.Entry {
|
|
for i := 0; i < len(keyValuePairs); i += 2 {
|
|
switch name := keyValuePairs[i].(type) {
|
|
case error:
|
|
logger = logger.WithField("error", name.Error())
|
|
case string:
|
|
if i+1 >= len(keyValuePairs) {
|
|
logger = logger.WithField(name, "!!!!Invalid number of arguments in log call!!!!")
|
|
} else {
|
|
switch v := keyValuePairs[i+1].(type) {
|
|
case time.Duration:
|
|
logger = logger.WithField(name, ShortDur(v))
|
|
case fmt.Stringer:
|
|
logger = logger.WithField(name, StringerValue(v))
|
|
case iter.Seq[string]:
|
|
logger = logger.WithField(name, formatSeq(v))
|
|
case []string:
|
|
logger = logger.WithField(name, formatSlice(v))
|
|
default:
|
|
logger = logger.WithField(name, v)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return logger
|
|
}
|
|
|
|
func extractLogger(ctx interface{}) (*logrus.Entry, error) {
|
|
switch ctx := ctx.(type) {
|
|
case *logrus.Entry:
|
|
return ctx, nil
|
|
case context.Context:
|
|
logger := ctx.Value(loggerCtxKey)
|
|
if logger != nil {
|
|
return logger.(*logrus.Entry), nil
|
|
}
|
|
return extractLogger(NewContext(ctx))
|
|
case *http.Request:
|
|
return extractLogger(ctx.Context())
|
|
}
|
|
return nil, errors.New("no logger found")
|
|
}
|
|
|
|
func createNewLogger() *logrus.Entry {
|
|
//logrus.SetFormatter(&logrus.TextFormatter{ForceColors: true, DisableTimestamp: false, FullTimestamp: true})
|
|
//l.Formatter = &logrus.TextFormatter{ForceColors: true, DisableTimestamp: false, FullTimestamp: true}
|
|
loggerMu.RLock()
|
|
defer loggerMu.RUnlock()
|
|
logger := logrus.NewEntry(defaultLogger)
|
|
return logger
|
|
}
|
|
|
|
func init() {
|
|
defaultLogger.Level = logrus.TraceLevel
|
|
_, file, _, ok := runtime.Caller(0)
|
|
if !ok {
|
|
return
|
|
}
|
|
rootPath = strings.TrimSuffix(file, "log/log.go")
|
|
}
|