Bump golangci-lint version in pipeline

This reverts commit e07152b695.

.github/FUNDING.yml

@@ -3,8 +3,8 @@
 github: deluan
 patreon: # Replace with a single Patreon username
 open_collective: # Replace with a single Open Collective username
-liberapay: deluan
 ko_fi: deluan
+liberapay: deluan
 tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
 community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
 issuehunt: # Replace with a single IssueHunt username

.github/workflows/pipeline.yml

@@ -19,9 +19,9 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v1
+        uses: golangci/golangci-lint-action@v2
         with:
-          version: v1.27
+          version: v1.32
           github-token: ${{ secrets.GITHUB_TOKEN }}
           args: --timeout 2m
 
@@ -33,7 +33,7 @@ jobs:
         run: sudo apt-get install libtag1-dev
 
       - name: Set up Go 1.15
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v2
         with:
           go-version: 1.15
         id: go
@@ -41,7 +41,7 @@ jobs:
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
 
-      - uses: actions/cache@v1
+      - uses: actions/cache@v2
         id: cache-go
         with:
           path: ~/go/pkg/mod
@@ -64,7 +64,7 @@ jobs:
         with:
           node-version: 14
 
-      - uses: actions/cache@v1
+      - uses: actions/cache@v2
         id: cache-npm
         with:
           path: ~/.npm
@@ -94,7 +94,7 @@ jobs:
 
   binaries:
     name: Binaries
-    needs: [js]
+    needs: [js, go, golangci-lint]
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
@@ -115,7 +115,7 @@ jobs:
 
       - name: Run GoReleaser - SNAPSHOT
         if: startsWith(github.ref, 'refs/tags/') != true
-        uses: docker://deluan/ci-goreleaser:1.15.2-1
+        uses: docker://deluan/ci-goreleaser:1.15.3-1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
@@ -123,7 +123,7 @@ jobs:
 
       - name: Run GoReleaser - RELEASE
         if: startsWith(github.ref, 'refs/tags/')
-        uses: docker://deluan/ci-goreleaser:1.15.2-1
+        uses: docker://deluan/ci-goreleaser:1.15.3-1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
@@ -152,7 +152,7 @@ jobs:
           buildx-version: latest
           qemu-version: latest
 
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         if: env.DOCKER_IMAGE != ''
 
       - uses: actions/download-artifact@v2

.golangci.yml

@@ -26,4 +26,4 @@ issues:
   exclude-rules:
     - linters:
         - gosec
-      text: "(G501|G401):"
+      text: "(G501|G401|G505):"

Makefile

@@ -33,13 +33,17 @@ testall: check_go_env test
 	@(cd ./ui && npm test -- --watchAll=false)
 .PHONY: testall
 
+lint:
+	golangci-lint run -v
+.PHONY: lint
+
 update-snapshots: check_go_env
 	UPDATE_SNAPSHOTS=true ginkgo ./server/subsonic/...
 .PHONY: update-snapshots
 
 migration:
 	@if [ -z "${name}" ]; then echo "Usage: make migration name=name_of_migration_file"; exit 1; fi
-	goose -dir db/migrations create ${name}
+	goose -dir db/migration create ${name}
 .PHONY: migration
 
 setup: download-deps
@@ -55,8 +59,6 @@ download-deps:
 .PHONY: download-deps
 
 setup-dev: setup setup-git
-	@echo Installing golangci-lint
-	@which golangci-lint || (echo "Installing GolangCI-Lint" && cd .. && GO111MODULE=on go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.27.0)
 .PHONY: setup-dev
 
 setup-git:
@@ -89,11 +91,7 @@ buildall: check_env
 	go build -ldflags="-X github.com/deluan/navidrome/consts.gitSha=$(GIT_SHA) -X github.com/deluan/navidrome/consts.gitTag=$(GIT_TAG)-SNAPSHOT" -tags=embed
 .PHONY: buildall
 
-pre-push:
-	golangci-lint run -v
-
-	@echo
-	make test
+pre-push: lint test
 .PHONY: pre-push
 
 release:
@@ -106,5 +104,5 @@ release:
 .PHONY: release
 
 snapshot:
-	 docker run -it -v $(PWD):/workspace -w /workspace deluan/ci-goreleaser:1.15.2-1 goreleaser release --rm-dist --skip-publish --snapshot
+	 docker run -it -v $(PWD):/workspace -w /workspace deluan/ci-goreleaser:1.15.3-1 goreleaser release --rm-dist --skip-publish --snapshot
 .PHONY: snapshot

README.md

@@ -2,11 +2,11 @@
 
 [![Last Release](https://img.shields.io/github/v/release/deluan/navidrome?label=latest&style=flat-square)](https://github.com/deluan/navidrome/releases)
 [![Build](https://img.shields.io/github/workflow/status/deluan/navidrome/Build?style=flat-square)](https://github.com/deluan/navidrome/actions)
-[![Downloads](https://img.shields.io/github/downloads/deluan/navidrome/total)](https://github.com/deluan/navidrome/releases/latest)
+[![Downloads](https://img.shields.io/github/downloads/deluan/navidrome/total?style=flat-square)](https://github.com/deluan/navidrome/releases/latest)
 [![Docker Pulls](https://img.shields.io/docker/pulls/deluan/navidrome?style=flat-square)](https://hub.docker.com/r/deluan/navidrome)
 [![Dev Chat](https://img.shields.io/discord/671335427726114836?label=chat&style=flat-square)](https://discord.gg/xh7j7yF)
 [![Subreddit](https://img.shields.io/reddit/subreddit-subscribers/navidrome?style=flat-square)](https://www.reddit.com/r/navidrome/)
-[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-v2.0-ff69b4.svg)](code_of_conduct.md)
+[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-v2.0-ff69b4.svg?style=flat-square)](code_of_conduct.md)
 
 ## [Check out our Live Demo!](https://www.navidrome.org/demo/)
 

cmd/root.go

@@ -3,10 +3,13 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/deluan/navidrome/conf"
 	"github.com/deluan/navidrome/consts"
 	"github.com/deluan/navidrome/db"
+	"github.com/deluan/navidrome/log"
+	"github.com/oklog/run"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -24,7 +27,7 @@ Complete documentation is available at https://www.navidrome.org/docs`,
 			preRun()
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			startServer()
+			runNavidrome()
 		},
 		Version: consts.Version(),
 	}
@@ -45,20 +48,61 @@ func preRun() {
 	conf.Load()
 }
 
-func startServer() {
+func runNavidrome() {
 	db.EnsureLatestVersion()
 
-	subsonic, err := CreateSubsonicAPIRouter()
-	if err != nil {
-		panic(fmt.Sprintf("Could not create the Subsonic API router. Aborting! err=%v", err))
+	var g run.Group
+	g.Add(startServer())
+	g.Add(startScanner())
+
+	if err := g.Run(); err != nil {
+		log.Error("Fatal error in Navidrome. Aborting", err)
 	}
-	a := CreateServer(conf.Server.MusicFolder)
-	a.MountRouter(consts.URLPathSubsonicAPI, subsonic)
-	a.MountRouter(consts.URLPathUI, CreateAppRouter())
-	a.Run(fmt.Sprintf("%s:%d", conf.Server.Address, conf.Server.Port))
 }
 
-// TODO: Implemement some struct tags to map flags to viper
+func startServer() (func() error, func(err error)) {
+	return func() error {
+			a := CreateServer(conf.Server.MusicFolder)
+			a.MountRouter(consts.URLPathSubsonicAPI, CreateSubsonicAPIRouter())
+			a.MountRouter(consts.URLPathUI, CreateAppRouter())
+			return a.Run(fmt.Sprintf("%s:%d", conf.Server.Address, conf.Server.Port))
+		}, func(err error) {
+			if err != nil {
+				log.Error("Shutting down Server due to error", err)
+			} else {
+				log.Info("Shutting down Server")
+			}
+		}
+}
+
+func startScanner() (func() error, func(err error)) {
+	interval := conf.Server.ScanInterval
+	log.Info("Starting scanner", "interval", interval.String())
+	scanner := GetScanner()
+	done := make(chan struct{})
+
+	return func() error {
+			if interval != 0 {
+				time.Sleep(2 * time.Second) // Wait 2 seconds before the first scan
+				scanner.Start(interval)
+			} else {
+				log.Warn("Periodic scan is DISABLED", "interval", interval)
+			}
+
+			<-done
+			return nil
+		}, func(err error) {
+			scanner.Stop()
+			done <- struct{}{}
+			if err != nil {
+				log.Error("Shutting down Scanner due to error", err)
+			} else {
+				log.Info("Shutting down Scanner")
+			}
+		}
+}
+
+// TODO: Implement some struct tags to map flags to viper
 func init() {
 	cobra.OnInitialize(func() {
 		conf.InitConfig(cfgFile)

cmd/scan.go

@@ -23,11 +23,10 @@ var scanCmd = &cobra.Command{
 }
 
 func runScanner() {
-	scanner := CreateScanner(conf.Server.MusicFolder)
-	err := scanner.RescanAll(fullRescan)
-	if err != nil {
-		log.Error("Error scanning media folder", "folder", conf.Server.MusicFolder, err)
-	}
+	conf.Server.DevPreCacheAlbumArtwork = false
+
+	scanner := GetScanner()
+	_ = scanner.RescanAll(fullRescan)
 	if fullRescan {
 		log.Info("Finished full rescan")
 	} else {

cmd/wire_gen.go

@@ -13,47 +13,63 @@ import (
 	"github.com/deluan/navidrome/server"
 	"github.com/deluan/navidrome/server/app"
 	"github.com/deluan/navidrome/server/subsonic"
-	"github.com/deluan/navidrome/server/subsonic/engine"
 	"github.com/google/wire"
+	"sync"
 )
 
 // Injectors from wire_injectors.go:
 
 func CreateServer(musicFolder string) *server.Server {
 	dataStore := persistence.New()
-	scannerScanner := scanner.New(dataStore)
-	serverServer := server.New(scannerScanner, dataStore)
+	serverServer := server.New(dataStore)
 	return serverServer
 }
 
-func CreateScanner(musicFolder string) *scanner.Scanner {
-	dataStore := persistence.New()
-	scannerScanner := scanner.New(dataStore)
-	return scannerScanner
-}
-
 func CreateAppRouter() *app.Router {
 	dataStore := persistence.New()
 	router := app.New(dataStore)
 	return router
 }
 
-func CreateSubsonicAPIRouter() (*subsonic.Router, error) {
+func CreateSubsonicAPIRouter() *subsonic.Router {
 	dataStore := persistence.New()
-	artworkCache := core.NewImageCache()
+	artworkCache := core.GetImageCache()
 	artwork := core.NewArtwork(dataStore, artworkCache)
-	nowPlayingRepository := engine.NewNowPlayingRepository()
-	listGenerator := engine.NewListGenerator(dataStore, nowPlayingRepository)
-	playlists := engine.NewPlaylists(dataStore)
 	transcoderTranscoder := transcoder.New()
-	transcodingCache := core.NewTranscodingCache()
+	transcodingCache := core.GetTranscodingCache()
 	mediaStreamer := core.NewMediaStreamer(dataStore, transcoderTranscoder, transcodingCache)
 	archiver := core.NewArchiver(dataStore)
-	players := engine.NewPlayers(dataStore)
-	router := subsonic.New(artwork, listGenerator, playlists, mediaStreamer, archiver, players, dataStore)
-	return router, nil
+	players := core.NewPlayers(dataStore)
+	client := core.LastFMNewClient()
+	spotifyClient := core.SpotifyNewClient()
+	externalInfo := core.NewExternalInfo(dataStore, client, spotifyClient)
+	scanner := GetScanner()
+	router := subsonic.New(dataStore, artwork, mediaStreamer, archiver, players, externalInfo, scanner)
+	return router
+}
+
+func createScanner() scanner.Scanner {
+	dataStore := persistence.New()
+	artworkCache := core.GetImageCache()
+	artwork := core.NewArtwork(dataStore, artworkCache)
+	cacheWarmer := core.NewCacheWarmer(artwork, artworkCache)
+	scannerScanner := scanner.New(dataStore, cacheWarmer)
+	return scannerScanner
 }
 
 // wire_injectors.go:
 
-var allProviders = wire.NewSet(engine.Set, core.Set, scanner.New, subsonic.New, app.New, persistence.New)
+var allProviders = wire.NewSet(core.Set, subsonic.New, app.New, persistence.New)
+
+// Scanner must be a Singleton
+var (
+	onceScanner     sync.Once
+	scannerInstance scanner.Scanner
+)
+
+func GetScanner() scanner.Scanner {
+	onceScanner.Do(func() {
+		scannerInstance = createScanner()
+	})
+	return scannerInstance
+}

cmd/wire_injectors.go

@@ -9,14 +9,12 @@ import (
 	"github.com/deluan/navidrome/server"
 	"github.com/deluan/navidrome/server/app"
 	"github.com/deluan/navidrome/server/subsonic"
-	"github.com/deluan/navidrome/server/subsonic/engine"
 	"github.com/google/wire"
+	"sync"
 )
 
 var allProviders = wire.NewSet(
-	engine.Set,
 	core.Set,
-	scanner.New,
 	subsonic.New,
 	app.New,
 	persistence.New,
@@ -29,16 +27,33 @@ func CreateServer(musicFolder string) *server.Server {
 	))
 }
 
-func CreateScanner(musicFolder string) *scanner.Scanner {
-	panic(wire.Build(
-		allProviders,
-	))
-}
-
 func CreateAppRouter() *app.Router {
 	panic(wire.Build(allProviders))
 }
 
-func CreateSubsonicAPIRouter() (*subsonic.Router, error) {
-	panic(wire.Build(allProviders))
+func CreateSubsonicAPIRouter() *subsonic.Router {
+	panic(wire.Build(
+		allProviders,
+		GetScanner,
+	))
+}
+
+// Scanner must be a Singleton
+var (
+	onceScanner     sync.Once
+	scannerInstance scanner.Scanner
+)
+
+func GetScanner() scanner.Scanner {
+	onceScanner.Do(func() {
+		scannerInstance = createScanner()
+	})
+	return scannerInstance
+}
+
+func createScanner() scanner.Scanner {
+	panic(wire.Build(
+		allProviders,
+		scanner.New,
+	))
 }

conf/configuration.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/deluan/navidrome/consts"
 	"github.com/deluan/navidrome/log"
+	"github.com/kr/pretty"
 	"github.com/spf13/viper"
 )
 
@@ -41,16 +42,32 @@ type configOptions struct {
 	AuthWindowLength time.Duration
 
 	Scanner scannerOptions
+	LastFM  lastfmOptions
+	Spotify spotifyOptions
 
 	// DevFlags. These are used to enable/disable debugging and incomplete features
 	DevLogSourceLine           bool
 	DevAutoCreateAdminPassword string
+	DevPreCacheAlbumArtwork    bool
+	DevDisableTrackCoverArt    bool
+	DevNewCacheLayout          bool
 }
 
 type scannerOptions struct {
 	Extractor string
 }
 
+type lastfmOptions struct {
+	ApiKey   string
+	Secret   string
+	Language string
+}
+
+type spotifyOptions struct {
+	ID     string
+	Secret string
+}
+
 var Server = &configOptions{}
 
 func LoadFromFile(confFile string) {
@@ -76,7 +93,9 @@ func Load() {
 
 	log.SetLevelString(Server.LogLevel)
 	log.SetLogSourceLine(Server.DevLogSourceLine)
-	log.Debug("Loaded configuration", "file", Server.ConfigFile, "config", fmt.Sprintf("%#v", Server))
+	if log.CurrentLevel() >= log.LevelDebug {
+		pretty.Printf("Loaded configuration from '%s': %# v\n", Server.ConfigFile, Server)
+	}
 }
 
 func init() {
@@ -107,11 +126,18 @@ func init() {
 	viper.SetDefault("authwindowlength", 20*time.Second)
 
 	viper.SetDefault("scanner.extractor", "taglib")
+	viper.SetDefault("lastfm.language", "en")
+	viper.SetDefault("lastfm.apikey", "")
+	viper.SetDefault("lastfm.secret", "")
+	viper.SetDefault("spotify.id", "")
+	viper.SetDefault("spotify.secret", "")
 
 	// DevFlags. These are used to enable/disable debugging and incomplete features
 	viper.SetDefault("devlogsourceline", false)
 	viper.SetDefault("devautocreateadminpassword", "")
-	viper.SetDefault("devoldscanner", false)
+	viper.SetDefault("devprecachealbumartwork", false)
+	viper.SetDefault("devnewcachelayout", false)
+	viper.SetDefault("devdisabletrackcoverart", false)
 }
 
 func InitConfig(cfgFile string) {

consts/consts.go

@@ -27,10 +27,13 @@ const (
 	RequestThrottleBacklogLimit   = 100
 	RequestThrottleBacklogTimeout = time.Minute
 
+	ArtistInfoTimeToLive = 1 * time.Hour
+
 	I18nFolder   = "i18n"
 	SkipScanFile = ".ndignore"
 
 	PlaceholderAlbumArt = "navidrome-600x600.png"
+	PlaceholderAvatar   = "logo-192x192.png"
 )
 
 // Cache options

consts/mime_types.go

@@ -22,6 +22,8 @@ func init() {
 		".m3u":  "audio/x-mpegurl",
 		".pls":  "audio/x-scpls",
 		".dsf":  "audio/dsd",
+		".wv":   "audio/x-wavpack",
+		".wvp":  "audio/x-wavpack",
 		".gif":  "image/gif",
 		".jpg":  "image/jpeg",
 		".jpeg": "image/jpeg",

core/artwork.go

@@ -12,8 +12,10 @@ import (
 	"io"
 	"os"
 	"strings"
+	"sync"
 	"time"
 
+	"github.com/deluan/navidrome/core/cache"
 	_ "golang.org/x/image/webp"
 
 	"github.com/deluan/navidrome/conf"
@@ -30,7 +32,7 @@ type Artwork interface {
 	Get(ctx context.Context, id string, size int, out io.Writer) error
 }
 
-type ArtworkCache FileCache
+type ArtworkCache cache.FileCache
 
 func NewArtwork(ds model.DataStore, cache ArtworkCache) Artwork {
 	return &artwork{ds: ds, cache: cache}
@@ -38,34 +40,40 @@ func NewArtwork(ds model.DataStore, cache ArtworkCache) Artwork {
 
 type artwork struct {
 	ds    model.DataStore
-	cache FileCache
+	cache cache.FileCache
 }
 
 type imageInfo struct {
-	c          *artwork
+	a          *artwork
+	id         string
 	path       string
 	size       int
 	lastUpdate time.Time
 }
 
-func (ci *imageInfo) String() string {
+func (ci *imageInfo) Key() string {
 	return fmt.Sprintf("%s.%d.%s.%d", ci.path, ci.size, ci.lastUpdate.Format(time.RFC3339Nano), conf.Server.CoverJpegQuality)
 }
 
-func (c *artwork) Get(ctx context.Context, id string, size int, out io.Writer) error {
-	path, lastUpdate, err := c.getImagePath(ctx, id)
+func (a *artwork) Get(ctx context.Context, id string, size int, out io.Writer) error {
+	path, lastUpdate, err := a.getImagePath(ctx, id)
 	if err != nil && err != model.ErrNotFound {
 		return err
 	}
 
+	if stat, err := os.Stat(path); err == nil {
+		lastUpdate = stat.ModTime()
+	}
+
 	info := &imageInfo{
-		c:          c,
+		a:          a,
+		id:         id,
 		path:       path,
 		size:       size,
 		lastUpdate: lastUpdate,
 	}
 
-	r, err := c.cache.Get(ctx, info)
+	r, err := a.cache.Get(ctx, info)
 	if err != nil {
 		log.Error(ctx, "Error accessing image cache", "path", path, "size", size, err)
 		return err
@@ -76,13 +84,13 @@ func (c *artwork) Get(ctx context.Context, id string, size int, out io.Writer) e
 	return err
 }
 
-func (c *artwork) getImagePath(ctx context.Context, id string) (path string, lastUpdated time.Time, err error) {
+func (a *artwork) getImagePath(ctx context.Context, id string) (path string, lastUpdated time.Time, err error) {
 	// If id is an album cover ID
 	if strings.HasPrefix(id, "al-") {
 		log.Trace(ctx, "Looking for album art", "id", id)
 		id = strings.TrimPrefix(id, "al-")
 		var al *model.Album
-		al, err = c.ds.Album(ctx).Get(id)
+		al, err = a.ds.Album(ctx).Get(id)
 		if err != nil {
 			return
 		}
@@ -94,29 +102,29 @@ func (c *artwork) getImagePath(ctx context.Context, id string) (path string, las
 
 	log.Trace(ctx, "Looking for media file art", "id", id)
 
-	// Check if id is a mediaFile cover id
+	// Check if id is a mediaFile id
 	var mf *model.MediaFile
-	mf, err = c.ds.MediaFile(ctx).Get(id)
+	mf, err = a.ds.MediaFile(ctx).Get(id)
 
 	// If it is not, may be an albumId
 	if err == model.ErrNotFound {
-		return c.getImagePath(ctx, "al-"+id)
+		return a.getImagePath(ctx, "al-"+id)
 	}
 	if err != nil {
 		return
 	}
 
-	// If it is a mediaFile and it has cover art, return it
-	if mf.HasCoverArt {
+	// If it is a mediaFile and it has cover art, return it (if feature is disabled, skip)
+	if !conf.Server.DevDisableTrackCoverArt && mf.HasCoverArt {
 		return mf.Path, mf.UpdatedAt, nil
 	}
 
 	// if the mediaFile does not have a coverArt, fallback to the album cover
 	log.Trace(ctx, "Media file does not contain art. Falling back to album art", "id", id, "albumId", "al-"+mf.AlbumID)
-	return c.getImagePath(ctx, "al-"+mf.AlbumID)
+	return a.getImagePath(ctx, "al-"+mf.AlbumID)
 }
 
-func (c *artwork) getArtwork(ctx context.Context, path string, size int) (reader io.Reader, err error) {
+func (a *artwork) getArtwork(ctx context.Context, id string, path string, size int) (reader io.Reader, err error) {
 	defer func() {
 		if err != nil {
 			log.Warn(ctx, "Error extracting image", "path", path, "size", size, err)
@@ -129,16 +137,23 @@ func (c *artwork) getArtwork(ctx context.Context, path string, size int) (reader
 	}
 
 	var data []byte
-	if utils.IsAudioFile(path) {
-		data, err = readFromTag(path)
-	} else {
-		data, err = readFromFile(path)
-	}
 
-	if err != nil {
-		return
-	} else if size > 0 {
-		data, err = resizeImage(bytes.NewReader(data), size)
+	if size == 0 {
+		// If requested original size, just read from the file
+		if utils.IsAudioFile(path) {
+			data, err = readFromTag(path)
+		} else {
+			data, err = readFromFile(path)
+		}
+	} else {
+		// If requested a resized image, get the original (possibly from cache) and resize it
+		a2 := NewArtwork(a.ds, a.cache)
+		buf := new(bytes.Buffer)
+		err = a2.Get(ctx, id, 0, buf)
+		if err != nil {
+			return
+		}
+		data, err = resizeImage(buf, size)
 	}
 
 	// Confirm the image is valid. Costly, but necessary
@@ -195,15 +210,23 @@ func readFromFile(path string) ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
-func NewImageCache() ArtworkCache {
-	return NewFileCache("Image", conf.Server.ImageCacheSize, consts.ImageCacheDir, consts.DefaultImageCacheMaxItems,
-		func(ctx context.Context, arg fmt.Stringer) (io.Reader, error) {
-			info := arg.(*imageInfo)
-			reader, err := info.c.getArtwork(ctx, info.path, info.size)
-			if err != nil {
-				log.Error(ctx, "Error loading artwork art", "path", info.path, "size", info.size, err)
-				return nil, err
-			}
-			return reader, nil
-		})
+var (
+	onceImageCache     sync.Once
+	instanceImageCache ArtworkCache
+)
+
+func GetImageCache() ArtworkCache {
+	onceImageCache.Do(func() {
+		instanceImageCache = cache.NewFileCache("Image", conf.Server.ImageCacheSize, consts.ImageCacheDir, consts.DefaultImageCacheMaxItems,
+			func(ctx context.Context, arg cache.Item) (io.Reader, error) {
+				info := arg.(*imageInfo)
+				reader, err := info.a.getArtwork(ctx, info.id, info.path, info.size)
+				if err != nil {
+					log.Error(ctx, "Error loading artwork art", "path", info.path, "size", info.size, err)
+					return nil, err
+				}
+				return reader, nil
+			})
+	})
+	return instanceImageCache
 }

core/artwork_test.go

@@ -5,12 +5,11 @@ import (
 	"context"
 	"image"
 	"io/ioutil"
-	"os"
 
 	"github.com/deluan/navidrome/conf"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
-	"github.com/deluan/navidrome/persistence"
+	"github.com/deluan/navidrome/tests"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -21,24 +20,27 @@ var _ = Describe("Artwork", func() {
 	ctx := log.NewContext(context.TODO())
 
 	BeforeEach(func() {
-		ds = &persistence.MockDataStore{MockedTranscoding: &mockTranscodingRepository{}}
-		ds.Album(ctx).(*persistence.MockAlbum).SetData(`[{"id": "222", "coverArtId": "123", "coverArtPath":"tests/fixtures/test.mp3"}, {"id": "333", "coverArtId": ""}, {"id": "444", "coverArtId": "444", "coverArtPath": "tests/fixtures/cover.jpg"}]`)
-		ds.MediaFile(ctx).(*persistence.MockMediaFile).SetData(`[{"id": "123", "albumId": "222", "path": "tests/fixtures/test.mp3", "hasCoverArt": true, "updatedAt":"2020-04-02T21:29:31.6377Z"},{"id": "456", "albumId": "222", "path": "tests/fixtures/test.ogg", "hasCoverArt": false, "updatedAt":"2020-04-02T21:29:31.6377Z"}]`)
+		ds = &tests.MockDataStore{MockedTranscoding: &tests.MockTranscodingRepository{}}
+		ds.Album(ctx).(*tests.MockAlbum).SetData(model.Albums{
+			{ID: "222", CoverArtId: "123", CoverArtPath: "tests/fixtures/test.mp3"},
+			{ID: "333", CoverArtId: ""},
+			{ID: "444", CoverArtId: "444", CoverArtPath: "tests/fixtures/cover.jpg"},
+		})
+		ds.MediaFile(ctx).(*tests.MockMediaFile).SetData(model.MediaFiles{
+			{ID: "123", AlbumID: "222", Path: "tests/fixtures/test.mp3", HasCoverArt: true},
+			{ID: "456", AlbumID: "222", Path: "tests/fixtures/test.ogg", HasCoverArt: false},
+		})
 	})
 
 	Context("Cache is configured", func() {
 		BeforeEach(func() {
 			conf.Server.DataFolder, _ = ioutil.TempDir("", "file_caches")
 			conf.Server.ImageCacheSize = "100MB"
-			cache := NewImageCache()
-			Eventually(func() bool { return cache.Ready() }).Should(BeTrue())
+			cache := GetImageCache()
+			Eventually(func() bool { return cache.Ready(context.TODO()) }).Should(BeTrue())
 			artwork = NewArtwork(ds, cache)
 		})
 
-		AfterEach(func() {
-			os.RemoveAll(conf.Server.DataFolder)
-		})
-
 		It("retrieves the external artwork art for an album", func() {
 			buf := new(bytes.Buffer)
 
@@ -125,14 +127,14 @@ var _ = Describe("Artwork", func() {
 
 		Context("Errors", func() {
 			It("returns err if gets error from album table", func() {
-				ds.Album(ctx).(*persistence.MockAlbum).SetError(true)
+				ds.Album(ctx).(*tests.MockAlbum).SetError(true)
 				buf := new(bytes.Buffer)
 
 				Expect(artwork.Get(ctx, "al-222", 0, buf)).To(MatchError("Error!"))
 			})
 
 			It("returns err if gets error from media_file table", func() {
-				ds.MediaFile(ctx).(*persistence.MockMediaFile).SetError(true)
+				ds.MediaFile(ctx).(*tests.MockMediaFile).SetError(true)
 				buf := new(bytes.Buffer)
 
 				Expect(artwork.Get(ctx, "123", 0, buf)).To(MatchError("Error!"))

core/cache/cache_suite_test.go

@@ -1,4 +1,4 @@
-package engine
+package cache
 
 import (
 	"testing"
@@ -9,9 +9,9 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-func TestEngine(t *testing.T) {
+func TestCache(t *testing.T) {
 	tests.Init(t, false)
 	log.SetLevel(log.LevelCritical)
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "Engine Suite")
+	RunSpecs(t, "Cache Suite")
 }

core/cache/file_caches.go

@@ -1,4 +1,4 @@
-package core
+package cache
 
 import (
 	"context"
@@ -15,11 +15,16 @@ import (
 	"github.com/dustin/go-humanize"
 )
 
-type ReadFunc func(ctx context.Context, arg fmt.Stringer) (io.Reader, error)
+type Item interface {
+	Key() string
+}
+
+type ReadFunc func(ctx context.Context, item Item) (io.Reader, error)
 
 type FileCache interface {
-	Get(ctx context.Context, arg fmt.Stringer) (*CachedStream, error)
-	Ready() bool
+	Get(ctx context.Context, item Item) (*CachedStream, error)
+	Ready(ctx context.Context) bool
+	Available(ctx context.Context) bool
 }
 
 func NewFileCache(name, cacheSize, cacheFolder string, maxItems int, getReader ReadFunc) *fileCache {
@@ -33,6 +38,7 @@ func NewFileCache(name, cacheSize, cacheFolder string, maxItems int, getReader R
 	}
 
 	go func() {
+		start := time.Now()
 		cache, err := newFSCache(fc.name, fc.cacheSize, fc.cacheFolder, fc.maxItems)
 		fc.mutex.Lock()
 		defer fc.mutex.Unlock()
@@ -40,9 +46,10 @@ func NewFileCache(name, cacheSize, cacheFolder string, maxItems int, getReader R
 			fc.cache = cache
 			fc.disabled = cache == nil
 		}
+		log.Info("Finished initializing cache", "cache", fc.name, "maxSize", fc.cacheSize, "elapsedTime", time.Since(start))
 		fc.ready = true
 		if fc.disabled {
-			log.Debug("Cache disabled", "cache", fc.name, "size", fc.cacheSize)
+			log.Debug("Cache DISABLED", "cache", fc.name, "size")
 		}
 	}()
 
@@ -61,13 +68,13 @@ type fileCache struct {
 	mutex       *sync.RWMutex
 }
 
-func (fc *fileCache) Ready() bool {
+func (fc *fileCache) Ready(ctx context.Context) bool {
 	fc.mutex.RLock()
 	defer fc.mutex.RUnlock()
 	return fc.ready
 }
 
-func (fc *fileCache) available(ctx context.Context) bool {
+func (fc *fileCache) Available(ctx context.Context) bool {
 	fc.mutex.RLock()
 	defer fc.mutex.RUnlock()
 
@@ -78,8 +85,8 @@ func (fc *fileCache) available(ctx context.Context) bool {
 	return fc.ready && !fc.disabled
 }
 
-func (fc *fileCache) Get(ctx context.Context, arg fmt.Stringer) (*CachedStream, error) {
-	if !fc.available(ctx) {
+func (fc *fileCache) Get(ctx context.Context, arg Item) (*CachedStream, error) {
+	if !fc.Available(ctx) {
 		reader, err := fc.getReader(ctx, arg)
 		if err != nil {
 			return nil, err
@@ -87,7 +94,7 @@ func (fc *fileCache) Get(ctx context.Context, arg fmt.Stringer) (*CachedStream,
 		return &CachedStream{Reader: reader}, nil
 	}
 
-	key := arg.String()
+	key := arg.Key()
 	r, w, err := fc.cache.Get(key)
 	if err != nil {
 		return nil, err
@@ -178,18 +185,21 @@ func newFSCache(name, cacheSize, cacheFolder string, maxItems int) (fscache.Cach
 		return nil, nil
 	}
 
-	start := time.Now()
 	lru := fscache.NewLRUHaunter(maxItems, int64(size), consts.DefaultCacheCleanUpInterval)
 	h := fscache.NewLRUHaunterStrategy(lru)
 	cacheFolder = filepath.Join(conf.Server.DataFolder, cacheFolder)
 
+	var fs fscache.FileSystem
 	log.Info(fmt.Sprintf("Creating %s cache", name), "path", cacheFolder, "maxSize", humanize.Bytes(size))
-	fs, err := fscache.NewFs(cacheFolder, 0755)
+	if conf.Server.DevNewCacheLayout {
+		fs, err = NewSpreadFS(cacheFolder, 0755)
+	} else {
+		fs, err = fscache.NewFs(cacheFolder, 0755)
+	}
 	if err != nil {
-		log.Error(fmt.Sprintf("Error initializing %s cache", name), err, "elapsedTime", time.Since(start))
+		log.Error(fmt.Sprintf("Error initializing %s cache", name), err)
 		return nil, err
 	}
-	log.Debug(fmt.Sprintf("%s cache initialized", name), "elapsedTime", time.Since(start))
 
 	return fscache.NewCacheWithHaunter(fs, h)
 }

core/cache/file_caches_test.go

@@ -1,8 +1,7 @@
-package core
+package cache
 
 import (
 	"context"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -17,7 +16,7 @@ import (
 // Call NewFileCache and wait for it to be ready
 func callNewFileCache(name, cacheSize, cacheFolder string, maxItems int, getReader ReadFunc) *fileCache {
 	fc := NewFileCache(name, cacheSize, cacheFolder, maxItems, getReader)
-	Eventually(func() bool { return fc.Ready() }).Should(BeTrue())
+	Eventually(func() bool { return fc.Ready(context.TODO()) }).Should(BeTrue())
 	return fc
 }
 
@@ -53,9 +52,9 @@ var _ = Describe("File Caches", func() {
 	Describe("FileCache", func() {
 		It("caches data if cache is enabled", func() {
 			called := false
-			fc := callNewFileCache("test", "1KB", "test", 0, func(ctx context.Context, arg fmt.Stringer) (io.Reader, error) {
+			fc := callNewFileCache("test", "1KB", "test", 0, func(ctx context.Context, arg Item) (io.Reader, error) {
 				called = true
-				return strings.NewReader(arg.String()), nil
+				return strings.NewReader(arg.Key()), nil
 			})
 			// First call is a MISS
 			s, err := fc.Get(context.TODO(), &testArg{"test"})
@@ -74,9 +73,9 @@ var _ = Describe("File Caches", func() {
 
 		It("does not cache data if cache is disabled", func() {
 			called := false
-			fc := callNewFileCache("test", "0", "test", 0, func(ctx context.Context, arg fmt.Stringer) (io.Reader, error) {
+			fc := callNewFileCache("test", "0", "test", 0, func(ctx context.Context, arg Item) (io.Reader, error) {
 				called = true
-				return strings.NewReader(arg.String()), nil
+				return strings.NewReader(arg.Key()), nil
 			})
 			// First call is a MISS
 			s, err := fc.Get(context.TODO(), &testArg{"test"})
@@ -97,4 +96,4 @@ var _ = Describe("File Caches", func() {
 
 type testArg struct{ s string }
 
-func (t *testArg) String() string { return t.s }
+func (t *testArg) Key() string { return t.s }

core/cache/spread_fs.go

@@ -0,0 +1,111 @@
+package cache
+
+import (
+	"crypto/sha1"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/djherbis/fscache"
+	"github.com/karrick/godirwalk"
+	"gopkg.in/djherbis/atime.v1"
+	"gopkg.in/djherbis/stream.v1"
+)
+
+type spreadFS struct {
+	root string
+	mode os.FileMode
+	init func() error
+}
+
+const keyFileExtension = ".key"
+
+// NewSpreadFS returns a FileSystem rooted at directory dir. It
+// Dir is created with perms if it doesn't exist.
+func NewSpreadFS(dir string, mode os.FileMode) (fscache.FileSystem, error) {
+	fs := &spreadFS{root: dir, mode: mode, init: func() error {
+		return os.MkdirAll(dir, mode)
+	}}
+	return fs, fs.init()
+}
+
+func (fs *spreadFS) Reload(f func(key string, name string)) error {
+	return godirwalk.Walk(fs.root, &godirwalk.Options{
+		Callback: func(absoluteFilePath string, de *godirwalk.Dirent) error {
+			path, err := filepath.Rel(fs.root, absoluteFilePath)
+			if err != nil {
+				return nil
+			}
+
+			// Skip if name is not in the format XX/XX/XXXXXXXXXXXX.key
+			parts := strings.Split(path, string(os.PathSeparator))
+			if len(parts) != 3 || len(parts[0]) != 2 || len(parts[1]) != 2 ||
+				filepath.Ext(path) != keyFileExtension {
+				return nil
+			}
+
+			keyFileName := absoluteFilePath
+			dataFileName := absoluteFilePath[0 : len(absoluteFilePath)-len(keyFileExtension)]
+
+			// Load the key from the key file. Remove and skip on error
+			key, err := ioutil.ReadFile(keyFileName)
+			if err != nil {
+				_ = fs.Remove(dataFileName)
+				return nil
+			}
+
+			// If the data file is not readable, remove and skip
+			file, err := os.Open(dataFileName)
+			defer func() { _ = file.Close() }()
+			if err != nil {
+				_ = fs.Remove(dataFileName)
+				return nil
+			}
+
+			f(string(key), dataFileName)
+			return nil
+		},
+		Unsorted: true,
+	})
+}
+
+func (fs *spreadFS) Create(name string) (stream.File, error) {
+	key := fmt.Sprintf("%x", sha1.Sum([]byte(name)))
+	path := fmt.Sprintf("%s%c%s", key[0:2], os.PathSeparator, key[2:4])
+	err := os.MkdirAll(filepath.Join(fs.root, path), fs.mode)
+	if err != nil {
+		return nil, err
+	}
+	absolutePath := filepath.Join(fs.root, path, key)
+	err = ioutil.WriteFile(absolutePath+keyFileExtension, []byte(name), 0600)
+	if err != nil {
+		return nil, err
+	}
+	return os.OpenFile(absolutePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+}
+
+func (fs *spreadFS) Open(name string) (stream.File, error) {
+	return os.Open(name)
+}
+
+func (fs *spreadFS) Remove(name string) error {
+	_ = os.Remove(name + keyFileExtension)
+	return os.Remove(name)
+}
+
+func (fs *spreadFS) Stat(name string) (fscache.FileInfo, error) {
+	stat, err := os.Stat(name)
+	if err != nil {
+		return fscache.FileInfo{}, err
+	}
+	return fscache.FileInfo{FileInfo: stat, Atime: atime.Get(stat)}, nil
+}
+
+func (fs *spreadFS) RemoveAll() error {
+	if err := os.RemoveAll(fs.root); err != nil {
+		return err
+	}
+	return fs.init()
+}

core/cache_warmer.go

@@ -0,0 +1,89 @@
+package core
+
+import (
+	"context"
+	"io/ioutil"
+	"time"
+
+	"github.com/deluan/navidrome/conf"
+	"github.com/deluan/navidrome/core/pool"
+	"github.com/deluan/navidrome/log"
+)
+
+type CacheWarmer interface {
+	AddAlbum(ctx context.Context, albumID string)
+	Flush(ctx context.Context)
+}
+
+func NewCacheWarmer(artwork Artwork, artworkCache ArtworkCache) CacheWarmer {
+	w := &warmer{
+		artwork:      artwork,
+		artworkCache: artworkCache,
+		albums:       map[string]struct{}{},
+	}
+	p, err := pool.NewPool("artwork", 3, &artworkItem{}, w.execute)
+	if err != nil {
+		log.Error(context.Background(), "Error creating pool for Album Artwork Cache Warmer", err)
+	} else {
+		w.pool = p
+	}
+
+	return w
+}
+
+type warmer struct {
+	pool         *pool.Pool
+	artwork      Artwork
+	artworkCache ArtworkCache
+	albums       map[string]struct{}
+}
+
+func (w *warmer) AddAlbum(ctx context.Context, albumID string) {
+	if albumID == "" {
+		return
+	}
+	w.albums[albumID] = struct{}{}
+}
+
+func (w *warmer) waitForCacheReady(ctx context.Context) {
+	tick := time.NewTicker(time.Second)
+	defer tick.Stop()
+	for {
+		<-tick.C
+		if w.artworkCache.Ready(ctx) {
+			return
+		}
+	}
+}
+
+func (w *warmer) Flush(ctx context.Context) {
+	w.waitForCacheReady(ctx)
+	if w.artworkCache.Available(ctx) {
+		if conf.Server.DevPreCacheAlbumArtwork {
+			if w.pool == nil || len(w.albums) == 0 {
+				return
+			}
+			log.Info(ctx, "Pre-caching album artworks", "numAlbums", len(w.albums))
+			for id := range w.albums {
+				w.pool.Submit(artworkItem{albumID: id})
+			}
+		}
+	} else {
+		log.Warn(ctx, "Pre-cache warmer is not available as ImageCache is DISABLED")
+	}
+	w.albums = map[string]struct{}{}
+}
+
+func (w *warmer) execute(workload interface{}) {
+	ctx := context.Background()
+	item := workload.(artworkItem)
+	log.Trace(ctx, "Pre-caching album artwork", "albumID", item.albumID)
+	err := w.artwork.Get(ctx, item.albumID, 0, ioutil.Discard)
+	if err != nil {
+		log.Warn("Error pre-caching artwork from album", "id", item.albumID, err)
+	}
+}
+
+type artworkItem struct {
+	albumID string
+}

core/core_suite_test.go

@@ -9,7 +9,7 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-func TestEngine(t *testing.T) {
+func TestCore(t *testing.T) {
 	tests.Init(t, false)
 	log.SetLevel(log.LevelCritical)
 	RegisterFailHandler(Fail)

core/external_info.go

@@ -0,0 +1,418 @@
+package core
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/Masterminds/squirrel"
+	"github.com/deluan/navidrome/consts"
+	"github.com/deluan/navidrome/core/lastfm"
+	"github.com/deluan/navidrome/core/spotify"
+	"github.com/deluan/navidrome/log"
+	"github.com/deluan/navidrome/model"
+	"github.com/microcosm-cc/bluemonday"
+	"github.com/xrash/smetrics"
+)
+
+const placeholderArtistImageSmallUrl = "https://lastfm.freetls.fastly.net/i/u/64s/2a96cbd8b46e442fc41c2b86b821562f.png"
+const placeholderArtistImageMediumUrl = "https://lastfm.freetls.fastly.net/i/u/174s/2a96cbd8b46e442fc41c2b86b821562f.png"
+const placeholderArtistImageLargeUrl = "https://lastfm.freetls.fastly.net/i/u/300x300/2a96cbd8b46e442fc41c2b86b821562f.png"
+
+type ExternalInfo interface {
+	UpdateArtistInfo(ctx context.Context, id string, count int, includeNotPresent bool) (*model.Artist, error)
+	SimilarSongs(ctx context.Context, id string, count int) (model.MediaFiles, error)
+	TopSongs(ctx context.Context, artist string, count int) (model.MediaFiles, error)
+}
+
+func NewExternalInfo(ds model.DataStore, lfm *lastfm.Client, spf *spotify.Client) ExternalInfo {
+	return &externalInfo{ds: ds, lfm: lfm, spf: spf}
+}
+
+type externalInfo struct {
+	ds  model.DataStore
+	lfm *lastfm.Client
+	spf *spotify.Client
+}
+
+const UnavailableArtistID = "-1"
+
+func (e *externalInfo) UpdateArtistInfo(ctx context.Context, id string, count int, includeNotPresent bool) (*model.Artist, error) {
+	artist, err := e.getArtist(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+
+	// If we have updated info, just return it
+	if time.Since(artist.ExternalInfoUpdatedAt) < consts.ArtistInfoTimeToLive {
+		log.Debug("Found cached ArtistInfo", "updatedAt", artist.ExternalInfoUpdatedAt, "name", artist.Name)
+		err := e.loadSimilar(ctx, artist, includeNotPresent)
+		return artist, err
+	}
+	log.Debug("ArtistInfo not cached", "updatedAt", artist.ExternalInfoUpdatedAt, "id", id)
+
+	// TODO Load from local: artist.jpg/png/webp, artist.json (with the remaining info)
+
+	var wg sync.WaitGroup
+	e.callArtistInfo(ctx, artist, &wg)
+	e.callArtistImages(ctx, artist, &wg)
+	e.callSimilarArtists(ctx, artist, count, &wg)
+	wg.Wait()
+
+	// Use placeholders if could not get from external sources
+	e.setBio(artist, "Biography not available")
+	e.setSmallImageUrl(artist, placeholderArtistImageSmallUrl)
+	e.setMediumImageUrl(artist, placeholderArtistImageMediumUrl)
+	e.setLargeImageUrl(artist, placeholderArtistImageLargeUrl)
+
+	artist.ExternalInfoUpdatedAt = time.Now()
+	err = e.ds.Artist(ctx).Put(artist)
+	if err != nil {
+		log.Error(ctx, "Error trying to update artistImageUrl", "id", id, err)
+	}
+
+	if !includeNotPresent {
+		similar := artist.SimilarArtists
+		artist.SimilarArtists = nil
+		for _, s := range similar {
+			if s.ID == UnavailableArtistID {
+				continue
+			}
+			artist.SimilarArtists = append(artist.SimilarArtists, s)
+		}
+	}
+
+	log.Trace(ctx, "ArtistInfo collected", "artist", artist)
+
+	return artist, nil
+}
+
+func (e *externalInfo) getArtist(ctx context.Context, id string) (*model.Artist, error) {
+	var entity interface{}
+	entity, err := GetEntityByID(ctx, e.ds, id)
+	if err != nil {
+		return nil, err
+	}
+
+	switch v := entity.(type) {
+	case *model.Artist:
+		return v, nil
+	case *model.MediaFile:
+		return e.ds.Artist(ctx).Get(v.ArtistID)
+	case *model.Album:
+		return e.ds.Artist(ctx).Get(v.AlbumArtistID)
+	}
+	return nil, model.ErrNotFound
+}
+
+// Replace some Unicode chars with their equivalent ASCII
+func clearName(name string) string {
+	name = strings.ReplaceAll(name, "–", "-")
+	name = strings.ReplaceAll(name, "‐", "-")
+	name = strings.ReplaceAll(name, "“", `"`)
+	name = strings.ReplaceAll(name, "”", `"`)
+	name = strings.ReplaceAll(name, "‘", `'`)
+	name = strings.ReplaceAll(name, "’", `'`)
+	return name
+}
+
+func (e *externalInfo) SimilarSongs(ctx context.Context, id string, count int) (model.MediaFiles, error) {
+	if e.lfm == nil {
+		log.Warn(ctx, "Last.FM client not configured")
+		return nil, model.ErrNotAvailable
+	}
+
+	artist, err := e.getArtist(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+
+	artists, err := e.similarArtists(ctx, clearName(artist.Name), count, false)
+	if err != nil {
+		return nil, err
+	}
+	ids := make([]string, len(artists)+1)
+	ids[0] = artist.ID
+	for i, a := range artists {
+		ids[i+1] = a.ID
+	}
+
+	return e.ds.MediaFile(ctx).GetAll(model.QueryOptions{
+		Filters: squirrel.Eq{"artist_id": ids},
+		Max:     count,
+		Sort:    "random()",
+	})
+}
+
+func (e *externalInfo) similarArtists(ctx context.Context, artistName string, count int, includeNotPresent bool) (model.Artists, error) {
+	var result model.Artists
+	var notPresent []string
+
+	log.Debug(ctx, "Calling Last.FM ArtistGetSimilar", "artist", artistName)
+	similar, err := e.lfm.ArtistGetSimilar(ctx, artistName, count)
+	if err != nil {
+		return nil, err
+	}
+
+	// First select artists that are present.
+	for _, s := range similar {
+		sa, err := e.findArtistByName(ctx, s.Name)
+		if err != nil {
+			notPresent = append(notPresent, s.Name)
+			continue
+		}
+		result = append(result, *sa)
+	}
+
+	// Then fill up with non-present artists
+	if includeNotPresent {
+		for _, s := range notPresent {
+			sa := model.Artist{ID: UnavailableArtistID, Name: s}
+			result = append(result, sa)
+		}
+	}
+
+	return result, nil
+}
+
+func (e *externalInfo) findArtistByName(ctx context.Context, artistName string) (*model.Artist, error) {
+	artists, err := e.ds.Artist(ctx).GetAll(model.QueryOptions{
+		Filters: squirrel.Like{"name": artistName},
+		Max:     1,
+	})
+	if err != nil {
+		return nil, err
+	}
+	if len(artists) == 0 {
+		return nil, model.ErrNotFound
+	}
+	return &artists[0], nil
+}
+
+func (e *externalInfo) TopSongs(ctx context.Context, artistName string, count int) (model.MediaFiles, error) {
+	if e.lfm == nil {
+		log.Warn(ctx, "Last.FM client not configured")
+		return nil, model.ErrNotAvailable
+	}
+	artist, err := e.findArtistByName(ctx, artistName)
+	if err != nil {
+		log.Error(ctx, "Artist not found", "name", artistName, err)
+		return nil, nil
+	}
+	artistName = clearName(artistName)
+
+	log.Debug(ctx, "Calling Last.FM ArtistGetTopTracks", "artist", artistName, "id", artist.ID)
+	tracks, err := e.lfm.ArtistGetTopTracks(ctx, artistName, count)
+	if err != nil {
+		return nil, err
+	}
+	var songs model.MediaFiles
+	for _, t := range tracks {
+		mf, err := e.findMatchingTrack(ctx, t.MBID, artist.ID, t.Name)
+		if err != nil {
+			continue
+		}
+		songs = append(songs, *mf)
+	}
+	return songs, nil
+}
+
+func (e *externalInfo) findMatchingTrack(ctx context.Context, mbid string, artistID, title string) (*model.MediaFile, error) {
+	if mbid != "" {
+		mfs, err := e.ds.MediaFile(ctx).GetAll(model.QueryOptions{
+			Filters: squirrel.Eq{"mbz_track_id": mbid},
+		})
+		if err == nil && len(mfs) > 0 {
+			return &mfs[0], nil
+		}
+	}
+	mfs, err := e.ds.MediaFile(ctx).GetAll(model.QueryOptions{
+		Filters: squirrel.And{
+			squirrel.Or{
+				squirrel.Eq{"artist_id": artistID},
+				squirrel.Eq{"album_artist_id": artistID},
+			},
+			squirrel.Like{"title": title},
+		},
+		Sort: "starred desc, rating desc, year asc",
+	})
+	if err != nil || len(mfs) == 0 {
+		return nil, model.ErrNotFound
+	}
+	return &mfs[0], nil
+}
+
+func (e *externalInfo) callArtistInfo(ctx context.Context, artist *model.Artist, wg *sync.WaitGroup) {
+	if e.lfm != nil {
+		name := clearName(artist.Name)
+		log.Debug(ctx, "Calling Last.FM ArtistGetInfo", "artist", name)
+		wg.Add(1)
+		go func() {
+			start := time.Now()
+			defer wg.Done()
+			lfmArtist, err := e.lfm.ArtistGetInfo(ctx, name)
+			if err != nil {
+				log.Error(ctx, "Error calling Last.FM", "artist", name, err)
+			} else {
+				log.Debug(ctx, "Got info from Last.FM", "artist", name, "info", lfmArtist.Bio.Summary, "elapsed", time.Since(start))
+			}
+			e.setBio(artist, lfmArtist.Bio.Summary)
+			e.setExternalUrl(artist, lfmArtist.URL)
+			e.setMbzID(artist, lfmArtist.MBID)
+		}()
+	}
+}
+
+func (e *externalInfo) searchArtist(ctx context.Context, name string) (*spotify.Artist, error) {
+	artists, err := e.spf.SearchArtists(ctx, name, 40)
+	if err != nil || len(artists) == 0 {
+		return nil, model.ErrNotFound
+	}
+	name = strings.ToLower(name)
+
+	// Sort results, prioritizing artists with images, with similar names and with high popularity, in this order
+	sort.Slice(artists, func(i, j int) bool {
+		ai := fmt.Sprintf("%-5t-%03d-%04d", len(artists[i].Images) == 0, smetrics.WagnerFischer(name, strings.ToLower(artists[i].Name), 1, 1, 2), 1000-artists[i].Popularity)
+		aj := fmt.Sprintf("%-5t-%03d-%04d", len(artists[j].Images) == 0, smetrics.WagnerFischer(name, strings.ToLower(artists[j].Name), 1, 1, 2), 1000-artists[j].Popularity)
+		return strings.Compare(ai, aj) < 0
+	})
+
+	// If the first one has the same name, that's the one
+	if strings.ToLower(artists[0].Name) != name {
+		return nil, model.ErrNotFound
+	}
+	return &artists[0], err
+}
+
+func (e *externalInfo) callSimilarArtists(ctx context.Context, artist *model.Artist, count int, wg *sync.WaitGroup) {
+	if e.lfm != nil {
+		name := clearName(artist.Name)
+		wg.Add(1)
+		go func() {
+			start := time.Now()
+			defer wg.Done()
+			similar, err := e.similarArtists(ctx, name, count, true)
+			if err != nil {
+				log.Error(ctx, "Error calling Last.FM", "artist", name, err)
+				return
+			}
+			log.Debug(ctx, "Got similar artists from Last.FM", "artist", name, "info", "elapsed", time.Since(start))
+			artist.SimilarArtists = similar
+		}()
+	}
+}
+
+func (e *externalInfo) callArtistImages(ctx context.Context, artist *model.Artist, wg *sync.WaitGroup) {
+	if e.spf != nil {
+		name := clearName(artist.Name)
+		log.Debug(ctx, "Calling Spotify SearchArtist", "artist", name)
+		wg.Add(1)
+		go func() {
+			start := time.Now()
+			defer wg.Done()
+
+			a, err := e.searchArtist(ctx, name)
+			if err != nil {
+				if err == model.ErrNotFound {
+					log.Warn(ctx, "Artist not found in Spotify", "artist", name)
+				} else {
+					log.Error(ctx, "Error calling Spotify", "artist", name, err)
+				}
+				return
+			}
+			spfImages := a.Images
+			log.Debug(ctx, "Got images from Spotify", "artist", name, "images", spfImages, "elapsed", time.Since(start))
+
+			sort.Slice(spfImages, func(i, j int) bool { return spfImages[i].Width > spfImages[j].Width })
+			if len(spfImages) >= 1 {
+				e.setLargeImageUrl(artist, spfImages[0].URL)
+			}
+			if len(spfImages) >= 2 {
+				e.setMediumImageUrl(artist, spfImages[1].URL)
+			}
+			if len(spfImages) >= 3 {
+				e.setSmallImageUrl(artist, spfImages[2].URL)
+			}
+		}()
+	}
+}
+
+func (e *externalInfo) setBio(artist *model.Artist, bio string) {
+	policy := bluemonday.UGCPolicy()
+	if artist.Biography == "" {
+		bio = policy.Sanitize(bio)
+		bio = strings.ReplaceAll(bio, "\n", " ")
+		artist.Biography = strings.ReplaceAll(bio, "<a ", "<a target='_blank' ")
+	}
+}
+
+func (e *externalInfo) setExternalUrl(artist *model.Artist, url string) {
+	if artist.ExternalUrl == "" {
+		artist.ExternalUrl = url
+	}
+}
+
+func (e *externalInfo) setMbzID(artist *model.Artist, mbID string) {
+	if artist.MbzArtistID == "" {
+		artist.MbzArtistID = mbID
+	}
+}
+
+func (e *externalInfo) setSmallImageUrl(artist *model.Artist, url string) {
+	if artist.SmallImageUrl == "" {
+		artist.SmallImageUrl = url
+	}
+}
+
+func (e *externalInfo) setMediumImageUrl(artist *model.Artist, url string) {
+	if artist.MediumImageUrl == "" {
+		artist.MediumImageUrl = url
+	}
+}
+
+func (e *externalInfo) setLargeImageUrl(artist *model.Artist, url string) {
+	if artist.LargeImageUrl == "" {
+		artist.LargeImageUrl = url
+	}
+}
+
+func (e *externalInfo) loadSimilar(ctx context.Context, artist *model.Artist, includeNotPresent bool) error {
+	var ids []string
+	for _, sa := range artist.SimilarArtists {
+		if sa.ID == UnavailableArtistID {
+			continue
+		}
+		ids = append(ids, sa.ID)
+	}
+
+	similar, err := e.ds.Artist(ctx).GetAll(model.QueryOptions{
+		Filters: squirrel.Eq{"id": ids},
+	})
+	if err != nil {
+		return err
+	}
+
+	// Use a map and iterate through original array, to keep the same order
+	artistMap := make(map[string]model.Artist)
+	for _, sa := range similar {
+		artistMap[sa.ID] = sa
+	}
+
+	var loaded model.Artists
+	for _, sa := range artist.SimilarArtists {
+		la, ok := artistMap[sa.ID]
+		if !ok {
+			if !includeNotPresent {
+				continue
+			}
+			la = sa
+			la.ID = UnavailableArtistID
+		}
+		loaded = append(loaded, la)
+	}
+	artist.SimilarArtists = loaded
+	return nil
+}

core/get_entity.go

@@ -0,0 +1,28 @@
+package core
+
+import (
+	"context"
+
+	"github.com/deluan/navidrome/model"
+)
+
+// TODO: Should the type be encoded in the ID?
+func GetEntityByID(ctx context.Context, ds model.DataStore, id string) (interface{}, error) {
+	ar, err := ds.Artist(ctx).Get(id)
+	if err == nil {
+		return ar, nil
+	}
+	al, err := ds.Album(ctx).Get(id)
+	if err == nil {
+		return al, nil
+	}
+	pls, err := ds.Playlist(ctx).Get(id)
+	if err == nil {
+		return pls, nil
+	}
+	mf, err := ds.MediaFile(ctx).Get(id)
+	if err == nil {
+		return mf, nil
+	}
+	return nil, err
+}

core/lastfm/client.go

@@ -0,0 +1,102 @@
+package lastfm
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+)
+
+const (
+	apiBaseUrl = "https://ws.audioscrobbler.com/2.0/"
+)
+
+type HttpClient interface {
+	Do(req *http.Request) (*http.Response, error)
+}
+
+func NewClient(apiKey string, lang string, hc HttpClient) *Client {
+	return &Client{apiKey, lang, hc}
+}
+
+type Client struct {
+	apiKey string
+	lang   string
+	hc     HttpClient
+}
+
+func (c *Client) makeRequest(params url.Values) (*Response, error) {
+	params.Add("format", "json")
+	params.Add("api_key", c.apiKey)
+
+	req, _ := http.NewRequest("GET", apiBaseUrl, nil)
+	req.URL.RawQuery = params.Encode()
+
+	resp, err := c.hc.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != 200 {
+		return nil, c.parseError(data)
+	}
+
+	var response Response
+	err = json.Unmarshal(data, &response)
+
+	return &response, err
+}
+
+func (c *Client) ArtistGetInfo(ctx context.Context, name string) (*Artist, error) {
+	params := url.Values{}
+	params.Add("method", "artist.getInfo")
+	params.Add("artist", name)
+	params.Add("lang", c.lang)
+	response, err := c.makeRequest(params)
+	if err != nil {
+		return nil, err
+	}
+	return &response.Artist, nil
+}
+
+func (c *Client) ArtistGetSimilar(ctx context.Context, name string, limit int) ([]Artist, error) {
+	params := url.Values{}
+	params.Add("method", "artist.getSimilar")
+	params.Add("artist", name)
+	params.Add("limit", strconv.Itoa(limit))
+	response, err := c.makeRequest(params)
+	if err != nil {
+		return nil, err
+	}
+	return response.SimilarArtists.Artists, nil
+}
+
+func (c *Client) ArtistGetTopTracks(ctx context.Context, name string, limit int) ([]Track, error) {
+	params := url.Values{}
+	params.Add("method", "artist.getTopTracks")
+	params.Add("artist", name)
+	params.Add("limit", strconv.Itoa(limit))
+	response, err := c.makeRequest(params)
+	if err != nil {
+		return nil, err
+	}
+	return response.TopTracks.Track, nil
+}
+
+func (c *Client) parseError(data []byte) error {
+	var e Error
+	err := json.Unmarshal(data, &e)
+	if err != nil {
+		return err
+	}
+	return fmt.Errorf("last.fm error(%d): %s", e.Code, e.Message)
+}

core/lastfm/client_test.go

@@ -0,0 +1,155 @@
+package lastfm
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"os"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Client", func() {
+	var httpClient *fakeHttpClient
+	var client *Client
+
+	BeforeEach(func() {
+		httpClient = &fakeHttpClient{}
+		client = NewClient("API_KEY", "pt", httpClient)
+	})
+
+	Describe("ArtistGetInfo", func() {
+		It("returns an artist for a successful response", func() {
+			f, _ := os.Open("tests/fixtures/lastfm.artist.getinfo.json")
+			httpClient.res = http.Response{Body: f, StatusCode: 200}
+
+			artist, err := client.ArtistGetInfo(context.TODO(), "U2")
+			Expect(err).To(BeNil())
+			Expect(artist.Name).To(Equal("U2"))
+			Expect(httpClient.savedRequest.URL.String()).To(Equal(apiBaseUrl + "?api_key=API_KEY&artist=U2&format=json&lang=pt&method=artist.getInfo"))
+		})
+
+		It("fails if Last.FM returns an error", func() {
+			httpClient.res = http.Response{
+				Body:       ioutil.NopCloser(bytes.NewBufferString(`{"error":3,"message":"Invalid Method - No method with that name in this package"}`)),
+				StatusCode: 400,
+			}
+
+			_, err := client.ArtistGetInfo(context.TODO(), "U2")
+			Expect(err).To(MatchError("last.fm error(3): Invalid Method - No method with that name in this package"))
+		})
+
+		It("fails if HttpClient.Do() returns error", func() {
+			httpClient.err = errors.New("generic error")
+
+			_, err := client.ArtistGetInfo(context.TODO(), "U2")
+			Expect(err).To(MatchError("generic error"))
+		})
+
+		It("fails if returned body is not a valid JSON", func() {
+			httpClient.res = http.Response{
+				Body:       ioutil.NopCloser(bytes.NewBufferString(`<xml>NOT_VALID_JSON</xml>`)),
+				StatusCode: 200,
+			}
+
+			_, err := client.ArtistGetInfo(context.TODO(), "U2")
+			Expect(err).To(MatchError("invalid character '<' looking for beginning of value"))
+		})
+
+	})
+
+	Describe("ArtistGetSimilar", func() {
+		It("returns an artist for a successful response", func() {
+			f, _ := os.Open("tests/fixtures/lastfm.artist.getsimilar.json")
+			httpClient.res = http.Response{Body: f, StatusCode: 200}
+
+			artists, err := client.ArtistGetSimilar(context.TODO(), "U2", 2)
+			Expect(err).To(BeNil())
+			Expect(len(artists)).To(Equal(2))
+			Expect(httpClient.savedRequest.URL.String()).To(Equal(apiBaseUrl + "?api_key=API_KEY&artist=U2&format=json&limit=2&method=artist.getSimilar"))
+		})
+
+		It("fails if Last.FM returns an error", func() {
+			httpClient.res = http.Response{
+				Body:       ioutil.NopCloser(bytes.NewBufferString(`{"error":3,"message":"Invalid Method - No method with that name in this package"}`)),
+				StatusCode: 400,
+			}
+
+			_, err := client.ArtistGetSimilar(context.TODO(), "U2", 2)
+			Expect(err).To(MatchError("last.fm error(3): Invalid Method - No method with that name in this package"))
+		})
+
+		It("fails if HttpClient.Do() returns error", func() {
+			httpClient.err = errors.New("generic error")
+
+			_, err := client.ArtistGetSimilar(context.TODO(), "U2", 2)
+			Expect(err).To(MatchError("generic error"))
+		})
+
+		It("fails if returned body is not a valid JSON", func() {
+			httpClient.res = http.Response{
+				Body:       ioutil.NopCloser(bytes.NewBufferString(`<xml>NOT_VALID_JSON</xml>`)),
+				StatusCode: 200,
+			}
+
+			_, err := client.ArtistGetSimilar(context.TODO(), "U2", 2)
+			Expect(err).To(MatchError("invalid character '<' looking for beginning of value"))
+		})
+	})
+
+	Describe("ArtistGetTopTracks", func() {
+		It("returns top tracks for a successful response", func() {
+			f, _ := os.Open("tests/fixtures/lastfm.artist.gettoptracks.json")
+			httpClient.res = http.Response{Body: f, StatusCode: 200}
+
+			tracks, err := client.ArtistGetTopTracks(context.TODO(), "U2", 2)
+			Expect(err).To(BeNil())
+			Expect(len(tracks)).To(Equal(2))
+			Expect(httpClient.savedRequest.URL.String()).To(Equal(apiBaseUrl + "?api_key=API_KEY&artist=U2&format=json&limit=2&method=artist.getTopTracks"))
+		})
+
+		It("fails if Last.FM returns an error", func() {
+			httpClient.res = http.Response{
+				Body:       ioutil.NopCloser(bytes.NewBufferString(`{"error":3,"message":"Invalid Method - No method with that name in this package"}`)),
+				StatusCode: 400,
+			}
+
+			_, err := client.ArtistGetTopTracks(context.TODO(), "U2", 2)
+			Expect(err).To(MatchError("last.fm error(3): Invalid Method - No method with that name in this package"))
+		})
+
+		It("fails if HttpClient.Do() returns error", func() {
+			httpClient.err = errors.New("generic error")
+
+			_, err := client.ArtistGetTopTracks(context.TODO(), "U2", 2)
+			Expect(err).To(MatchError("generic error"))
+		})
+
+		It("fails if returned body is not a valid JSON", func() {
+			httpClient.res = http.Response{
+				Body:       ioutil.NopCloser(bytes.NewBufferString(`<xml>NOT_VALID_JSON</xml>`)),
+				StatusCode: 200,
+			}
+
+			_, err := client.ArtistGetTopTracks(context.TODO(), "U2", 2)
+			Expect(err).To(MatchError("invalid character '<' looking for beginning of value"))
+		})
+	})
+})
+
+type fakeHttpClient struct {
+	res          http.Response
+	err          error
+	savedRequest *http.Request
+}
+
+func (c *fakeHttpClient) Do(req *http.Request) (*http.Response, error) {
+	c.savedRequest = req
+	if c.err != nil {
+		return nil, c.err
+	}
+	return &c.res, nil
+}

core/lastfm/lastfm_suite_test.go

@@ -0,0 +1,17 @@
+package lastfm
+
+import (
+	"testing"
+
+	"github.com/deluan/navidrome/log"
+	"github.com/deluan/navidrome/tests"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestLastFM(t *testing.T) {
+	tests.Init(t, false)
+	log.SetLevel(log.LevelCritical)
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "LastFM Test Suite")
+}

core/lastfm/responses.go

@@ -0,0 +1,58 @@
+package lastfm
+
+type Response struct {
+	Artist         Artist         `json:"artist"`
+	SimilarArtists SimilarArtists `json:"similarartists"`
+	TopTracks      TopTracks      `json:"toptracks"`
+}
+
+type Artist struct {
+	Name       string        `json:"name"`
+	MBID       string        `json:"mbid"`
+	URL        string        `json:"url"`
+	Image      []ArtistImage `json:"image"`
+	Streamable string        `json:"streamable"`
+	Stats      struct {
+		Listeners string `json:"listeners"`
+		Plays     string `json:"plays"`
+	} `json:"stats"`
+	Similar SimilarArtists `json:"similar"`
+	Tags    struct {
+		Tag []ArtistTag `json:"tag"`
+	} `json:"tags"`
+	Bio ArtistBio `json:"bio"`
+}
+
+type SimilarArtists struct {
+	Artists []Artist `json:"artist"`
+}
+
+type ArtistImage struct {
+	URL  string `json:"#text"`
+	Size string `json:"size"`
+}
+
+type ArtistTag struct {
+	Name string `json:"name"`
+	URL  string `json:"url"`
+}
+
+type ArtistBio struct {
+	Published string `json:"published"`
+	Summary   string `json:"summary"`
+	Content   string `json:"content"`
+}
+
+type Track struct {
+	Name string `json:"name"`
+	MBID string `json:"mbid"`
+}
+
+type TopTracks struct {
+	Track []Track `json:"track"`
+}
+
+type Error struct {
+	Code    int    `json:"error"`
+	Message string `json:"message"`
+}

core/lastfm/responses_test.go

@@ -0,0 +1,70 @@
+package lastfm
+
+import (
+	"encoding/json"
+	"io/ioutil"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("LastFM responses", func() {
+	Describe("Artist", func() {
+		It("parses the response correctly", func() {
+			var resp Response
+			body, _ := ioutil.ReadFile("tests/fixtures/lastfm.artist.getinfo.json")
+			err := json.Unmarshal(body, &resp)
+			Expect(err).To(BeNil())
+
+			Expect(resp.Artist.Name).To(Equal("U2"))
+			Expect(resp.Artist.MBID).To(Equal("a3cb23fc-acd3-4ce0-8f36-1e5aa6a18432"))
+			Expect(resp.Artist.URL).To(Equal("https://www.last.fm/music/U2"))
+			Expect(resp.Artist.Bio.Summary).To(ContainSubstring("U2 é uma das mais importantes bandas de rock de todos os tempos"))
+
+			similarArtists := []string{"Passengers", "INXS", "R.E.M.", "Simple Minds", "Bono"}
+			for i, similar := range similarArtists {
+				Expect(resp.Artist.Similar.Artists[i].Name).To(Equal(similar))
+			}
+		})
+	})
+
+	Describe("SimilarArtists", func() {
+		It("parses the response correctly", func() {
+			var resp Response
+			body, _ := ioutil.ReadFile("tests/fixtures/lastfm.artist.getsimilar.json")
+			err := json.Unmarshal(body, &resp)
+			Expect(err).To(BeNil())
+
+			Expect(resp.SimilarArtists.Artists).To(HaveLen(2))
+			Expect(resp.SimilarArtists.Artists[0].Name).To(Equal("Passengers"))
+			Expect(resp.SimilarArtists.Artists[1].Name).To(Equal("INXS"))
+		})
+	})
+
+	Describe("TopTracks", func() {
+		It("parses the response correctly", func() {
+			var resp Response
+			body, _ := ioutil.ReadFile("tests/fixtures/lastfm.artist.gettoptracks.json")
+			err := json.Unmarshal(body, &resp)
+			Expect(err).To(BeNil())
+
+			Expect(resp.TopTracks.Track).To(HaveLen(2))
+			Expect(resp.TopTracks.Track[0].Name).To(Equal("Beautiful Day"))
+			Expect(resp.TopTracks.Track[0].MBID).To(Equal("f7f264d0-a89b-4682-9cd7-a4e7c37637af"))
+			Expect(resp.TopTracks.Track[1].Name).To(Equal("With or Without You"))
+			Expect(resp.TopTracks.Track[1].MBID).To(Equal("6b9a509f-6907-4a6e-9345-2f12da09ba4b"))
+		})
+	})
+
+	Describe("Error", func() {
+		It("parses the error response correctly", func() {
+			var error Error
+			body := []byte(`{"error":3,"message":"Invalid Method - No method with that name in this package"}`)
+			err := json.Unmarshal(body, &error)
+			Expect(err).To(BeNil())
+
+			Expect(error.Code).To(Equal(3))
+			Expect(error.Message).To(Equal("Invalid Method - No method with that name in this package"))
+		})
+	})
+})

core/media_streamer.go

@@ -6,10 +6,12 @@ import (
 	"io"
 	"mime"
 	"os"
+	"sync"
 	"time"
 
 	"github.com/deluan/navidrome/conf"
 	"github.com/deluan/navidrome/consts"
+	"github.com/deluan/navidrome/core/cache"
 	"github.com/deluan/navidrome/core/transcoder"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
@@ -20,7 +22,7 @@ type MediaStreamer interface {
 	NewStream(ctx context.Context, id string, reqFormat string, reqBitRate int) (*Stream, error)
 }
 
-type TranscodingCache FileCache
+type TranscodingCache cache.FileCache
 
 func NewMediaStreamer(ds model.DataStore, ffm transcoder.Transcoder, cache TranscodingCache) MediaStreamer {
 	return &mediaStreamer{ds: ds, ffm: ffm, cache: cache}
@@ -29,7 +31,7 @@ func NewMediaStreamer(ds model.DataStore, ffm transcoder.Transcoder, cache Trans
 type mediaStreamer struct {
 	ds    model.DataStore
 	ffm   transcoder.Transcoder
-	cache FileCache
+	cache cache.FileCache
 }
 
 type streamJob struct {
@@ -39,7 +41,7 @@ type streamJob struct {
 	bitRate int
 }
 
-func (j *streamJob) String() string {
+func (j *streamJob) Key() string {
 	return fmt.Sprintf("%s.%s.%d.%s", j.mf.ID, j.mf.UpdatedAt.Format(time.RFC3339Nano), j.bitRate, j.format)
 }
 
@@ -166,21 +168,29 @@ func selectTranscodingOptions(ctx context.Context, ds model.DataStore, mf *model
 	return
 }
 
-func NewTranscodingCache() TranscodingCache {
-	return NewFileCache("Transcoding", conf.Server.TranscodingCacheSize,
-		consts.TranscodingCacheDir, consts.DefaultTranscodingCacheMaxItems,
-		func(ctx context.Context, arg fmt.Stringer) (io.Reader, error) {
-			job := arg.(*streamJob)
-			t, err := job.ms.ds.Transcoding(ctx).FindByFormat(job.format)
-			if err != nil {
-				log.Error(ctx, "Error loading transcoding command", "format", job.format, err)
-				return nil, os.ErrInvalid
-			}
-			out, err := job.ms.ffm.Start(ctx, t.Command, job.mf.Path, job.bitRate)
-			if err != nil {
-				log.Error(ctx, "Error starting transcoder", "id", job.mf.ID, err)
-				return nil, os.ErrInvalid
-			}
-			return out, nil
-		})
+var (
+	onceTranscodingCache     sync.Once
+	instanceTranscodingCache TranscodingCache
+)
+
+func GetTranscodingCache() TranscodingCache {
+	onceTranscodingCache.Do(func() {
+		instanceTranscodingCache = cache.NewFileCache("Transcoding", conf.Server.TranscodingCacheSize,
+			consts.TranscodingCacheDir, consts.DefaultTranscodingCacheMaxItems,
+			func(ctx context.Context, arg cache.Item) (io.Reader, error) {
+				job := arg.(*streamJob)
+				t, err := job.ms.ds.Transcoding(ctx).FindByFormat(job.format)
+				if err != nil {
+					log.Error(ctx, "Error loading transcoding command", "format", job.format, err)
+					return nil, os.ErrInvalid
+				}
+				out, err := job.ms.ffm.Start(ctx, t.Command, job.mf.Path, job.bitRate)
+				if err != nil {
+					log.Error(ctx, "Error starting transcoder", "id", job.mf.ID, err)
+					return nil, os.ErrInvalid
+				}
+				return out, nil
+			})
+	})
+	return instanceTranscodingCache
 }

core/media_streamer_test.go

@@ -4,14 +4,13 @@ import (
 	"context"
 	"io"
 	"io/ioutil"
-	"os"
 	"strings"
 
 	"github.com/deluan/navidrome/conf"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
 	"github.com/deluan/navidrome/model/request"
-	"github.com/deluan/navidrome/persistence"
+	"github.com/deluan/navidrome/tests"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -25,17 +24,15 @@ var _ = Describe("MediaStreamer", func() {
 	BeforeEach(func() {
 		conf.Server.DataFolder, _ = ioutil.TempDir("", "file_caches")
 		conf.Server.TranscodingCacheSize = "100MB"
-		ds = &persistence.MockDataStore{MockedTranscoding: &mockTranscodingRepository{}}
-		ds.MediaFile(ctx).(*persistence.MockMediaFile).SetData(`[{"id": "123", "path": "tests/fixtures/test.mp3", "suffix": "mp3", "bitRate": 128, "duration": 257.0}]`)
-		testCache := NewTranscodingCache()
-		Eventually(func() bool { return testCache.Ready() }).Should(BeTrue())
+		ds = &tests.MockDataStore{MockedTranscoding: &tests.MockTranscodingRepository{}}
+		ds.MediaFile(ctx).(*tests.MockMediaFile).SetData(model.MediaFiles{
+			{ID: "123", Path: "tests/fixtures/test.mp3", Suffix: "mp3", BitRate: 128, Duration: 257.0},
+		})
+		testCache := GetTranscodingCache()
+		Eventually(func() bool { return testCache.Ready(context.TODO()) }).Should(BeTrue())
 		streamer = NewMediaStreamer(ds, ffmpeg, testCache)
 	})
 
-	AfterEach(func() {
-		os.RemoveAll(conf.Server.DataFolder)
-	})
-
 	Context("NewStream", func() {
 		It("returns a seekable stream if format is 'raw'", func() {
 			s, err := streamer.NewStream(ctx, "123", "raw", 0)

core/nowplaying.go

@@ -1,4 +1,4 @@
-package engine
+package core
 
 import (
 	"container/list"
@@ -17,22 +17,10 @@ type NowPlayingInfo struct {
 }
 
 // This repo must have the semantics of a FIFO queue, for each playerId
-type NowPlayingRepository interface {
+type NowPlaying interface {
 	// Insert at the head of the queue
 	Enqueue(*NowPlayingInfo) error
 
-	// Removes and returns the element at the end of the queue
-	Dequeue(playerId int) (*NowPlayingInfo, error)
-
-	// Returns the element at the head of the queue (last inserted one)
-	Head(playerId int) (*NowPlayingInfo, error)
-
-	// Returns the element at the end of the queue (first inserted one)
-	Tail(playerId int) (*NowPlayingInfo, error)
-
-	// Size of the queue for the playerId
-	Count(playerId int) (int64, error)
-
 	// Returns all heads from all playerIds
 	GetAll() ([]*NowPlayingInfo, error)
 }
@@ -41,55 +29,17 @@ var playerMap = sync.Map{}
 
 type nowPlayingRepository struct{}
 
-func NewNowPlayingRepository() NowPlayingRepository {
+func NewNowPlayingRepository() NowPlaying {
 	r := &nowPlayingRepository{}
 	return r
 }
 
-func (r *nowPlayingRepository) getList(id int) *list.List {
-	l, _ := playerMap.LoadOrStore(id, list.New())
-	return l.(*list.List)
-}
-
 func (r *nowPlayingRepository) Enqueue(info *NowPlayingInfo) error {
 	l := r.getList(info.PlayerId)
 	l.PushFront(info)
 	return nil
 }
 
-func (r *nowPlayingRepository) Dequeue(playerId int) (*NowPlayingInfo, error) {
-	l := r.getList(playerId)
-	e := checkExpired(l, l.Back)
-	if e == nil {
-		return nil, nil
-	}
-	l.Remove(e)
-	return e.Value.(*NowPlayingInfo), nil
-}
-
-func (r *nowPlayingRepository) Head(playerId int) (*NowPlayingInfo, error) {
-	l := r.getList(playerId)
-	e := checkExpired(l, l.Front)
-	if e == nil {
-		return nil, nil
-	}
-	return e.Value.(*NowPlayingInfo), nil
-}
-
-func (r *nowPlayingRepository) Tail(playerId int) (*NowPlayingInfo, error) {
-	l := r.getList(playerId)
-	e := checkExpired(l, l.Back)
-	if e == nil {
-		return nil, nil
-	}
-	return e.Value.(*NowPlayingInfo), nil
-}
-
-func (r *nowPlayingRepository) Count(playerId int) (int64, error) {
-	l := r.getList(playerId)
-	return int64(l.Len()), nil
-}
-
 func (r *nowPlayingRepository) GetAll() ([]*NowPlayingInfo, error) {
 	var all []*NowPlayingInfo
 	playerMap.Range(func(playerId, l interface{}) bool {
@@ -103,6 +53,44 @@ func (r *nowPlayingRepository) GetAll() ([]*NowPlayingInfo, error) {
 	return all, nil
 }
 
+func (r *nowPlayingRepository) getList(id int) *list.List {
+	l, _ := playerMap.LoadOrStore(id, list.New())
+	return l.(*list.List)
+}
+
+func (r *nowPlayingRepository) dequeue(playerId int) (*NowPlayingInfo, error) {
+	l := r.getList(playerId)
+	e := checkExpired(l, l.Back)
+	if e == nil {
+		return nil, nil
+	}
+	l.Remove(e)
+	return e.Value.(*NowPlayingInfo), nil
+}
+
+func (r *nowPlayingRepository) head(playerId int) (*NowPlayingInfo, error) {
+	l := r.getList(playerId)
+	e := checkExpired(l, l.Front)
+	if e == nil {
+		return nil, nil
+	}
+	return e.Value.(*NowPlayingInfo), nil
+}
+
+func (r *nowPlayingRepository) tail(playerId int) (*NowPlayingInfo, error) {
+	l := r.getList(playerId)
+	e := checkExpired(l, l.Back)
+	if e == nil {
+		return nil, nil
+	}
+	return e.Value.(*NowPlayingInfo), nil
+}
+
+func (r *nowPlayingRepository) count(playerId int) (int64, error) {
+	l := r.getList(playerId)
+	return int64(l.Len()), nil
+}
+
 func checkExpired(l *list.List, f func() *list.Element) *list.Element {
 	for {
 		e := f()

core/nowplaying_test.go

@@ -1,4 +1,4 @@
-package engine
+package core
 
 import (
 	"sync"
@@ -8,27 +8,27 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("NowPlayingRepository", func() {
-	var repo NowPlayingRepository
+var _ = Describe("NowPlaying", func() {
+	var repo *nowPlayingRepository
 	var now = time.Now()
 	var past = time.Time{}
 
 	BeforeEach(func() {
 		playerMap = sync.Map{}
-		repo = NewNowPlayingRepository()
+		repo = NewNowPlayingRepository().(*nowPlayingRepository)
 	})
 
 	It("enqueues and dequeues records", func() {
 		Expect(repo.Enqueue(&NowPlayingInfo{PlayerId: 1, TrackID: "AAA", Start: now})).To(BeNil())
 		Expect(repo.Enqueue(&NowPlayingInfo{PlayerId: 1, TrackID: "BBB", Start: now})).To(BeNil())
 
-		Expect(repo.Tail(1)).To(Equal(&NowPlayingInfo{PlayerId: 1, TrackID: "AAA", Start: now}))
-		Expect(repo.Head(1)).To(Equal(&NowPlayingInfo{PlayerId: 1, TrackID: "BBB", Start: now}))
+		Expect(repo.tail(1)).To(Equal(&NowPlayingInfo{PlayerId: 1, TrackID: "AAA", Start: now}))
+		Expect(repo.head(1)).To(Equal(&NowPlayingInfo{PlayerId: 1, TrackID: "BBB", Start: now}))
 
-		Expect(repo.Count(1)).To(Equal(int64(2)))
+		Expect(repo.count(1)).To(Equal(int64(2)))
 
-		Expect(repo.Dequeue(1)).To(Equal(&NowPlayingInfo{PlayerId: 1, TrackID: "AAA", Start: now}))
-		Expect(repo.Count(1)).To(Equal(int64(1)))
+		Expect(repo.dequeue(1)).To(Equal(&NowPlayingInfo{PlayerId: 1, TrackID: "AAA", Start: now}))
+		Expect(repo.count(1)).To(Equal(int64(1)))
 	})
 
 	It("handles multiple players", func() {
@@ -43,11 +43,11 @@ var _ = Describe("NowPlayingRepository", func() {
 			{PlayerId: 2, TrackID: "DDD", Start: now},
 		}))
 
-		Expect(repo.Count(2)).To(Equal(int64(2)))
-		Expect(repo.Count(2)).To(Equal(int64(2)))
+		Expect(repo.count(2)).To(Equal(int64(2)))
+		Expect(repo.count(2)).To(Equal(int64(2)))
 
-		Expect(repo.Tail(1)).To(Equal(&NowPlayingInfo{PlayerId: 1, TrackID: "AAA", Start: now}))
-		Expect(repo.Head(2)).To(Equal(&NowPlayingInfo{PlayerId: 2, TrackID: "DDD", Start: now}))
+		Expect(repo.tail(1)).To(Equal(&NowPlayingInfo{PlayerId: 1, TrackID: "AAA", Start: now}))
+		Expect(repo.head(2)).To(Equal(&NowPlayingInfo{PlayerId: 2, TrackID: "DDD", Start: now}))
 	})
 
 	It("handles expired items", func() {

core/players.go

@@ -1,4 +1,4 @@
-package engine
+package core
 
 import (
 	"context"

core/players_test.go

@@ -1,4 +1,4 @@
-package engine
+package core
 
 import (
 	"context"
@@ -7,7 +7,7 @@ import (
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
 	"github.com/deluan/navidrome/model/request"
-	"github.com/deluan/navidrome/persistence"
+	"github.com/deluan/navidrome/tests"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -22,7 +22,7 @@ var _ = Describe("Players", func() {
 
 	BeforeEach(func() {
 		repo = &mockPlayerRepository{}
-		ds := &persistence.MockDataStore{MockedPlayer: repo, MockedTranscoding: &mockTranscodingRepository{}}
+		ds := &tests.MockDataStore{MockedPlayer: repo, MockedTranscoding: &tests.MockTranscodingRepository{}}
 		players = NewPlayers(ds)
 		beforeRegister = time.Now()
 	})

core/pool/pool.go

@@ -0,0 +1,122 @@
+package pool
+
+import (
+	"time"
+
+	"github.com/deluan/navidrome/log"
+)
+
+type Executor func(workload interface{})
+
+type Pool struct {
+	name          string
+	item          interface{}
+	workers       []worker
+	exec          Executor
+	logTicker     *time.Ticker
+	workerChannel chan chan work
+	queue         chan work // receives jobs to send to workers
+	end           chan bool // when receives bool stops workers
+	//queue *dque.DQue
+}
+
+// TODO This hardcoded value will go away when the queue is persisted in disk
+const bufferSize = 10000
+
+func NewPool(name string, workerCount int, item interface{}, exec Executor) (*Pool, error) {
+	p := &Pool{
+		name:  name,
+		item:  item,
+		exec:  exec,
+		queue: make(chan work, bufferSize),
+		end:   make(chan bool),
+	}
+
+	//q, err := dque.NewOrOpen(name, filepath.Join(conf.Server.DataFolder, "queues", name), 50, p.itemBuilder)
+	//if err != nil {
+	//	return nil, err
+	//}
+	//p.queue = q
+
+	p.workerChannel = make(chan chan work)
+	for i := 0; i < workerCount; i++ {
+		worker := worker{
+			p:             p,
+			id:            i,
+			channel:       make(chan work),
+			workerChannel: p.workerChannel,
+			end:           make(chan bool)}
+		worker.Start()
+		p.workers = append(p.workers, worker)
+	}
+
+	// start pool
+	go func() {
+		p.logTicker = time.NewTicker(10 * time.Second)
+		running := false
+		for {
+			select {
+			case <-p.logTicker.C:
+				if len(p.queue) > 0 {
+					log.Debug("Queue status", "pool", p.name, "items", len(p.queue))
+				} else {
+					if running {
+						log.Info("Queue empty", "pool", p.name)
+					}
+					running = false
+				}
+			case <-p.end:
+				for _, w := range p.workers {
+					w.Stop() // stop worker
+				}
+				return
+			case work := <-p.queue:
+				running = true
+				worker := <-p.workerChannel // wait for available channel
+				worker <- work              // dispatch work to worker
+			}
+		}
+	}()
+	return p, nil
+}
+
+func (p *Pool) Submit(workload interface{}) {
+	p.queue <- work{workload}
+}
+
+//func (p *Pool) itemBuilder() interface{} {
+//	t := reflect.TypeOf(p.item)
+//	return reflect.New(t).Interface()
+//}
+//
+type work struct {
+	workload interface{}
+}
+
+type worker struct {
+	id            int
+	p             *Pool
+	workerChannel chan chan work // used to communicate between dispatcher and workers
+	channel       chan work
+	end           chan bool
+}
+
+// start worker
+func (w *worker) Start() {
+	go func() {
+		for {
+			w.workerChannel <- w.channel // when the worker is available place channel in queue
+			select {
+			case job := <-w.channel: // worker has received job
+				w.p.exec(job.workload) // do work
+			case <-w.end:
+				return
+			}
+		}
+	}()
+}
+
+// end worker
+func (w *worker) Stop() {
+	w.end <- true
+}

core/pool/pool_test.go

@@ -0,0 +1,49 @@
+package pool
+
+import (
+	"testing"
+
+	"github.com/deluan/navidrome/log"
+	"github.com/deluan/navidrome/tests"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestCore(t *testing.T) {
+	tests.Init(t, false)
+	log.SetLevel(log.LevelCritical)
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Core Suite")
+}
+
+type testItem struct {
+	ID int
+}
+
+type results []int
+
+func (r results) Len() int { return len(r) }
+
+var processed results
+
+var _ = XDescribe("Pool", func() {
+	var pool *Pool
+
+	BeforeEach(func() {
+		processed = nil
+		pool, _ = NewPool("test", 2, &testItem{}, execute)
+	})
+
+	It("processes items", func() {
+		for i := 0; i < 5; i++ {
+			pool.Submit(&testItem{ID: i})
+		}
+		Eventually(processed.Len, "10s").Should(Equal(5))
+		Expect(processed).To(ContainElements(0, 1, 2, 3, 4))
+	})
+})
+
+func execute(workload interface{}) {
+	item := workload.(*testItem)
+	processed = append(processed, item.ID)
+}

core/spotify/client.go

@@ -0,0 +1,114 @@
+package spotify
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/deluan/navidrome/log"
+)
+
+const apiBaseUrl = "https://api.spotify.com/v1/"
+
+var (
+	ErrNotFound = errors.New("spotify: not found")
+)
+
+type HttpClient interface {
+	Do(req *http.Request) (*http.Response, error)
+}
+
+func NewClient(id, secret string, hc HttpClient) *Client {
+	return &Client{id, secret, hc}
+}
+
+type Client struct {
+	id     string
+	secret string
+	hc     HttpClient
+}
+
+func (c *Client) SearchArtists(ctx context.Context, name string, limit int) ([]Artist, error) {
+	token, err := c.authorize(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	params := url.Values{}
+	params.Add("type", "artist")
+	params.Add("q", name)
+	params.Add("offset", "0")
+	params.Add("limit", strconv.Itoa(limit))
+	req, _ := http.NewRequest("GET", apiBaseUrl+"search", nil)
+	req.URL.RawQuery = params.Encode()
+	req.Header.Add("Authorization", "Bearer "+token)
+
+	var results SearchResults
+	err = c.makeRequest(req, &results)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(results.Artists.Items) == 0 {
+		return nil, ErrNotFound
+	}
+	return results.Artists.Items, err
+}
+
+func (c *Client) authorize(ctx context.Context) (string, error) {
+	payload := url.Values{}
+	payload.Add("grant_type", "client_credentials")
+
+	req, _ := http.NewRequest("POST", "https://accounts.spotify.com/api/token", strings.NewReader(payload.Encode()))
+	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Add("Content-Length", strconv.Itoa(len(payload.Encode())))
+	auth := c.id + ":" + c.secret
+	req.Header.Add("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(auth)))
+
+	response := map[string]interface{}{}
+	err := c.makeRequest(req, &response)
+	if err != nil {
+		return "", err
+	}
+
+	if v, ok := response["access_token"]; ok {
+		return v.(string), nil
+	}
+	log.Error(ctx, "Invalid spotify response", "resp", response)
+	return "", errors.New("invalid response")
+}
+
+func (c *Client) makeRequest(req *http.Request, response interface{}) error {
+	resp, err := c.hc.Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != 200 {
+		return c.parseError(data)
+	}
+
+	return json.Unmarshal(data, response)
+}
+
+func (c *Client) parseError(data []byte) error {
+	var e Error
+	err := json.Unmarshal(data, &e)
+	if err != nil {
+		return err
+	}
+	return fmt.Errorf("spotify error(%s): %s", e.Code, e.Message)
+}

core/spotify/client_test.go

@@ -0,0 +1,131 @@
+package spotify
+
+import (
+	"bytes"
+	"context"
+	"io/ioutil"
+	"net/http"
+	"os"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Client", func() {
+	var httpClient *fakeHttpClient
+	var client *Client
+
+	BeforeEach(func() {
+		httpClient = &fakeHttpClient{}
+		client = NewClient("SPOTIFY_ID", "SPOTIFY_SECRET", httpClient)
+	})
+
+	Describe("ArtistImages", func() {
+		It("returns artist images from a successful request", func() {
+			f, _ := os.Open("tests/fixtures/spotify.search.artist.json")
+			httpClient.mock("https://api.spotify.com/v1/search", http.Response{Body: f, StatusCode: 200})
+			httpClient.mock("https://accounts.spotify.com/api/token", http.Response{
+				StatusCode: 200,
+				Body:       ioutil.NopCloser(bytes.NewBufferString(`{"access_token": "NEW_ACCESS_TOKEN","token_type": "Bearer","expires_in": 3600}`)),
+			})
+
+			artists, err := client.SearchArtists(context.TODO(), "U2", 10)
+			Expect(err).To(BeNil())
+			Expect(artists).To(HaveLen(20))
+			Expect(artists[0].Popularity).To(Equal(82))
+
+			images := artists[0].Images
+			Expect(images).To(HaveLen(3))
+			Expect(images[0].Width).To(Equal(640))
+			Expect(images[1].Width).To(Equal(320))
+			Expect(images[2].Width).To(Equal(160))
+		})
+
+		It("fails if artist was not found", func() {
+			httpClient.mock("https://api.spotify.com/v1/search", http.Response{
+				StatusCode: 200,
+				Body: ioutil.NopCloser(bytes.NewBufferString(`{
+						  "artists" : {
+							"href" : "https://api.spotify.com/v1/search?query=dasdasdas%2Cdna&type=artist&offset=0&limit=20",
+							"items" : [ ], "limit" : 20, "next" : null, "offset" : 0, "previous" : null, "total" : 0
+						  }}`)),
+			})
+			httpClient.mock("https://accounts.spotify.com/api/token", http.Response{
+				StatusCode: 200,
+				Body:       ioutil.NopCloser(bytes.NewBufferString(`{"access_token": "NEW_ACCESS_TOKEN","token_type": "Bearer","expires_in": 3600}`)),
+			})
+
+			_, err := client.SearchArtists(context.TODO(), "U2", 10)
+			Expect(err).To(MatchError(ErrNotFound))
+		})
+
+		It("fails if not able to authorize", func() {
+			f, _ := os.Open("tests/fixtures/spotify.search.artist.json")
+			httpClient.mock("https://api.spotify.com/v1/search", http.Response{Body: f, StatusCode: 200})
+			httpClient.mock("https://accounts.spotify.com/api/token", http.Response{
+				StatusCode: 400,
+				Body:       ioutil.NopCloser(bytes.NewBufferString(`{"error":"invalid_client","error_description":"Invalid client"}`)),
+			})
+
+			_, err := client.SearchArtists(context.TODO(), "U2", 10)
+			Expect(err).To(MatchError("spotify error(invalid_client): Invalid client"))
+		})
+	})
+
+	Describe("authorize", func() {
+		It("returns an access_token on successful authorization", func() {
+			httpClient.mock("https://accounts.spotify.com/api/token", http.Response{
+				StatusCode: 200,
+				Body:       ioutil.NopCloser(bytes.NewBufferString(`{"access_token": "NEW_ACCESS_TOKEN","token_type": "Bearer","expires_in": 3600}`)),
+			})
+
+			token, err := client.authorize(context.TODO())
+			Expect(err).To(BeNil())
+			Expect(token).To(Equal("NEW_ACCESS_TOKEN"))
+			auth := httpClient.lastRequest.Header.Get("Authorization")
+			Expect(auth).To(Equal("Basic U1BPVElGWV9JRDpTUE9USUZZX1NFQ1JFVA=="))
+		})
+
+		It("fails on unsuccessful authorization", func() {
+			httpClient.mock("https://accounts.spotify.com/api/token", http.Response{
+				StatusCode: 400,
+				Body:       ioutil.NopCloser(bytes.NewBufferString(`{"error":"invalid_client","error_description":"Invalid client"}`)),
+			})
+
+			_, err := client.authorize(context.TODO())
+			Expect(err).To(MatchError("spotify error(invalid_client): Invalid client"))
+		})
+
+		It("fails on invalid JSON response", func() {
+			httpClient.mock("https://accounts.spotify.com/api/token", http.Response{
+				StatusCode: 200,
+				Body:       ioutil.NopCloser(bytes.NewBufferString(`{NOT_VALID}`)),
+			})
+
+			_, err := client.authorize(context.TODO())
+			Expect(err).To(MatchError("invalid character 'N' looking for beginning of object key string"))
+		})
+	})
+})
+
+type fakeHttpClient struct {
+	responses   map[string]*http.Response
+	lastRequest *http.Request
+}
+
+func (c *fakeHttpClient) mock(url string, response http.Response) {
+	if c.responses == nil {
+		c.responses = make(map[string]*http.Response)
+	}
+	c.responses[url] = &response
+}
+
+func (c *fakeHttpClient) Do(req *http.Request) (*http.Response, error) {
+	c.lastRequest = req
+	u := req.URL
+	u.RawQuery = ""
+	if resp, ok := c.responses[u.String()]; ok {
+		return resp, nil
+	}
+	panic("URL not mocked: " + u.String())
+}

core/spotify/responses.go

@@ -0,0 +1,30 @@
+package spotify
+
+type SearchResults struct {
+	Artists ArtistsResult `json:"artists"`
+}
+
+type ArtistsResult struct {
+	HRef  string   `json:"href"`
+	Items []Artist `json:"items"`
+}
+
+type Artist struct {
+	Genres     []string `json:"genres"`
+	HRef       string   `json:"href"`
+	ID         string   `json:"id"`
+	Popularity int      `json:"popularity"`
+	Images     []Image  `json:"images"`
+	Name       string   `json:"name"`
+}
+
+type Image struct {
+	URL    string `json:"url"`
+	Width  int    `json:"width"`
+	Height int    `json:"height"`
+}
+
+type Error struct {
+	Code    string `json:"error"`
+	Message string `json:"error_description"`
+}

core/spotify/responses_test.go

@@ -0,0 +1,48 @@
+package spotify
+
+import (
+	"encoding/json"
+	"io/ioutil"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Responses", func() {
+	Describe("Search type=artist", func() {
+		It("parses the artist search result correctly ", func() {
+			var resp SearchResults
+			body, _ := ioutil.ReadFile("tests/fixtures/spotify.search.artist.json")
+			err := json.Unmarshal(body, &resp)
+			Expect(err).To(BeNil())
+
+			Expect(resp.Artists.Items).To(HaveLen(20))
+			u2 := resp.Artists.Items[0]
+			Expect(u2.Name).To(Equal("U2"))
+			Expect(u2.Genres).To(ContainElements("irish rock", "permanent wave", "rock"))
+			Expect(u2.ID).To(Equal("51Blml2LZPmy7TTiAg47vQ"))
+			Expect(u2.HRef).To(Equal("https://api.spotify.com/v1/artists/51Blml2LZPmy7TTiAg47vQ"))
+			Expect(u2.Images[0].URL).To(Equal("https://i.scdn.co/image/e22d5c0c8139b8439440a69854ed66efae91112d"))
+			Expect(u2.Images[0].Width).To(Equal(640))
+			Expect(u2.Images[0].Height).To(Equal(640))
+			Expect(u2.Images[1].URL).To(Equal("https://i.scdn.co/image/40d6c5c14355cfc127b70da221233315497ec91d"))
+			Expect(u2.Images[1].Width).To(Equal(320))
+			Expect(u2.Images[1].Height).To(Equal(320))
+			Expect(u2.Images[2].URL).To(Equal("https://i.scdn.co/image/7293d6752ae8a64e34adee5086858e408185b534"))
+			Expect(u2.Images[2].Width).To(Equal(160))
+			Expect(u2.Images[2].Height).To(Equal(160))
+		})
+	})
+
+	Describe("Error", func() {
+		It("parses the error response correctly", func() {
+			var errorResp Error
+			body := []byte(`{"error":"invalid_client","error_description":"Invalid client"}`)
+			err := json.Unmarshal(body, &errorResp)
+			Expect(err).To(BeNil())
+
+			Expect(errorResp.Code).To(Equal("invalid_client"))
+			Expect(errorResp.Message).To(Equal("Invalid client"))
+		})
+	})
+})

core/spotify/spotify_suite_test.go

@@ -0,0 +1,17 @@
+package spotify
+
+import (
+	"testing"
+
+	"github.com/deluan/navidrome/log"
+	"github.com/deluan/navidrome/tests"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestSpotify(t *testing.T) {
+	tests.Init(t, false)
+	log.SetLevel(log.LevelCritical)
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Spotify Test Suite")
+}

core/transcoder/ffmpeg.go

@@ -16,11 +16,10 @@ type Transcoder interface {
 }
 
 func New() Transcoder {
-	path, err := exec.LookPath("ffmpeg")
+	_, err := exec.LookPath("ffmpeg")
 	if err != nil {
 		log.Error("Unable to find ffmpeg", err)
 	}
-	log.Debug("Found ffmpeg", "path", path)
 	return &ffmpeg{}
 }
 

core/wire_providers.go

@@ -1,6 +1,11 @@
 package core
 
 import (
+	"net/http"
+
+	"github.com/deluan/navidrome/conf"
+	"github.com/deluan/navidrome/core/lastfm"
+	"github.com/deluan/navidrome/core/spotify"
 	"github.com/deluan/navidrome/core/transcoder"
 	"github.com/google/wire"
 )
@@ -8,8 +13,30 @@ import (
 var Set = wire.NewSet(
 	NewArtwork,
 	NewMediaStreamer,
-	NewTranscodingCache,
-	NewImageCache,
+	GetTranscodingCache,
+	GetImageCache,
 	NewArchiver,
+	NewNowPlayingRepository,
+	NewExternalInfo,
+	NewCacheWarmer,
+	NewPlayers,
+	LastFMNewClient,
+	SpotifyNewClient,
 	transcoder.New,
 )
+
+func LastFMNewClient() *lastfm.Client {
+	if conf.Server.LastFM.ApiKey == "" {
+		return nil
+	}
+
+	return lastfm.NewClient(conf.Server.LastFM.ApiKey, conf.Server.LastFM.Language, http.DefaultClient)
+}
+
+func SpotifyNewClient() *spotify.Client {
+	if conf.Server.Spotify.ID == "" || conf.Server.Spotify.Secret == "" {
+		return nil
+	}
+
+	return spotify.NewClient(conf.Server.Spotify.ID, conf.Server.Spotify.Secret, http.DefaultClient)
+}

db/migration/20201010162350_add_album_size.go

@@ -0,0 +1,30 @@
+package migration
+
+import (
+	"database/sql"
+
+	"github.com/pressly/goose"
+)
+
+func init() {
+	goose.AddMigration(Up20201010162350, Down20201010162350)
+}
+
+func Up20201010162350(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+alter table album
+	add size integer default 0 not null;
+create index if not exists album_size
+	on album(size);
+	`)
+	if err != nil {
+		return err
+	}
+	notice(tx, "A full rescan will be performed to calculate album sizes.")
+	return forceFullRescan(tx)
+}
+
+func Down20201010162350(tx *sql.Tx) error {
+	// This code is executed when the migration is rolled back.
+	return nil
+}

db/migration/20201012210022_add_artist_playlist_size.go

@@ -0,0 +1,41 @@
+package migration
+
+import (
+	"database/sql"
+
+	"github.com/pressly/goose"
+)
+
+func init() {
+	goose.AddMigration(Up20201012210022, Down20201012210022)
+}
+
+func Up20201012210022(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+alter table artist
+	add size integer default 0 not null;
+create index if not exists artist_size
+	on artist(size);
+
+alter table playlist
+	add size integer default 0 not null;
+create index if not exists playlist_size
+	on playlist(size);
+
+update playlist set size = ifnull((
+    select sum(size)
+    from media_file f
+             left join playlist_tracks pt on f.id = pt.media_file_id
+    where pt.playlist_id = playlist.id
+), 0);`)
+
+	if err != nil {
+		return err
+	}
+	notice(tx, "A full rescan will be performed to calculate artists (discographies) and playlists sizes.")
+	return forceFullRescan(tx)
+}
+
+func Down20201012210022(tx *sql.Tx) error {
+	return nil
+}

db/migration/20201021085410_add_mbids.go

@@ -0,0 +1,58 @@
+package migration
+
+import (
+	"database/sql"
+
+	"github.com/pressly/goose"
+)
+
+func init() {
+	goose.AddMigration(Up20201021085410, Down20201021085410)
+}
+
+func Up20201021085410(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+alter table media_file
+	add mbz_track_id varchar(255);
+alter table media_file
+	add mbz_album_id varchar(255);
+alter table media_file
+	add mbz_artist_id varchar(255);
+alter table media_file
+	add mbz_album_artist_id varchar(255);
+alter table media_file
+	add mbz_album_type varchar(255);
+alter table media_file
+	add mbz_album_comment varchar(255);
+alter table media_file
+	add catalog_num varchar(255);
+
+alter table album
+	add mbz_album_id varchar(255);
+alter table album
+	add mbz_album_artist_id varchar(255);
+alter table album
+	add mbz_album_type varchar(255);
+alter table album
+	add mbz_album_comment varchar(255);
+alter table album
+	add catalog_num varchar(255);
+
+create index if not exists album_mbz_album_type
+	on album (mbz_album_type);
+
+alter table artist
+	add mbz_artist_id varchar(255);
+
+`)
+	if err != nil {
+		return err
+	}
+	notice(tx, "A full rescan needs to be performed to import more tags")
+	return forceFullRescan(tx)
+}
+
+func Down20201021085410(tx *sql.Tx) error {
+	// This code is executed when the migration is rolled back.
+	return nil
+}

db/migration/20201021093209_add_media_file_indexes.go

@@ -0,0 +1,27 @@
+package migration
+
+import (
+	"database/sql"
+
+	"github.com/pressly/goose"
+)
+
+func init() {
+	goose.AddMigration(Up20201021093209, Down20201021093209)
+}
+
+func Up20201021093209(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+create index if not exists media_file_artist
+	on media_file (artist);
+create index if not exists media_file_album_artist
+	on media_file (album_artist);
+create index if not exists media_file_mbz_track_id
+	on media_file (mbz_track_id);
+`)
+	return err
+}
+
+func Down20201021093209(tx *sql.Tx) error {
+	return nil
+}

db/migration/20201021135455_add_media_file_artist_index.go

@@ -0,0 +1,23 @@
+package migration
+
+import (
+	"database/sql"
+
+	"github.com/pressly/goose"
+)
+
+func init() {
+	goose.AddMigration(Up20201021135455, Down20201021135455)
+}
+
+func Up20201021135455(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+create index if not exists media_file_artist_id
+	on media_file (artist_id);
+`)
+	return err
+}
+
+func Down20201021135455(tx *sql.Tx) error {
+	return nil
+}

db/migration/20201030162009_add_artist_info_table.go

@@ -0,0 +1,35 @@
+package migration
+
+import (
+	"database/sql"
+
+	"github.com/pressly/goose"
+)
+
+func init() {
+	goose.AddMigration(upAddArtistImageUrl, downAddArtistImageUrl)
+}
+
+func upAddArtistImageUrl(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+alter table artist
+	add biography varchar(255) default '' not null;
+alter table artist
+	add small_image_url varchar(255) default '' not null;
+alter table artist
+	add medium_image_url varchar(255) default '' not null;
+alter table artist
+	add large_image_url varchar(255) default '' not null;
+alter table artist
+	add similar_artists varchar(255) default '' not null;
+alter table artist
+	add external_url varchar(255) default '' not null;
+alter table artist
+	add external_info_updated_at datetime;
+`)
+	return err
+}
+
+func downAddArtistImageUrl(tx *sql.Tx) error {
+	return nil
+}

db/migration/migration.go

@@ -33,7 +33,7 @@ var once sync.Once
 
 func isDBInitialized(tx *sql.Tx) (initialized bool) {
 	once.Do(func() {
-		rows, err := tx.Query("select count(*) from property where id='" + consts.InitialSetupFlagKey + "'")
+		rows, err := tx.Query("select count(*) from property where id=?", consts.InitialSetupFlagKey)
 		checkErr(err)
 		initialized = checkCount(rows) > 0
 	})

go.mod

@@ -18,35 +18,35 @@ require (
 	github.com/go-chi/cors v1.1.1
 	github.com/go-chi/httprate v0.4.0
 	github.com/go-chi/jwtauth v4.0.4+incompatible
+	github.com/golangci/golangci-lint v1.32.1
 	github.com/google/uuid v1.1.2
 	github.com/google/wire v0.4.0
+	github.com/karrick/godirwalk v1.16.1
 	github.com/kennygrant/sanitize v0.0.0-20170120101633-6a0bfdde8629
+	github.com/kr/pretty v0.2.1
 	github.com/lib/pq v1.3.0 // indirect
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible
 	github.com/microcosm-cc/bluemonday v1.0.4
 	github.com/mitchellh/mapstructure v1.3.2 // indirect
-	github.com/onsi/ginkgo v1.14.1
-	github.com/onsi/gomega v1.10.2
+	github.com/oklog/run v1.1.0
+	github.com/onsi/ginkgo v1.14.2
+	github.com/onsi/gomega v1.10.3
 	github.com/pelletier/go-toml v1.8.0 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pressly/goose v2.6.0+incompatible
 	github.com/sirupsen/logrus v1.7.0
 	github.com/spf13/afero v1.3.1 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
-	github.com/spf13/cobra v1.0.0
+	github.com/spf13/cobra v1.1.1
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.7.1
 	github.com/unrolled/secure v1.0.8
+	github.com/xrash/smetrics v0.0.0-20200730060457-89a2a8a1fb0b
 	github.com/ziutek/mymysql v1.5.4 // indirect
 	golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8
-	golang.org/x/net v0.0.0-20200625001655-4c5254603344 // indirect
-	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect
-	golang.org/x/text v0.3.3 // indirect
-	golang.org/x/tools v0.0.0-20200117065230-39095c1d176c
+	golang.org/x/tools v0.0.0-20201013201025-64a9e34f3752
 	google.golang.org/protobuf v1.25.0 // indirect
-	gopkg.in/djherbis/atime.v1 v1.0.0 // indirect
-	gopkg.in/djherbis/stream.v1 v1.3.1 // indirect
+	gopkg.in/djherbis/atime.v1 v1.0.0
+	gopkg.in/djherbis/stream.v1 v1.3.1
 	gopkg.in/ini.v1 v1.57.0 // indirect
 )
 

go.sum

@@ -1,3 +1,5 @@
+4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a h1:wFEQiK85fRsEVF0CRrPAos5LoAryUsIX1kPW/WrIqFw=
+4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a/go.mod h1:wfdC5ZjKSPr7CybKEcgJhUOgeAQW1+7WcyK8OvUilfo=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
@@ -14,18 +16,25 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5 h1:XTrzB+F8+SpRmbhAH8HLxhiiG6nYNwaBZjrFps1oWEk=
+github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/Masterminds/squirrel v1.4.0 h1:he5i/EXixZxrBUWcxzDYMiju9WZ3ld/l7QBNuo/eN3w=
 github.com/Masterminds/squirrel v1.4.0/go.mod h1:yaPeOnPG5ZRwL9oKdTsO/prlkPbXWZlRVMQ/gGlzIuA=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us=
+github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
 github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
+github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/astaxie/beego v1.12.2 h1:CajUexhSX5ONWDiSCpeQBNVfTzOtPb9e9d+3vuU5FuU=
@@ -40,6 +49,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/bombsimon/wsl/v3 v3.1.0 h1:E5SRssoBgtVFPcYWUOFJEcgaySgdtTNYzsSKDOY7ss8=
+github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
 github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60=
 github.com/bradleyjkemp/cupaloy v2.3.0+incompatible h1:UafIjBvWQmS9i/xRg+CamMrnLTKNzo+bdmT/oH34c2Y=
 github.com/bradleyjkemp/cupaloy v2.3.0+incompatible/go.mod h1:Au1Xw1sgaJ5iSFktEhYsS0dbQiS1B0/XMXl+42y9Ilk=
@@ -56,9 +67,7 @@ github.com/chris-ramon/douceur v0.2.0/go.mod h1:wDW5xjJdeoMm1mRt4sD4c/LbF/mWdEpR
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
@@ -66,16 +75,21 @@ github.com/couchbase/go-couchbase v0.0.0-20200519150804-63f3cdb75e0d/go.mod h1:T
 github.com/couchbase/gomemcached v0.0.0-20200526233749-ec430f949808/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c=
 github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76 h1:Lgdd/Qp96Qj8jqLpq2cI1I1X7BJnu06efS+XkhRoLUQ=
 github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY=
+github.com/daixiang0/gci v0.2.4 h1:BUCKk5nlK2m+kRIsoj+wb/5hazHvHeZieBKWd9Afa8Q=
+github.com/daixiang0/gci v0.2.4/go.mod h1:+AV8KmHTGxxwp/pY84TLQfFKp2vuKXXJVzF3kD/hfR4=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/deluan/rest v0.0.0-20200327222046-b71e558c45d0 h1:qHX6TTBIsrpg0JkYNkoePIpstV37lhRVLj23bHUDNwk=
 github.com/deluan/rest v0.0.0-20200327222046-b71e558c45d0/go.mod h1:tSgDythFsl0QgS/PFWfIZqcJKnkADWneY80jaVRlqK8=
+github.com/denis-tingajkin/go-header v0.3.1 h1:ymEpSiFjeItCy1FOP+x0M2KdCELdEAHUsNa8F+hHc6w=
+github.com/denis-tingajkin/go-header v0.3.1/go.mod h1:sq/2IxMhaZX+RRcgHfCRx/m0M5na0fBt4/CRe7Lrji0=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
@@ -92,6 +106,8 @@ github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
+github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
@@ -108,15 +124,43 @@ github.com/go-chi/httprate v0.4.0 h1:M2qVV0w6ksgLs6L8lTrvqNeaVm0ZJNVdbYM8u2T8HaE
 github.com/go-chi/httprate v0.4.0/go.mod h1:7e7qjQtHzEbdyW5TYQrl4X2uNRCnlTajictc7B4ftgc=
 github.com/go-chi/jwtauth v4.0.4+incompatible h1:LGIxg6YfvSBzxU2BljXbrzVc1fMlgqSKBQgKOGAVtPY=
 github.com/go-chi/jwtauth v4.0.4+incompatible/go.mod h1:Q5EIArY/QnD6BdS+IyDw7B2m6iNbnPxtfd6/BcmtWbs=
+github.com/go-critic/go-critic v0.5.2 h1:3RJdgf6u4NZUumoP8nzbqiiNT8e1tC2Oc7jlgqre/IA=
+github.com/go-critic/go-critic v0.5.2/go.mod h1:cc0+HvdE3lFpqLecgqMaJcvWWH77sLdBp+wLGPM1Yyo=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
+github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
+github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
+github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
+github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
+github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
+github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
+github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
+github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
+github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
+github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
+github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
+github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
+github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
+github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
+github.com/go-toolsmith/typep v1.0.2 h1:8xdsa1+FSIH/RhEkgnD1j2CJOy5mNllW1Q9tRiYwvlk=
+github.com/go-toolsmith/typep v1.0.2/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
+github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b h1:khEcpUM4yFcxg4/FHQWkvVRmgijNXRfzkIDHh23ggEo=
+github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gofrs/flock v0.8.0 h1:MSdYClljsF3PbENUUEx85nkWfJSGfzYI9yEBZOJz6CY=
+github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -139,6 +183,36 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
+github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8=
+github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
+github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy7WKgLXmpQ5bHTrq5GDsp8R9Qs67g0=
+github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a h1:iR3fYXUjHCR97qWS8ch1y9zPNsgXThGwjKPrYfqMPks=
+github.com/golangci/gofmt v0.0.0-20190930125516-244bba706f1a/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.32.1 h1:XaDrjRo5VmoAwhCTKKlE2EpjWmrAoK2qaJ3xoooqFmw=
+github.com/golangci/golangci-lint v1.32.1/go.mod h1:8lqePWOtRXUYRU0BpoPyp+uZCYKMWxxCLEPBMto6HUg=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
+github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA=
+github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
+github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770 h1:EL/O5HGrF7Jaq0yNhBLucz9hTuRzj2LdwGBOaENgxIk=
+github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
+github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1aiNg8JbBMO2+E1WIQ+j/gL4SQqGPg=
+github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -149,6 +223,9 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -156,18 +233,26 @@ github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OI
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/subcommands v1.0.1 h1:/eqq+otEXm5vhfBrbREPCSVQbvofip6kIz+mX5TUH7k=
 github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/wire v0.4.0 h1:kXcsA/rIGzJImVqPdhfnr6q0xsS9gU0515q1EPpJ9fE=
 github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/gookit/color v1.2.5/go.mod h1:AhIE+pS6D4Ql0SQWbBeXPHw7gY0/sjHoA4s/n1KB7xg=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
 github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
-github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/gostaticanalysis/analysisutil v0.0.3 h1:iwp+5/UAyzQSFgQ4uR2sni99sJ8Eo9DEacKWM5pekIg=
+github.com/gostaticanalysis/analysisutil v0.0.3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/gostaticanalysis/analysisutil v0.1.0 h1:E4c8Y1EQURbBEAHoXc/jBTK7Np14ArT8NPUiSFOl9yc=
+github.com/gostaticanalysis/analysisutil v0.1.0/go.mod h1:dMhHRU9KTiDcuLGdy87/2gTR8WruwYZrKdRq9m1O6uw=
+github.com/gostaticanalysis/comment v1.3.0 h1:wTVgynbFu8/nz6SGgywA0TcyIoAVsYc7ai/Zp5xNGlw=
+github.com/gostaticanalysis/comment v1.3.0/go.mod h1:xMicKDx7XRXYdVwY9f9wQpDJVnqWxw9wCauCMKp+IBI=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
@@ -198,6 +283,12 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a h1:GmsqmapfzSJkm28dhRoHz2tLRbJmqhU86IPgBtN3mmk=
+github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s=
+github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3 h1:jNYPNLe3d8smommaoQlK7LOA5ESyUJJ+Wf79ZtA7Vp4=
+github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/jmoiron/sqlx v1.2.1-0.20190826204134-d7d95172beb5/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -205,21 +296,33 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
+github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kennygrant/sanitize v0.0.0-20170120101633-6a0bfdde8629 h1:m1E9veL+2sjZOMSM7y3a6jJ9fNVaGyIJCXYDPm9U+/0=
 github.com/kennygrant/sanitize v0.0.0-20170120101633-6a0bfdde8629/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kyoh86/exportloopref v0.1.7 h1:u+iHuTbkbTS2D/JP7fCuZDo/t3rBVGo3Hf58Rc+lQVY=
+github.com/kyoh86/exportloopref v0.1.7/go.mod h1:h1rDl2Kdj97+Kwh4gdz3ujE7XHmH51Q0lUiZ1z4NLj8=
 github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
 github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
 github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
@@ -228,16 +331,30 @@ github.com/ledisdb/ledisdb v0.0.0-20200510135210-d35789ec47e6/go.mod h1:n931TsDu
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
 github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
-github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/maratori/testpackage v1.0.1 h1:QtJ5ZjqapShm0w5DosRjg0PRlSdAdlx+W6cCKoALdbQ=
+github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU=
+github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUCe2BNSOz4tVy2yGyXhvYDvxGgeE=
+github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mbilski/exhaustivestruct v1.1.0 h1:4ykwscnAFeHJruT+EY3M3vdeP8uXMh0VV2E61iR7XD8=
+github.com/mbilski/exhaustivestruct v1.1.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc=
 github.com/microcosm-cc/bluemonday v1.0.4 h1:p0L+CTpo/PLFdkoPcJemLXG+fpMD7pYOoDEq1axMbGg=
 github.com/microcosm-cc/bluemonday v1.0.4/go.mod h1:8iwZnFn2CDDNZ0r6UXhF4xawGvzaqzCRa1n3/lO3W2w=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -245,6 +362,7 @@ github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceT
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
@@ -257,32 +375,47 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/moricho/tparallel v0.2.1 h1:95FytivzT6rYzdJLdtfn6m1bfFJylOJK41+lgv/EHf4=
+github.com/moricho/tparallel v0.2.1/go.mod h1:fXEIZxG2vdfl0ZF8b42f5a78EhjjD5mX8qUplsoSU4k=
+github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nakabonne/nestif v0.3.0 h1:+yOViDGhg8ygGrmII72nV9B/zGxY188TYpfolntsaPw=
+github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c=
+github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d h1:AREM5mwr4u1ORQBMvzfzBgpsctsbQikCVpvC+tX285E=
+github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nishanths/exhaustive v0.1.0 h1:kVlMw8h2LHPMGUVqUj6230oQjjTMFjwcZrnkhXzFfl8=
+github.com/nishanths/exhaustive v0.1.0/go.mod h1:S1j9110vxV1ECdCudXRkeMnFQ/DQk9ajLT0Uf2MYZQQ=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/ogier/pflag v0.0.1 h1:RW6JSWSu/RkSatfcLtogGfFgpim5p7ARQ10ECk5O750=
 github.com/ogier/pflag v0.0.1/go.mod h1:zkFki7tvTa0tafRvTBIZTvzYyAu6kQhPZFnshFFPE+g=
+github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
+github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
 github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
-github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4=
-github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
+github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M=
+github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs=
-github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA=
+github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.0.1/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw=
 github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs=
 github.com/peterh/liner v1.0.1-0.20171122030339-3681c2a91233/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc=
+github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d h1:CdDQnGF8Nq9ocOS/xlSptM1N3BbrA6/kmaep5ggwaIA=
+github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -291,6 +424,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/polyfloyd/go-errorlint v0.0.0-20201006195004-351e25ade6e3 h1:Amgs0nbayPhBNGh1qPqqr2e7B2qNAcBgRjnBH/lmn8k=
+github.com/polyfloyd/go-errorlint v0.0.0-20201006195004-351e25ade6e3/go.mod h1:wi9BfjxjF/bwiZ701TzmfKu6UKC357IOAtNr0Td0Lvw=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/pressly/goose v2.6.0+incompatible h1:3f8zIQ8rfgP9tyI0Hmcs2YNAqUCL1c+diLe3iU8Qd/k=
 github.com/pressly/goose v2.6.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8=
@@ -315,14 +450,35 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
 github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
+github.com/quasilyte/go-ruleguard v0.2.0 h1:UOVMyH2EKkxIfzrULvA9n/tO+HtEhqD9mrLSWMr5FwU=
+github.com/quasilyte/go-ruleguard v0.2.0/go.mod h1:2RT/tf0Ce0UDj5y243iWKosQogJd8+1G3Rs2fxmlYnw=
+github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95 h1:L8QM9bvf68pVdQ3bCFZMDmnt9yqcMBro1pC7F+IPYMY=
+github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryancurrah/gomodguard v1.1.0 h1:DWbye9KyMgytn8uYpuHkwf0RHqAYO6Ay/D0TbCpPtVU=
+github.com/ryancurrah/gomodguard v1.1.0/go.mod h1:4O8tr7hBODaGE6VIhfJDHcwzh5GUccKSJBU0UMXJFVM=
+github.com/ryanrolds/sqlclosecheck v0.3.0 h1:AZx+Bixh8zdUBxUA1NxbxVAS78vTPq4rCb8OUZI9xFw=
+github.com/ryanrolds/sqlclosecheck v0.3.0/go.mod h1:1gREqxyTGR3lVtpngyFo3hZAgk0KCtEdgEkHwDbigdA=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/securego/gosec/v2 v2.4.0 h1:ivAoWcY5DMs9n04Abc1VkqZBO0FL0h4ShTcVsC53lCE=
+github.com/securego/gosec/v2 v2.4.0/go.mod h1:0/Q4cjmlFDfDUj1+Fib61sc+U5IQb2w+Iv9/C3wPVko=
+github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=
+github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs=
 github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 h1:X+yvsM2yrEktyI+b2qND5gpH8YhURn0k8OCaeRnkINo=
 github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644/go.mod h1:nkxAfR/5quYxwPZhyDxgasBMnRtBZd0FCEpawpjMUFg=
+github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/siddontang/go v0.0.0-20170517070808-cb568a3e5cc0 h1:QIF48X1cihydXibm+4wfAc0r/qyPyuFiPFRNphdMpEE=
 github.com/siddontang/go v0.0.0-20170517070808-cb568a3e5cc0/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw=
 github.com/siddontang/goredis v0.0.0-20150324035039-760763f78400/go.mod h1:DDcKzU3qCuvj/tPnimWSsZZzvk9qvkvrIL5naVBPh5s=
 github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d h1:NVwnfyR3rENtlz62bcrkXME3INVUa4lcdGt+opvxExs=
@@ -330,6 +486,7 @@ github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
@@ -337,6 +494,10 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/sonatard/noctx v0.0.1 h1:VC1Qhl6Oxx9vvWo3UDgrGXYCeKCe3Wbw7qAWL6FrmTY=
+github.com/sonatard/noctx v0.0.1/go.mod h1:9D2D/EoULe8Yy2joDHJj7bv3sZoq9AaSb8B4lqBjiZI=
+github.com/sourcegraph/go-diff v0.6.1 h1:hmA1LzxW0n1c3Q4YbrFgg4P99GSnebYa3x8gr0HZqLQ=
+github.com/sourcegraph/go-diff v0.6.1/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
@@ -346,8 +507,8 @@ github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
-github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
+github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
+github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
@@ -356,34 +517,63 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
-github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk=
 github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/ssdb/gossdb v0.0.0-20180723034631-88f6b59b84ec/go.mod h1:QBvMkMya+gXctz3kmljlUCu/yB3GZ6oee+dUozsezQE=
+github.com/ssgreg/nlreturn/v2 v2.1.0 h1:6/s4Rc49L6Uo6RLjhWZGBpWWjfzk2yrf1nIW8m4wgVA=
+github.com/ssgreg/nlreturn/v2 v2.1.0/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/goleveldb v0.0.0-20160425020131-cfa635847112/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
 github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c h1:3eGShk3EQf5gJCYW+WzA0TEJQd37HLOmlYF7N0YJwv0=
 github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
+github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2 h1:Xr9gkxfOP0KQWXKNqmwe8vEeSUiUj4Rlee9CMVX2ZUQ=
+github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM=
+github.com/tetafro/godot v0.4.9 h1:dSOiuasshpevY73eeI3+zaqFnXSBKJ3mvxbyhh54VRo=
+github.com/tetafro/godot v0.4.9/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQxA5S+0=
+github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q=
+github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tomarrell/wrapcheck v0.0.0-20200807122107-df9e8bcb914d h1:3EZyvNUMsGD1QA8cu0STNn1L7I77rvhf2IhOcHYQhSw=
+github.com/tomarrell/wrapcheck v0.0.0-20200807122107-df9e8bcb914d/go.mod h1:yiFB6fFoV7saXirUGfuK+cPtUh4NX/Hf5y2WC2lehu0=
+github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa h1:RC4maTWLKKwb7p1cnoygsbKIgNlJqSYBeAFON3Ar8As=
+github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig=
 github.com/ugorji/go v0.0.0-20171122102828-84cb69a8af83/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
-github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/ultraware/funlen v0.0.3 h1:5ylVWm8wsNwH5aWo9438pwvsK0QiqVuUrt9bn7S/iLA=
+github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
+github.com/ultraware/whitespace v0.0.4 h1:If7Va4cM03mpgrNH9k49/VOicWpGoG70XPBFFODYDsg=
+github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
 github.com/unrolled/secure v1.0.8 h1:JaMvKbe4CRt8oyxVXn+xY+6jlqd7pyJNSVkmsBxxQsM=
 github.com/unrolled/secure v1.0.8/go.mod h1:fO+mEan+FLB0CdEnHf6Q4ZZVNqG+5fuLFnP8p0BXDPI=
 github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc=
 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
+github.com/uudashr/gocognit v1.0.1 h1:MoG2fZ0b/Eo7NXoIwCVFLG5JED3qgQz5/NEE+rOsjPs=
+github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA=
+github.com/valyala/quicktemplate v1.6.3/go.mod h1:fwPzK2fHuYEODzJ9pkw0ipCPNHZ2tD5KW4lOuSdPKzY=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/wader/tag v0.0.0-20200426234345-d072771f6a51 h1:WAxntH7YQD6fIboAvewi7eU+2PQ7Y1K9OOXh67CM4bY=
 github.com/wader/tag v0.0.0-20200426234345-d072771f6a51/go.mod h1:f3YqVk9PEeVf7T4JQ2+TdRqqjTg2fkaROZv0EMQOuKo=
 github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/xrash/smetrics v0.0.0-20200730060457-89a2a8a1fb0b h1:tnWgqoOBmInkt5pbLjagwNVjjT4RdJhFHzL1ebCSRh8=
+github.com/xrash/smetrics v0.0.0-20200730060457-89a2a8a1fb0b/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
+github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/gopher-lua v0.0.0-20171031051903-609c9cd26973/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU=
 github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs=
 github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
@@ -424,6 +614,9 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -438,14 +631,18 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 h1:wBouT66WTYFXdxfVdz9sVWARVd/2vfGcmI45D2gj45M=
+golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -456,6 +653,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -465,6 +663,7 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
@@ -478,13 +677,18 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634 h1:bNEHhJCnrwMKNMmOx3yAynp5vs5/gRy+XWFtZFu7NBM=
+golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -495,12 +699,18 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190307163923-6a08e3108db3/go.mod h1:25r3+/G6/xytQM8iWZKq3Hn0kr0rgFKPUNVEL/dr3z4=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b h1:NVD8gBK33xpdqCaZVVtd6OFJp+3dxkXuz7+U7KaVN6s=
@@ -512,15 +722,41 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200117065230-39095c1d176c h1:FodBYPZKH5tAN2O60HlglMwXGAeV/4k+NKbli79M/2c=
 golang.org/x/tools v0.0.0-20200117065230-39095c1d176c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117220505-0cba7a3a9ee9/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200321224714-0d839f3cf2ed/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200324003944-a576cf524670/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
+golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200414032229-332987a829c3/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200422022333-3d57cf2e726e/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200624225443-88f3c62a19ff/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200625211823-6506e20df31f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200701041122-1837592efa10/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200724022722-7017fd6b1305/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200731060945-b5fad4ed8dd6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200812195022-5ae4c3c160a0 h1:SQvH+DjrwqD1hyyQU+K7JegHz1KEZgEwt17p9d6R2eg=
+golang.org/x/tools v0.0.0-20200812195022-5ae4c3c160a0/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
+golang.org/x/tools v0.0.0-20200831203904-5a2aa26beb65/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
+golang.org/x/tools v0.0.0-20201001104356-43ebab892c4c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20201002184944-ecd9fd270d5d/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20201011145850-ed2f50202694/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20201013201025-64a9e34f3752 h1:2ntEwh02rqo2jSsrYmp4yKHHjh0CbXP3ZtSUetSB+q8=
+golang.org/x/tools v0.0.0-20201013201025-64a9e34f3752/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -542,7 +778,6 @@ google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvx
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
@@ -589,9 +824,21 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+honnef.co/go/tools v0.0.1-2020.1.6 h1:W18jzjh8mfPez+AwGLxmOImucz/IFjpNlrKVnaj2YVc=
+honnef.co/go/tools v0.0.1-2020.1.6/go.mod h1:pyyisuGw24ruLjrr1ddx39WE0y9OooInRzEYLhQB2YY=
+mvdan.cc/gofumpt v0.0.0-20200802201014-ab5a8192947d h1:t8TAw9WgTLghti7RYkpPmqk4JtQ3+wcP5GgZqgWeWLQ=
+mvdan.cc/gofumpt v0.0.0-20200802201014-ab5a8192947d/go.mod h1:bzrjFmaD6+xqohD3KYP0H2FEuxknnBmyyOxdhLdaIws=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
+mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7 h1:kAREL6MPwpsk1/PQPFD3Eg7WAQR5mPTWZJaBiG5LDbY=
+mvdan.cc/unparam v0.0.0-20200501210554-b37ab49443f7/go.mod h1:HGC5lll35J70Y5v7vCGb9oLhHoScFwkHDJm/05RdSTc=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

model/album.go

@@ -9,9 +9,9 @@ type Album struct {
 	Name                 string    `json:"name"`
 	CoverArtPath         string    `json:"coverArtPath"`
 	CoverArtId           string    `json:"coverArtId"`
-	ArtistID             string    `json:"artistId"      orm:"pk;column(artist_id)"`
+	ArtistID             string    `json:"artistId"      orm:"column(artist_id)"`
 	Artist               string    `json:"artist"`
-	AlbumArtistID        string    `json:"albumArtistId" orm:"pk;column(album_artist_id)"`
+	AlbumArtistID        string    `json:"albumArtistId" orm:"column(album_artist_id)"`
 	AlbumArtist          string    `json:"albumArtist"`
 	MaxYear              int       `json:"maxYear"`
 	MinYear              int       `json:"minYear"`
@@ -25,8 +25,14 @@ type Album struct {
 	SortAlbumArtistName  string    `json:"sortAlbumArtistName"`
 	OrderAlbumName       string    `json:"orderAlbumName"`
 	OrderAlbumArtistName string    `json:"orderAlbumArtistName"`
+	CatalogNum           string    `json:"catalogNum"`
+	MbzAlbumID           string    `json:"mbzAlbumId"         orm:"column(mbz_album_id)"`
+	MbzAlbumArtistID     string    `json:"mbzAlbumArtistId"   orm:"column(mbz_album_artist_id)"`
+	MbzAlbumType         string    `json:"mbzAlbumType"`
+	MbzAlbumComment      string    `json:"mbzAlbumComment"`
 	CreatedAt            time.Time `json:"createdAt"`
 	UpdatedAt            time.Time `json:"updatedAt"`
+	Size                 int64     `json:"size"`
 }
 
 type Albums []Album

model/artist.go

@@ -1,15 +1,36 @@
 package model
 
+import "time"
+
 type Artist struct {
 	Annotations
 
-	ID              string `json:"id"          orm:"column(id)"`
-	Name            string `json:"name"`
-	AlbumCount      int    `json:"albumCount"`
-	SongCount       int    `json:"songCount"`
-	FullText        string `json:"fullText"`
-	SortArtistName  string `json:"sortArtistName"`
-	OrderArtistName string `json:"orderArtistName"`
+	ID                    string    `json:"id"               orm:"column(id)"`
+	Name                  string    `json:"name"`
+	AlbumCount            int       `json:"albumCount"`
+	SongCount             int       `json:"songCount"`
+	FullText              string    `json:"fullText"`
+	SortArtistName        string    `json:"sortArtistName"`
+	OrderArtistName       string    `json:"orderArtistName"`
+	Size                  int64     `json:"size"`
+	MbzArtistID           string    `json:"mbzArtistId"      orm:"column(mbz_artist_id)"`
+	Biography             string    `json:"biography"`
+	SmallImageUrl         string    `json:"smallImageUrl"`
+	MediumImageUrl        string    `json:"mediumImageUrl"`
+	LargeImageUrl         string    `json:"largeImageUrl"`
+	ExternalUrl           string    `json:"externalUrl"      orm:"column(external_url)"`
+	SimilarArtists        Artists   `json:"-"   orm:"-"`
+	ExternalInfoUpdatedAt time.Time `json:"externalInfoUpdatedAt"`
+}
+
+func (a Artist) ArtistImageUrl() string {
+	if a.MediumImageUrl != "" {
+		return a.MediumImageUrl
+	}
+	if a.LargeImageUrl != "" {
+		return a.LargeImageUrl
+	}
+	return a.SmallImageUrl
 }
 
 type Artists []Artist
@@ -25,6 +46,7 @@ type ArtistRepository interface {
 	Exists(id string) (bool, error)
 	Put(m *Artist) error
 	Get(id string) (*Artist, error)
+	GetAll(options ...QueryOptions) (Artists, error)
 	GetStarred(options ...QueryOptions) (Artists, error)
 	Search(q string, offset int, size int) (Artists, error)
 	Refresh(ids ...string) error

model/artist_info.go

@@ -0,0 +1,13 @@
+package model
+
+type ArtistInfo struct {
+	ID             string
+	Name           string
+	MBID           string
+	Biography      string
+	SmallImageUrl  string
+	MediumImageUrl string
+	LargeImageUrl  string
+	LastFMUrl      string
+	SimilarArtists Artists
+}

model/errors.go

@@ -6,4 +6,5 @@ var (
 	ErrNotFound      = errors.New("data not found")
 	ErrInvalidAuth   = errors.New("invalid authentication")
 	ErrNotAuthorized = errors.New("not authorized")
+	ErrNotAvailable  = errors.New("functionality not available")
 )

model/mediafile.go

@@ -37,6 +37,13 @@ type MediaFile struct {
 	OrderArtistName      string    `json:"orderArtistName"`
 	OrderAlbumArtistName string    `json:"orderAlbumArtistName"`
 	Compilation          bool      `json:"compilation"`
+	CatalogNum           string    `json:"catalogNum"`
+	MbzTrackID           string    `json:"mbzTrackId"         orm:"column(mbz_track_id)"`
+	MbzAlbumID           string    `json:"mbzAlbumId"         orm:"column(mbz_album_id)"`
+	MbzArtistID          string    `json:"mbzArtistId"        orm:"column(mbz_artist_id)"`
+	MbzAlbumArtistID     string    `json:"mbzAlbumArtistId"   orm:"column(mbz_album_artist_id)"`
+	MbzAlbumType         string    `json:"mbzAlbumType"`
+	MbzAlbumComment      string    `json:"mbzAlbumComment"`
 	CreatedAt            time.Time `json:"createdAt"`
 	UpdatedAt            time.Time `json:"updatedAt"`
 }

model/mediafolder.go

@@ -9,6 +9,6 @@ type MediaFolder struct {
 type MediaFolders []MediaFolder
 
 type MediaFolderRepository interface {
-	Get(id string) (*MediaFolder, error)
+	Get(id int32) (*MediaFolder, error)
 	GetAll() (MediaFolders, error)
 }

model/playlist.go

@@ -9,6 +9,7 @@ type Playlist struct {
 	Name      string     `json:"name"`
 	Comment   string     `json:"comment"`
 	Duration  float32    `json:"duration"`
+	Size      int64      `json:"size"`
 	SongCount int        `json:"songCount"`
 	Owner     string     `json:"owner"`
 	Public    bool       `json:"public"`

persistence/album_repository.go

@@ -161,10 +161,12 @@ func (r *albumRepository) refresh(ids ...string) error {
 	sel := Select(`f.album_id as id, f.album as name, f.artist, f.album_artist, f.artist_id, f.album_artist_id, 
 		f.sort_album_name, f.sort_artist_name, f.sort_album_artist_name,
 		f.order_album_name, f.order_album_artist_name, f.path,
-		f.compilation, f.genre, max(f.year) as max_year, sum(f.duration) as duration, 
+		group_concat(f.mbz_album_id, ' ') as mbz_album_id, f.mbz_album_artist_id, f.mbz_album_type, f.mbz_album_comment, 
+		f.catalog_num, f.compilation, f.genre, max(f.year) as max_year, sum(f.duration) as duration, 
 		count(f.id) as song_count, a.id as current_id, 
 		group_concat(f.disc_subtitle, ' ') as disc_subtitles,
-		group_concat(f.artist, ' ') as song_artists, group_concat(f.year, ' ') as years`).
+		group_concat(f.artist, ' ') as song_artists, group_concat(f.year, ' ') as years,
+		sum(f.size) as size`).
 		From("media_file f").
 		LeftJoin("album a on f.album_id = a.id").
 		Where(Eq{"f.album_id": ids}).GroupBy("f.album_id")
@@ -209,6 +211,7 @@ func (r *albumRepository) refresh(ids ...string) error {
 			al.AlbumArtistID = al.ArtistID
 		}
 		al.MinYear = getMinYear(al.Years)
+		al.MbzAlbumID = getMbzId(r.ctx, al.MbzAlbumID, r.tableName, al.Name)
 		al.UpdatedAt = time.Now()
 		if al.CurrentId != "" {
 			toUpdate++

persistence/artist_repository.go

@@ -2,6 +2,8 @@ package persistence
 
 import (
 	"context"
+	"fmt"
+	"net/url"
 	"sort"
 	"strings"
 
@@ -20,6 +22,11 @@ type artistRepository struct {
 	indexGroups utils.IndexGroups
 }
 
+type dbArtist struct {
+	model.Artist
+	SimilarArtists string `json:"similarArtists"`
+}
+
 func NewArtistRepository(ctx context.Context, o orm.Ormer) model.ArtistRepository {
 	r := &artistRepository{}
 	r.ctx = ctx
@@ -50,29 +57,70 @@ func (r *artistRepository) Exists(id string) (bool, error) {
 
 func (r *artistRepository) Put(a *model.Artist) error {
 	a.FullText = getFullText(a.Name, a.SortArtistName)
-	_, err := r.put(a.ID, a)
+	dba := r.fromModel(a)
+	_, err := r.put(dba.ID, dba)
 	return err
 }
 
 func (r *artistRepository) Get(id string) (*model.Artist, error) {
 	sel := r.selectArtist().Where(Eq{"id": id})
-	var res model.Artists
-	if err := r.queryAll(sel, &res); err != nil {
+	var dba []dbArtist
+	if err := r.queryAll(sel, &dba); err != nil {
 		return nil, err
 	}
-	if len(res) == 0 {
+	if len(dba) == 0 {
 		return nil, model.ErrNotFound
 	}
+	res := r.toModels(dba)
 	return &res[0], nil
 }
 
 func (r *artistRepository) GetAll(options ...model.QueryOptions) (model.Artists, error) {
 	sel := r.selectArtist(options...)
-	res := model.Artists{}
-	err := r.queryAll(sel, &res)
+	var dba []dbArtist
+	err := r.queryAll(sel, &dba)
+	res := r.toModels(dba)
 	return res, err
 }
 
+func (r *artistRepository) toModels(dba []dbArtist) model.Artists {
+	res := model.Artists{}
+	for i := range dba {
+		a := dba[i]
+		res = append(res, *r.toModel(&a))
+	}
+	return res
+}
+
+func (r *artistRepository) toModel(dba *dbArtist) *model.Artist {
+	a := dba.Artist
+	a.SimilarArtists = nil
+	for _, s := range strings.Split(dba.SimilarArtists, ";") {
+		fields := strings.Split(s, ":")
+		if len(fields) != 2 {
+			continue
+		}
+		name, _ := url.QueryUnescape(fields[1])
+		a.SimilarArtists = append(a.SimilarArtists, model.Artist{
+			ID:   fields[0],
+			Name: name,
+		})
+	}
+	return &a
+}
+
+func (r *artistRepository) fromModel(a *model.Artist) *dbArtist {
+	dba := &dbArtist{Artist: *a}
+	var sa []string
+
+	for _, s := range a.SimilarArtists {
+		sa = append(sa, fmt.Sprintf("%s:%s", s.ID, url.QueryEscape(s.Name)))
+	}
+
+	dba.SimilarArtists = strings.Join(sa, ";")
+	return dba
+}
+
 func (r *artistRepository) getIndexKey(a *model.Artist) string {
 	name := strings.ToLower(utils.NoArticle(a.Name))
 	for k, v := range r.indexGroups {
@@ -86,9 +134,7 @@ func (r *artistRepository) getIndexKey(a *model.Artist) string {
 
 // TODO Cache the index (recalculate when there are changes to the DB)
 func (r *artistRepository) GetIndex() (model.ArtistIndexes, error) {
-	sq := r.selectArtist().OrderBy("order_artist_name")
-	var all model.Artists
-	err := r.queryAll(sq, &all)
+	all, err := r.GetAll(model.QueryOptions{Sort: "order_artist_name"})
 	if err != nil {
 		return nil, err
 	}
@@ -132,8 +178,9 @@ func (r *artistRepository) refresh(ids ...string) error {
 	}
 	var artists []refreshArtist
 	sel := Select("f.album_artist_id as id", "f.album_artist as name", "count(*) as album_count", "a.id as current_id",
+		"group_concat(f.mbz_album_artist_id , ' ') as mbz_artist_id",
 		"f.sort_album_artist_name as sort_artist_name", "f.order_album_artist_name as order_artist_name",
-		"sum(f.song_count) as song_count").
+		"sum(f.song_count) as song_count", "sum(f.size) as size").
 		From("album f").
 		LeftJoin("artist a on f.album_artist_id = a.id").
 		Where(Eq{"f.album_artist_id": ids}).
@@ -151,6 +198,7 @@ func (r *artistRepository) refresh(ids ...string) error {
 		} else {
 			toInsert++
 		}
+		ar.MbzArtistID = getMbzId(r.ctx, ar.MbzArtistID, r.tableName, ar.Name)
 		err := r.Put(&ar.Artist)
 		if err != nil {
 			return err
@@ -167,8 +215,9 @@ func (r *artistRepository) refresh(ids ...string) error {
 
 func (r *artistRepository) GetStarred(options ...model.QueryOptions) (model.Artists, error) {
 	sq := r.selectArtist(options...).Where("starred = true")
-	starred := model.Artists{}
-	err := r.queryAll(sq, &starred)
+	var dba []dbArtist
+	err := r.queryAll(sq, &dba)
+	starred := r.toModels(dba)
 	return starred, err
 }
 
@@ -184,9 +233,12 @@ func (r *artistRepository) purgeEmpty() error {
 }
 
 func (r *artistRepository) Search(q string, offset int, size int) (model.Artists, error) {
-	results := model.Artists{}
-	err := r.doSearch(q, offset, size, &results, "name")
-	return results, err
+	var dba []dbArtist
+	err := r.doSearch(q, offset, size, &dba, "name")
+	if err != nil {
+		return nil, err
+	}
+	return r.toModels(dba), nil
 }
 
 func (r *artistRepository) Count(options ...rest.QueryOptions) (int64, error) {

persistence/artist_repository_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/deluan/navidrome/model/request"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gstruct"
 )
 
 var _ = Describe("ArtistRepository", func() {
@@ -69,4 +70,26 @@ var _ = Describe("ArtistRepository", func() {
 			}))
 		})
 	})
+
+	Describe("dbArtist mapping", func() {
+		var a *model.Artist
+		BeforeEach(func() {
+			a = &model.Artist{ID: "1", Name: "Van Halen", SimilarArtists: []model.Artist{
+				{ID: "2", Name: "AC/DC"}, {ID: "-1", Name: "Test;With:Sep,Chars"},
+			}}
+		})
+		It("maps fields", func() {
+			dba := repo.(*artistRepository).fromModel(a)
+			actual := repo.(*artistRepository).toModel(dba)
+			Expect(*actual).To(MatchFields(IgnoreExtras, Fields{
+				"ID":   Equal(a.ID),
+				"Name": Equal(a.Name),
+			}))
+			Expect(actual.SimilarArtists).To(HaveLen(2))
+			Expect(actual.SimilarArtists[0].ID).To(Equal("2"))
+			Expect(actual.SimilarArtists[0].Name).To(Equal("AC/DC"))
+			Expect(actual.SimilarArtists[1].ID).To(Equal("-1"))
+			Expect(actual.SimilarArtists[1].Name).To(Equal("Test;With:Sep,Chars"))
+		})
+	})
 })

persistence/helpers.go

@@ -1,12 +1,15 @@
 package persistence
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"regexp"
 	"strings"
 
 	"github.com/Masterminds/squirrel"
+	"github.com/deluan/navidrome/consts"
+	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
 	"github.com/deluan/navidrome/utils"
 )
@@ -55,3 +58,32 @@ func (e existsCond) ToSql() (string, []interface{}, error) {
 	sql = fmt.Sprintf("exists (select 1 from %s where %s)", e.subTable, sql)
 	return sql, args, err
 }
+
+func getMbzId(ctx context.Context, mbzIDS, entityName, name string) string {
+	ids := strings.Fields(mbzIDS)
+	if len(ids) == 0 {
+		return ""
+	}
+	idCounts := map[string]int{}
+	for _, id := range ids {
+		if c, ok := idCounts[id]; ok {
+			idCounts[id] = c + 1
+		} else {
+			idCounts[id] = 1
+		}
+	}
+
+	var topKey string
+	var topCount int
+	for k, v := range idCounts {
+		if v > topCount {
+			topKey = k
+			topCount = v
+		}
+	}
+
+	if len(idCounts) > 1 && name != consts.VariousArtists {
+		log.Warn(ctx, "Multiple MBIDs found for "+entityName, "name", name, "mbids", idCounts)
+	}
+	return topKey
+}

persistence/helpers_test.go

@@ -1,6 +1,7 @@
 package persistence
 
 import (
+	"context"
 	"time"
 
 	"github.com/Masterminds/squirrel"
@@ -61,4 +62,16 @@ var _ = Describe("Helpers", func() {
 			Expect(err).To(BeNil())
 		})
 	})
+
+	Describe("getMbzId", func() {
+		It(`returns "" when no ids are passed`, func() {
+			Expect(getMbzId(context.TODO(), " ", "", "")).To(Equal(""))
+		})
+		It(`returns the only id passed`, func() {
+			Expect(getMbzId(context.TODO(), "1234 ", "", "")).To(Equal("1234"))
+		})
+		It(`returns the id with higher frequency`, func() {
+			Expect(getMbzId(context.TODO(), "1 2 3 4 1", "", "")).To(Equal("1"))
+		})
+	})
 })

persistence/mediafolders_repository.go

@@ -16,7 +16,7 @@ func NewMediaFolderRepository(ctx context.Context, o orm.Ormer) model.MediaFolde
 	return &mediaFolderRepository{ctx}
 }
 
-func (r *mediaFolderRepository) Get(id string) (*model.MediaFolder, error) {
+func (r *mediaFolderRepository) Get(id int32) (*model.MediaFolder, error) {
 	mediaFolder := hardCoded()
 	return &mediaFolder, nil
 }

persistence/playlist_track_repository.go

@@ -140,19 +140,21 @@ func (r *playlistTrackRepository) Update(mediaFileIds []string) error {
 }
 
 func (r *playlistTrackRepository) updateStats() error {
-	// Get total playlist duration and count
-	statsSql := Select("sum(duration) as duration", "count(*) as count").From("media_file").
+	// Get total playlist duration, size and count
+	statsSql := Select("sum(duration) as duration", "sum(size) as size", "count(*) as count").
+		From("media_file").
 		Join("playlist_tracks f on f.media_file_id = media_file.id").
 		Where(Eq{"playlist_id": r.playlistId})
-	var res struct{ Duration, Count float32 }
+	var res struct{ Duration, Size, Count float32 }
 	err := r.queryOne(statsSql, &res)
 	if err != nil {
 		return err
 	}
 
-	// Update playlist's total duration and count
+	// Update playlist's total duration, size and count
 	upd := Update("playlist").
 		Set("duration", res.Duration).
+		Set("size", res.Size).
 		Set("song_count", res.Count).
 		Set("updated_at", time.Now()).
 		Where(Eq{"id": r.playlistId})

persistence/sql_restful.go

@@ -21,6 +21,9 @@ func (r sqlRestful) parseRestFilters(options rest.QueryOptions) Sqlizer {
 	}
 	filters := And{}
 	for f, v := range options.Filters {
+		if v == "" {
+			continue
+		}
 		if ff, ok := r.filterMappings[f]; ok {
 			filters = append(filters, ff(f, v))
 		} else if f == "id" {

resources/i18n/cs.json

@@ -275,7 +275,8 @@
                 "defaultView": "Výchozí stránka"
             }
         },
-        "albumList": "Alba"
+        "albumList": "Alba",
+        "about": "O"
     },
     "player": {
         "playListsText": "Fronta",
@@ -301,5 +302,12 @@
             "singleLoop": "Opakovat jednou",
             "shufflePlay": "Zamíchat"
         }
+    },
+    "about": {
+        "links": {
+            "homepage": "Domovská stránka",
+            "source": "Zdrojový kód",
+            "featureRequests": "Požadavky o funkce"
+        }
     }
 }

resources/i18n/es.json

@@ -4,7 +4,7 @@
         "song": {
             "name": "Canción ||||  Canciones",
             "fields": {
-                "albumArtist": "Artista del Álbum",
+                "albumArtist": "Artista del álbum",
                 "duration": "Duración",
                 "trackNumber": "#",
                 "playCount": "Reproducciones",
@@ -18,14 +18,14 @@
                 "size": "Tamaño del archivo",
                 "updatedAt": "Actualizado el",
                 "bitRate": "Tasa de bits",
-                "discSubtitle": "Subtítulo del Disco",
+                "discSubtitle": "Subtítulo del disco",
                 "starred": "Favorito"
             },
             "actions": {
-                "addToQueue": "Reproducir Después",
-                "playNow": "Reproducir Ahora",
+                "addToQueue": "Reproducir después",
+                "playNow": "Reproducir ahora",
                 "addToPlaylist": "Agregar a la lista de reproducción",
-                "shuffleAll": "Todas Aleatorias",
+                "shuffleAll": "Todas aleatorias",
                 "download": "Descarga",
                 "playNext": "Siguiente"
             }
@@ -33,7 +33,7 @@
         "album": {
             "name": "Álbum |||| Álbumes",
             "fields": {
-                "albumArtist": "Artista del Álbum",
+                "albumArtist": "Artista del álbum",
                 "artist": "Artista",
                 "duration": "Duración",
                 "songCount": "Canciones",
@@ -46,8 +46,8 @@
             },
             "actions": {
                 "playAll": "Reproducir",
-                "playNext": "Reproducir Siguiente",
-                "addToQueue": "Reproducir Después",
+                "playNext": "Reproducir siguiente",
+                "addToQueue": "Reproducir después",
                 "shuffle": "Aletorio",
                 "addToPlaylist": "Agregar a la lista",
                 "download": "Descargar"
@@ -57,7 +57,7 @@
                 "random": "Aleatorio",
                 "recentlyAdded": "Recientes",
                 "recentlyPlayed": "Recientes",
-                "mostPlayed": "Más Reproducidos",
+                "mostPlayed": "Más reproducidos",
                 "starred": "Favoritos"
             }
         },
@@ -74,8 +74,8 @@
             "name": "Usuario |||| Usuarios",
             "fields": {
                 "userName": "Nombre de usuario",
-                "isAdmin": "Es Administrador",
-                "lastLoginAt": "Último Acceso el",
+                "isAdmin": "Es administrador",
+                "lastLoginAt": "Último acceso el",
                 "updatedAt": "Actualizado el",
                 "name": "Nombre",
                 "password": "Contraseña",
@@ -127,7 +127,7 @@
         "auth": {
             "welcome1": "¡Gracias por instalar Navidrome!",
             "welcome2": "Para empezar, crea un usuario administrador",
-            "confirmPassword": "Confirme la Contraseña",
+            "confirmPassword": "Confirme la contraseña",
             "buttonCreateAdmin": "Crear Admin",
             "auth_check_error": "Por favor inicie sesión para continuar",
             "user_menu": "Perfil",
@@ -135,7 +135,7 @@
             "password": "Contraseña",
             "sign_in": "Acceder",
             "sign_in_error": "La autenticación falló, por favor, vuelva a intentarlo",
-            "logout": "Cerrar Sesión"
+            "logout": "Cerrar sesión"
         },
         "validation": {
             "invalidChars": "Por favor use solo letras y números",
@@ -153,7 +153,7 @@
         "action": {
             "add_filter": "Añadir filtro",
             "add": "Añadir",
-            "back": "Ir Atrás",
+            "back": "Ir atrás",
             "bulk_actions": "1 elemento seleccionado |||| %{smart_count} elementos selecccionados",
             "cancel": "Cancelar",
             "clear_input_value": "Limpiar valor",
@@ -196,12 +196,12 @@
         },
         "input": {
             "file": {
-                "upload_several": "Arrastre algunos archivos para subir, o haga clic para seleccionar uno.",
-                "upload_single": "Arrastre un archivo para subir, o haga clic para seleccionarlo."
+                "upload_several": "Arrastre algunos archivos para subir o haga clic para seleccionar uno.",
+                "upload_single": "Arrastre un archivo para subir o haga clic para seleccionarlo."
             },
             "image": {
-                "upload_several": "Arrastre algunas imagénes para subir, o haga clic para seleccionar una.",
-                "upload_single": "Arrastre alguna imagen para subir, o haga clic para seleccionarla."
+                "upload_several": "Arrastre algunas imagénes para subir o haga clic para seleccionar una.",
+                "upload_single": "Arrastre alguna imagen para subir o haga clic para seleccionarla."
             },
             "references": {
                 "all_missing": "No se pueden encontrar datos de referencias.",
@@ -225,7 +225,7 @@
             "invalid_form": "El formulario no es válido. Por favor verifique si hay errores",
             "loading": "La página se está cargando, espere un momento por favor",
             "no": "No",
-            "not_found": "O bien escribió una URL incorrecta, o siguió un enlace incorrecto.",
+            "not_found": "O bien escribió una URL incorrecta o siguió un enlace incorrecto.",
             "yes": "Sí",
             "unsaved_changes": "Algunos de sus cambios no se guardaron. ¿Está seguro que quiere ignorarlos?"
         },
@@ -275,16 +275,17 @@
                 "defaultView": "Vista por defecto"
             }
         },
-        "albumList": "Álbumes"
+        "albumList": "Álbumes",
+        "about": "Acerca de"
     },
     "player": {
-        "playListsText": "Lista de Reproducción",
+        "playListsText": "Lista de reproducción",
         "openText": "Abrir",
         "closeText": "Cerrar",
         "notContentText": "Sin música",
         "clickToPlayText": "Clic para reproducir",
         "clickToPauseText": "Clic para pausar",
-        "nextTrackText": "Pista Siguiente",
+        "nextTrackText": "Pista siguiente",
         "previousTrackText": "Pista anterior",
         "reloadText": "Refrescar",
         "volumeText": "Volumen",
@@ -298,8 +299,15 @@
         "playModeText": {
             "order": "En orden",
             "orderLoop": "Repetir",
-            "singleLoop": "Repetir Una",
+            "singleLoop": "Repetir una",
             "shufflePlay": "Aleatorio"
         }
+    },
+    "about": {
+        "links": {
+            "homepage": "Página de inicio",
+            "source": "Código fuente",
+            "featureRequests": "Pedir funcionalidad"
+        }
     }
-}
+}

resources/i18n/nl.json

@@ -206,7 +206,7 @@
             "references": {
                 "all_missing": "De gerefereerde elementen konden niet gevonden worden.",
                 "many_missing": "Een of meer van de gerefereerde elementen is niet meer beschikbaar.",
-                "single_missing": "Een van de gerefereerde elementen is niet meer beschikbaar"
+                "single_missing": "Een van de bijbehorende elementen is niet meer beschikbaar"
             },
             "password": {
                 "toggle_visible": "Verberg wachtwoord",
@@ -255,7 +255,7 @@
     },
     "message": {
         "note": "Notitie",
-        "transcodingDisabled": "Het wijzigen van de transcoderingsconfiguratie via de web interface is om veiligheidsredenen uitgeschakeld. Als je transcoderopties wilt wijzigen (bewerken of toevoegen), start de server opnieuw op met de %{config} configuratie-optie.",
+        "transcodingDisabled": "Het wijzigen van de transcodersconfiguratie via de web interface is om veiligheidsredenen uitgeschakeld. Als je transcoderopties wilt wijzigen (bewerken of toevoegen), start de server opnieuw op met de %{config} configuratie-optie.",
         "transcodingEnabled": "Navidrome werkt momenteel met %{config}, waardoor het mogelijk is om systeemopdrachten uit te voeren vanuit de transcoderinstellingen via de web interface. We raden aan dit om veiligheidsredenen uit te schakelen en alleen in te schakelen bij het configureren van transcoderopties.",
         "songsAddedToPlaylist": "1 nummer toegevoegd aan afspeellijst |||| %{smart_count} nummers toegevoegd aan afspeellijst",
         "noPlaylistsAvailable": "Geen beschikbaar",
@@ -275,7 +275,8 @@
                 "defaultView": "Standaard Weergave"
             }
         },
-        "albumList": "Albums"
+        "albumList": "Albums",
+        "about": "Over"
     },
     "player": {
         "playListsText": "Afspeellijst afspelen",
@@ -301,5 +302,12 @@
             "singleLoop": "Herhaal Eenmalig",
             "shufflePlay": "Shuffle"
         }
+    },
+    "about": {
+        "links": {
+            "homepage": "Thuispagina",
+            "source": "Broncode",
+            "featureRequests": "Functie verzoeken"
+        }
     }
 }

resources/i18n/pl.json

@@ -19,7 +19,7 @@
                 "updatedAt": "Zaktualizowano",
                 "bitRate": "Szybkość transmisji danych",
                 "discSubtitle": "Podtytuł Płyty",
-                "starred": "Oznaczone gwiazdką"
+                "starred": "Ulubione"
             },
             "actions": {
                 "addToQueue": "Odtwarzaj Później",
@@ -275,7 +275,8 @@
                 "defaultView": "Widok Podstawowy"
             }
         },
-        "albumList": "Albumy"
+        "albumList": "Albumy",
+        "about": "O aplikacji"
     },
     "player": {
         "playListsText": "Kolejka Odtwarzania",
@@ -301,5 +302,12 @@
             "singleLoop": "Powtórz Raz",
             "shufflePlay": "Odtwarzaj losowo"
         }
+    },
+    "about": {
+        "links": {
+            "homepage": "Strona główna",
+            "source": "Kod źródłowy",
+            "featureRequests": "Prośby o nowe funkcjonalności"
+        }
     }
 }

resources/i18n/ru.json

@@ -0,0 +1,313 @@
+{
+    "languageName": "Pусский",
+    "resources": {
+        "song": {
+            "name": "Трека |||| Треки",
+            "fields": {
+                "albumArtist": "Исполнитель",
+                "duration": "Длительность",
+                "trackNumber": "#",
+                "playCount": "Проиграно раз",
+                "title": "Название",
+                "artist": "Исполнитель",
+                "album": "Альбом",
+                "path": "Путь",
+                "genre": "Стиль",
+                "compilation": "Сборник",
+                "year": "Год",
+                "size": "Размер",
+                "updatedAt": "Обновлено",
+                "bitRate": "Битрейт",
+                "discSubtitle": "Название диска",
+                "starred": "Избранные"
+            },
+            "actions": {
+                "addToQueue": "В очередь",
+                "playNow": "Играть сейчас",
+                "addToPlaylist": "Добавить в плейлист",
+                "shuffleAll": "Перемешать все",
+                "download": "Скачать",
+                "playNext": "Следующий"
+            }
+        },
+        "album": {
+            "name": "Альбом |||| Альбомы",
+            "fields": {
+                "albumArtist": "Исполнитель",
+                "artist": "Исполнитель",
+                "duration": "Длительность",
+                "songCount": "Треков",
+                "playCount": "Проиграно раз",
+                "name": "Название",
+                "genre": "Стиль",
+                "compilation": "Сборник",
+                "year": "Год",
+                "updatedAt": "Обновлено"
+            },
+            "actions": {
+                "playAll": "Воспроизведение",
+                "playNext": "Следующий",
+                "addToQueue": "В очередь",
+                "shuffle": "Перемешать",
+                "addToPlaylist": "Добавить в плейлист",
+                "download": "Скачать"
+            },
+            "lists": {
+                "all": "Все",
+                "random": "Случайные",
+                "recentlyAdded": "Свежие",
+                "recentlyPlayed": "Проигранные",
+                "mostPlayed": "Популярные",
+                "starred": "Избранные"
+            }
+        },
+        "artist": {
+            "name": "Исполнитель |||| Исполнители",
+            "fields": {
+                "name": "Название",
+                "albumCount": "Количество альбомов",
+                "songCount": "Количество треков",
+                "playCount": "Проиграно раз"
+            }
+        },
+        "user": {
+            "name": "Пользователь |||| Пользователи",
+            "fields": {
+                "userName": "Логин",
+                "isAdmin": "Администратор",
+                "lastLoginAt": "Последний вход",
+                "updatedAt": "Обновлено",
+                "name": "Имя",
+                "password": "Пароль",
+                "createdAt": "Создано"
+            }
+        },
+        "player": {
+            "name": "Плеер |||| Плееры",
+            "fields": {
+                "name": "Имя",
+                "transcodingId": "Транскодирование",
+                "maxBitRate": "Макс. Битрейт",
+                "client": "Клиент",
+                "userName": "Пользователь",
+                "lastSeen": "Был на сайте"
+            }
+        },
+        "transcoding": {
+            "name": "Транскодирование  |||| Транскодирование",
+            "fields": {
+                "name": "Наименование",
+                "targetFormat": "Целевой формат",
+                "defaultBitRate": "Битрейт по умолчанию",
+                "command": "Команда"
+            }
+        },
+        "playlist": {
+            "name": "Плейлистов |||| Плейлисты",
+            "fields": {
+                "name": "Название",
+                "duration": "Длительность",
+                "owner": "Владелец",
+                "public": "Публичный",
+                "updatedAt": "Обновлено",
+                "createdAt": "Создано",
+                "songCount": "Треков",
+                "comment": "Комментарий",
+                "sync": "Автоимпорт",
+                "path": "Импортировать из"
+            },
+            "actions": {
+                "selectPlaylist": "Выберите плейлист",
+                "addNewPlaylist": "Создать \"%{name}\"",
+                "export": "Экспорт"
+            }
+        }
+    },
+    "ra": {
+        "auth": {
+            "welcome1": "Спасибо за установку Navidrome!",
+            "welcome2": "Для начала создайте Администратора",
+            "confirmPassword": "Подтвердить Пароль",
+            "buttonCreateAdmin": "Создать Администратора",
+            "auth_check_error": "Пожалуйста, авторизуйтесь для продолжения работы",
+            "user_menu": "Профиль",
+            "username": "Имя пользователя",
+            "password": "Пароль",
+            "sign_in": "Войти",
+            "sign_in_error": "Ошибка аутентификации, попробуйте снова",
+            "logout": "Выйти"
+        },
+        "validation": {
+            "invalidChars": "Пожалуйста используйте только буквы и цифры",
+            "passwordDoesNotMatch": "Пароли не совпадают",
+            "required": "Обязательно для заполнения",
+            "minLength": "Минимальное кол-во символов %{min}",
+            "maxLength": "Максимальное кол-во символов %{max}",
+            "minValue": "Минимальное значение %{min}",
+            "maxValue": "Значение может быть %{max} или меньше",
+            "number": "Должно быть цифрой",
+            "email": "Некорректный email",
+            "oneOf": "Должно быть одним из: %{options}",
+            "regex": "Должно быть в формате (regexp): %{pattern}"
+        },
+        "action": {
+            "add_filter": "Фильтр",
+            "add": "Добавить",
+            "back": "Назад",
+            "bulk_actions": "1 выбран |||| %{smart_count} выбрано |||| %{smart_count} выбрано",
+            "cancel": "Отмена",
+            "clear_input_value": "Очистить",
+            "clone": "Дублировать",
+            "confirm": "Подтвердить",
+            "create": "Создать",
+            "delete": "Удалить",
+            "edit": "Редактировать",
+            "export": "Экспорт",
+            "list": "Список",
+            "refresh": "Обновить",
+            "remove_filter": "Убрать фильтр",
+            "remove": "Удалить",
+            "save": "Сохранить",
+            "search": "Поиск",
+            "show": "Просмотр",
+            "sort": "Сортировать",
+            "undo": "Отменить",
+            "expand": "Раскрыть",
+            "close": "Закрыть",
+            "open_menu": "Открыть меню",
+            "close_menu": "Закрыть меню",
+            "unselect": "Отменить выделение"
+        },
+        "boolean": {
+            "true": "Да",
+            "false": "Нет"
+        },
+        "page": {
+            "create": "Создать %{name}",
+            "dashboard": "Главная",
+            "edit": "%{name} #%{id}",
+            "error": "Что-то пошло не так",
+            "list": "%{name}",
+            "loading": "Загрузка",
+            "not_found": "Не найдено",
+            "show": "%{name} #%{id}",
+            "empty": "Нет %{name}.",
+            "invite": "Хотите создать?"
+        },
+        "input": {
+            "file": {
+                "upload_several": "Перетащите файлы сюда или нажмите для выбора.",
+                "upload_single": "Перетащите файл сюда или нажмите для выбора."
+            },
+            "image": {
+                "upload_several": "Перетащите изображения сюда или нажмите для выбора.",
+                "upload_single": "Перетащите изображение сюда или нажмите для выбора."
+            },
+            "references": {
+                "all_missing": "Связанных данных не найдено",
+                "many_missing": "Некоторые из связанных данных не доступны",
+                "single_missing": "Связанный объект не доступен"
+            },
+            "password": {
+                "toggle_visible": "Скрыть пароль",
+                "toggle_hidden": "Показать пароль"
+            }
+        },
+        "message": {
+            "about": "Справка",
+            "are_you_sure": "Вы уверены?",
+            "bulk_delete_content": "Вы уверены, что хотите удалить %{name}? |||| Вы уверены, что хотите удалить объекты, кол-вом %{smart_count} ? |||| Вы уверены, что хотите удалить объекты, кол-вом %{smart_count} ?",
+            "bulk_delete_title": "Удалить %{name} |||| Удалить %{smart_count} %{name} |||| Удалить %{smart_count} %{name}",
+            "delete_content": "Вы уверены что хотите удалить этот объект",
+            "delete_title": "Удалить %{name} #%{id}",
+            "details": "Описание",
+            "error": "При выполнении запроса возникла ошибка, и он не может быть завершен",
+            "invalid_form": "Форма заполнена неверно, проверьте, пожалуйста, ошибки",
+            "loading": "Идет загрузка, пожалуйста, подождите...",
+            "no": "Нет",
+            "not_found": "Ошибка URL или вы следуете по неверной ссылке",
+            "yes": "Да",
+            "unsaved_changes": "Некоторые из ваших изменений не сохранены. Продолжить без сохранения?"
+        },
+        "navigation": {
+            "no_results": "Результатов не найдено",
+            "no_more_results": "Страница %{page} выходит за пределы нумерации, попробуйте предыдущую",
+            "page_out_of_boundaries": "Страница %{page} вне границ",
+            "page_out_from_end": "Невозможно переместиться дальше последней страницы",
+            "page_out_from_begin": "Номер страницы не может быть меньше 1",
+            "page_range_info": "%{offsetBegin}-%{offsetEnd} из %{total}",
+            "page_rows_per_page": "Строк на странице:",
+            "next": "Следующая",
+            "prev": "Предыдущая"
+        },
+        "notification": {
+            "updated": "Элемент обновлен |||| %{smart_count} обновлено |||| %{smart_count} обновлено",
+            "created": "Элемент создан",
+            "deleted": "Элемент удален |||| %{smart_count} удалено |||| %{smart_count} удалено",
+            "bad_item": "Элемент не валиден",
+            "item_doesnt_exist": "Элемент не существует",
+            "http_error": "Ошибка сервера",
+            "data_provider_error": "Ошибка dataProvider, проверьте консоль",
+            "i18n_error": "Не удалось загрузить перевод для указанного языка",
+            "canceled": "Операция отменена",
+            "logged_out": "Ваша сессия завершена, попробуйте переподключиться/войти снова"
+        }
+    },
+    "message": {
+        "note": "ПРИМЕЧАНИЕ",
+        "transcodingDisabled": "Изменение настроек транскодирования через веб интерфейс, отключено по соображениям безопасности. Если вы хотите изменить или добавить опции транскодирования, перезапустите сервер с %{config} опцией конфигурации.",
+        "transcodingEnabled": "Navidrome работает с настройками %{config}, позволяющими запускать команды с настройками транскодирования через веб интерфейс. В целях безопасности, мы рекомендуем отключить эту возможность.",
+        "songsAddedToPlaylist": "Один трек добавлен в плейлист ||||   %{smart_count} треков добавлено в плейлист",
+        "noPlaylistsAvailable": "Недоступно",
+        "delete_user_title": "Удалить пользователя '%{name}'",
+        "delete_user_content": "Вы уверены, что вы хотите удалить пользователя и все его данные (включая плейлисты и настройки)?"
+    },
+    "menu": {
+        "library": "Библиотека",
+        "settings": "Настройки",
+        "version": "Версия",
+        "theme": "Тема",
+        "personal": {
+            "name": "Личные",
+            "options": {
+                "theme": "Тема",
+                "language": "Язык",
+                "defaultView": "Вид по умолчанию"
+            }
+        },
+        "albumList": "Альбомы",
+        "about": "О нас"
+    },
+    "player": {
+        "playListsText": "Очередь воспроизведения",
+        "openText": "Открыть",
+        "closeText": "Закрыть",
+        "notContentText": "Не музыка",
+        "clickToPlayText": "Играть",
+        "clickToPauseText": "Пауза",
+        "nextTrackText": "Следующий",
+        "previousTrackText": "Предыдущий",
+        "reloadText": "Перезагрузить",
+        "volumeText": "Громкость",
+        "toggleLyricText": "Посмотреть текст",
+        "toggleMiniModeText": "Минимизировать",
+        "destroyText": "Выключить",
+        "downloadText": "Скачать",
+        "removeAudioListsText": "Удалить список воспроизведения",
+        "clickToDeleteText": "Нажмите для удаления %{name}",
+        "emptyLyricText": "Без текста",
+        "playModeText": {
+            "order": "По порядку",
+            "orderLoop": "Повторять",
+            "singleLoop": "Повторить один раз",
+            "shufflePlay": "Перемешать"
+        }
+    },
+    "about": {
+        "links": {
+            "homepage": "Главная",
+            "source": "Код",
+            "featureRequests": "Предложения"
+        }
+    }
+}

scanner/mapping.go

@@ -52,6 +52,13 @@ func (s *mediaFileMapper) toMediaFile(md metadata.Metadata) model.MediaFile {
 	mf.OrderAlbumName = sanitizeFieldForSorting(mf.Album)
 	mf.OrderArtistName = sanitizeFieldForSorting(mf.Artist)
 	mf.OrderAlbumArtistName = sanitizeFieldForSorting(mf.AlbumArtist)
+	mf.CatalogNum = md.CatalogNum()
+	mf.MbzTrackID = md.MbzTrackID()
+	mf.MbzAlbumID = md.MbzAlbumID()
+	mf.MbzArtistID = md.MbzArtistID()
+	mf.MbzAlbumArtistID = md.MbzAlbumArtistID()
+	mf.MbzAlbumType = md.MbzAlbumType()
+	mf.MbzAlbumComment = md.MbzAlbumComment()
 
 	// TODO Get Creation time. https://github.com/djherbis/times ?
 	mf.CreatedAt = md.ModificationTime()

scanner/metadata/ffmpeg.go

@@ -21,7 +21,7 @@ func (m *ffmpegMetadata) BitRate() int      { return m.parseInt("bitrate") }
 func (m *ffmpegMetadata) HasPicture() bool {
 	return m.getTag("has_picture", "metadata_block_picture") != ""
 }
-func (m *ffmpegMetadata) DiscNumber() (int, int) { return m.parseTuple("tpa", "disc") }
+func (m *ffmpegMetadata) DiscNumber() (int, int) { return m.parseTuple("tpa", "disc", "discnumber") }
 
 type ffmpegExtractor struct{}
 

scanner/metadata/ffmpeg_test.go

@@ -52,6 +52,34 @@ var _ = Describe("ffmpegExtractor", func() {
 	})
 
 	Context("extractMetadata", func() {
+		It("extracts MusicBrainz custom tags", func() {
+			const output = `
+Input #0, ape, from './Capture/02 01 - Symphony No. 5 in C minor, Op. 67 I. Allegro con brio - Ludwig van Beethoven.ape':
+  Metadata:
+    ALBUM           : Forever Classics
+    ARTIST          : Ludwig van Beethoven
+    TITLE           : Symphony No. 5 in C minor, Op. 67: I. Allegro con brio
+    MUSICBRAINZ_ALBUMSTATUS: official
+    MUSICBRAINZ_ALBUMTYPE: album
+    MusicBrainz_AlbumComment: MP3
+    Musicbrainz_Albumid: 71eb5e4a-90e2-4a31-a2d1-a96485fcb667
+    musicbrainz_trackid: ffe06940-727a-415a-b608-b7e45737f9d8
+    Musicbrainz_Artistid: 1f9df192-a621-4f54-8850-2c5373b7eac9
+    Musicbrainz_Albumartistid: 89ad4ac3-39f7-470e-963a-56509c546377
+    Musicbrainz_Releasegroupid: 708b1ae1-2d3d-34c7-b764-2732b154f5b6
+    musicbrainz_releasetrackid: 6fee2e35-3049-358f-83be-43b36141028b
+    CatalogNumber   : PLD 1201
+`
+			md, _ := e.extractMetadata("tests/fixtures/test.mp3", output)
+			Expect(md.CatalogNum()).To(Equal("PLD 1201"))
+			Expect(md.MbzTrackID()).To(Equal("ffe06940-727a-415a-b608-b7e45737f9d8"))
+			Expect(md.MbzAlbumID()).To(Equal("71eb5e4a-90e2-4a31-a2d1-a96485fcb667"))
+			Expect(md.MbzArtistID()).To(Equal("1f9df192-a621-4f54-8850-2c5373b7eac9"))
+			Expect(md.MbzAlbumArtistID()).To(Equal("89ad4ac3-39f7-470e-963a-56509c546377"))
+			Expect(md.MbzAlbumType()).To(Equal("album"))
+			Expect(md.MbzAlbumComment()).To(Equal("MP3"))
+		})
+
 		It("detects embedded cover art correctly", func() {
 			const output = `
 Input #0, mp3, from '/Users/deluan/Music/iTunes/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/09 Pablo's Blues.mp3':

scanner/metadata/metadata.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/deluan/navidrome/conf"
 	"github.com/deluan/navidrome/log"
+	"github.com/google/uuid"
 )
 
 type Extractor interface {
@@ -51,6 +52,13 @@ type Metadata interface {
 	HasPicture() bool
 	Comment() string
 	Compilation() bool
+	CatalogNum() string
+	MbzTrackID() string
+	MbzAlbumID() string
+	MbzArtistID() string
+	MbzAlbumArtistID() string
+	MbzAlbumType() string
+	MbzAlbumComment() string
 	Duration() float32
 	BitRate() int
 	ModificationTime() time.Time
@@ -87,6 +95,25 @@ func (m *baseMetadata) DiscNumber() (int, int)  { return m.parseTuple("disc", "d
 func (m *baseMetadata) DiscSubtitle() string {
 	return m.getTag("tsst", "discsubtitle", "setsubtitle")
 }
+func (m *baseMetadata) CatalogNum() string { return m.getTag("catalognumber") }
+func (m *baseMetadata) MbzTrackID() string {
+	return m.getMbzID("musicbrainz_trackid", "musicbrainz track id")
+}
+func (m *baseMetadata) MbzAlbumID() string {
+	return m.getMbzID("musicbrainz_albumid", "musicbrainz album id")
+}
+func (m *baseMetadata) MbzArtistID() string {
+	return m.getMbzID("musicbrainz_artistid", "musicbrainz artist id")
+}
+func (m *baseMetadata) MbzAlbumArtistID() string {
+	return m.getMbzID("musicbrainz_albumartistid", "musicbrainz album artist id")
+}
+func (m *baseMetadata) MbzAlbumType() string {
+	return m.getTag("musicbrainz_albumtype", "musicbrainz album type")
+}
+func (m *baseMetadata) MbzAlbumComment() string {
+	return m.getTag("musicbrainz_albumcomment", "musicbrainz album comment")
+}
 
 func (m *baseMetadata) ModificationTime() time.Time { return m.fileInfo.ModTime() }
 func (m *baseMetadata) Size() int64                 { return m.fileInfo.Size() }
@@ -132,6 +159,20 @@ func (m *baseMetadata) parseYear(tags ...string) int {
 	return 0
 }
 
+func (m *baseMetadata) getMbzID(tags ...string) string {
+	var value string
+	for _, t := range tags {
+		if v, ok := m.tags[t]; ok {
+			value = v
+			break
+		}
+	}
+	if _, err := uuid.Parse(value); err != nil {
+		return ""
+	}
+	return value
+}
+
 func (m *baseMetadata) getTag(tags ...string) string {
 	for _, t := range tags {
 		if v, ok := m.tags[t]; ok {

scanner/metadata/metadata_test.go

@@ -6,7 +6,7 @@ import (
 )
 
 var _ = Describe("ffmpegMetadata", func() {
-	Context("parseYear", func() {
+	Describe("parseYear", func() {
 		It("parses the year correctly", func() {
 			var examples = map[string]int{
 				"1985":         1985,
@@ -31,4 +31,33 @@ var _ = Describe("ffmpegMetadata", func() {
 			Expect(md.Year()).To(Equal(0))
 		})
 	})
+
+	Describe("getMbzID", func() {
+		It("return a valid MBID", func() {
+			md := &baseMetadata{}
+			md.tags = map[string]string{
+				"musicbrainz_trackid":       "8f84da07-09a0-477b-b216-cc982dabcde1",
+				"musicbrainz_albumid":       "f68c985d-f18b-4f4a-b7f0-87837cf3fbf9",
+				"musicbrainz_artistid":      "89ad4ac3-39f7-470e-963a-56509c546377",
+				"musicbrainz_albumartistid": "ada7a83c-e3e1-40f1-93f9-3e73dbc9298a",
+			}
+			Expect(md.MbzTrackID()).To(Equal("8f84da07-09a0-477b-b216-cc982dabcde1"))
+			Expect(md.MbzAlbumID()).To(Equal("f68c985d-f18b-4f4a-b7f0-87837cf3fbf9"))
+			Expect(md.MbzArtistID()).To(Equal("89ad4ac3-39f7-470e-963a-56509c546377"))
+			Expect(md.MbzAlbumArtistID()).To(Equal("ada7a83c-e3e1-40f1-93f9-3e73dbc9298a"))
+		})
+		It("return empty string for invalid MBID", func() {
+			md := &baseMetadata{}
+			md.tags = map[string]string{
+				"musicbrainz_trackid":       "11406732-6",
+				"musicbrainz_albumid":       "11406732",
+				"musicbrainz_artistid":      "200455",
+				"musicbrainz_albumartistid": "194",
+			}
+			Expect(md.MbzTrackID()).To(Equal(""))
+			Expect(md.MbzAlbumID()).To(Equal(""))
+			Expect(md.MbzArtistID()).To(Equal(""))
+			Expect(md.MbzAlbumArtistID()).To(Equal(""))
+		})
+	})
 })

scanner/metadata/taglib.go

@@ -59,6 +59,11 @@ func (e *taglibExtractor) extractMetadata(filePath string) (*taglibMetadata, err
 }
 
 func hasEmbeddedImage(path string) bool {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Error("Panic while checking for images. Please report this error with a copy of the file", "path", path, r)
+		}
+	}()
 	f, err := os.Open(path)
 	if err != nil {
 		log.Warn("Error opening file", "filePath", path, err)

scanner/playlist_sync.go

@@ -24,8 +24,8 @@ func newPlaylistSync(ds model.DataStore) *playlistSync {
 	return &playlistSync{ds: ds}
 }
 
-func (s *playlistSync) processPlaylists(ctx context.Context, dir string) int {
-	count := 0
+func (s *playlistSync) processPlaylists(ctx context.Context, dir string) int64 {
+	var count int64
 	files, err := ioutil.ReadDir(dir)
 	if err != nil {
 		log.Error(ctx, "Error reading files", "dir", dir, err)

scanner/playlist_sync_test.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/deluan/navidrome/model"
-	"github.com/deluan/navidrome/persistence"
+	"github.com/deluan/navidrome/tests"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -15,7 +15,7 @@ var _ = Describe("playlistSync", func() {
 		var ps *playlistSync
 		ctx := context.TODO()
 		BeforeEach(func() {
-			ds = &persistence.MockDataStore{
+			ds = &tests.MockDataStore{
 				MockedMediaFile: &mockedMediaFile{},
 			}
 			ps = newPlaylistSync(ds)

scanner/refresh_buffer.go

@@ -25,8 +25,12 @@ func newRefreshBuffer(ctx context.Context, ds model.DataStore) *refreshBuffer {
 }
 
 func (f *refreshBuffer) accumulate(mf model.MediaFile) {
-	f.album[mf.AlbumID] = struct{}{}
-	f.artist[mf.AlbumArtistID] = struct{}{}
+	if mf.AlbumID != "" {
+		f.album[mf.AlbumID] = struct{}{}
+	}
+	if mf.AlbumArtistID != "" {
+		f.artist[mf.AlbumArtistID] = struct{}{}
+	}
 }
 
 type refreshCallbackFunc = func(ids ...string) error

scanner/scanner.go

@@ -5,27 +5,100 @@ import (
 	"errors"
 	"fmt"
 	"strconv"
+	"sync"
+	"sync/atomic"
 	"time"
 
+	"github.com/deluan/navidrome/core"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
+	"github.com/deluan/navidrome/utils"
 )
 
-type Scanner struct {
-	folders map[string]FolderScanner
-	ds      model.DataStore
+type Scanner interface {
+	Start(interval time.Duration)
+	Stop()
+	RescanAll(fullRescan bool) error
+	Status(mediaFolder string) (*StatusInfo, error)
+	Scanning() bool
 }
 
-func New(ds model.DataStore) *Scanner {
-	s := &Scanner{ds: ds, folders: map[string]FolderScanner{}}
+type StatusInfo struct {
+	MediaFolder string
+	Scanning    bool
+	LastScan    time.Time
+	Count       uint32
+}
+
+var (
+	ErrAlreadyScanning = errors.New("already scanning")
+	ErrScanError       = errors.New("scan error")
+)
+
+type FolderScanner interface {
+	Scan(ctx context.Context, lastModifiedSince time.Time, progress chan uint32) error
+}
+
+var isScanning utils.AtomicBool
+
+type scanner struct {
+	folders     map[string]FolderScanner
+	status      map[string]*scanStatus
+	lock        *sync.RWMutex
+	ds          model.DataStore
+	cacheWarmer core.CacheWarmer
+	done        chan bool
+	scan        chan bool
+}
+
+type scanStatus struct {
+	active     bool
+	count      uint32
+	lastUpdate time.Time
+}
+
+func New(ds model.DataStore, cacheWarmer core.CacheWarmer) Scanner {
+	s := &scanner{
+		ds:          ds,
+		cacheWarmer: cacheWarmer,
+		folders:     map[string]FolderScanner{},
+		status:      map[string]*scanStatus{},
+		lock:        &sync.RWMutex{},
+		done:        make(chan bool),
+		scan:        make(chan bool),
+	}
 	s.loadFolders()
 	return s
 }
 
-func (s *Scanner) Rescan(mediaFolder string, fullRescan bool) error {
+func (s *scanner) Start(interval time.Duration) {
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		err := s.RescanAll(false)
+		if err != nil {
+			log.Error(err)
+		}
+		select {
+		case <-ticker.C:
+			continue
+		case <-s.done:
+			return
+		}
+	}
+}
+
+func (s *scanner) Stop() {
+	s.done <- true
+}
+
+func (s *scanner) rescan(mediaFolder string, fullRescan bool) error {
 	folderScanner := s.folders[mediaFolder]
 	start := time.Now()
 
+	s.setStatusStart(mediaFolder)
+	defer s.setStatusEnd(mediaFolder, start)
+
 	lastModifiedSince := time.Time{}
 	if !fullRescan {
 		lastModifiedSince = s.getLastModifiedSince(mediaFolder)
@@ -34,7 +107,19 @@ func (s *Scanner) Rescan(mediaFolder string, fullRescan bool) error {
 		log.Debug("Scanning folder (full scan)", "folder", mediaFolder)
 	}
 
-	err := folderScanner.Scan(log.NewContext(context.TODO()), lastModifiedSince)
+	progress := make(chan uint32)
+	go func() {
+		for {
+			count, more := <-progress
+			if !more {
+				break
+			}
+			atomic.AddUint32(&s.status[mediaFolder].count, count)
+		}
+	}()
+
+	err := folderScanner.Scan(log.NewContext(context.TODO()), lastModifiedSince, progress)
+	close(progress)
 	if err != nil {
 		log.Error("Error importing MediaFolder", "folder", mediaFolder, err)
 	}
@@ -43,22 +128,72 @@ func (s *Scanner) Rescan(mediaFolder string, fullRescan bool) error {
 	return err
 }
 
-func (s *Scanner) RescanAll(fullRescan bool) error {
+func (s *scanner) RescanAll(fullRescan bool) error {
+	if s.Scanning() {
+		log.Debug("Scanner already running, ignoring request for rescan.")
+		return ErrAlreadyScanning
+	}
+	isScanning.Set(true)
+	defer func() { isScanning.Set(false) }()
+
+	defer s.cacheWarmer.Flush(context.Background())
 	var hasError bool
 	for folder := range s.folders {
-		err := s.Rescan(folder, fullRescan)
+		err := s.rescan(folder, fullRescan)
 		hasError = hasError || err != nil
 	}
 	if hasError {
 		log.Error("Errors while scanning media. Please check the logs")
-		return errors.New("errors while scanning media")
+		return ErrScanError
 	}
 	return nil
 }
 
-func (s *Scanner) Status() []StatusInfo { return nil }
+func (s *scanner) getStatus(folder string) *scanStatus {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	if status, ok := s.status[folder]; ok {
+		return status
+	}
+	return nil
+}
 
-func (s *Scanner) getLastModifiedSince(folder string) time.Time {
+func (s *scanner) setStatusStart(folder string) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	if status, ok := s.status[folder]; ok {
+		status.active = true
+		status.count = 0
+	}
+}
+
+func (s *scanner) setStatusEnd(folder string, lastUpdate time.Time) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	if status, ok := s.status[folder]; ok {
+		status.active = false
+		status.lastUpdate = lastUpdate
+	}
+}
+
+func (s *scanner) Scanning() bool {
+	return isScanning.Get()
+}
+
+func (s *scanner) Status(mediaFolder string) (*StatusInfo, error) {
+	status := s.getStatus(mediaFolder)
+	if status == nil {
+		return nil, errors.New("mediaFolder not found")
+	}
+	return &StatusInfo{
+		MediaFolder: mediaFolder,
+		Scanning:    status.active,
+		LastScan:    status.lastUpdate,
+		Count:       status.count,
+	}, nil
+}
+
+func (s *scanner) getLastModifiedSince(folder string) time.Time {
 	ms, err := s.ds.Property(context.TODO()).Get(model.PropLastScan + "-" + folder)
 	if err != nil {
 		return time.Time{}
@@ -70,32 +205,26 @@ func (s *Scanner) getLastModifiedSince(folder string) time.Time {
 	return time.Unix(0, i*int64(time.Millisecond))
 }
 
-func (s *Scanner) updateLastModifiedSince(folder string, t time.Time) {
+func (s *scanner) updateLastModifiedSince(folder string, t time.Time) {
 	millis := t.UnixNano() / int64(time.Millisecond)
 	if err := s.ds.Property(context.TODO()).Put(model.PropLastScan+"-"+folder, fmt.Sprint(millis)); err != nil {
 		log.Error("Error updating DB after scan", err)
 	}
 }
 
-func (s *Scanner) loadFolders() {
+func (s *scanner) loadFolders() {
 	fs, _ := s.ds.MediaFolder(context.TODO()).GetAll()
 	for _, f := range fs {
 		log.Info("Configuring Media Folder", "name", f.Name, "path", f.Path)
 		s.folders[f.Path] = s.newScanner(f)
+		s.status[f.Path] = &scanStatus{
+			active:     false,
+			count:      0,
+			lastUpdate: s.getLastModifiedSince(f.Path),
+		}
 	}
 }
 
-func (s *Scanner) newScanner(f model.MediaFolder) FolderScanner {
-	return NewTagScanner(f.Path, s.ds)
-}
-
-type Status int
-
-type StatusInfo struct {
-	MediaFolder string
-	Status      Status
-}
-
-type FolderScanner interface {
-	Scan(ctx context.Context, lastModifiedSince time.Time) error
+func (s *scanner) newScanner(f model.MediaFolder) FolderScanner {
+	return NewTagScanner(f.Path, s.ds, s.cacheWarmer)
 }

scanner/tag_scanner.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/deluan/navidrome/conf"
+	"github.com/deluan/navidrome/core"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
 	"github.com/deluan/navidrome/model/request"
@@ -17,42 +18,48 @@ import (
 )
 
 type TagScanner struct {
-	rootFolder string
-	ds         model.DataStore
-	mapper     *mediaFileMapper
-	plsSync    *playlistSync
-	cnt        *counters
+	rootFolder  string
+	ds          model.DataStore
+	mapper      *mediaFileMapper
+	plsSync     *playlistSync
+	cnt         *counters
+	cacheWarmer core.CacheWarmer
 }
 
-func NewTagScanner(rootFolder string, ds model.DataStore) *TagScanner {
+func NewTagScanner(rootFolder string, ds model.DataStore, cacheWarmer core.CacheWarmer) *TagScanner {
 	return &TagScanner{
-		rootFolder: rootFolder,
-		mapper:     newMediaFileMapper(rootFolder),
-		plsSync:    newPlaylistSync(ds),
-		ds:         ds,
+		rootFolder:  rootFolder,
+		mapper:      newMediaFileMapper(rootFolder),
+		plsSync:     newPlaylistSync(ds),
+		ds:          ds,
+		cacheWarmer: cacheWarmer,
 	}
 }
 
-type counters struct {
-	added   int64
-	updated int64
-	deleted int64
-}
+type (
+	counters struct {
+		added     int64
+		updated   int64
+		deleted   int64
+		playlists int64
+	}
+	dirMap map[string]dirStats
+)
 
 const (
 	// filesBatchSize used for batching file metadata extraction
 	filesBatchSize = 100
 )
 
-// Scanner algorithm overview:
-// Load all directories under the music folder, with their ModTime (self or any non-dir children, whichever is newer)
+// TagScanner algorithm overview:
 // Load all directories from the DB
-// Compare both collections to find changed folders (based on lastModifiedSince) and deleted folders
-// For each deleted folder: delete all files from DB whose path starts with the delete folder path (non-recursively)
+// Traverse the music folder, collecting each subfolder's ModTime (self or any non-dir children, whichever is newer)
 // For each changed folder: get all files from DB whose path starts with the changed folder (non-recursively), check each file:
 //	    if file in folder is newer, update the one in DB
 //      if file in folder does not exists in DB, add it
 // 	    for each file in the DB that is not found in the folder, delete it from DB
+// Compare directories in the fs with the ones in the DB to find deleted folders
+// For each deleted folder: delete all files from DB whose path starts with the delete folder path (non-recursively)
 // Create new albums/artists, update counters:
 //      collect all albumIDs and artistIDs from previous steps
 //	    refresh the collected albums and artists with the metadata from the mediafiles
@@ -60,62 +67,74 @@ const (
 //      If the playlist is not in the DB, import it, setting sync = true
 //      If the playlist is in the DB and sync == true, import it, or else skip it
 // Delete all empty albums, delete all empty artists, clean-up playlists
-func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) error {
+func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time, progress chan uint32) error {
 	ctx = s.withAdminUser(ctx)
-
 	start := time.Now()
-	allFSDirs, err := s.getDirTree(ctx)
-	if err != nil {
-		return err
-	}
 
 	allDBDirs, err := s.getDBDirTree(ctx)
 	if err != nil {
 		return err
 	}
 
-	changedDirs := s.getChangedDirs(ctx, allFSDirs, allDBDirs, lastModifiedSince)
-	deletedDirs := s.getDeletedDirs(ctx, allFSDirs, allDBDirs)
+	allFSDirs := dirMap{}
+	var changedDirs []string
+	s.cnt = &counters{}
 
-	if len(changedDirs)+len(deletedDirs) == 0 {
+	foldersFound, walkerError := s.getRootFolderWalker(ctx)
+	for {
+		folderStats, more := <-foldersFound
+		if !more {
+			break
+		}
+		progress <- folderStats.AudioFilesCount
+		allFSDirs[folderStats.Path] = folderStats
+
+		if s.folderHasChanged(ctx, folderStats, allDBDirs, lastModifiedSince) {
+			changedDirs = append(changedDirs, folderStats.Path)
+			log.Debug("Processing changed folder", "dir", folderStats.Path)
+			err := s.processChangedDir(ctx, folderStats.Path)
+			if err != nil {
+				log.Error("Error updating folder in the DB", "dir", folderStats.Path, err)
+			}
+		}
+	}
+
+	if err := <-walkerError; err != nil {
+		log.Error("Scan was interrupted by error. See errors above", err)
+		return err
+	}
+
+	// If the media folder is empty, abort to avoid deleting all data
+	if len(allFSDirs) <= 1 {
+		log.Error(ctx, "Media Folder is empty. Aborting scan.", "folder", s.rootFolder)
+		return nil
+	}
+
+	deletedDirs := s.getDeletedDirs(ctx, allFSDirs, allDBDirs)
+	if len(deletedDirs)+len(changedDirs) == 0 {
 		log.Debug(ctx, "No changes found in Music Folder", "folder", s.rootFolder)
 		return nil
 	}
 
-	if log.CurrentLevel() >= log.LevelTrace {
-		log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs),
-			"changed", strings.Join(changedDirs, ";"), "deleted", strings.Join(deletedDirs, ";"))
-	} else {
-		log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs))
-	}
-
-	s.cnt = &counters{}
-
 	for _, dir := range deletedDirs {
 		err := s.processDeletedDir(ctx, dir)
 		if err != nil {
-			log.Error("Error removing deleted folder from DB", "path", dir, err)
-		}
-	}
-	for _, dir := range changedDirs {
-		err := s.processChangedDir(ctx, dir)
-		if err != nil {
-			log.Error("Error updating folder in the DB", "path", dir, err)
+			log.Error("Error removing deleted folder from DB", "dir", dir, err)
 		}
 	}
 
-	plsCount := 0
+	s.cnt.playlists = 0
 	if conf.Server.AutoImportPlaylists {
-		// Now that all mediafiles are imported/updated, search for and import playlists
+		// Now that all mediafiles are imported/updated, search for and import/update playlists
 		u, _ := request.UserFrom(ctx)
 		for _, dir := range changedDirs {
 			info := allFSDirs[dir]
-			if info.hasPlaylist {
+			if info.HasPlaylist {
 				if !u.IsAdmin {
 					log.Warn("Playlists will not be imported, as there are no admin users yet, "+
 						"Please create an admin user first, and then update the playlists for them to be imported", "dir", dir)
 				} else {
-					plsCount = s.plsSync.processPlaylists(ctx, dir)
+					s.cnt.playlists = s.plsSync.processPlaylists(ctx, dir)
 				}
 			}
 		}
@@ -125,20 +144,25 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) erro
 
 	err = s.ds.GC(log.NewContext(ctx), s.rootFolder)
 	log.Info("Finished processing Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start),
-		"added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", plsCount)
+		"added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", s.cnt.playlists)
 
 	return err
 }
 
-func (s *TagScanner) getDirTree(ctx context.Context) (dirMap, error) {
+func (s *TagScanner) getRootFolderWalker(ctx context.Context) (walkResults, chan error) {
 	start := time.Now()
 	log.Trace(ctx, "Loading directory tree from music folder", "folder", s.rootFolder)
-	dirs, err := loadDirTree(ctx, s.rootFolder)
-	if err != nil {
-		return nil, err
-	}
-	log.Debug("Directory tree loaded from music folder", "total", len(dirs), "elapsed", time.Since(start))
-	return dirs, nil
+	results := make(chan dirStats, 5000)
+	walkerError := make(chan error)
+	go func() {
+		err := walkDirTree(ctx, s.rootFolder, results)
+		if err != nil {
+			log.Error("There were errors reading directories from filesystem", err)
+		}
+		walkerError <- err
+		log.Debug("Finished reading directories from filesystem", "elapsed", time.Since(start))
+	}()
+	return results, walkerError
 }
 
 func (s *TagScanner) getDBDirTree(ctx context.Context) (map[string]struct{}, error) {
@@ -159,21 +183,10 @@ func (s *TagScanner) getDBDirTree(ctx context.Context) (map[string]struct{}, err
 	return resp, nil
 }
 
-func (s *TagScanner) getChangedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}, lastModified time.Time) []string {
-	start := time.Now()
-	log.Trace(ctx, "Checking for changed folders")
-	var changed []string
-
-	for d, info := range fsDirs {
-		_, inDB := dbDirs[d]
-		if (!inDB && (info.hasAudioFiles)) || info.modTime.After(lastModified) {
-			changed = append(changed, d)
-		}
-	}
-
-	sort.Strings(changed)
-	log.Debug(ctx, "Finished changed folders check", "total", len(changed), "elapsed", time.Since(start))
-	return changed
+func (s *TagScanner) folderHasChanged(ctx context.Context, folder dirStats, dbDirs map[string]struct{}, lastModified time.Time) bool {
+	_, inDB := dbDirs[folder.Path]
+	// If is a new folder with at least one song OR it was modified after lastModified
+	return (!inDB && (folder.AudioFilesCount > 0)) || folder.ModTime.After(lastModified)
 }
 
 func (s *TagScanner) getDeletedDirs(ctx context.Context, fsDirs dirMap, dbDirs map[string]struct{}) []string {
@@ -209,10 +222,11 @@ func (s *TagScanner) processDeletedDir(ctx context.Context, dir string) error {
 
 	for _, t := range mfs {
 		buffer.accumulate(t)
+		s.cacheWarmer.AddAlbum(ctx, t.AlbumID)
 	}
 
 	err = buffer.flush()
-	log.Info(ctx, "Finished processing deleted folder", "path", dir, "purged", len(mfs), "elapsed", time.Since(start))
+	log.Info(ctx, "Finished processing deleted folder", "dir", dir, "purged", len(mfs), "elapsed", time.Since(start))
 	return err
 }
 
@@ -285,6 +299,11 @@ func (s *TagScanner) processChangedDir(ctx context.Context, dir string) error {
 		}
 	}
 
+	// Pre cache all changed album artwork
+	for albumID := range buffer.album {
+		s.cacheWarmer.AddAlbum(ctx, albumID)
+	}
+
 	err = buffer.flush()
 	log.Info(ctx, "Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks,
 		"purged", numPurgedTracks, "elapsed", time.Since(start))

scanner/walk_dir_tree.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/deluan/navidrome/consts"
@@ -13,49 +14,53 @@ import (
 )
 
 type (
-	dirMapValue struct {
-		modTime       time.Time
-		hasImages     bool
-		hasPlaylist   bool
-		hasAudioFiles bool
+	dirStats struct {
+		Path            string
+		ModTime         time.Time
+		HasImages       bool
+		HasPlaylist     bool
+		AudioFilesCount uint32
 	}
-	dirMap = map[string]dirMapValue
+	walkResults = chan dirStats
 )
 
-func loadDirTree(ctx context.Context, rootFolder string) (dirMap, error) {
-	newMap := make(dirMap)
-	err := loadMap(ctx, rootFolder, rootFolder, newMap)
+func walkDirTree(ctx context.Context, rootFolder string, results walkResults) error {
+	err := walkFolder(ctx, rootFolder, rootFolder, results)
 	if err != nil {
 		log.Error(ctx, "Error loading directory tree", err)
 	}
-	return newMap, err
+	close(results)
+	return err
 }
 
-func loadMap(ctx context.Context, rootPath string, currentFolder string, dirMap dirMap) error {
-	children, dirMapValue, err := loadDir(ctx, currentFolder)
+func walkFolder(ctx context.Context, rootPath string, currentFolder string, results walkResults) error {
+	children, stats, err := loadDir(ctx, currentFolder)
 	if err != nil {
 		return err
 	}
 	for _, c := range children {
-		err := loadMap(ctx, rootPath, c, dirMap)
+		err := walkFolder(ctx, rootPath, c, results)
 		if err != nil {
 			return err
 		}
 	}
 
 	dir := filepath.Clean(currentFolder)
-	dirMap[dir] = dirMapValue
+	log.Trace(ctx, "Found directory", "dir", dir, "audioCount", stats.AudioFilesCount,
+		"hasImages", stats.HasImages, "HasPlaylist", stats.HasPlaylist)
+	stats.Path = dir
+	results <- stats
 
 	return nil
 }
 
-func loadDir(ctx context.Context, dirPath string) (children []string, info dirMapValue, err error) {
+func loadDir(ctx context.Context, dirPath string) (children []string, stats dirStats, err error) {
 	dirInfo, err := os.Stat(dirPath)
 	if err != nil {
 		log.Error(ctx, "Error stating dir", "path", dirPath, err)
 		return
 	}
-	info.modTime = dirInfo.ModTime()
+	stats.ModTime = dirInfo.ModTime()
 
 	files, err := ioutil.ReadDir(dirPath)
 	if err != nil {
@@ -66,17 +71,21 @@ func loadDir(ctx context.Context, dirPath string) (children []string, info dirMa
 		isDir, err := isDirOrSymlinkToDir(dirPath, f)
 		// Skip invalid symlinks
 		if err != nil {
+			log.Error(ctx, "Invalid symlink", "dir", dirPath)
 			continue
 		}
 		if isDir && !isDirIgnored(dirPath, f) && isDirReadable(dirPath, f) {
 			children = append(children, filepath.Join(dirPath, f.Name()))
 		} else {
-			if f.ModTime().After(info.modTime) {
-				info.modTime = f.ModTime()
+			if f.ModTime().After(stats.ModTime) {
+				stats.ModTime = f.ModTime()
+			}
+			if utils.IsAudioFile(f.Name()) {
+				stats.AudioFilesCount++
+			} else {
+				stats.HasPlaylist = stats.HasPlaylist || utils.IsPlaylist(f.Name())
+				stats.HasImages = stats.HasImages || utils.IsImageFile(f.Name())
 			}
-			info.hasImages = info.hasImages || utils.IsImageFile(f.Name())
-			info.hasPlaylist = info.hasPlaylist || utils.IsPlaylist(f.Name())
-			info.hasAudioFiles = info.hasAudioFiles || utils.IsAudioFile(f.Name())
 		}
 	}
 	return
@@ -105,6 +114,9 @@ func isDirOrSymlinkToDir(baseDir string, dirInfo os.FileInfo) (bool, error) {
 // isDirIgnored returns true if the directory represented by dirInfo contains an
 // `ignore` file (named after consts.SkipScanFile)
 func isDirIgnored(baseDir string, dirInfo os.FileInfo) bool {
+	if strings.HasPrefix(dirInfo.Name(), ".") {
+		return true
+	}
 	_, err := os.Stat(filepath.Join(baseDir, dirInfo.Name(), consts.SkipScanFile))
 	return err == nil
 }

scanner/walk_dir_tree_test.go

@@ -1,14 +1,46 @@
 package scanner
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	. "github.com/onsi/gomega/gstruct"
 )
 
 var _ = Describe("load_tree", func() {
+
+	Describe("walkDirTree", func() {
+		It("reads all info correctly", func() {
+			var collected = dirMap{}
+			results := make(walkResults, 5000)
+			var err error
+			go func() {
+				err = walkDirTree(context.TODO(), "tests/fixtures", results)
+			}()
+
+			for {
+				stats, more := <-results
+				if !more {
+					break
+				}
+				collected[stats.Path] = stats
+			}
+
+			Expect(err).To(BeNil())
+			Expect(collected["tests/fixtures"]).To(MatchFields(IgnoreExtras, Fields{
+				"HasImages":       BeTrue(),
+				"HasPlaylist":     BeFalse(),
+				"AudioFilesCount": BeNumerically("==", 4),
+			}))
+			Expect(collected["tests/fixtures/playlists"].HasPlaylist).To(BeTrue())
+			Expect(collected).To(HaveKey("tests/fixtures/symlink2dir"))
+			Expect(collected).To(HaveKey("tests/fixtures/empty_folder"))
+		})
+	})
+
 	Describe("isDirOrSymlinkToDir", func() {
 		It("returns true for normal dirs", func() {
 			dir, _ := os.Stat("tests/fixtures")
@@ -38,5 +70,9 @@ var _ = Describe("load_tree", func() {
 			dir, _ := os.Stat(filepath.Join(baseDir, "ignored_folder"))
 			Expect(isDirIgnored(baseDir, dir)).To(BeTrue())
 		})
+		It("returns true when folder name starts with a `.`", func() {
+			dir, _ := os.Stat(filepath.Join(baseDir, ".hidden_folder"))
+			Expect(isDirIgnored(baseDir, dir)).To(BeTrue())
+		})
 	})
 })

server/app/serve_index_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/deluan/navidrome/conf"
 	"github.com/deluan/navidrome/consts"
 	"github.com/deluan/navidrome/model"
-	"github.com/deluan/navidrome/persistence"
+	"github.com/deluan/navidrome/tests"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -22,7 +22,7 @@ var _ = Describe("serveIndex", func() {
 	fs := http.Dir("tests/fixtures")
 
 	BeforeEach(func() {
-		ds = &persistence.MockDataStore{MockedUser: mockUser}
+		ds = &tests.MockDataStore{MockedUser: mockUser}
 		conf.Server.UILoginBackgroundURL = ""
 	})
 

server/initial_setup.go

@@ -3,6 +3,7 @@ package server
 import (
 	"context"
 	"fmt"
+	"os/exec"
 	"time"
 
 	"github.com/deluan/navidrome/conf"
@@ -79,3 +80,26 @@ func createJWTSecret(ds model.DataStore) error {
 	}
 	return err
 }
+
+func checkFfmpegInstallation() {
+	path, err := exec.LookPath("ffmpeg")
+	if err == nil {
+		log.Info("Found ffmpeg", "path", path)
+		return
+	}
+	log.Warn("Unable to find ffmpeg. Transcoding will fail if used", err)
+	if conf.Server.Scanner.Extractor == "ffmpeg" {
+		log.Warn("ffmpeg cannot be used for metadata extraction. Falling back to taglib")
+		conf.Server.Scanner.Extractor = "taglib"
+	}
+}
+
+func checkExternalCredentials() {
+	if conf.Server.LastFM.ApiKey == "" || conf.Server.LastFM.Secret == "" {
+		log.Info("Last.FM integration not available: missing ApiKey/Secret")
+	}
+
+	if conf.Server.Spotify.ID == "" || conf.Server.Spotify.Secret == "" {
+		log.Info("Spotify integration is not enabled: artist images will not be available")
+	}
+}

server/middlewares.go

@@ -70,11 +70,11 @@ func robotsTXT(fs http.FileSystem) func(next http.Handler) http.Handler {
 
 func secureMiddleware() func(h http.Handler) http.Handler {
 	sec := secure.New(secure.Options{
-		ContentTypeNosniff:    true,
-		FrameDeny:             true,
-		ReferrerPolicy:        "same-origin",
-		FeaturePolicy:         "autoplay 'none'; camera: 'none'; display-capture 'none'; microphone: 'none'; usb: 'none'",
-		ContentSecurityPolicy: "script-src 'self' 'unsafe-inline'",
+		ContentTypeNosniff: true,
+		FrameDeny:          true,
+		ReferrerPolicy:     "same-origin",
+		FeaturePolicy:      "autoplay 'none'; camera: 'none'; display-capture 'none'; microphone: 'none'; usb: 'none'",
+		//ContentSecurityPolicy: "script-src 'self' 'unsafe-inline'",
 	})
 	return sec.Handler
 }

server/server.go

@@ -3,14 +3,12 @@ package server
 import (
 	"net/http"
 	"path"
-	"time"
 
 	"github.com/deluan/navidrome/assets"
 	"github.com/deluan/navidrome/conf"
 	"github.com/deluan/navidrome/consts"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
-	"github.com/deluan/navidrome/scanner"
 	"github.com/go-chi/chi"
 	"github.com/go-chi/chi/middleware"
 	"github.com/go-chi/cors"
@@ -22,16 +20,16 @@ type Handler interface {
 }
 
 type Server struct {
-	Scanner *scanner.Scanner
-	router  *chi.Mux
-	ds      model.DataStore
+	router *chi.Mux
+	ds     model.DataStore
 }
 
-func New(scanner *scanner.Scanner, ds model.DataStore) *Server {
-	a := &Server{Scanner: scanner, ds: ds}
+func New(ds model.DataStore) *Server {
+	a := &Server{ds: ds}
 	initialSetup(ds)
 	a.initRoutes()
-	a.initScanner()
+	checkFfmpegInstallation()
+	checkExternalCredentials()
 	return a
 }
 
@@ -45,9 +43,9 @@ func (a *Server) MountRouter(urlPath string, subRouter Handler) {
 	})
 }
 
-func (a *Server) Run(addr string) {
+func (a *Server) Run(addr string) error {
 	log.Info("Navidrome server is accepting requests", "address", addr)
-	log.Error(http.ListenAndServe(addr, a.router))
+	return http.ListenAndServe(addr, a.router)
 }
 
 func (a *Server) initRoutes() {
@@ -70,22 +68,3 @@ func (a *Server) initRoutes() {
 
 	a.router = r
 }
-
-func (a *Server) initScanner() {
-	interval := conf.Server.ScanInterval
-	if interval == 0 {
-		log.Warn("Scanner is disabled", "interval", conf.Server.ScanInterval)
-		return
-	}
-	log.Info("Starting scanner", "interval", interval.String())
-	go func() {
-		time.Sleep(2 * time.Second)
-		for {
-			err := a.Scanner.RescanAll(false)
-			if err != nil {
-				log.Error("Error scanning media folder", "folder", conf.Server.MusicFolder, err)
-			}
-			time.Sleep(interval)
-		}
-	}()
-}

server/subsonic/album_lists.go

@@ -1,139 +1,159 @@
 package subsonic
 
 import (
+	"context"
 	"errors"
 	"net/http"
+	"time"
 
+	"github.com/deluan/navidrome/core"
 	"github.com/deluan/navidrome/log"
-	"github.com/deluan/navidrome/server/subsonic/engine"
+	"github.com/deluan/navidrome/model"
+	"github.com/deluan/navidrome/server/subsonic/filter"
 	"github.com/deluan/navidrome/server/subsonic/responses"
 	"github.com/deluan/navidrome/utils"
 )
 
 type AlbumListController struct {
-	listGen engine.ListGenerator
+	ds         model.DataStore
+	nowPlaying core.NowPlaying
 }
 
-func NewAlbumListController(listGen engine.ListGenerator) *AlbumListController {
+func NewAlbumListController(ds model.DataStore, nowPlaying core.NowPlaying) *AlbumListController {
 	c := &AlbumListController{
-		listGen: listGen,
+		ds:         ds,
+		nowPlaying: nowPlaying,
 	}
 	return c
 }
 
-func (c *AlbumListController) getNewAlbumList(r *http.Request) (engine.Entries, error) {
-	typ, err := requiredParamString(r, "type", "Required string parameter 'type' is not present")
+func (c *AlbumListController) getAlbumList(r *http.Request) (model.Albums, error) {
+	typ, err := requiredParamString(r, "type")
 	if err != nil {
 		return nil, err
 	}
 
-	var filter engine.ListFilter
+	var opts filter.Options
 	switch typ {
 	case "newest":
-		filter = engine.ByNewest()
+		opts = filter.AlbumsByNewest()
 	case "recent":
-		filter = engine.ByRecent()
+		opts = filter.AlbumsByRecent()
 	case "random":
-		filter = engine.ByRandom()
+		opts = filter.AlbumsByRandom()
 	case "alphabeticalByName":
-		filter = engine.ByName()
+		opts = filter.AlbumsByName()
 	case "alphabeticalByArtist":
-		filter = engine.ByArtist()
+		opts = filter.AlbumsByArtist()
 	case "frequent":
-		filter = engine.ByFrequent()
+		opts = filter.AlbumsByFrequent()
 	case "starred":
-		filter = engine.ByStarred()
+		opts = filter.AlbumsByStarred()
 	case "highest":
-		filter = engine.ByRating()
+		opts = filter.AlbumsByRating()
 	case "byGenre":
-		filter = engine.ByGenre(utils.ParamString(r, "genre"))
+		opts = filter.AlbumsByGenre(utils.ParamString(r, "genre"))
 	case "byYear":
-		filter = engine.ByYear(utils.ParamInt(r, "fromYear", 0), utils.ParamInt(r, "toYear", 0))
+		opts = filter.AlbumsByYear(utils.ParamInt(r, "fromYear", 0), utils.ParamInt(r, "toYear", 0))
 	default:
 		log.Error(r, "albumList type not implemented", "type", typ)
-		return nil, errors.New("Not implemented!")
+		return nil, errors.New("not implemented")
 	}
 
-	offset := utils.ParamInt(r, "offset", 0)
-	size := utils.MinInt(utils.ParamInt(r, "size", 10), 500)
+	opts.Offset = utils.ParamInt(r, "offset", 0)
+	opts.Max = utils.MinInt(utils.ParamInt(r, "size", 10), 500)
+	albums, err := c.ds.Album(r.Context()).GetAll(model.QueryOptions(opts))
 
-	albums, err := c.listGen.GetAlbums(r.Context(), offset, size, filter)
 	if err != nil {
 		log.Error(r, "Error retrieving albums", "error", err)
-		return nil, errors.New("Internal Error")
+		return nil, errors.New("internal Error")
 	}
 
 	return albums, nil
 }
 
 func (c *AlbumListController) GetAlbumList(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
-	albums, err := c.getNewAlbumList(r)
+	albums, err := c.getAlbumList(r)
 	if err != nil {
 		return nil, newError(responses.ErrorGeneric, err.Error())
 	}
 
 	response := newResponse()
-	response.AlbumList = &responses.AlbumList{Album: toChildren(r.Context(), albums)}
+	response.AlbumList = &responses.AlbumList{Album: childrenFromAlbums(r.Context(), albums)}
 	return response, nil
 }
 
 func (c *AlbumListController) GetAlbumList2(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
-	albums, err := c.getNewAlbumList(r)
+	albums, err := c.getAlbumList(r)
 	if err != nil {
 		return nil, newError(responses.ErrorGeneric, err.Error())
 	}
 
 	response := newResponse()
-	response.AlbumList2 = &responses.AlbumList{Album: toAlbums(r.Context(), albums)}
+	response.AlbumList2 = &responses.AlbumList{Album: childrenFromAlbums(r.Context(), albums)}
 	return response, nil
 }
 
 func (c *AlbumListController) GetStarred(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
-	artists, albums, mediaFiles, err := c.listGen.GetAllStarred(r.Context())
+	ctx := r.Context()
+	options := model.QueryOptions{Sort: "starred_at", Order: "desc"}
+	artists, err := c.ds.Artist(ctx).GetStarred(options)
 	if err != nil {
-		log.Error(r, "Error retrieving starred media", "error", err)
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		log.Error(r, "Error retrieving starred artists", "error", err)
+		return nil, err
+	}
+	albums, err := c.ds.Album(ctx).GetStarred(options)
+	if err != nil {
+		log.Error(r, "Error retrieving starred albums", "error", err)
+		return nil, err
+	}
+	mediaFiles, err := c.ds.MediaFile(ctx).GetStarred(options)
+	if err != nil {
+		log.Error(r, "Error retrieving starred mediaFiles", "error", err)
+		return nil, err
 	}
 
 	response := newResponse()
 	response.Starred = &responses.Starred{}
-	response.Starred.Artist = toArtists(artists)
-	response.Starred.Album = toChildren(r.Context(), albums)
-	response.Starred.Song = toChildren(r.Context(), mediaFiles)
+	response.Starred.Artist = toArtists(ctx, artists)
+	response.Starred.Album = childrenFromAlbums(r.Context(), albums)
+	response.Starred.Song = childrenFromMediaFiles(r.Context(), mediaFiles)
 	return response, nil
 }
 
 func (c *AlbumListController) GetStarred2(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
-	artists, albums, mediaFiles, err := c.listGen.GetAllStarred(r.Context())
+	resp, err := c.GetStarred(w, r)
 	if err != nil {
-		log.Error(r, "Error retrieving starred media", "error", err)
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 
 	response := newResponse()
-	response.Starred2 = &responses.Starred{}
-	response.Starred2.Artist = toArtists(artists)
-	response.Starred2.Album = toAlbums(r.Context(), albums)
-	response.Starred2.Song = toChildren(r.Context(), mediaFiles)
+	response.Starred2 = resp.Starred
 	return response, nil
 }
 
 func (c *AlbumListController) GetNowPlaying(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
-	npInfos, err := c.listGen.GetNowPlaying(r.Context())
+	ctx := r.Context()
+	npInfo, err := c.nowPlaying.GetAll()
 	if err != nil {
 		log.Error(r, "Error retrieving now playing list", "error", err)
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 
 	response := newResponse()
 	response.NowPlaying = &responses.NowPlaying{}
-	response.NowPlaying.Entry = make([]responses.NowPlayingEntry, len(npInfos))
-	for i, entry := range npInfos {
-		response.NowPlaying.Entry[i].Child = toChild(r.Context(), entry)
-		response.NowPlaying.Entry[i].UserName = entry.UserName
-		response.NowPlaying.Entry[i].MinutesAgo = entry.MinutesAgo
-		response.NowPlaying.Entry[i].PlayerId = entry.PlayerId
-		response.NowPlaying.Entry[i].PlayerName = entry.PlayerName
+	response.NowPlaying.Entry = make([]responses.NowPlayingEntry, len(npInfo))
+	for i, np := range npInfo {
+		mf, err := c.ds.MediaFile(ctx).Get(np.TrackID)
+		if err != nil {
+			return nil, err
+		}
+
+		response.NowPlaying.Entry[i].Child = childFromMediaFile(ctx, *mf)
+		response.NowPlaying.Entry[i].UserName = np.Username
+		response.NowPlaying.Entry[i].MinutesAgo = int(time.Since(np.Start).Minutes())
+		response.NowPlaying.Entry[i].PlayerId = np.PlayerId
+		response.NowPlaying.Entry[i].PlayerName = np.PlayerName
 	}
 	return response, nil
 }
@@ -144,15 +164,15 @@ func (c *AlbumListController) GetRandomSongs(w http.ResponseWriter, r *http.Requ
 	fromYear := utils.ParamInt(r, "fromYear", 0)
 	toYear := utils.ParamInt(r, "toYear", 0)
 
-	songs, err := c.listGen.GetSongs(r.Context(), 0, size, engine.SongsByRandom(genre, fromYear, toYear))
+	songs, err := c.getSongs(r.Context(), 0, size, filter.SongsByRandom(genre, fromYear, toYear))
 	if err != nil {
 		log.Error(r, "Error retrieving random songs", "error", err)
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 
 	response := newResponse()
 	response.RandomSongs = &responses.Songs{}
-	response.RandomSongs.Songs = toChildren(r.Context(), songs)
+	response.RandomSongs.Songs = childrenFromMediaFiles(r.Context(), songs)
 	return response, nil
 }
 
@@ -161,14 +181,20 @@ func (c *AlbumListController) GetSongsByGenre(w http.ResponseWriter, r *http.Req
 	offset := utils.MinInt(utils.ParamInt(r, "offset", 0), 500)
 	genre := utils.ParamString(r, "genre")
 
-	songs, err := c.listGen.GetSongs(r.Context(), offset, count, engine.SongsByGenre(genre))
+	songs, err := c.getSongs(r.Context(), offset, count, filter.SongsByGenre(genre))
 	if err != nil {
 		log.Error(r, "Error retrieving random songs", "error", err)
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 
 	response := newResponse()
 	response.SongsByGenre = &responses.Songs{}
-	response.SongsByGenre.Songs = toChildren(r.Context(), songs)
+	response.SongsByGenre.Songs = childrenFromMediaFiles(r.Context(), songs)
 	return response, nil
 }
+
+func (c *AlbumListController) getSongs(ctx context.Context, offset, size int, opts filter.Options) (model.MediaFiles, error) {
+	opts.Offset = offset
+	opts.Max = size
+	return c.ds.MediaFile(ctx).GetAll(model.QueryOptions(opts))
+}

server/subsonic/album_lists_test.go

@@ -2,103 +2,90 @@ package subsonic
 
 import (
 	"context"
-	"errors"
 	"net/http/httptest"
 
-	"github.com/deluan/navidrome/server/subsonic/engine"
+	"github.com/deluan/navidrome/log"
+	"github.com/deluan/navidrome/model"
+	"github.com/deluan/navidrome/tests"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
-type fakeListGen struct {
-	engine.ListGenerator
-	data       engine.Entries
-	err        error
-	recvOffset int
-	recvSize   int
-}
-
-func (lg *fakeListGen) GetAlbums(ctx context.Context, offset int, size int, filter engine.ListFilter) (engine.Entries, error) {
-	if lg.err != nil {
-		return nil, lg.err
-	}
-	lg.recvOffset = offset
-	lg.recvSize = size
-	return lg.data, nil
-}
-
 var _ = Describe("AlbumListController", func() {
 	var controller *AlbumListController
-	var listGen *fakeListGen
+	var ds model.DataStore
+	var mockRepo *tests.MockAlbum
 	var w *httptest.ResponseRecorder
+	ctx := log.NewContext(context.TODO())
 
 	BeforeEach(func() {
-		listGen = &fakeListGen{}
-		controller = NewAlbumListController(listGen)
+		ds = &tests.MockDataStore{}
+		mockRepo = ds.Album(ctx).(*tests.MockAlbum)
+		controller = NewAlbumListController(ds, nil)
 		w = httptest.NewRecorder()
 	})
 
 	Describe("GetAlbumList", func() {
 		It("should return list of the type specified", func() {
 			r := newGetRequest("type=newest", "offset=10", "size=20")
-			listGen.data = engine.Entries{
-				{Id: "1"}, {Id: "2"},
-			}
+			mockRepo.SetData(model.Albums{
+				{ID: "1"}, {ID: "2"},
+			})
 			resp, err := controller.GetAlbumList(w, r)
 
 			Expect(err).To(BeNil())
 			Expect(resp.AlbumList.Album[0].Id).To(Equal("1"))
 			Expect(resp.AlbumList.Album[1].Id).To(Equal("2"))
-			Expect(listGen.recvOffset).To(Equal(10))
-			Expect(listGen.recvSize).To(Equal(20))
+			Expect(mockRepo.Options.Offset).To(Equal(10))
+			Expect(mockRepo.Options.Max).To(Equal(20))
 		})
 
 		It("should fail if missing type parameter", func() {
 			r := newGetRequest()
 			_, err := controller.GetAlbumList(w, r)
 
-			Expect(err).To(MatchError("Required string parameter 'type' is not present"))
+			Expect(err).To(MatchError("required 'type' parameter is missing"))
 		})
 
 		It("should return error if call fails", func() {
-			listGen.err = errors.New("some issue")
+			mockRepo.SetError(true)
 			r := newGetRequest("type=newest")
 
 			_, err := controller.GetAlbumList(w, r)
 
-			Expect(err).To(MatchError("Internal Error"))
+			Expect(err).ToNot(BeNil())
 		})
 	})
 
 	Describe("GetAlbumList2", func() {
 		It("should return list of the type specified", func() {
 			r := newGetRequest("type=newest", "offset=10", "size=20")
-			listGen.data = engine.Entries{
-				{Id: "1"}, {Id: "2"},
-			}
+			mockRepo.SetData(model.Albums{
+				{ID: "1"}, {ID: "2"},
+			})
 			resp, err := controller.GetAlbumList2(w, r)
 
 			Expect(err).To(BeNil())
 			Expect(resp.AlbumList2.Album[0].Id).To(Equal("1"))
 			Expect(resp.AlbumList2.Album[1].Id).To(Equal("2"))
-			Expect(listGen.recvOffset).To(Equal(10))
-			Expect(listGen.recvSize).To(Equal(20))
+			Expect(mockRepo.Options.Offset).To(Equal(10))
+			Expect(mockRepo.Options.Max).To(Equal(20))
 		})
 
 		It("should fail if missing type parameter", func() {
 			r := newGetRequest()
 			_, err := controller.GetAlbumList2(w, r)
 
-			Expect(err).To(MatchError("Required string parameter 'type' is not present"))
+			Expect(err).To(MatchError("required 'type' parameter is missing"))
 		})
 
 		It("should return error if call fails", func() {
-			listGen.err = errors.New("some issue")
+			mockRepo.SetError(true)
 			r := newGetRequest("type=newest")
 
 			_, err := controller.GetAlbumList2(w, r)
 
-			Expect(err).To(MatchError("Internal Error"))
+			Expect(err).ToNot(BeNil())
 		})
 	})
 })

server/subsonic/api.go

@@ -11,34 +11,40 @@ import (
 	"github.com/deluan/navidrome/core"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
-	"github.com/deluan/navidrome/server/subsonic/engine"
+	"github.com/deluan/navidrome/scanner"
 	"github.com/deluan/navidrome/server/subsonic/responses"
 	"github.com/deluan/navidrome/utils"
 	"github.com/go-chi/chi"
 	"github.com/go-chi/chi/middleware"
 )
 
-const Version = "1.12.0"
+const Version = "1.16.1"
 
-type Handler = func(http.ResponseWriter, *http.Request) (*responses.Subsonic, error)
+type handler = func(http.ResponseWriter, *http.Request) (*responses.Subsonic, error)
 
 type Router struct {
-	Artwork       core.Artwork
-	ListGenerator engine.ListGenerator
-	Playlists     engine.Playlists
-	Streamer      core.MediaStreamer
-	Archiver      core.Archiver
-	Players       engine.Players
-	DataStore     model.DataStore
+	DataStore    model.DataStore
+	Artwork      core.Artwork
+	Streamer     core.MediaStreamer
+	Archiver     core.Archiver
+	Players      core.Players
+	ExternalInfo core.ExternalInfo
+	Scanner      scanner.Scanner
 
 	mux http.Handler
 }
 
-func New(artwork core.Artwork, listGenerator engine.ListGenerator,
-	playlists engine.Playlists, streamer core.MediaStreamer,
-	archiver core.Archiver, players engine.Players, ds model.DataStore) *Router {
-	r := &Router{Artwork: artwork, ListGenerator: listGenerator, Playlists: playlists,
-		Streamer: streamer, Archiver: archiver, Players: players, DataStore: ds}
+func New(ds model.DataStore, artwork core.Artwork, streamer core.MediaStreamer, archiver core.Archiver, players core.Players,
+	externalInfo core.ExternalInfo, scanner scanner.Scanner) *Router {
+	r := &Router{
+		DataStore:    ds,
+		Artwork:      artwork,
+		Streamer:     streamer,
+		Archiver:     archiver,
+		Players:      players,
+		ExternalInfo: externalInfo,
+		Scanner:      scanner,
+	}
 	r.mux = r.routes()
 	return r
 }
@@ -61,105 +67,117 @@ func (api *Router) routes() http.Handler {
 	r.Group(func(r chi.Router) {
 		c := initSystemController(api)
 		withPlayer := r.With(getPlayer(api.Players))
-		H(withPlayer, "ping", c.Ping)
-		H(withPlayer, "getLicense", c.GetLicense)
+		h(withPlayer, "ping", c.Ping)
+		h(withPlayer, "getLicense", c.GetLicense)
 	})
 	r.Group(func(r chi.Router) {
 		c := initBrowsingController(api)
 		withPlayer := r.With(getPlayer(api.Players))
-		H(withPlayer, "getMusicFolders", c.GetMusicFolders)
-		H(withPlayer, "getIndexes", c.GetIndexes)
-		H(withPlayer, "getArtists", c.GetArtists)
-		H(withPlayer, "getGenres", c.GetGenres)
-		H(withPlayer, "getMusicDirectory", c.GetMusicDirectory)
-		H(withPlayer, "getArtist", c.GetArtist)
-		H(withPlayer, "getAlbum", c.GetAlbum)
-		H(withPlayer, "getSong", c.GetSong)
-		H(withPlayer, "getArtistInfo", c.GetArtistInfo)
-		H(withPlayer, "getArtistInfo2", c.GetArtistInfo2)
-		H(withPlayer, "getTopSongs", c.GetTopSongs)
+		h(withPlayer, "getMusicFolders", c.GetMusicFolders)
+		h(withPlayer, "getIndexes", c.GetIndexes)
+		h(withPlayer, "getArtists", c.GetArtists)
+		h(withPlayer, "getGenres", c.GetGenres)
+		h(withPlayer, "getMusicDirectory", c.GetMusicDirectory)
+		h(withPlayer, "getArtist", c.GetArtist)
+		h(withPlayer, "getAlbum", c.GetAlbum)
+		h(withPlayer, "getSong", c.GetSong)
+		h(withPlayer, "getArtistInfo", c.GetArtistInfo)
+		h(withPlayer, "getArtistInfo2", c.GetArtistInfo2)
+		h(withPlayer, "getTopSongs", c.GetTopSongs)
+		h(withPlayer, "getSimilarSongs", c.GetSimilarSongs)
+		h(withPlayer, "getSimilarSongs2", c.GetSimilarSongs2)
 	})
 	r.Group(func(r chi.Router) {
 		c := initAlbumListController(api)
 		withPlayer := r.With(getPlayer(api.Players))
-		H(withPlayer, "getAlbumList", c.GetAlbumList)
-		H(withPlayer, "getAlbumList2", c.GetAlbumList2)
-		H(withPlayer, "getStarred", c.GetStarred)
-		H(withPlayer, "getStarred2", c.GetStarred2)
-		H(withPlayer, "getNowPlaying", c.GetNowPlaying)
-		H(withPlayer, "getRandomSongs", c.GetRandomSongs)
-		H(withPlayer, "getSongsByGenre", c.GetSongsByGenre)
+		h(withPlayer, "getAlbumList", c.GetAlbumList)
+		h(withPlayer, "getAlbumList2", c.GetAlbumList2)
+		h(withPlayer, "getStarred", c.GetStarred)
+		h(withPlayer, "getStarred2", c.GetStarred2)
+		h(withPlayer, "getNowPlaying", c.GetNowPlaying)
+		h(withPlayer, "getRandomSongs", c.GetRandomSongs)
+		h(withPlayer, "getSongsByGenre", c.GetSongsByGenre)
 	})
 	r.Group(func(r chi.Router) {
 		c := initMediaAnnotationController(api)
-		H(r, "setRating", c.SetRating)
-		H(r, "star", c.Star)
-		H(r, "unstar", c.Unstar)
-		H(r, "scrobble", c.Scrobble)
+		h(r, "setRating", c.SetRating)
+		h(r, "star", c.Star)
+		h(r, "unstar", c.Unstar)
+		h(r, "scrobble", c.Scrobble)
 	})
 	r.Group(func(r chi.Router) {
 		c := initPlaylistsController(api)
 		withPlayer := r.With(getPlayer(api.Players))
-		H(withPlayer, "getPlaylists", c.GetPlaylists)
-		H(withPlayer, "getPlaylist", c.GetPlaylist)
-		H(withPlayer, "createPlaylist", c.CreatePlaylist)
-		H(withPlayer, "deletePlaylist", c.DeletePlaylist)
-		H(withPlayer, "updatePlaylist", c.UpdatePlaylist)
+		h(withPlayer, "getPlaylists", c.GetPlaylists)
+		h(withPlayer, "getPlaylist", c.GetPlaylist)
+		h(withPlayer, "createPlaylist", c.CreatePlaylist)
+		h(withPlayer, "deletePlaylist", c.DeletePlaylist)
+		h(withPlayer, "updatePlaylist", c.UpdatePlaylist)
 	})
 	r.Group(func(r chi.Router) {
 		c := initBookmarksController(api)
 		withPlayer := r.With(getPlayer(api.Players))
-		H(withPlayer, "getBookmarks", c.GetBookmarks)
-		H(withPlayer, "createBookmark", c.CreateBookmark)
-		H(withPlayer, "deleteBookmark", c.DeleteBookmark)
-		H(withPlayer, "getPlayQueue", c.GetPlayQueue)
-		H(withPlayer, "savePlayQueue", c.SavePlayQueue)
+		h(withPlayer, "getBookmarks", c.GetBookmarks)
+		h(withPlayer, "createBookmark", c.CreateBookmark)
+		h(withPlayer, "deleteBookmark", c.DeleteBookmark)
+		h(withPlayer, "getPlayQueue", c.GetPlayQueue)
+		h(withPlayer, "savePlayQueue", c.SavePlayQueue)
 	})
 	r.Group(func(r chi.Router) {
 		c := initSearchingController(api)
 		withPlayer := r.With(getPlayer(api.Players))
-		H(withPlayer, "search2", c.Search2)
-		H(withPlayer, "search3", c.Search3)
+		h(withPlayer, "search2", c.Search2)
+		h(withPlayer, "search3", c.Search3)
 	})
 	r.Group(func(r chi.Router) {
 		c := initUsersController(api)
-		H(r, "getUser", c.GetUser)
+		h(r, "getUser", c.GetUser)
+		h(r, "getUsers", c.GetUsers)
+	})
+	r.Group(func(r chi.Router) {
+		c := initLibraryScanningController(api)
+		h(r, "getScanStatus", c.GetScanStatus)
+		h(r, "startScan", c.StartScan)
 	})
 	r.Group(func(r chi.Router) {
 		c := initMediaRetrievalController(api)
 		// configure request throttling
 		maxRequests := utils.MaxInt(2, runtime.NumCPU())
 		withThrottle := r.With(middleware.ThrottleBacklog(maxRequests, consts.RequestThrottleBacklogLimit, consts.RequestThrottleBacklogTimeout))
-		H(withThrottle, "getAvatar", c.GetAvatar)
-		H(withThrottle, "getCoverArt", c.GetCoverArt)
+		h(withThrottle, "getAvatar", c.GetAvatar)
+		h(withThrottle, "getCoverArt", c.GetCoverArt)
 	})
 	r.Group(func(r chi.Router) {
 		c := initStreamController(api)
 		withPlayer := r.With(getPlayer(api.Players))
-		H(withPlayer, "stream", c.Stream)
-		H(withPlayer, "download", c.Download)
+		h(withPlayer, "stream", c.Stream)
+		h(withPlayer, "download", c.Download)
 	})
 
 	// Deprecated/Out of scope endpoints
-	HGone(r, "getChatMessages")
-	HGone(r, "addChatMessage")
-	HGone(r, "getVideos")
-	HGone(r, "getVideoInfo")
-	HGone(r, "getCaptions")
+	h410(r, "getChatMessages")
+	h410(r, "addChatMessage")
+	h410(r, "getVideos")
+	h410(r, "getVideoInfo")
+	h410(r, "getCaptions")
 	return r
 }
 
 // Add the Subsonic handler, with and without `.view` extension
 // Ex: if path = `ping` it will create the routes `/ping` and `/ping.view`
-func H(r chi.Router, path string, f Handler) {
+func h(r chi.Router, path string, f handler) {
 	handle := func(w http.ResponseWriter, r *http.Request) {
 		res, err := f(w, r)
 		if err != nil {
-			SendError(w, r, err)
+			// If it is not a Subsonic error, convert it to an ErrorGeneric
+			if _, ok := err.(subError); !ok {
+				err = newError(responses.ErrorGeneric, "Internal Error")
+			}
+			sendError(w, r, err)
 			return
 		}
 		if res != nil {
-			SendResponse(w, r, res)
+			sendResponse(w, r, res)
 		}
 	}
 	r.HandleFunc("/"+path, handle)
@@ -167,7 +185,7 @@ func H(r chi.Router, path string, f Handler) {
 }
 
 // Add a handler that returns 410 - Gone. Used to signal that an endpoint will not be implemented
-func HGone(r chi.Router, path string) {
+func h410(r chi.Router, path string) {
 	handle := func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(410)
 		_, _ = w.Write([]byte("This endpoint will not be implemented"))
@@ -176,19 +194,19 @@ func HGone(r chi.Router, path string) {
 	r.HandleFunc("/"+path+".view", handle)
 }
 
-func SendError(w http.ResponseWriter, r *http.Request, err error) {
+func sendError(w http.ResponseWriter, r *http.Request, err error) {
 	response := newResponse()
 	code := responses.ErrorGeneric
-	if e, ok := err.(SubsonicError); ok {
+	if e, ok := err.(subError); ok {
 		code = e.code
 	}
 	response.Status = "fail"
 	response.Error = &responses.Error{Code: code, Message: err.Error()}
 
-	SendResponse(w, r, response)
+	sendResponse(w, r, response)
 }
 
-func SendResponse(w http.ResponseWriter, r *http.Request, payload *responses.Subsonic) {
+func sendResponse(w http.ResponseWriter, r *http.Request, payload *responses.Subsonic) {
 	f := utils.ParamString(r, "f")
 	var response []byte
 	switch f {

server/subsonic/bookmarks.go

@@ -24,7 +24,7 @@ func (c *BookmarksController) GetBookmarks(w http.ResponseWriter, r *http.Reques
 	repo := c.ds.MediaFile(r.Context())
 	bmks, err := repo.GetBookmarks()
 	if err != nil {
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 
 	response := newResponse()
@@ -44,7 +44,7 @@ func (c *BookmarksController) GetBookmarks(w http.ResponseWriter, r *http.Reques
 }
 
 func (c *BookmarksController) CreateBookmark(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
-	id, err := requiredParamString(r, "id", "id parameter required")
+	id, err := requiredParamString(r, "id")
 	if err != nil {
 		return nil, err
 	}
@@ -55,13 +55,13 @@ func (c *BookmarksController) CreateBookmark(w http.ResponseWriter, r *http.Requ
 	repo := c.ds.MediaFile(r.Context())
 	err = repo.AddBookmark(id, comment, position)
 	if err != nil {
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 	return newResponse(), nil
 }
 
 func (c *BookmarksController) DeleteBookmark(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
-	id, err := requiredParamString(r, "id", "id parameter required")
+	id, err := requiredParamString(r, "id")
 	if err != nil {
 		return nil, err
 	}
@@ -69,7 +69,7 @@ func (c *BookmarksController) DeleteBookmark(w http.ResponseWriter, r *http.Requ
 	repo := c.ds.MediaFile(r.Context())
 	err = repo.DeleteBookmark(id)
 	if err != nil {
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 	return newResponse(), nil
 }
@@ -80,7 +80,7 @@ func (c *BookmarksController) GetPlayQueue(w http.ResponseWriter, r *http.Reques
 	repo := c.ds.PlayQueue(r.Context())
 	pq, err := repo.Retrieve(user.ID)
 	if err != nil {
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 
 	response := newResponse()
@@ -96,7 +96,7 @@ func (c *BookmarksController) GetPlayQueue(w http.ResponseWriter, r *http.Reques
 }
 
 func (c *BookmarksController) SavePlayQueue(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
-	ids, err := requiredParamStrings(r, "id", "id parameter required")
+	ids, err := requiredParamStrings(r, "id")
 	if err != nil {
 		return nil, err
 	}
@@ -125,7 +125,7 @@ func (c *BookmarksController) SavePlayQueue(w http.ResponseWriter, r *http.Reque
 	repo := c.ds.PlayQueue(r.Context())
 	err = repo.Store(pq)
 	if err != nil {
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 	return newResponse(), nil
 }

server/subsonic/browsing.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/deluan/navidrome/conf"
+	"github.com/deluan/navidrome/core"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
 	"github.com/deluan/navidrome/server/subsonic/responses"
@@ -17,10 +18,11 @@ import (
 
 type BrowsingController struct {
 	ds model.DataStore
+	ei core.ExternalInfo
 }
 
-func NewBrowsingController(ds model.DataStore) *BrowsingController {
-	return &BrowsingController{ds: ds}
+func NewBrowsingController(ds model.DataStore, ei core.ExternalInfo) *BrowsingController {
+	return &BrowsingController{ds: ds, ei: ei}
 }
 
 func (c *BrowsingController) GetMusicFolders(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
@@ -35,17 +37,17 @@ func (c *BrowsingController) GetMusicFolders(w http.ResponseWriter, r *http.Requ
 	return response, nil
 }
 
-func (c *BrowsingController) getArtistIndex(ctx context.Context, mediaFolderId string, ifModifiedSince time.Time) (*responses.Indexes, error) {
-	folder, err := c.ds.MediaFolder(ctx).Get(mediaFolderId)
+func (c *BrowsingController) getArtistIndex(ctx context.Context, mediaFolderId int, ifModifiedSince time.Time) (*responses.Indexes, error) {
+	folder, err := c.ds.MediaFolder(ctx).Get(int32(mediaFolderId))
 	if err != nil {
 		log.Error(ctx, "Error retrieving MediaFolder", "id", mediaFolderId, err)
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 
 	l, err := c.ds.Property(ctx).DefaultGet(model.PropLastScan+"-"+folder.Path, "-1")
 	if err != nil {
 		log.Error(ctx, "Error retrieving LastScan property", err)
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 
 	var indexes model.ArtistIndexes
@@ -55,7 +57,7 @@ func (c *BrowsingController) getArtistIndex(ctx context.Context, mediaFolderId s
 		indexes, err = c.ds.Artist(ctx).GetIndex()
 		if err != nil {
 			log.Error(ctx, "Error retrieving Indexes", err)
-			return nil, newError(responses.ErrorGeneric, "Internal Error")
+			return nil, err
 		}
 	}
 
@@ -67,18 +69,13 @@ func (c *BrowsingController) getArtistIndex(ctx context.Context, mediaFolderId s
 	res.Index = make([]responses.Index, len(indexes))
 	for i, idx := range indexes {
 		res.Index[i].Name = idx.ID
-		res.Index[i].Artists = make([]responses.Artist, len(idx.Artists))
-		for j, a := range idx.Artists {
-			res.Index[i].Artists[j].Id = a.ID
-			res.Index[i].Artists[j].Name = a.Name
-			res.Index[i].Artists[j].AlbumCount = a.AlbumCount
-		}
+		res.Index[i].Artists = toArtists(ctx, idx.Artists)
 	}
 	return res, nil
 }
 
 func (c *BrowsingController) GetIndexes(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
-	musicFolderId := utils.ParamString(r, "musicFolderId")
+	musicFolderId := utils.ParamInt(r, "musicFolderId", 0)
 	ifModifiedSince := utils.ParamTime(r, "ifModifiedSince", time.Time{})
 
 	res, err := c.getArtistIndex(r.Context(), musicFolderId, ifModifiedSince)
@@ -92,7 +89,7 @@ func (c *BrowsingController) GetIndexes(w http.ResponseWriter, r *http.Request)
 }
 
 func (c *BrowsingController) GetArtists(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
-	musicFolderId := utils.ParamString(r, "musicFolderId")
+	musicFolderId := utils.ParamInt(r, "musicFolderId", 0)
 	res, err := c.getArtistIndex(r.Context(), musicFolderId, time.Time{})
 	if err != nil {
 		return nil, err
@@ -107,14 +104,14 @@ func (c *BrowsingController) GetMusicDirectory(w http.ResponseWriter, r *http.Re
 	id := utils.ParamString(r, "id")
 	ctx := r.Context()
 
-	entity, err := getEntityByID(ctx, c.ds, id)
+	entity, err := core.GetEntityByID(ctx, c.ds, id)
 	switch {
 	case err == model.ErrNotFound:
 		log.Error(r, "Requested ID not found ", "id", id)
 		return nil, newError(responses.ErrorDataNotFound, "Directory not found")
 	case err != nil:
 		log.Error(err)
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 
 	var dir *responses.Directory
@@ -131,7 +128,7 @@ func (c *BrowsingController) GetMusicDirectory(w http.ResponseWriter, r *http.Re
 
 	if err != nil {
 		log.Error(err)
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 
 	response := newResponse()
@@ -150,13 +147,13 @@ func (c *BrowsingController) GetArtist(w http.ResponseWriter, r *http.Request) (
 		return nil, newError(responses.ErrorDataNotFound, "Artist not found")
 	case err != nil:
 		log.Error(ctx, "Error retrieving artist", "id", id, err)
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 
 	albums, err := c.ds.Album(ctx).FindByArtist(id)
 	if err != nil {
 		log.Error(ctx, "Error retrieving albums by artist", "id", id, "name", artist.Name, err)
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 
 	response := newResponse()
@@ -175,13 +172,13 @@ func (c *BrowsingController) GetAlbum(w http.ResponseWriter, r *http.Request) (*
 		return nil, newError(responses.ErrorDataNotFound, "Album not found")
 	case err != nil:
 		log.Error(ctx, "Error retrieving album", "id", id, err)
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 
 	mfs, err := c.ds.MediaFile(ctx).FindByAlbum(id)
 	if err != nil {
 		log.Error(ctx, "Error retrieving tracks from album", "id", id, "name", album.Name, err)
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 
 	response := newResponse()
@@ -200,7 +197,7 @@ func (c *BrowsingController) GetSong(w http.ResponseWriter, r *http.Request) (*r
 		return nil, newError(responses.ErrorDataNotFound, "Song not found")
 	case err != nil:
 		log.Error(r, "Error retrieving MediaFile", "id", id, err)
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 
 	response := newResponse()
@@ -214,7 +211,7 @@ func (c *BrowsingController) GetGenres(w http.ResponseWriter, r *http.Request) (
 	genres, err := c.ds.Genre(ctx).GetAll()
 	if err != nil {
 		log.Error(r, err)
-		return nil, newError(responses.ErrorGeneric, "Internal Error")
+		return nil, err
 	}
 	for i, g := range genres {
 		if strings.TrimSpace(g.Name) == "" {
@@ -230,36 +227,106 @@ func (c *BrowsingController) GetGenres(w http.ResponseWriter, r *http.Request) (
 	return response, nil
 }
 
-const placeholderArtistImageSmallUrl = "https://lastfm.freetls.fastly.net/i/u/64s/2a96cbd8b46e442fc41c2b86b821562f.png"
-const placeholderArtistImageMediumUrl = "https://lastfm.freetls.fastly.net/i/u/174s/2a96cbd8b46e442fc41c2b86b821562f.png"
-const placeholderArtistImageLargeUrl = "https://lastfm.freetls.fastly.net/i/u/300x300/2a96cbd8b46e442fc41c2b86b821562f.png"
-
-// TODO Integrate with Last.FM
 func (c *BrowsingController) GetArtistInfo(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
+	ctx := r.Context()
+	id, err := requiredParamString(r, "id")
+	if err != nil {
+		return nil, err
+	}
+	count := utils.ParamInt(r, "count", 20)
+	includeNotPresent := utils.ParamBool(r, "includeNotPresent", false)
+
+	artist, err := c.ei.UpdateArtistInfo(ctx, id, count, includeNotPresent)
+	if err != nil {
+		return nil, err
+	}
+
 	response := newResponse()
 	response.ArtistInfo = &responses.ArtistInfo{}
-	response.ArtistInfo.Biography = "Biography not available"
-	response.ArtistInfo.SmallImageUrl = placeholderArtistImageSmallUrl
-	response.ArtistInfo.MediumImageUrl = placeholderArtistImageMediumUrl
-	response.ArtistInfo.LargeImageUrl = placeholderArtistImageLargeUrl
+	response.ArtistInfo.Biography = artist.Biography
+	response.ArtistInfo.SmallImageUrl = artist.SmallImageUrl
+	response.ArtistInfo.MediumImageUrl = artist.MediumImageUrl
+	response.ArtistInfo.LargeImageUrl = artist.LargeImageUrl
+	response.ArtistInfo.LastFmUrl = artist.ExternalUrl
+	response.ArtistInfo.MusicBrainzID = artist.MbzArtistID
+	for _, s := range artist.SimilarArtists {
+		similar := toArtist(ctx, s)
+		response.ArtistInfo.SimilarArtist = append(response.ArtistInfo.SimilarArtist, similar)
+	}
 	return response, nil
 }
 
-// TODO Integrate with Last.FM
 func (c *BrowsingController) GetArtistInfo2(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
+	info, err := c.GetArtistInfo(w, r)
+	if err != nil {
+		return nil, err
+	}
+
 	response := newResponse()
 	response.ArtistInfo2 = &responses.ArtistInfo2{}
-	response.ArtistInfo2.Biography = "Biography not available"
-	response.ArtistInfo2.SmallImageUrl = placeholderArtistImageSmallUrl
-	response.ArtistInfo2.MediumImageUrl = placeholderArtistImageSmallUrl
-	response.ArtistInfo2.LargeImageUrl = placeholderArtistImageSmallUrl
+	response.ArtistInfo2.ArtistInfoBase = info.ArtistInfo.ArtistInfoBase
+	for _, s := range info.ArtistInfo.SimilarArtist {
+		similar := responses.ArtistID3{}
+		similar.Id = s.Id
+		similar.Name = s.Name
+		similar.AlbumCount = s.AlbumCount
+		similar.Starred = s.Starred
+		similar.ArtistImageUrl = s.ArtistImageUrl
+		response.ArtistInfo2.SimilarArtist = append(response.ArtistInfo2.SimilarArtist, similar)
+	}
 	return response, nil
 }
 
-// TODO Integrate with Last.FM
-func (c *BrowsingController) GetTopSongs(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
+func (c *BrowsingController) GetSimilarSongs(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
+	ctx := r.Context()
+	id, err := requiredParamString(r, "id")
+	if err != nil {
+		return nil, err
+	}
+	count := utils.ParamInt(r, "count", 50)
+
+	songs, err := c.ei.SimilarSongs(ctx, id, count)
+	if err != nil {
+		return nil, err
+	}
+
 	response := newResponse()
-	response.TopSongs = &responses.TopSongs{}
+	response.SimilarSongs = &responses.SimilarSongs{
+		Song: childrenFromMediaFiles(ctx, songs),
+	}
+	return response, nil
+}
+
+func (c *BrowsingController) GetSimilarSongs2(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
+	res, err := c.GetSimilarSongs(w, r)
+	if err != nil {
+		return nil, err
+	}
+
+	response := newResponse()
+	response.SimilarSongs2 = &responses.SimilarSongs2{
+		Song: res.SimilarSongs.Song,
+	}
+	return response, nil
+}
+
+func (c *BrowsingController) GetTopSongs(w http.ResponseWriter, r *http.Request) (*responses.Subsonic, error) {
+	ctx := r.Context()
+	artist, err := requiredParamString(r, "artist")
+	if err != nil {
+		return nil, err
+	}
+	count := utils.ParamInt(r, "count", 50)
+
+	songs, err := c.ei.TopSongs(ctx, artist, count)
+	if err != nil {
+		return nil, err
+	}
+
+	response := newResponse()
+	response.TopSongs = &responses.TopSongs{
+		Song: childrenFromMediaFiles(ctx, songs),
+	}
 	return response, nil
 }
 
@@ -284,16 +351,10 @@ func (c *BrowsingController) buildArtistDirectory(ctx context.Context, artist *m
 }
 
 func (c *BrowsingController) buildArtist(ctx context.Context, artist *model.Artist, albums model.Albums) *responses.ArtistWithAlbumsID3 {
-	dir := &responses.ArtistWithAlbumsID3{}
-	dir.Id = artist.ID
-	dir.Name = artist.Name
-	dir.AlbumCount = artist.AlbumCount
-	if artist.Starred {
-		dir.Starred = &artist.StarredAt
-	}
-
-	dir.Album = childrenFromAlbums(ctx, albums)
-	return dir
+	a := &responses.ArtistWithAlbumsID3{}
+	a.ArtistID3 = toArtistID3(ctx, *artist)
+	a.Album = childrenFromAlbums(ctx, albums)
+	return a
 }
 
 func (c *BrowsingController) buildAlbumDirectory(ctx context.Context, album *model.Album) (*responses.Directory, error) {

server/subsonic/engine/common.go

@@ -1,160 +0,0 @@
-package engine
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/deluan/navidrome/consts"
-	"github.com/deluan/navidrome/model"
-)
-
-type Entry struct {
-	Id               string
-	Title            string
-	IsDir            bool
-	Parent           string
-	Album            string
-	Year             int
-	Artist           string
-	Genre            string
-	CoverArt         string
-	Starred          time.Time
-	Track            int
-	Duration         int
-	Size             int64
-	Suffix           string
-	BitRate          int
-	ContentType      string
-	Path             string
-	PlayCount        int32
-	DiscNumber       int
-	Created          time.Time
-	AlbumId          string
-	ArtistId         string
-	Type             string
-	UserRating       int
-	SongCount        int
-	UserName         string
-	MinutesAgo       int
-	PlayerId         int
-	PlayerName       string
-	AlbumCount       int
-	BookmarkPosition int64
-	AbsolutePath     string
-}
-
-type Entries []Entry
-
-func FromArtist(ar *model.Artist) Entry {
-	e := Entry{}
-	e.Id = ar.ID
-	e.Title = ar.Name
-	e.AlbumCount = ar.AlbumCount
-	e.IsDir = true
-	if ar.Starred {
-		e.Starred = ar.StarredAt
-	}
-	return e
-}
-
-func FromAlbum(al *model.Album) Entry {
-	e := Entry{}
-	e.Id = al.ID
-	e.Title = al.Name
-	e.IsDir = true
-	e.Parent = al.AlbumArtistID
-	e.Album = al.Name
-	e.Year = al.MaxYear
-	e.Artist = al.AlbumArtist
-	e.Genre = al.Genre
-	e.CoverArt = al.CoverArtId
-	e.Created = al.CreatedAt
-	e.AlbumId = al.ID
-	e.ArtistId = al.AlbumArtistID
-	e.Duration = int(al.Duration)
-	e.SongCount = al.SongCount
-	if al.Starred {
-		e.Starred = al.StarredAt
-	}
-	e.PlayCount = int32(al.PlayCount)
-	e.UserRating = al.Rating
-	return e
-}
-
-func FromMediaFile(mf *model.MediaFile) Entry {
-	e := Entry{}
-	e.Id = mf.ID
-	e.Title = mf.Title
-	e.IsDir = false
-	e.Parent = mf.AlbumID
-	e.Album = mf.Album
-	e.Year = mf.Year
-	e.Artist = mf.Artist
-	e.Genre = mf.Genre
-	e.Track = mf.TrackNumber
-	e.Duration = int(mf.Duration)
-	e.Size = mf.Size
-	e.Suffix = mf.Suffix
-	e.BitRate = mf.BitRate
-	if mf.HasCoverArt {
-		e.CoverArt = mf.ID
-	} else {
-		e.CoverArt = "al-" + mf.AlbumID
-	}
-	e.ContentType = mf.ContentType()
-	e.AbsolutePath = mf.Path
-	// Creates a "pseudo" Path, to avoid sending absolute paths to the client
-	if mf.Path != "" {
-		e.Path = fmt.Sprintf("%s/%s/%s.%s", realArtistName(mf), mf.Album, mf.Title, mf.Suffix)
-	}
-	e.DiscNumber = mf.DiscNumber
-	e.Created = mf.CreatedAt
-	e.AlbumId = mf.AlbumID
-	e.ArtistId = mf.ArtistID
-	e.Type = "music"
-	e.PlayCount = int32(mf.PlayCount)
-	if mf.Starred {
-		e.Starred = mf.StarredAt
-	}
-	e.UserRating = mf.Rating
-	e.BookmarkPosition = mf.BookmarkPosition
-	return e
-}
-
-func realArtistName(mf *model.MediaFile) string {
-	switch {
-	case mf.Compilation:
-		return consts.VariousArtists
-	case mf.AlbumArtist != "":
-		return mf.AlbumArtist
-	}
-
-	return mf.Artist
-}
-
-func FromAlbums(albums model.Albums) Entries {
-	entries := make(Entries, len(albums))
-	for i := range albums {
-		al := albums[i]
-		entries[i] = FromAlbum(&al)
-	}
-	return entries
-}
-
-func FromMediaFiles(mfs model.MediaFiles) Entries {
-	entries := make(Entries, len(mfs))
-	for i := range mfs {
-		mf := mfs[i]
-		entries[i] = FromMediaFile(&mf)
-	}
-	return entries
-}
-
-func FromArtists(ars model.Artists) Entries {
-	entries := make(Entries, len(ars))
-	for i := range ars {
-		ar := ars[i]
-		entries[i] = FromArtist(&ar)
-	}
-	return entries
-}

server/subsonic/engine/list_generator.go

@@ -1,186 +0,0 @@
-package engine
-
-import (
-	"context"
-	"time"
-
-	"github.com/Masterminds/squirrel"
-	"github.com/deluan/navidrome/model"
-)
-
-type ListGenerator interface {
-	GetAllStarred(ctx context.Context) (artists Entries, albums Entries, mediaFiles Entries, err error)
-	GetNowPlaying(ctx context.Context) (Entries, error)
-	GetSongs(ctx context.Context, offset, size int, filter ListFilter) (Entries, error)
-	GetAlbums(ctx context.Context, offset, size int, filter ListFilter) (Entries, error)
-}
-
-func NewListGenerator(ds model.DataStore, npRepo NowPlayingRepository) ListGenerator {
-	return &listGenerator{ds, npRepo}
-}
-
-type ListFilter model.QueryOptions
-
-func ByNewest() ListFilter {
-	return ListFilter{Sort: "createdAt", Order: "desc"}
-}
-
-func ByRecent() ListFilter {
-	return ListFilter{Sort: "playDate", Order: "desc", Filters: squirrel.Gt{"play_date": time.Time{}}}
-}
-
-func ByFrequent() ListFilter {
-	return ListFilter{Sort: "playCount", Order: "desc", Filters: squirrel.Gt{"play_count": 0}}
-}
-
-func ByRandom() ListFilter {
-	return ListFilter{Sort: "random()"}
-}
-
-func ByName() ListFilter {
-	return ListFilter{Sort: "name"}
-}
-
-func ByArtist() ListFilter {
-	return ListFilter{Sort: "artist"}
-}
-
-func ByStarred() ListFilter {
-	return ListFilter{Sort: "starred_at", Order: "desc", Filters: squirrel.Eq{"starred": true}}
-}
-
-func ByRating() ListFilter {
-	return ListFilter{Sort: "Rating", Order: "desc", Filters: squirrel.Gt{"rating": 0}}
-}
-
-func ByGenre(genre string) ListFilter {
-	return ListFilter{
-		Sort:    "genre asc, name asc",
-		Filters: squirrel.Eq{"genre": genre},
-	}
-}
-
-func ByYear(fromYear, toYear int) ListFilter {
-	return ListFilter{
-		Sort: "max_year, name",
-		Filters: squirrel.Or{
-			squirrel.And{
-				squirrel.GtOrEq{"min_year": fromYear},
-				squirrel.LtOrEq{"min_year": toYear},
-			},
-			squirrel.And{
-				squirrel.GtOrEq{"max_year": fromYear},
-				squirrel.LtOrEq{"max_year": toYear},
-			},
-		},
-	}
-}
-
-func SongsByGenre(genre string) ListFilter {
-	return ListFilter{
-		Sort:    "genre asc, title asc",
-		Filters: squirrel.Eq{"genre": genre},
-	}
-}
-
-func SongsByRandom(genre string, fromYear, toYear int) ListFilter {
-	options := ListFilter{
-		Sort: "random()",
-	}
-	ff := squirrel.And{}
-	if genre != "" {
-		ff = append(ff, squirrel.Eq{"genre": genre})
-	}
-	if fromYear != 0 {
-		ff = append(ff, squirrel.GtOrEq{"year": fromYear})
-	}
-	if toYear != 0 {
-		ff = append(ff, squirrel.LtOrEq{"year": toYear})
-	}
-	options.Filters = ff
-	return options
-}
-
-type listGenerator struct {
-	ds     model.DataStore
-	npRepo NowPlayingRepository
-}
-
-func (g *listGenerator) GetSongs(ctx context.Context, offset, size int, filter ListFilter) (Entries, error) {
-	qo := model.QueryOptions(filter)
-	qo.Offset = offset
-	qo.Max = size
-	mediaFiles, err := g.ds.MediaFile(ctx).GetAll(qo)
-	if err != nil {
-		return nil, err
-	}
-
-	return FromMediaFiles(mediaFiles), nil
-}
-
-func (g *listGenerator) GetAlbums(ctx context.Context, offset, size int, filter ListFilter) (Entries, error) {
-	qo := model.QueryOptions(filter)
-	qo.Offset = offset
-	qo.Max = size
-	albums, err := g.ds.Album(ctx).GetAll(qo)
-	if err != nil {
-		return nil, err
-	}
-
-	return FromAlbums(albums), nil
-}
-
-func (g *listGenerator) GetStarred(ctx context.Context, offset int, size int) (Entries, error) {
-	qo := model.QueryOptions{Offset: offset, Max: size, Sort: "starred_at", Order: "desc"}
-	albums, err := g.ds.Album(ctx).GetStarred(qo)
-	if err != nil {
-		return nil, err
-	}
-
-	return FromAlbums(albums), nil
-}
-
-func (g *listGenerator) GetAllStarred(ctx context.Context) (artists Entries, albums Entries, mediaFiles Entries, err error) {
-	options := model.QueryOptions{Sort: "starred_at", Order: "desc"}
-
-	ars, err := g.ds.Artist(ctx).GetStarred(options)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	als, err := g.ds.Album(ctx).GetStarred(options)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	mfs, err := g.ds.MediaFile(ctx).GetStarred(options)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	artists = FromArtists(ars)
-	albums = FromAlbums(als)
-	mediaFiles = FromMediaFiles(mfs)
-
-	return
-}
-
-func (g *listGenerator) GetNowPlaying(ctx context.Context) (Entries, error) {
-	npInfo, err := g.npRepo.GetAll()
-	if err != nil {
-		return nil, err
-	}
-	entries := make(Entries, len(npInfo))
-	for i, np := range npInfo {
-		mf, err := g.ds.MediaFile(ctx).Get(np.TrackID)
-		if err != nil {
-			return nil, err
-		}
-		entries[i] = FromMediaFile(mf)
-		entries[i].UserName = np.Username
-		entries[i].MinutesAgo = int(time.Since(np.Start).Minutes())
-		entries[i].PlayerId = np.PlayerId
-		entries[i].PlayerName = np.PlayerName
-	}
-	return entries, nil
-}

server/subsonic/engine/mock_transcoding_repo_test.go

@@ -1,22 +0,0 @@
-package engine
-
-import "github.com/deluan/navidrome/model"
-
-type mockTranscodingRepository struct {
-	model.TranscodingRepository
-}
-
-func (m *mockTranscodingRepository) Get(id string) (*model.Transcoding, error) {
-	return &model.Transcoding{ID: id, TargetFormat: "mp3", DefaultBitRate: 160}, nil
-}
-
-func (m *mockTranscodingRepository) FindByFormat(format string) (*model.Transcoding, error) {
-	switch format {
-	case "mp3":
-		return &model.Transcoding{ID: "mp31", TargetFormat: "mp3", DefaultBitRate: 160}, nil
-	case "oga":
-		return &model.Transcoding{ID: "oga1", TargetFormat: "oga", DefaultBitRate: 128}, nil
-	default:
-		return nil, model.ErrNotFound
-	}
-}

server/subsonic/engine/playlists.go

@@ -1,151 +0,0 @@
-package engine
-
-import (
-	"context"
-	"time"
-
-	"github.com/deluan/navidrome/model"
-	"github.com/deluan/navidrome/model/request"
-	"github.com/deluan/navidrome/utils"
-)
-
-type Playlists interface {
-	GetAll(ctx context.Context) (model.Playlists, error)
-	Get(ctx context.Context, id string) (*PlaylistInfo, error)
-	Create(ctx context.Context, playlistId, name string, ids []string) error
-	Delete(ctx context.Context, playlistId string) error
-	Update(ctx context.Context, playlistId string, name *string, idsToAdd []string, idxToRemove []int) error
-}
-
-func NewPlaylists(ds model.DataStore) Playlists {
-	return &playlists{ds}
-}
-
-type playlists struct {
-	ds model.DataStore
-}
-
-func (p *playlists) Create(ctx context.Context, playlistId, name string, ids []string) error {
-	return p.ds.WithTx(func(tx model.DataStore) error {
-		owner := p.getUser(ctx)
-		var pls *model.Playlist
-		var err error
-
-		// If playlistID is present, override tracks
-		if playlistId != "" {
-			pls, err = tx.Playlist(ctx).Get(playlistId)
-			if err != nil {
-				return err
-			}
-			if owner != pls.Owner {
-				return model.ErrNotAuthorized
-			}
-			pls.Tracks = nil
-		} else {
-			pls = &model.Playlist{
-				Name:  name,
-				Owner: owner,
-			}
-		}
-		for _, id := range ids {
-			pls.Tracks = append(pls.Tracks, model.MediaFile{ID: id})
-		}
-
-		return tx.Playlist(ctx).Put(pls)
-	})
-}
-
-func (p *playlists) getUser(ctx context.Context) string {
-	user, ok := request.UserFrom(ctx)
-	if ok {
-		return user.UserName
-	}
-	return ""
-}
-
-func (p *playlists) Delete(ctx context.Context, playlistId string) error {
-	return p.ds.WithTx(func(tx model.DataStore) error {
-		pls, err := tx.Playlist(ctx).Get(playlistId)
-		if err != nil {
-			return err
-		}
-
-		owner := p.getUser(ctx)
-		if owner != pls.Owner {
-			return model.ErrNotAuthorized
-		}
-		return tx.Playlist(ctx).Delete(playlistId)
-	})
-}
-
-func (p *playlists) Update(ctx context.Context, playlistId string, name *string, idsToAdd []string, idxToRemove []int) error {
-	return p.ds.WithTx(func(tx model.DataStore) error {
-		pls, err := tx.Playlist(ctx).Get(playlistId)
-		if err != nil {
-			return err
-		}
-
-		owner := p.getUser(ctx)
-		if owner != pls.Owner {
-			return model.ErrNotAuthorized
-		}
-
-		if name != nil {
-			pls.Name = *name
-		}
-		newTracks := model.MediaFiles{}
-		for i, t := range pls.Tracks {
-			if utils.IntInSlice(i, idxToRemove) {
-				continue
-			}
-			newTracks = append(newTracks, t)
-		}
-
-		for _, id := range idsToAdd {
-			newTracks = append(newTracks, model.MediaFile{ID: id})
-		}
-		pls.Tracks = newTracks
-
-		return tx.Playlist(ctx).Put(pls)
-	})
-}
-
-func (p *playlists) GetAll(ctx context.Context) (model.Playlists, error) {
-	return p.ds.Playlist(ctx).GetAll()
-}
-
-type PlaylistInfo struct {
-	Id        string
-	Name      string
-	Entries   Entries
-	SongCount int
-	Duration  int
-	Public    bool
-	Owner     string
-	Comment   string
-	Created   time.Time
-	Changed   time.Time
-}
-
-func (p *playlists) Get(ctx context.Context, id string) (*PlaylistInfo, error) {
-	pl, err := p.ds.Playlist(ctx).Get(id)
-	if err != nil {
-		return nil, err
-	}
-
-	// TODO Use model.Playlist when got rid of Entries
-	plsInfo := &PlaylistInfo{
-		Id:        pl.ID,
-		Name:      pl.Name,
-		SongCount: pl.SongCount,
-		Duration:  int(pl.Duration),
-		Public:    pl.Public,
-		Owner:     pl.Owner,
-		Comment:   pl.Comment,
-		Changed:   pl.UpdatedAt,
-		Created:   pl.CreatedAt,
-	}
-
-	plsInfo.Entries = FromMediaFiles(pl.Tracks)
-	return plsInfo, nil
-}

server/subsonic/engine/wire_providers.go

@@ -1,12 +0,0 @@
-package engine
-
-import (
-	"github.com/google/wire"
-)
-
-var Set = wire.NewSet(
-	NewListGenerator,
-	NewPlaylists,
-	NewNowPlayingRepository,
-	NewPlayers,
-)

server/subsonic/filter/filters.go

@@ -0,0 +1,95 @@
+package filter
+
+import (
+	"time"
+
+	"github.com/Masterminds/squirrel"
+	"github.com/deluan/navidrome/model"
+)
+
+type Options model.QueryOptions
+
+func AlbumsByNewest() Options {
+	return Options{Sort: "createdAt", Order: "desc"}
+}
+
+func AlbumsByRecent() Options {
+	return Options{Sort: "playDate", Order: "desc", Filters: squirrel.Gt{"play_date": time.Time{}}}
+}
+
+func AlbumsByFrequent() Options {
+	return Options{Sort: "playCount", Order: "desc", Filters: squirrel.Gt{"play_count": 0}}
+}
+
+func AlbumsByRandom() Options {
+	return Options{Sort: "random()"}
+}
+
+func AlbumsByName() Options {
+	return Options{Sort: "name"}
+}
+
+func AlbumsByArtist() Options {
+	return Options{Sort: "artist"}
+}
+
+func AlbumsByStarred() Options {
+	return Options{Sort: "starred_at", Order: "desc", Filters: squirrel.Eq{"starred": true}}
+}
+
+func AlbumsByRating() Options {
+	return Options{Sort: "Rating", Order: "desc", Filters: squirrel.Gt{"rating": 0}}
+}
+
+func AlbumsByGenre(genre string) Options {
+	return Options{
+		Sort:    "genre asc, name asc",
+		Filters: squirrel.Eq{"genre": genre},
+	}
+}
+
+func AlbumsByYear(fromYear, toYear int) Options {
+	sortOption := "max_year, name"
+	if fromYear > toYear {
+		fromYear, toYear = toYear, fromYear
+		sortOption = "max_year desc, name"
+	}
+	return Options{
+		Sort: sortOption,
+		Filters: squirrel.Or{
+			squirrel.And{
+				squirrel.GtOrEq{"min_year": fromYear},
+				squirrel.LtOrEq{"min_year": toYear},
+			},
+			squirrel.And{
+				squirrel.GtOrEq{"max_year": fromYear},
+				squirrel.LtOrEq{"max_year": toYear},
+			},
+		},
+	}
+}
+
+func SongsByGenre(genre string) Options {
+	return Options{
+		Sort:    "genre asc, title asc",
+		Filters: squirrel.Eq{"genre": genre},
+	}
+}
+
+func SongsByRandom(genre string, fromYear, toYear int) Options {
+	options := Options{
+		Sort: "random()",
+	}
+	ff := squirrel.And{}
+	if genre != "" {
+		ff = append(ff, squirrel.Eq{"genre": genre})
+	}
+	if fromYear != 0 {
+		ff = append(ff, squirrel.GtOrEq{"year": fromYear})
+	}
+	if toYear != 0 {
+		ff = append(ff, squirrel.LtOrEq{"year": toYear})
+	}
+	options.Filters = ff
+	return options
+}