Change the default scanner to use new implementation

Bumps [lodash](https://github.com/lodash/lodash) from 4.17.15 to 4.17.19.
- [Release notes](https://github.com/lodash/lodash/releases)
- [Commits](https://github.com/lodash/lodash/compare/4.17.15...4.17.19)

Signed-off-by: dependabot[bot] <support@github.com>

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: deluan
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+liberapay: deluan
+ko_fi: deluan
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/pipeline.dockerfile

@@ -6,7 +6,7 @@ ARG TARGETPLATFORM
 RUN echo "Target Platform = ${TARGETPLATFORM}"
 
 COPY dist .
-RUN if [ "$TARGETPLATFORM" = "linux/amd64" ];  then cp navidrome_linux_musl_amd64_linux_amd64/navidrome /navidrome; fi
+RUN if [ "$TARGETPLATFORM" = "linux/amd64" ];  then cp navidrome_linux_amd64_linux_amd64/navidrome /navidrome; fi
 RUN if [ "$TARGETPLATFORM" = "linux/arm64" ];  then cp navidrome_linux_arm64_linux_arm64/navidrome /navidrome; fi
 RUN if [ "$TARGETPLATFORM" = "linux/arm/v6" ]; then cp navidrome_linux_arm_linux_arm_6/navidrome /navidrome; fi
 RUN if [ "$TARGETPLATFORM" = "linux/arm/v7" ]; then cp navidrome_linux_arm_linux_arm_7/navidrome /navidrome; fi
@@ -27,11 +27,9 @@ COPY --from=copy-binary /navidrome /app/
 VOLUME ["/data", "/music"]
 ENV ND_MUSICFOLDER /music
 ENV ND_DATAFOLDER /data
-ENV ND_SCANINTERVAL 1m
-ENV ND_TRANSCODINGCACHESIZE 100MB
-ENV ND_SESSIONTIMEOUT 30m
 ENV ND_LOGLEVEL info
 ENV ND_PORT 4533
+ENV GODEBUG "asyncpreemptoff=1"
 
 EXPOSE ${ND_PORT}
 HEALTHCHECK CMD wget -O- http://localhost:${ND_PORT}/ping || exit 1

.github/workflows/pipeline.yml

@@ -15,9 +15,9 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v0.1.7
+        uses: golangci/golangci-lint-action@v1
         with:
-          version: v1.26
+          version: v1.27
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
   go:
@@ -60,7 +60,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v1
         with:
-          node-version: 13
+          node-version: 14
 
       - uses: actions/cache@v1
         id: cache-npm
@@ -108,7 +108,7 @@ jobs:
 
       - name: Run GoReleaser - SNAPSHOT
         if: startsWith(github.ref, 'refs/tags/') != true
-        uses: docker://deluan/ci-goreleaser:1.14.1-1
+        uses: docker://deluan/ci-goreleaser:1.14.4-2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
@@ -116,7 +116,7 @@ jobs:
 
       - name: Run GoReleaser - RELEASE
         if: startsWith(github.ref, 'refs/tags/')
-        uses: docker://deluan/ci-goreleaser:1.14.1-1
+        uses: docker://deluan/ci-goreleaser:1.14.4-2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:

.gitignore

@@ -1,5 +1,6 @@
 .DS_Store
 .idea
+.vscode
 .envrc
 /navidrome
 /iTunes*.xml
@@ -15,10 +16,10 @@ Jamstash-master
 testDB
 navidrome.db
 *.swp
-*_gen.go
+embedded_gen.go
 dist
 music
-docker-compose.override.yml
+docker-compose.yml
 navidrome.db-shm
 navidrome.db-wal
 tags

.goreleaser.yml

@@ -19,7 +19,7 @@ builds:
     flags:
       - -tags=embed
     ldflags:
-      - -X github.com/deluan/navidrome/consts.gitSha={{.ShortCommit}} -X github.com/deluan/navidrome/consts.gitTag={{.Version}}
+      - -s -w -X github.com/deluan/navidrome/consts.gitSha={{.ShortCommit}} -X github.com/deluan/navidrome/consts.gitTag={{.Version}}
 
   - id: navidrome_linux_amd64
     env:
@@ -32,43 +32,30 @@ builds:
       - -tags=embed
     ldflags:
       - "-extldflags '-static'"
-      - -X github.com/deluan/navidrome/consts.gitSha={{.ShortCommit}} -X github.com/deluan/navidrome/consts.gitTag={{.Version}}
-
-  - id: navidrome_linux_musl_amd64
-    env:
-      - CGO_ENABLED=1
-      - CC=musl-gcc
-    goos:
-      - linux
-    goarch:
-      - amd64
-    flags:
-      - -tags=embed
-    ldflags:
-      - "-extldflags '-static'"
-      - -X github.com/deluan/navidrome/consts.gitSha={{.ShortCommit}} -X github.com/deluan/navidrome/consts.gitTag={{.Version}}
+      - -s -w -X github.com/deluan/navidrome/consts.gitSha={{.ShortCommit}} -X github.com/deluan/navidrome/consts.gitTag={{.Version}}
 
   - id: navidrome_linux_arm
     env:
       - CGO_ENABLED=1
-      - CC=arm-linux-gnueabi-gcc
+      - CC=arm-linux-gnueabi-gcc-5
     goos:
       - linux
     goarch:
       - arm
     goarm:
+      - 5
       - 6
       - 7
     flags:
       - -tags=embed
     ldflags:
       - "-extldflags '-static'"
-      - -X github.com/deluan/navidrome/consts.gitSha={{.ShortCommit}} -X github.com/deluan/navidrome/consts.gitTag={{.Version}}
+      - -s -w -X github.com/deluan/navidrome/consts.gitSha={{.ShortCommit}} -X github.com/deluan/navidrome/consts.gitTag={{.Version}}
 
   - id: navidrome_linux_arm64
     env:
       - CGO_ENABLED=1
-      - CC=aarch64-linux-gnu-gcc
+      - CC=aarch64-linux-gnu-gcc-5
     goos:
       - linux
     goarch:
@@ -77,7 +64,7 @@ builds:
       - -tags=embed
     ldflags:
       - "-extldflags '-static'"
-      - -X github.com/deluan/navidrome/consts.gitSha={{.ShortCommit}} -X github.com/deluan/navidrome/consts.gitTag={{.Version}}
+      - -s -w -X github.com/deluan/navidrome/consts.gitSha={{.ShortCommit}} -X github.com/deluan/navidrome/consts.gitTag={{.Version}}
 
   - id: navidrome_windows_i686
     env:
@@ -92,7 +79,7 @@ builds:
       - -tags=embed
     ldflags:
       - "-extldflags '-static'"
-      - -X github.com/deluan/navidrome/consts.gitSha={{.ShortCommit}} -X github.com/deluan/navidrome/consts.gitTag={{.Version}}
+      - -s -w -X github.com/deluan/navidrome/consts.gitSha={{.ShortCommit}} -X github.com/deluan/navidrome/consts.gitTag={{.Version}}
 
   - id: navidrome_windows_x64
     env:
@@ -107,25 +94,10 @@ builds:
       - -tags=embed
     ldflags:
       - "-extldflags '-static'"
-      - -X github.com/deluan/navidrome/consts.gitSha={{.ShortCommit}} -X github.com/deluan/navidrome/consts.gitTag={{.Version}}
+      - -s -w -X github.com/deluan/navidrome/consts.gitSha={{.ShortCommit}} -X github.com/deluan/navidrome/consts.gitTag={{.Version}}
 
 archives:
-  - id: musl
-    builds:
-      - navidrome_linux_musl_amd64
-    name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_musl_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
-    replacements:
-      linux: Linux
-      amd64: x86_64
-  - id: default
-    builds:
-      - navidrome_darwin
-      - navidrome_linux_amd64
-      - navidrome_linux_arm
-      - navidrome_linux_arm64
-      - navidrome_windows_i686
-      - navidrome_windows_x64
-    format_overrides:
+  - format_overrides:
       - goos: windows
         format: zip
     replacements:

.nvmrc

@@ -1 +1 @@
-v13
+v14

Dockerfile

@@ -1,66 +0,0 @@
-#####################################################
-### Build UI bundles
-FROM node:13-alpine AS jsbuilder
-WORKDIR /src
-COPY ui/package.json ui/package-lock.json ./
-RUN npm ci
-COPY ui/ .
-RUN npm run build
-
-
-#####################################################
-### Build executable
-FROM golang:1.14-alpine AS gobuilder
-
-# Download build tools
-RUN mkdir -p /src/ui/build
-RUN apk add -U --no-cache build-base git
-RUN go get -u github.com/go-bindata/go-bindata/...
-
-# Download project dependencies
-WORKDIR /src
-COPY go.mod go.sum ./
-RUN go mod download
-
-# Copy source, test it
-COPY . .
-RUN go test ./...
-
-# Copy UI bundle, build executable
-COPY --from=jsbuilder /src/build/* /src/ui/build/
-COPY --from=jsbuilder /src/build/static/css/* /src/ui/build/static/css/
-COPY --from=jsbuilder /src/build/static/js/* /src/ui/build/static/js/
-RUN rm -rf /src/build/css /src/build/js
-RUN GIT_TAG=$(git describe --tags `git rev-list --tags --max-count=1`) && \
-    GIT_TAG=${GIT_TAG#"tags/"} && \
-    GIT_SHA=$(git rev-parse --short HEAD) && \
-    echo "Building version: ${GIT_TAG} (${GIT_SHA})" && \
-    go-bindata -fs -prefix resources -tags embed -ignore="\\\*.go" -pkg resources -o resources/embedded_gen.go resources/...
-    go-bindata -fs -prefix ui/build -tags embed -nocompress -pkg assets -o assets/embedded_gen.go ui/build/... && \
-    go build -ldflags="-X github.com/deluan/navidrome/consts.gitSha=${GIT_SHA} -X github.com/deluan/navidrome/consts.gitTag=${GIT_TAG}" -tags=embed
-
-#####################################################
-### Build Final Image
-FROM alpine as release
-LABEL maintainer="deluan@navidrome.org"
-
-COPY --from=gobuilder /src/navidrome /app/
-
-# Install ffmpeg and output build config
-RUN apk add --no-cache ffmpeg
-RUN ffmpeg -buildconf
-
-VOLUME ["/data", "/music"]
-ENV ND_MUSICFOLDER /music
-ENV ND_DATAFOLDER /data
-ENV ND_SCANINTERVAL 1m
-ENV ND_TRANSCODINGCACHESIZE 100MB
-ENV ND_SESSIONTIMEOUT 30m
-ENV ND_LOGLEVEL info
-ENV ND_PORT 4533
-
-EXPOSE ${ND_PORT}
-HEALTHCHECK CMD wget -O- http://localhost:${ND_PORT}/ping || exit 1
-WORKDIR /app
-
-ENTRYPOINT ["/app/navidrome"]

Makefile

@@ -37,21 +37,30 @@ update-snapshots: check_go_env
 	UPDATE_SNAPSHOTS=true ginkgo ./server/subsonic/...
 .PHONY: update-snapshots
 
+create-migration:
+	@if [ -z "${name}" ]; then echo "Usage: make create-migration name=name_of_migration_file"; exit 1; fi
+	goose -dir db/migration create ${name}
+.PHONY: create-migration
+
 setup:
 	@which go-bindata || (echo "Installing BinData"  && GO111MODULE=off go get -u github.com/go-bindata/go-bindata/...)
 	go mod download
 	@(cd ./ui && npm ci)
 .PHONY: setup
 
-setup-dev: setup
+setup-dev: setup setup-git
 	@which wire          || (echo "Installing Wire"          && GO111MODULE=off go get -u github.com/google/wire/cmd/wire)
 	@which ginkgo        || (echo "Installing Ginkgo"        && GO111MODULE=off go get -u github.com/onsi/ginkgo/ginkgo)
 	@which goose         || (echo "Installing Goose"         && GO111MODULE=off go get -u github.com/pressly/goose/cmd/goose)
 	@which reflex        || (echo "Installing Reflex"        && GO111MODULE=off go get -u github.com/cespare/reflex)
-	@which golangci-lint || (echo "Installing GolangCI-Lint" && cd .. && GO111MODULE=on go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.26.0)
-	@which lefthook      || (echo "Installing Lefthook"      && GO111MODULE=off go get -u github.com/Arkweid/lefthook)
-	@lefthook install
-.PHONY: setup
+	@which golangci-lint || (echo "Installing GolangCI-Lint" && cd .. && GO111MODULE=on go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.27.0)
+	@which goimports     || (echo "Installing goimports"     && GO111MODULE=off go get -u golang.org/x/tools/cmd/goimports)
+.PHONY: setup-dev
+
+setup-git:
+	@mkdir -p .git/hooks
+	(cd .git/hooks && ln -sf ../../git/* .)
+.PHONY: setup-git
 
 Jamstash-master:
 	wget -N https://github.com/tsquillario/Jamstash/archive/master.zip
@@ -61,12 +70,7 @@ Jamstash-master:
 	rm -rf Jamstash-master/node_modules Jamstash-master/bower_components
 
 check_env: check_go_env check_node_env
-.PHONE: check_env
-
-check_hooks:
-	@lefthook add pre-commit
-	@lefthook add pre-push
-.PHONE: check_hooks
+.PHONY: check_env
 
 check_go_env:
 	@(hash go) || (echo "\nERROR: GO environment not setup properly!\n"; exit 1)
@@ -99,5 +103,5 @@ release:
 .PHONY: release
 
 snapshot:
-	 docker run -it -v $(PWD):/workspace -w /workspace deluan/ci-goreleaser:1.14.1-1 goreleaser release --rm-dist --skip-publish --snapshot
+	 docker run -it -v $(PWD):/workspace -w /workspace deluan/ci-goreleaser:1.14.4-2 goreleaser release --rm-dist --skip-publish --snapshot
 .PHONY: snapshot

README.md

@@ -2,6 +2,7 @@
 
 [![Last Release](https://img.shields.io/github/v/release/deluan/navidrome?label=latest&style=flat-square)](https://github.com/deluan/navidrome/releases)
 [![Build](https://img.shields.io/github/workflow/status/deluan/navidrome/Build?style=flat-square)](https://github.com/deluan/navidrome/actions)
+[![Downloads](https://img.shields.io/github/downloads/deluan/navidrome/total)](https://github.com/deluan/navidrome/releases/latest)
 [![Docker Pulls](https://img.shields.io/docker/pulls/deluan/navidrome?style=flat-square)](https://hub.docker.com/r/deluan/navidrome)
 [![Dev Chat](https://img.shields.io/discord/671335427726114836?label=chat&style=flat-square)](https://discord.gg/xh7j7yF)
 [![Subreddit](https://img.shields.io/reddit/subreddit-subscribers/navidrome?style=flat-square)](https://www.reddit.com/r/navidrome/)
@@ -17,8 +18,32 @@ please fill a [GitHub issue](https://github.com/deluan/navidrome/issues) or join
 [themes](https://www.navidrome.org/docs/developers/creating-themes)), please join the chat in our 
 [Discord server](https://discord.gg/xh7j7yF). 
 
+## Demo Site
+
+To see Navidrome in action, check out our [live demo](https://www.navidrome.org/demo/)
+
+## Installation
+
+See instructions in the [project's website](https://www.navidrome.org/docs/installation/)
+
+## Features
+ 
+ - Handles very **large music collections**
+ - Streams virtually **any audio format** available
+ - Reads and uses all your beautifully curated **metadata**
+ - Great support for **Box Sets** (multi-disc albums)
+ - **Multi-user**, each user has their own play counts, playlists, favourites, etc...
+ - Very **low resource usage**
+ - **Multi-platform**, runs on macOS, Linux and Windows. **Docker** images are also provided
+ - Ready to use **Raspberry Pi** binaries and Docker images available
+ - Automatically **monitors your library** for changes, importing new files and reloading new metadata 
+ - **Themeable**, modern and responsive **Web interface** based on [Material UI](https://material-ui.com)
+ - **Compatible** with all Subsonic/Madsonic/Airsonic [clients](https://www.navidrome.org/docs/overview/#apps)
+ - **Transcoding** on the fly. Can be set per user/player. **Opus encoding is supported**
+ - Translated to **various languages**
+
 ## Documentation
-All documentation can be found in the project's homepage: https://www.navidrome.org/docs. 
+All documentation can be found in the project's website: https://www.navidrome.org/docs. 
 Here are some useful direct links:
 
 - [Overview](https://www.navidrome.org/docs/overview/)

cmd/root.go

@@ -0,0 +1,96 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/deluan/navidrome/conf"
+	"github.com/deluan/navidrome/consts"
+	"github.com/deluan/navidrome/db"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var (
+	cfgFile  string
+	noBanner bool
+
+	rootCmd = &cobra.Command{
+		Use:   "navidrome",
+		Short: "Navidrome is a self-hosted music server and streamer",
+		Long: `Navidrome is a self-hosted music server and streamer.
+Complete documentation is available at https://www.navidrome.org/docs`,
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			preRun()
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			startServer()
+		},
+		Version: consts.Version(),
+	}
+)
+
+func Execute() {
+	rootCmd.SetVersionTemplate(`{{println .Version}}`)
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func preRun() {
+	if !noBanner {
+		println(consts.Banner())
+	}
+	conf.Load()
+}
+
+func startServer() {
+	db.EnsureLatestVersion()
+
+	subsonic, err := CreateSubsonicAPIRouter()
+	if err != nil {
+		panic(fmt.Sprintf("Could not create the Subsonic API router. Aborting! err=%v", err))
+	}
+	a := CreateServer(conf.Server.MusicFolder)
+	a.MountRouter(consts.URLPathSubsonicAPI, subsonic)
+	a.MountRouter(consts.URLPathUI, CreateAppRouter())
+	a.Run(fmt.Sprintf("%s:%d", conf.Server.Address, conf.Server.Port))
+}
+
+// TODO: Implemement some struct tags to map flags to viper
+func init() {
+	cobra.OnInitialize(func() {
+		conf.InitConfig(cfgFile)
+	})
+
+	rootCmd.PersistentFlags().StringVarP(&cfgFile, "configfile", "c", "", `config file (default "./navidrome.toml")`)
+	rootCmd.PersistentFlags().BoolVarP(&noBanner, "nobanner", "n", false, `don't show banner`)
+	rootCmd.PersistentFlags().String("musicfolder", viper.GetString("musicfolder"), "folder where your music is stored")
+	rootCmd.PersistentFlags().String("datafolder", viper.GetString("datafolder"), "folder to store application data (DB, cache...), needs write access")
+	rootCmd.PersistentFlags().StringP("loglevel", "l", viper.GetString("loglevel"), "log level, possible values: error, info, debug, trace")
+
+	_ = viper.BindPFlag("musicfolder", rootCmd.PersistentFlags().Lookup("musicfolder"))
+	_ = viper.BindPFlag("datafolder", rootCmd.PersistentFlags().Lookup("datafolder"))
+	_ = viper.BindPFlag("loglevel", rootCmd.PersistentFlags().Lookup("loglevel"))
+
+	rootCmd.Flags().StringP("address", "a", viper.GetString("address"), "IP address to bind")
+	rootCmd.Flags().IntP("port", "p", viper.GetInt("port"), "HTTP port Navidrome will use")
+	rootCmd.Flags().Duration("sessiontimeout", viper.GetDuration("sessiontimeout"), "how long Navidrome will wait before closing web ui idle sessions")
+	rootCmd.Flags().Duration("scaninterval", viper.GetDuration("scaninterval"), "how frequently to scan for changes in your music library")
+	rootCmd.Flags().String("baseurl", viper.GetString("baseurl"), "base URL (only the path part) to configure Navidrome behind a proxy (ex: /music)")
+	rootCmd.Flags().String("uiloginbackgroundurl", viper.GetString("uiloginbackgroundurl"), "URL to a backaground image used in the Login page")
+	rootCmd.Flags().Bool("enabletranscodingconfig", viper.GetBool("enabletranscodingconfig"), "enables transcoding configuration in the UI")
+	rootCmd.Flags().String("transcodingcachesize", viper.GetString("transcodingcachesize"), "size of transcoding cache")
+	rootCmd.Flags().String("imagecachesize", viper.GetString("imagecachesize"), "size of image (art work) cache. set to 0 to disable cache")
+
+	_ = viper.BindPFlag("address", rootCmd.Flags().Lookup("address"))
+	_ = viper.BindPFlag("port", rootCmd.Flags().Lookup("port"))
+	_ = viper.BindPFlag("sessiontimeout", rootCmd.Flags().Lookup("sessiontimeout"))
+	_ = viper.BindPFlag("scaninterval", rootCmd.Flags().Lookup("scaninterval"))
+	_ = viper.BindPFlag("baseurl", rootCmd.Flags().Lookup("baseurl"))
+	_ = viper.BindPFlag("uiloginbackgroundurl", rootCmd.Flags().Lookup("uiloginbackgroundurl"))
+	_ = viper.BindPFlag("enabletranscodingconfig", rootCmd.Flags().Lookup("enabletranscodingconfig"))
+	_ = viper.BindPFlag("transcodingcachesize", rootCmd.Flags().Lookup("transcodingcachesize"))
+	_ = viper.BindPFlag("imagecachesize", rootCmd.Flags().Lookup("imagecachesize"))
+}

cmd/scan.go

@@ -0,0 +1,36 @@
+package cmd
+
+import (
+	"github.com/deluan/navidrome/conf"
+	"github.com/deluan/navidrome/log"
+	"github.com/spf13/cobra"
+)
+
+var fullRescan bool
+
+func init() {
+	scanCmd.Flags().BoolVarP(&fullRescan, "full", "f", false, "check all subfolders, ignoring timestamps")
+	rootCmd.AddCommand(scanCmd)
+}
+
+var scanCmd = &cobra.Command{
+	Use:   "scan",
+	Short: "Scan music folder",
+	Long:  "Scan music folder for updates",
+	Run: func(cmd *cobra.Command, args []string) {
+		runScanner()
+	},
+}
+
+func runScanner() {
+	scanner := CreateScanner(conf.Server.MusicFolder)
+	err := scanner.RescanAll(fullRescan)
+	if err != nil {
+		log.Error("Error scanning media folder", "folder", conf.Server.MusicFolder, err)
+	}
+	if fullRescan {
+		log.Info("Finished full rescan")
+	} else {
+		log.Info("Finished rescan")
+	}
+}

cmd/wire_gen.go

@@ -3,11 +3,12 @@
 //go:generate wire
 //+build !wireinject
 
-package main
+package cmd
 
 import (
+	"github.com/deluan/navidrome/core"
+	"github.com/deluan/navidrome/core/transcoder"
 	"github.com/deluan/navidrome/engine"
-	"github.com/deluan/navidrome/engine/transcoder"
 	"github.com/deluan/navidrome/persistence"
 	"github.com/deluan/navidrome/scanner"
 	"github.com/deluan/navidrome/server"
@@ -25,6 +26,12 @@ func CreateServer(musicFolder string) *server.Server {
 	return serverServer
 }
 
+func CreateScanner(musicFolder string) *scanner.Scanner {
+	dataStore := persistence.New()
+	scannerScanner := scanner.New(dataStore)
+	return scannerScanner
+}
+
 func CreateAppRouter() *app.Router {
 	dataStore := persistence.New()
 	router := app.New(dataStore)
@@ -34,11 +41,11 @@ func CreateAppRouter() *app.Router {
 func CreateSubsonicAPIRouter() (*subsonic.Router, error) {
 	dataStore := persistence.New()
 	browser := engine.NewBrowser(dataStore)
-	imageCache, err := engine.NewImageCache()
+	imageCache, err := core.NewImageCache()
 	if err != nil {
 		return nil, err
 	}
-	cover := engine.NewCover(dataStore, imageCache)
+	cover := core.NewCover(dataStore, imageCache)
 	nowPlayingRepository := engine.NewNowPlayingRepository()
 	listGenerator := engine.NewListGenerator(dataStore, nowPlayingRepository)
 	users := engine.NewUsers(dataStore)
@@ -47,11 +54,11 @@ func CreateSubsonicAPIRouter() (*subsonic.Router, error) {
 	scrobbler := engine.NewScrobbler(dataStore, nowPlayingRepository)
 	search := engine.NewSearch(dataStore)
 	transcoderTranscoder := transcoder.New()
-	transcodingCache, err := engine.NewTranscodingCache()
+	transcodingCache, err := core.NewTranscodingCache()
 	if err != nil {
 		return nil, err
 	}
-	mediaStreamer := engine.NewMediaStreamer(dataStore, transcoderTranscoder, transcodingCache)
+	mediaStreamer := core.NewMediaStreamer(dataStore, transcoderTranscoder, transcodingCache)
 	players := engine.NewPlayers(dataStore)
 	router := subsonic.New(browser, cover, listGenerator, users, playlists, ratings, scrobbler, search, mediaStreamer, players)
 	return router, nil
@@ -59,4 +66,4 @@ func CreateSubsonicAPIRouter() (*subsonic.Router, error) {
 
 // wire_injectors.go:
 
-var allProviders = wire.NewSet(engine.Set, scanner.New, subsonic.New, app.New, persistence.New)
+var allProviders = wire.NewSet(engine.Set, core.Set, scanner.New, subsonic.New, app.New, persistence.New)

cmd/wire_injectors.go

@@ -1,8 +1,9 @@
 //+build wireinject
 
-package main
+package cmd
 
 import (
+	"github.com/deluan/navidrome/core"
 	"github.com/deluan/navidrome/engine"
 	"github.com/deluan/navidrome/persistence"
 	"github.com/deluan/navidrome/scanner"
@@ -14,6 +15,7 @@ import (
 
 var allProviders = wire.NewSet(
 	engine.Set,
+	core.Set,
 	scanner.New,
 	subsonic.New,
 	app.New,
@@ -27,6 +29,12 @@ func CreateServer(musicFolder string) *server.Server {
 	))
 }
 
+func CreateScanner(musicFolder string) *scanner.Scanner {
+	panic(wire.Build(
+		allProviders,
+	))
+}
+
 func CreateAppRouter() *app.Router {
 	panic(wire.Build(allProviders))
 }

conf/configuration.go

@@ -1,124 +1,124 @@
 package conf
 
 import (
-	"flag"
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
+	"time"
 
 	"github.com/deluan/navidrome/consts"
 	"github.com/deluan/navidrome/log"
-	"github.com/koding/multiconfig"
+	"github.com/spf13/viper"
 )
 
-type nd struct {
-	ConfigFile     string `default:"./navidrome.toml"`
-	Port           string `default:"4533"`
-	MusicFolder    string `default:"./music"`
-	DataFolder     string `default:"./"`
-	ScanInterval   string `default:"1m"`
-	DbPath         string ``
-	LogLevel       string `default:"info"`
-	SessionTimeout string `default:"30m"`
-	BaseURL        string `default:""`
+type configOptions struct {
+	ConfigFile              string
+	Address                 string
+	Port                    int
+	MusicFolder             string
+	DataFolder              string
+	DbPath                  string
+	LogLevel                string
+	ScanInterval            time.Duration
+	SessionTimeout          time.Duration
+	BaseURL                 string
+	UILoginBackgroundURL    string
+	EnableTranscodingConfig bool
+	TranscodingCacheSize    string
+	ImageCacheSize          string
 
-	UILoginBackgroundURL string `default:"https://source.unsplash.com/random/1600x900?music"`
-
-	IgnoredArticles string `default:"The El La Los Las Le Les Os As O A"`
-	IndexGroups     string `default:"A B C D E F G H I J K L M N O P Q R S T U V W X-Z(XYZ) [Unknown]([)"`
-
-	EnableTranscodingConfig bool   `default:"false"`
-	TranscodingCacheSize    string `default:"100MB"` // in MB
-	ImageCacheSize          string `default:"100MB"` // in MB
-	ProbeCommand            string `default:"ffmpeg %s -f ffmetadata"`
+	IgnoredArticles  string
+	IndexGroups      string
+	ProbeCommand     string
+	CoverArtPriority string
+	CoverJpegQuality int
+	UIWelcomeMessage string
+	GATrackingID     string
+	AuthRequestLimit int
+	AuthWindowLength time.Duration
 
 	// DevFlags. These are used to enable/disable debugging and incomplete features
-	DevLogSourceLine           bool   `default:"false"`
-	DevAutoCreateAdminPassword string `default:""`
-	DevEnableUIPlaylists       bool   `default:"true"`
+	DevLogSourceLine           bool
+	DevAutoCreateAdminPassword string
+	DevOldScanner              bool
 }
 
-var Server = &nd{}
+var Server = &configOptions{}
 
-// TODO refactor configuration and use something different. Maybe https://github.com/spf13/cobra
-// This function loads the config just load the ConfigFile. This is very cumbersome, but doesn't
-// seem there's a simpler way to do thiswith multiconfig. Time to replace this library?
-func configFile() string {
-	conf := &nd{}
-	loader := multiconfig.MultiLoader(
-		&multiconfig.TagLoader{},
-		&multiconfig.EnvironmentLoader{},
-		&multiconfig.FlagLoader{},
-	)
-	d := &multiconfig.DefaultLoader{}
-	d.Loader = loader
-	d.Validator = multiconfig.MultiValidator(&multiconfig.RequiredValidator{})
-	if err := d.Load(conf); err != nil {
-		return consts.LocalConfigFile
-	}
-	if _, err := os.Stat(conf.ConfigFile); err != nil {
-		return consts.LocalConfigFile
-	}
-	return conf.ConfigFile
-}
-
-func newWithPath(path string, skipFlags ...bool) *multiconfig.DefaultLoader {
-	var loaders []multiconfig.Loader
-
-	// Read default values defined via tag fields "default"
-	loaders = append(loaders, &multiconfig.TagLoader{})
-
-	if _, err := os.Stat(path); err == nil {
-		if strings.HasSuffix(path, "toml") {
-			loaders = append(loaders, &multiconfig.TOMLLoader{Path: path})
-		}
-
-		if strings.HasSuffix(path, "json") {
-			loaders = append(loaders, &multiconfig.JSONLoader{Path: path})
-		}
-
-		if strings.HasSuffix(path, "yml") || strings.HasSuffix(path, "yaml") {
-			loaders = append(loaders, &multiconfig.YAMLLoader{Path: path})
-		}
-	}
-
-	e := &multiconfig.EnvironmentLoader{}
-	loaders = append(loaders, e)
-	if len(skipFlags) == 0 || !skipFlags[0] {
-		f := &multiconfig.FlagLoader{}
-		loaders = append(loaders, f)
-	}
-
-	loader := multiconfig.MultiLoader(loaders...)
-
-	d := &multiconfig.DefaultLoader{}
-	d.Loader = loader
-	d.Validator = multiconfig.MultiValidator(&multiconfig.RequiredValidator{})
-	return d
-}
-
-func LoadFromFile(confFile string, skipFlags ...bool) {
-	m := newWithPath(confFile, skipFlags...)
-	err := m.Load(Server)
-	if err == flag.ErrHelp {
-		os.Exit(1)
-	}
-	if err != nil {
-		fmt.Printf("Error trying to load config '%s'. Error: %v", confFile, err)
-		os.Exit(2)
-	}
-	if Server.DbPath == "" {
-		Server.DbPath = filepath.Join(Server.DataFolder, consts.DefaultDbPath)
-	}
-	if os.Getenv("PORT") != "" {
-		Server.Port = os.Getenv("PORT")
-	}
-	log.SetLevelString(Server.LogLevel)
-	log.SetLogSourceLine(Server.DevLogSourceLine)
-	log.Debug("Loaded configuration", "file", confFile, "config", fmt.Sprintf("%#v", Server))
+func LoadFromFile(confFile string) {
+	viper.SetConfigFile(confFile)
+	Load()
 }
 
 func Load() {
-	LoadFromFile(configFile())
+	err := viper.Unmarshal(&Server)
+	if err != nil {
+		fmt.Println("Error parsing config:", err)
+		os.Exit(1)
+	}
+	err = os.MkdirAll(Server.DataFolder, os.ModePerm)
+	if err != nil {
+		fmt.Println("Error creating data path:", "path", Server.DataFolder, err)
+		os.Exit(1)
+	}
+	Server.ConfigFile = viper.GetViper().ConfigFileUsed()
+	if Server.DbPath == "" {
+		Server.DbPath = filepath.Join(Server.DataFolder, consts.DefaultDbPath)
+	}
+
+	log.SetLevelString(Server.LogLevel)
+	log.SetLogSourceLine(Server.DevLogSourceLine)
+	log.Debug("Loaded configuration", "file", Server.ConfigFile, "config", fmt.Sprintf("%#v", Server))
+}
+
+func init() {
+	viper.SetDefault("musicfolder", "./music")
+	viper.SetDefault("datafolder", "./")
+	viper.SetDefault("loglevel", "info")
+	viper.SetDefault("address", "0.0.0.0")
+	viper.SetDefault("port", 4533)
+	viper.SetDefault("sessiontimeout", consts.DefaultSessionTimeout)
+	viper.SetDefault("scaninterval", time.Minute)
+	viper.SetDefault("baseurl", "")
+	viper.SetDefault("uiloginbackgroundurl", "")
+	viper.SetDefault("enabletranscodingconfig", false)
+	viper.SetDefault("transcodingcachesize", "100MB")
+	viper.SetDefault("imagecachesize", "100MB")
+
+	// Config options only valid for file/env configuration
+	viper.SetDefault("ignoredarticles", "The El La Los Las Le Les Os As O A")
+	viper.SetDefault("indexgroups", "A B C D E F G H I J K L M N O P Q R S T U V W X-Z(XYZ) [Unknown]([)")
+	viper.SetDefault("probecommand", "ffmpeg %s -f ffmetadata")
+	viper.SetDefault("coverartpriority", "embedded, cover.*, folder.*, front.*")
+	viper.SetDefault("coverjpegquality", 75)
+	viper.SetDefault("uiwelcomemessage", "")
+	viper.SetDefault("gatrackingid", "")
+	viper.SetDefault("authrequestlimit", 5)
+	viper.SetDefault("authwindowlength", 20*time.Second)
+
+	// DevFlags. These are used to enable/disable debugging and incomplete features
+	viper.SetDefault("devlogsourceline", false)
+	viper.SetDefault("devautocreateadminpassword", "")
+	viper.SetDefault("devoldscanner", false)
+}
+
+func InitConfig(cfgFile string) {
+	if cfgFile != "" {
+		// Use config file from the flag.
+		viper.SetConfigFile(cfgFile)
+	} else {
+		// Search config in local directory with name "navidrome" (without extension).
+		viper.AddConfigPath(".")
+		viper.SetConfigName("navidrome")
+	}
+
+	_ = viper.BindEnv("port")
+	viper.SetEnvPrefix("ND")
+	viper.AutomaticEnv()
+
+	err := viper.ReadInConfig()
+	if cfgFile != "" && err != nil {
+		fmt.Println("Navidrome could not open config file: ", err)
+		os.Exit(1)
+	}
 }

consts/consts.go

@@ -11,13 +11,13 @@ const (
 	AppName = "navidrome"
 
 	LocalConfigFile     = "./navidrome.toml"
-	DefaultDbPath       = "navidrome.db?cache=shared&_busy_timeout=15000&_journal_mode=WAL"
+	DefaultDbPath       = "navidrome.db?cache=shared&_busy_timeout=15000&_journal_mode=WAL&_foreign_keys=on"
 	InitialSetupFlagKey = "InitialSetup"
 
 	UIAuthorizationHeader = "X-ND-Authorization"
 	JWTSecretKey          = "JWTSecret"
 	JWTIssuer             = "ND"
-	DefaultSessionTimeout = 30 * time.Minute
+	DefaultSessionTimeout = 24 * time.Hour
 
 	DevInitialUserName = "admin"
 	DevInitialName     = "Dev Admin"
@@ -28,7 +28,10 @@ const (
 	RequestThrottleBacklogLimit   = 100
 	RequestThrottleBacklogTimeout = time.Minute
 
-	I18nFolder = "i18n"
+	I18nFolder   = "i18n"
+	SkipScanFile = ".ndignore"
+
+	PlaceholderAlbumArt = "navidrome-600x600.png"
 )
 
 // Cache options

consts/mime_types.go

@@ -19,6 +19,9 @@ func init() {
 		".shn":  "audio/x-shn",
 		".aif":  "audio/x-aiff",
 		".aiff": "audio/x-aiff",
+		".m3u":  "audio/x-mpegurl",
+		".pls":  "audio/x-scpls",
+		".dsf":  "audio/dsd",
 		".gif":  "image/gif",
 		".jpg":  "image/jpeg",
 		".jpeg": "image/jpeg",

contrib/navidrome.service

@@ -5,6 +5,9 @@ Description=Navidrome Music Server and Streamer compatible with Subsonic/Airsoni
 After=remote-fs.target network.target
 AssertPathExists=/var/lib/navidrome
 
+[Install]
+WantedBy=multi-user.target
+
 [Service]
 User=navidrome
 Group=navidrome

core/auth/auth.go

@@ -44,11 +44,7 @@ func CreateToken(u *model.User) (string, error) {
 
 func getSessionTimeOut() time.Duration {
 	if sessionTimeOut == 0 {
-		if to, err := time.ParseDuration(conf.Server.SessionTimeout); err != nil {
-			sessionTimeOut = consts.DefaultSessionTimeout
-		} else {
-			sessionTimeOut = to
-		}
+		sessionTimeOut = conf.Server.SessionTimeout
 		log.Info("Setting Session Timeout", "value", sessionTimeOut)
 	}
 	return sessionTimeOut

core/auth/auth_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/deluan/navidrome/engine/auth"
+	"github.com/deluan/navidrome/core/auth"
 	"github.com/deluan/navidrome/log"
 	"github.com/dgrijalva/jwt-go"
 	. "github.com/onsi/ginkgo"

core/common.go

@@ -0,0 +1,15 @@
+package core
+
+import (
+	"context"
+
+	"github.com/deluan/navidrome/model/request"
+)
+
+func userName(ctx context.Context) string {
+	if user, ok := request.UserFrom(ctx); !ok {
+		return "UNKNOWN"
+	} else {
+		return user.UserName
+	}
+}

core/core_suite_test.go

@@ -0,0 +1,35 @@
+package core
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/deluan/navidrome/log"
+	"github.com/deluan/navidrome/tests"
+	"github.com/djherbis/fscache"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestEngine(t *testing.T) {
+	tests.Init(t, false)
+	log.SetLevel(log.LevelCritical)
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Core Suite")
+}
+
+var testCache fscache.Cache
+var testCacheDir string
+
+var _ = Describe("Core Suite Setup", func() {
+	BeforeSuite(func() {
+		testCacheDir, _ = ioutil.TempDir("", "core_test_cache")
+		fs, _ := fscache.NewFs(testCacheDir, 0755)
+		testCache, _ = fscache.NewCache(fs, nil)
+	})
+
+	AfterSuite(func() {
+		os.RemoveAll(testCacheDir)
+	})
+})

core/cover.go

@@ -1,4 +1,4 @@
-package engine
+package core
 
 import (
 	"bytes"
@@ -19,6 +19,7 @@ import (
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
 	"github.com/deluan/navidrome/resources"
+	"github.com/deluan/navidrome/utils"
 	"github.com/dhowden/tag"
 	"github.com/disintegration/imaging"
 	"github.com/djherbis/fscache"
@@ -40,7 +41,6 @@ type cover struct {
 }
 
 func (c *cover) Get(ctx context.Context, id string, size int, out io.Writer) error {
-	id = strings.TrimPrefix(id, "al-")
 	path, lastUpdate, err := c.getCoverPath(ctx, id)
 	if err != nil && err != model.ErrNotFound {
 		return err
@@ -87,11 +87,10 @@ func (c *cover) Get(ctx context.Context, id string, size int, out io.Writer) err
 }
 
 func (c *cover) getCoverPath(ctx context.Context, id string) (path string, lastUpdated time.Time, err error) {
-	var found bool
-	if found, err = c.ds.Album(ctx).Exists(id); err != nil {
-		return
-	}
-	if found {
+	// If id is an album cover ID
+	if strings.HasPrefix(id, "al-") {
+		log.Trace(ctx, "Looking for album art", "id", id)
+		id = strings.TrimPrefix(id, "al-")
 		var al *model.Album
 		al, err = c.ds.Album(ctx).Get(id)
 		if err != nil {
@@ -99,10 +98,12 @@ func (c *cover) getCoverPath(ctx context.Context, id string) (path string, lastU
 		}
 		if al.CoverArtId == "" {
 			err = model.ErrNotFound
-			return
 		}
-		id = al.CoverArtId
+		return al.CoverArtPath, al.UpdatedAt, err
 	}
+
+	log.Trace(ctx, "Looking for media file art", "id", id)
+	// if id is a mediafile cover id
 	var mf *model.MediaFile
 	mf, err = c.ds.MediaFile(ctx).Get(id)
 	if err != nil {
@@ -111,24 +112,38 @@ func (c *cover) getCoverPath(ctx context.Context, id string) (path string, lastU
 	if mf.HasCoverArt {
 		return mf.Path, mf.UpdatedAt, nil
 	}
-	return "", time.Time{}, model.ErrNotFound
+
+	// if the mediafile does not have a coverArt, fallback to the album cover
+	log.Trace(ctx, "Media file does not contain art. Falling back to album art", "id", id, "albumId", "al-"+mf.AlbumID)
+	return c.getCoverPath(ctx, "al-"+mf.AlbumID)
 }
 
 func imageCacheKey(path string, size int, lastUpdate time.Time) string {
-	return fmt.Sprintf("%s.%d.%s", path, size, lastUpdate.Format(time.RFC3339Nano))
+	return fmt.Sprintf("%s.%d.%s.%d", path, size, lastUpdate.Format(time.RFC3339Nano), conf.Server.CoverJpegQuality)
 }
 
 func (c *cover) getCover(ctx context.Context, path string, size int) (reader io.Reader, err error) {
 	defer func() {
 		if err != nil {
 			log.Warn(ctx, "Error extracting image", "path", path, "size", size, err)
-			reader, err = resources.AssetFile().Open("navidrome-310x310.png")
+			reader, err = resources.AssetFile().Open(consts.PlaceholderAlbumArt)
 		}
 	}()
-	var data []byte
-	data, err = readFromTag(path)
 
-	if err == nil && size > 0 {
+	if path == "" {
+		return nil, errors.New("empty path given for cover")
+	}
+
+	var data []byte
+	if utils.IsAudioFile(path) {
+		data, err = readFromTag(path)
+	} else {
+		data, err = readFromFile(path)
+	}
+
+	if err != nil {
+		return
+	} else if size > 0 {
 		data, err = resizeImage(bytes.NewReader(data), size)
 	}
 
@@ -148,7 +163,7 @@ func resizeImage(reader io.Reader, size int) ([]byte, error) {
 	}
 	m := imaging.Resize(img, size, size, imaging.Lanczos)
 	buf := new(bytes.Buffer)
-	err = jpeg.Encode(buf, m, &jpeg.Options{Quality: 75})
+	err = jpeg.Encode(buf, m, &jpeg.Options{Quality: conf.Server.CoverJpegQuality})
 	return buf.Bytes(), err
 }
 
@@ -171,6 +186,21 @@ func readFromTag(path string) ([]byte, error) {
 	return picture.Data, nil
 }
 
+func readFromFile(path string) ([]byte, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	defer f.Close()
+	var buf bytes.Buffer
+	if _, err := buf.ReadFrom(f); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
 func NewImageCache() (ImageCache, error) {
 	return newFileCache("Image", conf.Server.ImageCacheSize, consts.ImageCacheDir, consts.DefaultImageCacheMaxItems)
 }

core/cover_test.go

@@ -1,4 +1,4 @@
-package engine
+package core
 
 import (
 	"bytes"
@@ -19,8 +19,8 @@ var _ = Describe("Cover", func() {
 
 	BeforeEach(func() {
 		ds = &persistence.MockDataStore{MockedTranscoding: &mockTranscodingRepository{}}
-		ds.Album(ctx).(*persistence.MockAlbum).SetData(`[{"id": "222", "coverArtId": "123"}, {"id": "333", "coverArtId": ""}]`)
-		ds.MediaFile(ctx).(*persistence.MockMediaFile).SetData(`[{"id": "123", "path": "tests/fixtures/test.mp3", "hasCoverArt": true, "updatedAt":"2020-04-02T21:29:31.6377Z"}]`)
+		ds.Album(ctx).(*persistence.MockAlbum).SetData(`[{"id": "222", "coverArtId": "123", "coverArtPath":"tests/fixtures/test.mp3"}, {"id": "333", "coverArtId": ""}, {"id": "444", "coverArtId": "444", "coverArtPath": "tests/fixtures/cover.jpg"}]`)
+		ds.MediaFile(ctx).(*persistence.MockMediaFile).SetData(`[{"id": "123", "albumId": "222", "path": "tests/fixtures/test.mp3", "hasCoverArt": true, "updatedAt":"2020-04-02T21:29:31.6377Z"},{"id": "456", "albumId": "222", "path": "tests/fixtures/test.ogg", "hasCoverArt": false, "updatedAt":"2020-04-02T21:29:31.6377Z"}]`)
 	})
 
 	Context("Cache is configured", func() {
@@ -28,17 +28,17 @@ var _ = Describe("Cover", func() {
 			cover = NewCover(ds, testCache)
 		})
 
-		It("retrieves the original cover art from an album", func() {
+		It("retrieves the external cover art for an album", func() {
 			buf := new(bytes.Buffer)
 
-			Expect(cover.Get(ctx, "222", 0, buf)).To(BeNil())
+			Expect(cover.Get(ctx, "al-444", 0, buf)).To(BeNil())
 
 			_, format, err := image.Decode(bytes.NewReader(buf.Bytes()))
 			Expect(err).To(BeNil())
 			Expect(format).To(Equal("jpeg"))
 		})
 
-		It("accepts albumIds with 'al-' prefix", func() {
+		It("retrieves the embedded cover art for an album", func() {
 			buf := new(bytes.Buffer)
 
 			Expect(cover.Get(ctx, "al-222", 0, buf)).To(BeNil())
@@ -51,7 +51,7 @@ var _ = Describe("Cover", func() {
 		It("returns the default cover if album does not have cover", func() {
 			buf := new(bytes.Buffer)
 
-			Expect(cover.Get(ctx, "333", 0, buf)).To(BeNil())
+			Expect(cover.Get(ctx, "al-333", 0, buf)).To(BeNil())
 
 			_, format, err := image.Decode(bytes.NewReader(buf.Bytes()))
 			Expect(err).To(BeNil())
@@ -61,7 +61,7 @@ var _ = Describe("Cover", func() {
 		It("returns the default cover if album is not found", func() {
 			buf := new(bytes.Buffer)
 
-			Expect(cover.Get(ctx, "444", 0, buf)).To(BeNil())
+			Expect(cover.Get(ctx, "al-0101", 0, buf)).To(BeNil())
 
 			_, format, err := image.Decode(bytes.NewReader(buf.Bytes()))
 			Expect(err).To(BeNil())
@@ -80,6 +80,16 @@ var _ = Describe("Cover", func() {
 			Expect(img.Bounds().Size().Y).To(Equal(600))
 		})
 
+		It("retrieves the album cover art if media_file does not have one", func() {
+			buf := new(bytes.Buffer)
+
+			Expect(cover.Get(ctx, "456", 0, buf)).To(BeNil())
+
+			_, format, err := image.Decode(bytes.NewReader(buf.Bytes()))
+			Expect(err).To(BeNil())
+			Expect(format).To(Equal("jpeg"))
+		})
+
 		It("resized cover art as requested", func() {
 			buf := new(bytes.Buffer)
 
@@ -97,7 +107,7 @@ var _ = Describe("Cover", func() {
 				ds.Album(ctx).(*persistence.MockAlbum).SetError(true)
 				buf := new(bytes.Buffer)
 
-				Expect(cover.Get(ctx, "222", 0, buf)).To(MatchError("Error!"))
+				Expect(cover.Get(ctx, "al-222", 0, buf)).To(MatchError("Error!"))
 			})
 
 			It("returns err if gets error from media_file table", func() {
@@ -116,7 +126,7 @@ var _ = Describe("Cover", func() {
 		It("retrieves the original cover art from an album", func() {
 			buf := new(bytes.Buffer)
 
-			Expect(cover.Get(ctx, "222", 0, buf)).To(BeNil())
+			Expect(cover.Get(ctx, "al-222", 0, buf)).To(BeNil())
 
 			_, format, err := image.Decode(bytes.NewReader(buf.Bytes()))
 			Expect(err).To(BeNil())

core/file_caches.go

@@ -1,4 +1,4 @@
-package engine
+package core
 
 import (
 	"fmt"

core/file_caches_test.go

@@ -1,4 +1,4 @@
-package engine
+package core
 
 import (
 	"io/ioutil"

core/media_streamer.go

@@ -1,4 +1,4 @@
-package engine
+package core
 
 import (
 	"context"
@@ -10,7 +10,7 @@ import (
 
 	"github.com/deluan/navidrome/conf"
 	"github.com/deluan/navidrome/consts"
-	"github.com/deluan/navidrome/engine/transcoder"
+	"github.com/deluan/navidrome/core/transcoder"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
 	"github.com/deluan/navidrome/model/request"
@@ -41,9 +41,11 @@ func (ms *mediaStreamer) NewStream(ctx context.Context, id string, reqFormat str
 
 	var format string
 	var bitRate int
+	var cached bool
 	defer func() {
-		log.Info("Streaming file", "title", mf.Title, "artist", mf.Artist, "format", format,
-			"bitRate", bitRate, "user", userName(ctx), "transcoding", format != "raw", "originalFormat", mf.Suffix)
+		log.Info("Streaming file", "title", mf.Title, "artist", mf.Artist, "format", format, "cached", cached,
+			"bitRate", bitRate, "user", userName(ctx), "transcoding", format != "raw",
+			"originalFormat", mf.Suffix, "originalBitRate", mf.BitRate)
 	}()
 
 	format, bitRate = selectTranscodingOptions(ctx, ms.ds, mf, reqFormat, reqBitRate)
@@ -76,8 +78,10 @@ func (ms *mediaStreamer) NewStream(ctx context.Context, id string, reqFormat str
 		return nil, err
 	}
 
+	cached = w == nil
+
 	// If this is a brand new transcoding request, not in the cache, start transcoding
-	if w != nil {
+	if !cached {
 		log.Trace(ctx, "Cache miss. Starting new transcoding session", "id", mf.ID)
 		t, err := ms.ds.Transcoding(ctx).FindByFormat(format)
 		if err != nil {
@@ -93,7 +97,7 @@ func (ms *mediaStreamer) NewStream(ctx context.Context, id string, reqFormat str
 	}
 
 	// If it is in the cache, check if the stream is done being written. If so, return a ReaderSeeker
-	if w == nil {
+	if cached {
 		size := getFinalCachedSize(r)
 		if size > 0 {
 			log.Debug(ctx, "Streaming cached file", "id", mf.ID, "path", mf.Path,

core/media_streamer_test.go

@@ -1,4 +1,4 @@
-package engine
+package core
 
 import (
 	"context"

core/mock_transcoding_repo_test.go

@@ -0,0 +1,22 @@
+package core
+
+import "github.com/deluan/navidrome/model"
+
+type mockTranscodingRepository struct {
+	model.TranscodingRepository
+}
+
+func (m *mockTranscodingRepository) Get(id string) (*model.Transcoding, error) {
+	return &model.Transcoding{ID: id, TargetFormat: "mp3", DefaultBitRate: 160}, nil
+}
+
+func (m *mockTranscodingRepository) FindByFormat(format string) (*model.Transcoding, error) {
+	switch format {
+	case "mp3":
+		return &model.Transcoding{ID: "mp31", TargetFormat: "mp3", DefaultBitRate: 160}, nil
+	case "oga":
+		return &model.Transcoding{ID: "oga1", TargetFormat: "oga", DefaultBitRate: 128}, nil
+	default:
+		return nil, model.ErrNotFound
+	}
+}

core/wire_providers.go

@@ -0,0 +1,14 @@
+package core
+
+import (
+	"github.com/deluan/navidrome/core/transcoder"
+	"github.com/google/wire"
+)
+
+var Set = wire.NewSet(
+	NewCover,
+	NewMediaStreamer,
+	NewTranscodingCache,
+	NewImageCache,
+	transcoder.New,
+)

db/db.go

@@ -27,7 +27,7 @@ func Db() *sql.DB {
 		var err error
 		Path = conf.Server.DbPath
 		if Path == ":memory:" {
-			Path = "file::memory:?cache=shared"
+			Path = "file::memory:?cache=shared&_foreign_keys=on"
 			conf.Server.DbPath = Path
 		}
 		log.Debug("Opening DataBase", "dbPath", Path, "driver", Driver)

db/migration/20200516140647_add_playlist_tracks_table.go

@@ -87,7 +87,7 @@ func Up20200516140647UpdatePlaylistTracks(tx *sql.Tx, id string, tracks string)
 		return err
 	}
 	for i, trackId := range trackList {
-		_, err := stmt.Exec(id, trackId, i)
+		_, err := stmt.Exec(id, trackId, i+1)
 		if err != nil {
 			log.Error("Error adding track to playlist", "playlistId", id, "trackId", trackId, err)
 		}

db/migration/20200608153717_referential_integrity.go

@@ -0,0 +1,137 @@
+package migration
+
+import (
+	"database/sql"
+
+	"github.com/pressly/goose"
+)
+
+func init() {
+	goose.AddMigration(Up20200608153717, Down20200608153717)
+}
+
+func Up20200608153717(tx *sql.Tx) error {
+	// First delete dangling players
+	_, err := tx.Exec(`
+delete from player where user_name not in (select user_name from user)`)
+	if err != nil {
+		return err
+	}
+
+	// Also delete dangling players
+	_, err = tx.Exec(`
+delete from playlist where owner not in (select user_name from user)`)
+	if err != nil {
+		return err
+	}
+
+	// Also delete dangling playlist tracks
+	_, err = tx.Exec(`
+delete from playlist_tracks where playlist_id not in (select id from playlist)`)
+	if err != nil {
+		return err
+	}
+
+	// Add foreign key to player table
+	err = updatePlayer_20200608153717(tx)
+	if err != nil {
+		return err
+	}
+
+	// Add foreign key to playlist table
+	err = updatePlaylist_20200608153717(tx)
+	if err != nil {
+		return err
+	}
+
+	// Add foreign keys to playlist_tracks table
+	return updatePlaylistTracks_20200608153717(tx)
+}
+
+func updatePlayer_20200608153717(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+create table player_dg_tmp
+(
+	id varchar(255) not null
+		primary key,
+	name varchar not null
+		unique,
+	type varchar,
+	user_name varchar not null
+		references user (user_name)
+			on update cascade on delete cascade,
+	client varchar not null,
+	ip_address varchar,
+	last_seen timestamp,
+	max_bit_rate int default 0,
+	transcoding_id varchar null
+);
+
+insert into player_dg_tmp(id, name, type, user_name, client, ip_address, last_seen, max_bit_rate, transcoding_id) select id, name, type, user_name, client, ip_address, last_seen, max_bit_rate, transcoding_id from player;
+
+drop table player;
+
+alter table player_dg_tmp rename to player;
+`)
+	return err
+}
+
+func updatePlaylist_20200608153717(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+create table playlist_dg_tmp
+(
+	id varchar(255) not null
+		primary key,
+	name varchar(255) default '' not null,
+	comment varchar(255) default '' not null,
+	duration real default 0 not null,
+	song_count integer default 0 not null,
+	owner varchar(255) default '' not null
+		constraint playlist_user_user_name_fk
+			references user (user_name)
+				on update cascade on delete cascade,
+	public bool default FALSE not null,
+	created_at datetime,
+	updated_at datetime
+);
+
+insert into playlist_dg_tmp(id, name, comment, duration, song_count, owner, public, created_at, updated_at) select id, name, comment, duration, song_count, owner, public, created_at, updated_at from playlist;
+
+drop table playlist;
+
+alter table playlist_dg_tmp rename to playlist;
+
+create index playlist_name
+	on playlist (name);
+`)
+	return err
+}
+
+func updatePlaylistTracks_20200608153717(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+create table playlist_tracks_dg_tmp
+(
+	id integer default 0 not null,
+	playlist_id varchar(255) not null
+		constraint playlist_tracks_playlist_id_fk
+			references playlist
+				on update cascade on delete cascade,
+	media_file_id varchar(255) not null
+);
+
+insert into playlist_tracks_dg_tmp(id, playlist_id, media_file_id) select id, playlist_id, media_file_id from playlist_tracks;
+
+drop table playlist_tracks;
+
+alter table playlist_tracks_dg_tmp rename to playlist_tracks;
+
+create unique index playlist_tracks_pos
+	on playlist_tracks (playlist_id, id);
+
+`)
+	return err
+}
+
+func Down20200608153717(tx *sql.Tx) error {
+	return nil
+}

db/migration/20200706231659_add_default_transcodings.go

@@ -0,0 +1,43 @@
+package migration
+
+import (
+	"database/sql"
+
+	"github.com/deluan/navidrome/consts"
+	"github.com/google/uuid"
+	"github.com/pressly/goose"
+)
+
+func init() {
+	goose.AddMigration(upAddDefaultTranscodings, downAddDefaultTranscodings)
+}
+
+func upAddDefaultTranscodings(tx *sql.Tx) error {
+	row := tx.QueryRow("SELECT COUNT(*) FROM transcoding")
+	var count int
+	err := row.Scan(&count)
+	if err != nil {
+		return err
+	}
+	if count > 0 {
+		return nil
+	}
+
+	stmt, err := tx.Prepare("insert into transcoding (id, name, target_format, default_bit_rate, command) values (?, ?, ?, ?, ?)")
+	if err != nil {
+		return err
+	}
+
+	for _, t := range consts.DefaultTranscodings {
+		r, _ := uuid.NewRandom()
+		_, err := stmt.Exec(r.String(), t["name"], t["targetFormat"], t["defaultBitRate"], t["command"])
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func downAddDefaultTranscodings(tx *sql.Tx) error {
+	return nil
+}

db/migration/20200710211442_add_playlist_path.go

@@ -0,0 +1,27 @@
+package migration
+
+import (
+	"database/sql"
+
+	"github.com/pressly/goose"
+)
+
+func init() {
+	goose.AddMigration(upAddPlaylistPath, downAddPlaylistPath)
+}
+
+func upAddPlaylistPath(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+alter table playlist
+	add path string default '' not null;
+
+alter table playlist
+	add sync bool default false not null;
+`)
+
+	return err
+}
+
+func downAddPlaylistPath(tx *sql.Tx) error {
+	return nil
+}

docker-compose.yml

@@ -1,21 +0,0 @@
-# This is just an example. Customize it to your needs.
-
-version: "3"
-services:
-  navidrome:
-    image: deluan/navidrome:latest
-    ports:
-      - "4533:4533"
-    environment:
-      # All options with their default values:
-      ND_MUSICFOLDER: /music
-      ND_DATAFOLDER: /data
-      ND_SCANINTERVAL: 1m
-      ND_LOGLEVEL: info
-      ND_PORT: 4533
-      ND_TRANSCODINGCACHESIZE: 100MB
-      ND_SESSIONTIMEOUT: 30m
-      ND_BASEURL: ""
-    volumes:
-      - "./data:/data"
-      - "./music:/music"

engine/browser.go

@@ -61,7 +61,7 @@ type DirectoryInfo struct {
 	Entries    Entries
 	Parent     string
 	Starred    time.Time
-	PlayCount  int32
+	PlayCount  int64
 	UserRating int
 	AlbumCount int
 	CoverArt   string
@@ -138,7 +138,7 @@ func (b *browser) buildArtistDir(a *model.Artist, albums model.Albums) *Director
 	for i := range albums {
 		al := albums[i]
 		dir.Entries[i] = FromAlbum(&al)
-		dir.PlayCount += int32(al.PlayCount)
+		dir.PlayCount += al.PlayCount
 	}
 	return dir
 }
@@ -156,7 +156,7 @@ func (b *browser) buildAlbumDir(al *model.Album, tracks model.MediaFiles) *Direc
 		Year:       al.MaxYear,
 		Genre:      al.Genre,
 		CoverArt:   al.CoverArtId,
-		PlayCount:  int32(al.PlayCount),
+		PlayCount:  al.PlayCount,
 		UserRating: al.Rating,
 	}
 

engine/common.go

@@ -23,7 +23,7 @@ type Entry struct {
 	Starred     time.Time
 	Track       int
 	Duration    int
-	Size        int
+	Size        int64
 	Suffix      string
 	BitRate     int
 	ContentType string
@@ -101,6 +101,8 @@ func FromMediaFile(mf *model.MediaFile) Entry {
 	e.BitRate = mf.BitRate
 	if mf.HasCoverArt {
 		e.CoverArt = mf.ID
+	} else {
+		e.CoverArt = "al-" + mf.AlbumID
 	}
 	e.ContentType = mf.ContentType()
 	e.AbsolutePath = mf.Path

engine/engine_suite_test.go

@@ -1,13 +1,10 @@
 package engine
 
 import (
-	"io/ioutil"
-	"os"
 	"testing"
 
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/tests"
-	"github.com/djherbis/fscache"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -18,18 +15,3 @@ func TestEngine(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Engine Suite")
 }
-
-var testCache fscache.Cache
-var testCacheDir string
-
-var _ = Describe("Engine Suite Setup", func() {
-	BeforeSuite(func() {
-		testCacheDir, _ = ioutil.TempDir("", "engine_test_cache")
-		fs, _ := fscache.NewFs(testCacheDir, 0755)
-		testCache, _ = fscache.NewCache(fs, nil)
-	})
-
-	AfterSuite(func() {
-		os.RemoveAll(testCacheDir)
-	})
-})

engine/players.go

@@ -38,7 +38,7 @@ func (p *players) Register(ctx context.Context, id, client, typ, ip string) (*mo
 	if err != nil || id == "" {
 		plr, err = p.ds.Player(ctx).FindByName(client, userName)
 		if err == nil {
-			log.Debug("Found player by name", "id", plr.ID, "client", client, "userName", userName)
+			log.Debug("Found player by name", "id", plr.ID, "client", client, "username", userName)
 		} else {
 			r, _ := uuid.NewRandom()
 			plr = &model.Player{
@@ -47,7 +47,7 @@ func (p *players) Register(ctx context.Context, id, client, typ, ip string) (*mo
 				UserName: userName,
 				Client:   client,
 			}
-			log.Info("Registering new player", "id", plr.ID, "client", client, "userName", userName)
+			log.Info("Registering new player", "id", plr.ID, "client", client, "username", userName)
 		}
 	}
 	plr.LastSeen = time.Now()

engine/scrobbler.go

@@ -43,7 +43,11 @@ func (s *scrobbler) Register(ctx context.Context, playerId int, trackId string,
 		return err
 	})
 
-	log.Info("Scrobbled", "title", mf.Title, "artist", mf.Artist, "user", userName(ctx))
+	if err != nil {
+		log.Error("Error while scrobbling", "trackId", trackId, err)
+	} else {
+		log.Info("Scrobbled", "title", mf.Title, "artist", mf.Artist, "user", userName(ctx))
+	}
 
 	return mf, err
 }

engine/users.go

@@ -7,7 +7,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/deluan/navidrome/engine/auth"
+	"github.com/deluan/navidrome/core/auth"
 	"github.com/deluan/navidrome/model"
 )
 

engine/users_test.go

@@ -3,7 +3,7 @@ package engine
 import (
 	"context"
 
-	"github.com/deluan/navidrome/engine/auth"
+	"github.com/deluan/navidrome/core/auth"
 	"github.com/deluan/navidrome/model"
 	"github.com/deluan/navidrome/persistence"
 	. "github.com/onsi/ginkgo"

engine/wire_providers.go

@@ -1,13 +1,11 @@
 package engine
 
 import (
-	"github.com/deluan/navidrome/engine/transcoder"
 	"github.com/google/wire"
 )
 
 var Set = wire.NewSet(
 	NewBrowser,
-	NewCover,
 	NewListGenerator,
 	NewPlaylists,
 	NewRatings,
@@ -15,9 +13,5 @@ var Set = wire.NewSet(
 	NewSearch,
 	NewNowPlayingRepository,
 	NewUsers,
-	NewMediaStreamer,
-	transcoder.New,
-	NewTranscodingCache,
-	NewImageCache,
 	NewPlayers,
 )

git/pre-commit

@@ -0,0 +1,32 @@
+#!/bin/sh
+# Copyright 2012 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+# git gofmt pre-commit hook
+#
+# To use, store as .git/hooks/pre-commit inside your repository and make sure
+# it has execute permissions.
+#
+# This script does not handle file names that contain spaces.
+
+if which goimports > /dev/null; then
+  gofmtcmd=goimports
+else
+  gofmtcmd=gofmt
+fi
+
+gofiles=$(git diff --cached --name-only --diff-filter=ACM | grep '.go$')
+[ -z "$gofiles" ] && exit 0
+
+unformatted=$($gofmtcmd -l $gofiles)
+[ -z "$unformatted" ] && exit 0
+
+# Some files are not gofmt'd. Print message and fail.
+
+echo >&2 "Go files must be formatted with '$gofmtcmd'. Please run:"
+for fn in $unformatted; do
+	echo >&2 "  $gofmtcmd -w $PWD/$fn"
+done
+
+exit 1

git/pre-push

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -e
+
+golangci-lint run
+make test

go.mod

@@ -3,9 +3,8 @@ module github.com/deluan/navidrome
 go 1.14
 
 require (
-	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/Masterminds/squirrel v1.4.0
-	github.com/astaxie/beego v1.12.1
+	github.com/astaxie/beego v1.12.2
 	github.com/bradleyjkemp/cupaloy v2.3.0+incompatible
 	github.com/deluan/rest v0.0.0-20200327222046-b71e558c45d0
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
@@ -13,32 +12,36 @@ require (
 	github.com/disintegration/imaging v1.6.2
 	github.com/djherbis/fscache v0.10.1
 	github.com/dustin/go-humanize v1.0.0
-	github.com/fatih/camelcase v0.0.0-20160318181535-f6a740d52f96 // indirect
-	github.com/fatih/structs v1.0.0 // indirect
-	github.com/go-chi/chi v4.1.1+incompatible
+	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/go-chi/cors v1.1.1
+	github.com/go-chi/httprate v0.4.0
 	github.com/go-chi/jwtauth v4.0.4+incompatible
-	github.com/go-sql-driver/mysql v1.5.0 // indirect
-	github.com/golang/protobuf v1.3.1 // indirect
 	github.com/google/uuid v1.1.1
 	github.com/google/wire v0.4.0
 	github.com/kennygrant/sanitize v0.0.0-20170120101633-6a0bfdde8629
-	github.com/koding/multiconfig v0.0.0-20170327155832-26b6dfd3a84a
-	github.com/kr/pretty v0.1.0 // indirect
 	github.com/lib/pq v1.3.0 // indirect
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible
-	github.com/onsi/ginkgo v1.12.0
-	github.com/onsi/gomega v1.10.0
+	github.com/microcosm-cc/bluemonday v1.0.3
+	github.com/mitchellh/mapstructure v1.3.2 // indirect
+	github.com/onsi/ginkgo v1.14.0
+	github.com/onsi/gomega v1.10.1
+	github.com/pelletier/go-toml v1.8.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pressly/goose v2.6.0+incompatible
 	github.com/sirupsen/logrus v1.6.0
-	github.com/stretchr/testify v1.4.0 // indirect
-	golang.org/x/net v0.0.0-20200202094626-16171245cfb2 // indirect
-	golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 // indirect
-	golang.org/x/text v0.3.2 // indirect
-	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	github.com/spf13/afero v1.3.1 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/cobra v1.0.0
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.7.0
+	golang.org/x/net v0.0.0-20200625001655-4c5254603344 // indirect
+	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect
+	golang.org/x/text v0.3.3 // indirect
+	google.golang.org/protobuf v1.25.0 // indirect
 	gopkg.in/djherbis/atime.v1 v1.0.0 // indirect
 	gopkg.in/djherbis/stream.v1 v1.3.1 // indirect
-	gopkg.in/yaml.v2 v2.2.8 // indirect
+	gopkg.in/ini.v1 v1.57.0 // indirect
 )
+
+replace github.com/dhowden/tag => github.com/wader/tag v0.0.0-20200426234345-d072771f6a51

go.sum

@@ -1,21 +1,69 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
 github.com/Masterminds/squirrel v1.4.0 h1:he5i/EXixZxrBUWcxzDYMiju9WZ3ld/l7QBNuo/eN3w=
 github.com/Masterminds/squirrel v1.4.0/go.mod h1:yaPeOnPG5ZRwL9oKdTsO/prlkPbXWZlRVMQ/gGlzIuA=
-github.com/OwnLocal/goes v1.0.0/go.mod h1:8rIFjBGTue3lCU0wplczcUgt9Gxgrkkrw7etMIcn8TM=
-github.com/astaxie/beego v1.12.1 h1:dfpuoxpzLVgclveAXe4PyNKqkzgm5zF4tgF2B3kkM2I=
-github.com/astaxie/beego v1.12.1/go.mod h1:kPBWpSANNbSdIqOc8SUL9h+1oyBMZhROeYsXQDbidWQ=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
+github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/astaxie/beego v1.12.2 h1:CajUexhSX5ONWDiSCpeQBNVfTzOtPb9e9d+3vuU5FuU=
+github.com/astaxie/beego v1.12.2/go.mod h1:TMcqhsbhN3UFpN+RCfysaxPAbrhox6QSS3NIAEp/uzE=
+github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
+github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd/go.mod h1:1b+Y/CofkYwXMUU0OhQqGvsY2Bvgr4j6jfT699wyZKQ=
 github.com/beego/x2j v0.0.0-20131220205130-a0352aadc542/go.mod h1:kSeGC/p1AbBiEp5kat81+DSQrZenVBZXklMLaELspWU=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60=
 github.com/bradleyjkemp/cupaloy v2.3.0+incompatible h1:UafIjBvWQmS9i/xRg+CamMrnLTKNzo+bdmT/oH34c2Y=
 github.com/bradleyjkemp/cupaloy v2.3.0+incompatible/go.mod h1:Au1Xw1sgaJ5iSFktEhYsS0dbQiS1B0/XMXl+42y9Ilk=
 github.com/casbin/casbin v1.7.0/go.mod h1:c67qKN6Oum3UF5Q1+BByfFxkwKvhwW57ITjqwtzR1KE=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chris-ramon/douceur v0.2.0 h1:IDMEdxlEUUBYBKE4z/mJnFyVXox+MjuEVDJNN27glkU=
+github.com/chris-ramon/douceur v0.2.0/go.mod h1:wDW5xjJdeoMm1mRt4sD4c/LbF/mWdEpRXQKjTR8nIBE=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
-github.com/couchbase/go-couchbase v0.0.0-20181122212707-3e9b6e1258bb/go.mod h1:TWI8EKQMs5u5jLKW/tsb9VwauIrMIxQG1r5fMsswK5U=
-github.com/couchbase/gomemcached v0.0.0-20181122193126-5125a94a666c/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/couchbase/go-couchbase v0.0.0-20200519150804-63f3cdb75e0d/go.mod h1:TWI8EKQMs5u5jLKW/tsb9VwauIrMIxQG1r5fMsswK5U=
+github.com/couchbase/gomemcached v0.0.0-20200526233749-ec430f949808/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c=
 github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76 h1:Lgdd/Qp96Qj8jqLpq2cI1I1X7BJnu06efS+XkhRoLUQ=
 github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
@@ -26,8 +74,7 @@ github.com/deluan/rest v0.0.0-20200327222046-b71e558c45d0 h1:qHX6TTBIsrpg0JkYNko
 github.com/deluan/rest v0.0.0-20200327222046-b71e558c45d0/go.mod h1:tSgDythFsl0QgS/PFWfIZqcJKnkADWneY80jaVRlqK8=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dhowden/tag v0.0.0-20200412032933-5d76b8eaae27 h1:Z6xaGRBbqfLR797upHuzQ6w4zg33BLKfAKtVCcmMDgg=
-github.com/dhowden/tag v0.0.0-20200412032933-5d76b8eaae27/go.mod h1:SniNVYuaD1jmdEEvi+7ywb1QFR7agjeTdGKyFb0p7Rw=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
 github.com/djherbis/fscache v0.10.1 h1:hDv+RGyvD+UDKyRYuLoVNbuRTnf2SrA2K3VyR1br9lk=
@@ -36,52 +83,131 @@ github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 h1:aaQcKT9WumO6JEJcRyTqFVq4XUZiUcKR2/GI31TOcz8=
 github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
+github.com/elastic/go-elasticsearch/v6 v6.8.5/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI=
 github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
-github.com/fatih/camelcase v0.0.0-20160318181535-f6a740d52f96 h1:5e8GDOdG6jKeeqNGbR+tlmqhf4vQVs3atTTMEWeEcAk=
-github.com/fatih/camelcase v0.0.0-20160318181535-f6a740d52f96/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
-github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU=
-github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/go-chi/chi v4.1.1+incompatible h1:MmTgB0R8Bt/jccxp+t6S/1VGIKdJw5J74CK/c9tTfA4=
-github.com/go-chi/chi v4.1.1+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/glendc/gopher-json v0.0.0-20170414221815-dc4743023d0c/go.mod h1:Gja1A+xZ9BoviGJNA2E9vFkPjjsl+CoJxSXiQM1UXtw=
+github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
+github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
 github.com/go-chi/cors v1.1.1 h1:eHuqxsIw89iXcWnWUN8R72JMibABJTN/4IOYI5WERvw=
 github.com/go-chi/cors v1.1.1/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I=
+github.com/go-chi/httprate v0.4.0 h1:M2qVV0w6ksgLs6L8lTrvqNeaVm0ZJNVdbYM8u2T8HaE=
+github.com/go-chi/httprate v0.4.0/go.mod h1:7e7qjQtHzEbdyW5TYQrl4X2uNRCnlTajictc7B4ftgc=
 github.com/go-chi/jwtauth v4.0.4+incompatible h1:LGIxg6YfvSBzxU2BljXbrzVc1fMlgqSKBQgKOGAVtPY=
 github.com/go-chi/jwtauth v4.0.4+incompatible/go.mod h1:Q5EIArY/QnD6BdS+IyDw7B2m6iNbnPxtfd6/BcmtWbs=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
-github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
-github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/wire v0.4.0 h1:kXcsA/rIGzJImVqPdhfnr6q0xsS9gU0515q1EPpJ9fE=
 github.com/google/wire v0.4.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
+github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kennygrant/sanitize v0.0.0-20170120101633-6a0bfdde8629 h1:m1E9veL+2sjZOMSM7y3a6jJ9fNVaGyIJCXYDPm9U+/0=
 github.com/kennygrant/sanitize v0.0.0-20170120101633-6a0bfdde8629/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
-github.com/koding/multiconfig v0.0.0-20170327155832-26b6dfd3a84a h1:KZAp4Cn6Wybs23MKaIrKyb/6+qs2rncDspTuRYwOmvU=
-github.com/koding/multiconfig v0.0.0-20170327155832-26b6dfd3a84a/go.mod h1:Y2SaZf2Rzd0pXkLVhLlCiAXFCLSXAIbTKDivVgff/AM=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -91,34 +217,106 @@ github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq
 github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o=
 github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk=
 github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6FmdpVm2joNMFikkuWg0EoCKLGUMNw=
+github.com/ledisdb/ledisdb v0.0.0-20200510135210-d35789ec47e6/go.mod h1:n931TsDuKuq+uX4v1fulaMbA/7ZLLhjc85h7chZGBCQ=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
 github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/microcosm-cc/bluemonday v1.0.3 h1:EjVH7OqbU219kdm8acbveoclh2zZFqPJTJw6VUlTLAQ=
+github.com/microcosm-cc/bluemonday v1.0.3/go.mod h1:8iwZnFn2CDDNZ0r6UXhF4xawGvzaqzCRa1n3/lO3W2w=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.3.2 h1:mRS76wmkOn3KkKAyXDu42V+6ebnXWIztFSYGN7GeoRg=
+github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU=
 github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
+github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA=
+github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
-github.com/onsi/gomega v1.10.0 h1:Gwkk+PTu/nfOwNMtUB/mRUv0X7ewW5dO4AERT1ThVKo=
-github.com/onsi/gomega v1.10.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
+github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pelletier/go-toml v1.0.1/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw=
+github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs=
+github.com/peterh/liner v1.0.1-0.20171122030339-3681c2a91233/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/pressly/goose v2.6.0+incompatible h1:3f8zIQ8rfgP9tyI0Hmcs2YNAqUCL1c+diLe3iU8Qd/k=
 github.com/pressly/goose v2.6.0+incompatible/go.mod h1:m+QHWCqxR3k8D9l7qfzuC/djtlfzxr34mozWDYEu1z8=
-github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM=
-github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw=
-github.com/siddontang/ledisdb v0.0.0-20181029004158-becf5f38d373 h1:p6IxqQMjab30l4lb9mmkIkkcE1yv6o0SKbPhW5pxqHI=
-github.com/siddontang/ledisdb v0.0.0-20181029004158-becf5f38d373/go.mod h1:mF1DpOSOUiJRMR+FDqaqu3EBqrybQtrDDszLUZ6oxPg=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.0 h1:wCi7urQOGBsYcQROHqpUUX4ct84xp40t9R9JX0FuA/U=
+github.com/prometheus/client_golang v1.7.0/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 h1:X+yvsM2yrEktyI+b2qND5gpH8YhURn0k8OCaeRnkINo=
+github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644/go.mod h1:nkxAfR/5quYxwPZhyDxgasBMnRtBZd0FCEpawpjMUFg=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/siddontang/go v0.0.0-20170517070808-cb568a3e5cc0/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw=
+github.com/siddontang/goredis v0.0.0-20150324035039-760763f78400/go.mod h1:DDcKzU3qCuvj/tPnimWSsZZzvk9qvkvrIL5naVBPh5s=
 github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d h1:NVwnfyR3rENtlz62bcrkXME3INVUa4lcdGt+opvxExs=
 github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z92TR1JKMkLLoaOQk++LVnOKL3ScbJ8GNGA=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
@@ -127,72 +325,255 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.3.1 h1:GPTpEAuNr98px18yNQ66JllNil98wfRZ/5Ukny8FeQA=
+github.com/spf13/afero v1.3.1/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
+github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
+github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8=
+github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
+github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
+github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM=
+github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/ssdb/gossdb v0.0.0-20180723034631-88f6b59b84ec/go.mod h1:QBvMkMya+gXctz3kmljlUCu/yB3GZ6oee+dUozsezQE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/syndtr/goleveldb v0.0.0-20160425020131-cfa635847112/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
 github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c h1:3eGShk3EQf5gJCYW+WzA0TEJQd37HLOmlYF7N0YJwv0=
 github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/ugorji/go v0.0.0-20171122102828-84cb69a8af83/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/wader/tag v0.0.0-20200426234345-d072771f6a51 h1:WAxntH7YQD6fIboAvewi7eU+2PQ7Y1K9OOXh67CM4bY=
+github.com/wader/tag v0.0.0-20200426234345-d072771f6a51/go.mod h1:f3YqVk9PEeVf7T4JQ2+TdRqqjTg2fkaROZv0EMQOuKo=
 github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yuin/gopher-lua v0.0.0-20171031051903-609c9cd26973/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
+golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
+golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b h1:NVD8gBK33xpdqCaZVVtd6OFJp+3dxkXuz7+U7KaVN6s=
 golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200117065230-39095c1d176c h1:FodBYPZKH5tAN2O60HlglMwXGAeV/4k+NKbli79M/2c=
 golang.org/x/tools v0.0.0-20200117065230-39095c1d176c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/djherbis/atime.v1 v1.0.0 h1:eMRqB/JrLKocla2PBPKgQYg/p5UG4L6AUAs92aP7F60=
 gopkg.in/djherbis/atime.v1 v1.0.0/go.mod h1:hQIUStKmJfvf7xdh/wtK84qe+DsTV5LnA9lzxxtPpJ8=
 gopkg.in/djherbis/stream.v1 v1.3.1 h1:uGfmsOY1qqMjQQphhRBSGLyA9qumJ56exkRu9ASTjCw=
 gopkg.in/djherbis/stream.v1 v1.3.1/go.mod h1:aEV8CBVRmSpLamVJfM903Npic1IKmb2qS30VAZ+sssg=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
+gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

lefthook.yml

@@ -1,17 +0,0 @@
-pre-push:
-  parallel: true
-  commands:
-    unit-tests:
-      tags: tests
-      run: go test ./...
-    lint:
-      tags: tests
-      run: golangci-lint run
-
-pre-commit:
-  parallel: false
-  commands:
-    gofmt:
-      tags: style
-      glob: "*.go"
-      run: gofmt -w {staged_files}; git add {staged_files}

main.go

@@ -1,25 +1,7 @@
 package main
 
-import (
-	"fmt"
-
-	"github.com/deluan/navidrome/conf"
-	"github.com/deluan/navidrome/consts"
-	"github.com/deluan/navidrome/db"
-)
+import "github.com/deluan/navidrome/cmd"
 
 func main() {
-	println(consts.Banner())
-
-	conf.Load()
-	db.EnsureLatestVersion()
-
-	subsonic, err := CreateSubsonicAPIRouter()
-	if err != nil {
-		panic(fmt.Sprintf("Could not create the Subsonic API router. Aborting! err=%v", err))
-	}
-	a := CreateServer(conf.Server.MusicFolder)
-	a.MountRouter(consts.URLPathSubsonicAPI, subsonic)
-	a.MountRouter(consts.URLPathUI, CreateAppRouter())
-	a.Run(":" + conf.Server.Port)
+	cmd.Execute()
 }

model/album.go

@@ -3,6 +3,8 @@ package model
 import "time"
 
 type Album struct {
+	Annotations
+
 	ID                   string    `json:"id"            orm:"column(id)"`
 	Name                 string    `json:"name"`
 	CoverArtPath         string    `json:"coverArtPath"`
@@ -25,13 +27,6 @@ type Album struct {
 	OrderAlbumArtistName string    `json:"orderAlbumArtistName"`
 	CreatedAt            time.Time `json:"createdAt"`
 	UpdatedAt            time.Time `json:"updatedAt"`
-
-	// Annotations
-	PlayCount int       `json:"playCount"   orm:"-"`
-	PlayDate  time.Time `json:"playDate"    orm:"-"`
-	Rating    int       `json:"rating"      orm:"-"`
-	Starred   bool      `json:"starred"     orm:"-"`
-	StarredAt time.Time `json:"starredAt"   orm:"-"`
 }
 
 type Albums []Album
@@ -46,6 +41,9 @@ type AlbumRepository interface {
 	GetStarred(options ...QueryOptions) (Albums, error)
 	Search(q string, offset int, size int) (Albums, error)
 	Refresh(ids ...string) error
-	PurgeEmpty() error
 	AnnotatedRepository
 }
+
+func (a Album) GetAnnotations() Annotations {
+	return a.Annotations
+}

model/annotation.go

@@ -2,6 +2,18 @@ package model
 
 import "time"
 
+type Annotations struct {
+	PlayCount int64     `json:"playCount"`
+	PlayDate  time.Time `json:"playDate"`
+	Rating    int       `json:"rating"`
+	Starred   bool      `json:"starred"`
+	StarredAt time.Time `json:"starredAt"`
+}
+
+type AnnotatedModel interface {
+	GetAnnotations() Annotations
+}
+
 type AnnotatedRepository interface {
 	IncPlayCount(itemID string, ts time.Time) error
 	SetStar(starred bool, itemIDs ...string) error

model/artist.go

@@ -1,8 +1,8 @@
 package model
 
-import "time"
-
 type Artist struct {
+	Annotations
+
 	ID              string `json:"id"          orm:"column(id)"`
 	Name            string `json:"name"`
 	AlbumCount      int    `json:"albumCount"`
@@ -10,13 +10,6 @@ type Artist struct {
 	FullText        string `json:"fullText"`
 	SortArtistName  string `json:"sortArtistName"`
 	OrderArtistName string `json:"orderArtistName"`
-
-	// Annotations
-	PlayCount int       `json:"playCount"   orm:"-"`
-	PlayDate  time.Time `json:"playDate"    orm:"-"`
-	Rating    int       `json:"rating"      orm:"-"`
-	Starred   bool      `json:"starred"     orm:"-"`
-	StarredAt time.Time `json:"starredAt"   orm:"-"`
 }
 
 type Artists []Artist
@@ -36,6 +29,9 @@ type ArtistRepository interface {
 	Search(q string, offset int, size int) (Artists, error)
 	Refresh(ids ...string) error
 	GetIndex() (ArtistIndexes, error)
-	PurgeEmpty() error
 	AnnotatedRepository
 }
+
+func (a Artist) GetAnnotations() Annotations {
+	return a.Annotations
+}

model/mediafile.go

@@ -6,13 +6,15 @@ import (
 )
 
 type MediaFile struct {
+	Annotations
+
 	ID                   string    `json:"id"            orm:"pk;column(id)"`
 	Path                 string    `json:"path"`
 	Title                string    `json:"title"`
 	Album                string    `json:"album"`
 	ArtistID             string    `json:"artistId"      orm:"pk;column(artist_id)"`
 	Artist               string    `json:"artist"`
-	AlbumArtistID        string    `json:"albumArtistId"`
+	AlbumArtistID        string    `json:"albumArtistId" orm:"pk;column(album_artist_id)"`
 	AlbumArtist          string    `json:"albumArtist"`
 	AlbumID              string    `json:"albumId"       orm:"pk;column(album_id)"`
 	HasCoverArt          bool      `json:"hasCoverArt"`
@@ -20,7 +22,7 @@ type MediaFile struct {
 	DiscNumber           int       `json:"discNumber"`
 	DiscSubtitle         string    `json:"discSubtitle"`
 	Year                 int       `json:"year"`
-	Size                 int       `json:"size"`
+	Size                 int64     `json:"size"`
 	Suffix               string    `json:"suffix"`
 	Duration             float32   `json:"duration"`
 	BitRate              int       `json:"bitRate"`
@@ -36,13 +38,6 @@ type MediaFile struct {
 	Compilation          bool      `json:"compilation"`
 	CreatedAt            time.Time `json:"createdAt"`
 	UpdatedAt            time.Time `json:"updatedAt"`
-
-	// Annotations
-	PlayCount int       `json:"playCount"   orm:"-"`
-	PlayDate  time.Time `json:"playDate"    orm:"-"`
-	Rating    int       `json:"rating"      orm:"-"`
-	Starred   bool      `json:"starred"     orm:"-"`
-	StarredAt time.Time `json:"starredAt"   orm:"-"`
 }
 
 func (mf *MediaFile) ContentType() string {
@@ -58,12 +53,18 @@ type MediaFileRepository interface {
 	Get(id string) (*MediaFile, error)
 	GetAll(options ...QueryOptions) (MediaFiles, error)
 	FindByAlbum(albumId string) (MediaFiles, error)
-	FindByPath(path string) (MediaFiles, error)
+	FindAllByPath(path string) (MediaFiles, error)
+	FindByPath(path string) (*MediaFile, error)
+	FindPathsRecursively(basePath string) ([]string, error)
 	GetStarred(options ...QueryOptions) (MediaFiles, error)
 	GetRandom(options ...QueryOptions) (MediaFiles, error)
 	Search(q string, offset int, size int) (MediaFiles, error)
 	Delete(id string) error
-	DeleteByPath(path string) error
+	DeleteByPath(path string) (int64, error)
 
 	AnnotatedRepository
 }
+
+func (mf MediaFile) GetAnnotations() Annotations {
+	return mf.Annotations
+}

model/playlist.go

@@ -13,6 +13,8 @@ type Playlist struct {
 	Owner     string     `json:"owner"`
 	Public    bool       `json:"public"`
 	Tracks    MediaFiles `json:"tracks,omitempty"`
+	Path      string     `json:"path"`
+	Sync      bool       `json:"sync"`
 	CreatedAt time.Time  `json:"createdAt"`
 	UpdatedAt time.Time  `json:"updatedAt"`
 }
@@ -25,6 +27,7 @@ type PlaylistRepository interface {
 	Put(pls *Playlist) error
 	Get(id string) (*Playlist, error)
 	GetAll(options ...QueryOptions) (Playlists, error)
+	FindByPath(path string) (*Playlist, error)
 	Delete(id string) error
 	Tracks(playlistId string) PlaylistTrackRepository
 }
@@ -43,4 +46,5 @@ type PlaylistTrackRepository interface {
 	Add(mediaFileIds []string) error
 	Update(mediaFileIds []string) error
 	Delete(id string) error
+	Reorder(pos int, newPos int) error
 }

model/request/request.go

@@ -2,6 +2,7 @@ package request
 
 import (
 	"context"
+	"strings"
 
 	"github.com/deluan/navidrome/model"
 )
@@ -22,7 +23,7 @@ func WithUser(ctx context.Context, u model.User) context.Context {
 }
 
 func WithUsername(ctx context.Context, username string) context.Context {
-	return context.WithValue(ctx, Username, username)
+	return context.WithValue(ctx, Username, strings.ToLower(username))
 }
 
 func WithClient(ctx context.Context, client string) context.Context {

model/user.go

@@ -22,6 +22,7 @@ type UserRepository interface {
 	CountAll(...QueryOptions) (int64, error)
 	Get(id string) (*User, error)
 	Put(*User) error
+	FindFirstAdmin() (*User, error)
 	// FindByUsername must be case-insensitive
 	FindByUsername(username string) (*User, error)
 	UpdateLastLoginAt(id string) error

persistence/album_repository.go

@@ -2,6 +2,8 @@ package persistence
 
 import (
 	"context"
+	"os"
+	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
@@ -9,9 +11,11 @@ import (
 
 	. "github.com/Masterminds/squirrel"
 	"github.com/astaxie/beego/orm"
+	"github.com/deluan/navidrome/conf"
 	"github.com/deluan/navidrome/consts"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
+	"github.com/deluan/navidrome/utils"
 	"github.com/deluan/rest"
 )
 
@@ -26,8 +30,10 @@ func NewAlbumRepository(ctx context.Context, o orm.Ormer) model.AlbumRepository
 	r.ormer = o
 	r.tableName = "album"
 	r.sortMappings = map[string]string{
-		"artist": "compilation asc, order_album_artist_name asc, order_album_name asc",
-		"random": "RANDOM()",
+		"name":     "order_album_name",
+		"artist":   "compilation asc, order_album_artist_name asc, order_album_name asc",
+		"random":   "RANDOM()",
+		"max_year": "max_year asc, name, order_album_name asc",
 	}
 	r.filterMappings = map[string]filterFunc{
 		"name":        fullTextFilter,
@@ -74,12 +80,14 @@ func (r *albumRepository) selectAlbum(options ...model.QueryOptions) SelectBuild
 
 func (r *albumRepository) Get(id string) (*model.Album, error) {
 	sq := r.selectAlbum().Where(Eq{"id": id})
-	var res model.Album
-	err := r.queryOne(sq, &res)
-	if err != nil {
+	var res model.Albums
+	if err := r.queryAll(sq, &res); err != nil {
 		return nil, err
 	}
-	return &res, nil
+	if len(res) == 0 {
+		return nil, model.ErrNotFound
+	}
+	return &res[0], nil
 }
 
 func (r *albumRepository) FindByArtist(artistId string) (model.Albums, error) {
@@ -105,37 +113,76 @@ func (r *albumRepository) GetRandom(options ...model.QueryOptions) (model.Albums
 	return results, err
 }
 
+// Return a map of mediafiles that have embedded covers for the given album ids
+func (r *albumRepository) getEmbeddedCovers(ids []string) (map[string]model.MediaFile, error) {
+	var mfs model.MediaFiles
+	coverSql := Select("album_id", "id", "path").Distinct().From("media_file").
+		Where(And{Eq{"has_cover_art": true}, Eq{"album_id": ids}}).
+		GroupBy("album_id")
+	err := r.queryAll(coverSql, &mfs)
+	if err != nil {
+		return nil, err
+	}
+
+	result := map[string]model.MediaFile{}
+	for _, mf := range mfs {
+		result[mf.AlbumID] = mf
+	}
+	return result, nil
+}
+
 func (r *albumRepository) Refresh(ids ...string) error {
 	type refreshAlbum struct {
 		model.Album
 		CurrentId     string
-		HasCoverArt   bool
 		SongArtists   string
 		Years         string
 		DiscSubtitles string
+		Path          string
 	}
 	var albums []refreshAlbum
-	sel := Select(`album_id as id, album as name, f.artist, f.album_artist, f.artist_id, f.album_artist_id, 
+	sel := Select(`f.album_id as id, f.album as name, f.artist, f.album_artist, f.artist_id, f.album_artist_id, 
 		f.sort_album_name, f.sort_artist_name, f.sort_album_artist_name,
-		f.order_album_name, f.order_album_artist_name,
+		f.order_album_name, f.order_album_artist_name, f.path,
 		f.compilation, f.genre, max(f.year) as max_year, sum(f.duration) as duration, 
-		count(*) as song_count, a.id as current_id, f.id as cover_art_id, f.path as cover_art_path, f.has_cover_art, 
+		count(f.id) as song_count, a.id as current_id, 
 		group_concat(f.disc_subtitle, ' ') as disc_subtitles,
 		group_concat(f.artist, ' ') as song_artists, group_concat(f.year, ' ') as years`).
 		From("media_file f").
 		LeftJoin("album a on f.album_id = a.id").
-		Where(Eq{"f.album_id": ids}).GroupBy("album_id").OrderBy("f.id")
+		Where(Eq{"f.album_id": ids}).GroupBy("f.album_id")
 	err := r.queryAll(sel, &albums)
 	if err != nil {
 		return err
 	}
 
+	covers, err := r.getEmbeddedCovers(ids)
+	if err != nil {
+		return nil
+	}
+
 	toInsert := 0
 	toUpdate := 0
 	for _, al := range albums {
-		if !al.HasCoverArt {
-			al.CoverArtId = ""
+		embedded, hasCoverArt := covers[al.ID]
+		if hasCoverArt {
+			al.CoverArtId = embedded.ID
+			al.CoverArtPath = embedded.Path
 		}
+
+		if !hasCoverArt || !strings.HasPrefix(conf.Server.CoverArtPriority, "embedded") {
+			if path := getCoverFromPath(al.Path, al.CoverArtPath); path != "" {
+				al.CoverArtId = "al-" + al.ID
+				al.CoverArtPath = path
+			}
+		}
+
+		if al.CoverArtId != "" {
+			log.Trace(r.ctx, "Found album art", "id", al.ID, "name", al.Name, "coverArtPath", al.CoverArtPath, "coverArtId", al.CoverArtId, "hasCoverArt", hasCoverArt)
+		} else {
+			log.Trace(r.ctx, "Could not find album art", "id", al.ID, "name", al.Name)
+		}
+
 		if al.Compilation {
 			al.AlbumArtist = consts.VariousArtists
 			al.AlbumArtistID = consts.VariousArtistsID
@@ -180,7 +227,44 @@ func getMinYear(years string) int {
 	return 0
 }
 
-func (r *albumRepository) PurgeEmpty() error {
+// GetCoverFromPath accepts a path to a file, and returns a path to an eligible cover image from the
+// file's directory (as configured with CoverArtPriority). If no cover file is found, among
+// available choices, or an error occurs, an empty string is returned. If HasEmbeddedCover is true,
+// and 'embedded' is matched among eligible choices, GetCoverFromPath will return early with an
+// empty path.
+func getCoverFromPath(mediaPath string, embeddedPath string) string {
+	n, err := os.Open(filepath.Dir(mediaPath))
+	if err != nil {
+		return ""
+	}
+
+	defer n.Close()
+	names, err := n.Readdirnames(-1)
+	if err != nil {
+		return ""
+	}
+
+	for _, p := range strings.Split(conf.Server.CoverArtPriority, ",") {
+		pat := strings.ToLower(strings.TrimSpace(p))
+		if pat == "embedded" {
+			if embeddedPath != "" {
+				return ""
+			}
+			continue
+		}
+
+		for _, name := range names {
+			match, _ := filepath.Match(pat, strings.ToLower(name))
+			if match && utils.IsImageFile(name) {
+				return filepath.Join(filepath.Dir(mediaPath), name)
+			}
+		}
+	}
+
+	return ""
+}
+
+func (r *albumRepository) purgeEmpty() error {
 	del := Delete(r.tableName).Where("id not in (select distinct(album_id) from media_file)")
 	c, err := r.executeSQL(del)
 	if err == nil {

persistence/album_repository_test.go

@@ -2,8 +2,12 @@ package persistence
 
 import (
 	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 
 	"github.com/astaxie/beego/orm"
+	"github.com/deluan/navidrome/conf"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
 	"github.com/deluan/navidrome/model/request"
@@ -86,4 +90,52 @@ var _ = Describe("AlbumRepository", func() {
 			Expect(getMinYear("2000 0 1800")).To(Equal(1800))
 		})
 	})
+
+	Describe("getCoverFromPath", func() {
+		testFolder, _ := ioutil.TempDir("", "album_persistence_tests")
+		if err := os.MkdirAll(testFolder, 0777); err != nil {
+			panic(err)
+		}
+		if _, err := os.Create(filepath.Join(testFolder, "Cover.jpeg")); err != nil {
+			panic(err)
+		}
+		if _, err := os.Create(filepath.Join(testFolder, "FRONT.PNG")); err != nil {
+			panic(err)
+		}
+
+		testPath := filepath.Join(testFolder, "somefile.test")
+		embeddedPath := filepath.Join(testFolder, "somefile.mp3")
+		It("returns audio file for embedded cover", func() {
+			conf.Server.CoverArtPriority = "embedded, cover.*, front.*"
+			Expect(getCoverFromPath(testPath, embeddedPath)).To(Equal(""))
+		})
+
+		It("returns external file when no embedded cover exists", func() {
+			conf.Server.CoverArtPriority = "embedded, cover.*, front.*"
+			Expect(getCoverFromPath(testPath, "")).To(Equal(filepath.Join(testFolder, "Cover.jpeg")))
+		})
+
+		It("returns embedded cover even if not first choice", func() {
+			conf.Server.CoverArtPriority = "something.png, embedded, cover.*, front.*"
+			Expect(getCoverFromPath(testPath, embeddedPath)).To(Equal(""))
+		})
+
+		It("returns first correct match case-insensitively", func() {
+			conf.Server.CoverArtPriority = "embedded, cover.jpg, front.svg, front.png"
+			Expect(getCoverFromPath(testPath, "")).To(Equal(filepath.Join(testFolder, "FRONT.PNG")))
+		})
+
+		It("returns match for embedded pattern", func() {
+			conf.Server.CoverArtPriority = "embedded, cover.jp?g, front.png"
+			Expect(getCoverFromPath(testPath, "")).To(Equal(filepath.Join(testFolder, "Cover.jpeg")))
+		})
+
+		It("returns empty string if no match was found", func() {
+			conf.Server.CoverArtPriority = "embedded, cover.jpg, front.apng"
+			Expect(getCoverFromPath(testPath, "")).To(Equal(""))
+		})
+
+		// Reset configuration to default.
+		conf.Server.CoverArtPriority = "embedded, cover.*, front.*"
+	})
 })

persistence/artist_repository.go

@@ -55,9 +55,14 @@ func (r *artistRepository) Put(a *model.Artist) error {
 
 func (r *artistRepository) Get(id string) (*model.Artist, error) {
 	sel := r.selectArtist().Where(Eq{"id": id})
-	var res model.Artist
-	err := r.queryOne(sel, &res)
-	return &res, err
+	var res model.Artists
+	if err := r.queryAll(sel, &res); err != nil {
+		return nil, err
+	}
+	if len(res) == 0 {
+		return nil, model.ErrNotFound
+	}
+	return &res[0], nil
 }
 
 func (r *artistRepository) GetAll(options ...model.QueryOptions) (model.Artists, error) {
@@ -155,7 +160,7 @@ func (r *artistRepository) GetStarred(options ...model.QueryOptions) (model.Arti
 	return starred, err
 }
 
-func (r *artistRepository) PurgeEmpty() error {
+func (r *artistRepository) purgeEmpty() error {
 	del := Delete(r.tableName).Where("id not in (select distinct(album_artist_id) from album)")
 	c, err := r.executeSQL(del)
 	if err == nil {

persistence/mediafile_repository.go

@@ -2,8 +2,9 @@ package persistence
 
 import (
 	"context"
+	"fmt"
 	"os"
-	"strings"
+	"path/filepath"
 
 	. "github.com/Masterminds/squirrel"
 	"github.com/astaxie/beego/orm"
@@ -28,13 +29,14 @@ func NewMediaFileRepository(ctx context.Context, o orm.Ormer) *mediaFileReposito
 		"random": "RANDOM()",
 	}
 	r.filterMappings = map[string]filterFunc{
-		"title": fullTextFilter,
+		"title":   fullTextFilter,
+		"starred": booleanFilter,
 	}
 	return r
 }
 
 func (r mediaFileRepository) CountAll(options ...model.QueryOptions) (int64, error) {
-	return r.count(Select(), options...)
+	return r.count(r.newSelectWithAnnotation("media_file.id"), options...)
 }
 
 func (r mediaFileRepository) Exists(id string) (bool, error) {
@@ -54,9 +56,14 @@ func (r mediaFileRepository) selectMediaFile(options ...model.QueryOptions) Sele
 
 func (r mediaFileRepository) Get(id string) (*model.MediaFile, error) {
 	sel := r.selectMediaFile().Where(Eq{"id": id})
-	var res model.MediaFile
-	err := r.queryOne(sel, &res)
-	return &res, err
+	var res model.MediaFiles
+	if err := r.queryAll(sel, &res); err != nil {
+		return nil, err
+	}
+	if len(res) == 0 {
+		return nil, model.ErrNotFound
+	}
+	return &res[0], nil
 }
 
 func (r mediaFileRepository) GetAll(options ...model.QueryOptions) (model.MediaFiles, error) {
@@ -73,25 +80,45 @@ func (r mediaFileRepository) FindByAlbum(albumId string) (model.MediaFiles, erro
 	return res, err
 }
 
-func (r mediaFileRepository) FindByPath(path string) (model.MediaFiles, error) {
-	sel := r.selectMediaFile().Where(Like{"path": path + "%"})
-	res := model.MediaFiles{}
-	err := r.queryAll(sel, &res)
-	if err != nil {
+func (r mediaFileRepository) FindByPath(path string) (*model.MediaFile, error) {
+	sel := r.selectMediaFile().Where(Eq{"path": path})
+	var res model.MediaFiles
+	if err := r.queryAll(sel, &res); err != nil {
 		return nil, err
 	}
-
-	// Only return mediafiles that are direct child of requested path
-	filtered := model.MediaFiles{}
-	path = strings.ToLower(path) + string(os.PathSeparator)
-	for _, mf := range res {
-		filename := strings.TrimPrefix(strings.ToLower(mf.Path), path)
-		if len(strings.Split(filename, string(os.PathSeparator))) > 1 {
-			continue
-		}
-		filtered = append(filtered, mf)
+	if len(res) == 0 {
+		return nil, model.ErrNotFound
 	}
-	return filtered, nil
+	return &res[0], nil
+}
+
+// FindAllByPath only return mediafiles that are direct children of requested path
+func (r mediaFileRepository) FindAllByPath(path string) (model.MediaFiles, error) {
+	// Query by path based on https://stackoverflow.com/a/13911906/653632
+	sel0 := r.selectMediaFile().Columns(fmt.Sprintf("substr(path, %d) AS item", len(path)+2)).
+		Where(pathStartsWith(path))
+	sel := r.newSelect().Columns("*", "item NOT GLOB '*"+string(os.PathSeparator)+"*' AS isLast").
+		Where(Eq{"isLast": 1}).FromSelect(sel0, "sel0")
+
+	res := model.MediaFiles{}
+	err := r.queryAll(sel, &res)
+	return res, err
+}
+
+func pathStartsWith(path string) Sqlizer {
+	cleanPath := filepath.Clean(path)
+	substr := fmt.Sprintf("substr(path, 1, %d)", len(cleanPath))
+	return Eq{substr: cleanPath}
+}
+
+// FindPathsRecursively returns a list of all subfolders of basePath, recursively
+func (r mediaFileRepository) FindPathsRecursively(basePath string) ([]string, error) {
+	// Query based on https://stackoverflow.com/a/38330814/653632
+	sel := r.newSelect().Columns(fmt.Sprintf("distinct rtrim(path, replace(path, '%s', ''))", string(os.PathSeparator))).
+		Where(pathStartsWith(basePath))
+	var res []string
+	err := r.queryAll(sel, &res)
+	return res, err
 }
 
 func (r mediaFileRepository) GetStarred(options ...model.QueryOptions) (model.MediaFiles, error) {
@@ -114,22 +141,14 @@ func (r mediaFileRepository) Delete(id string) error {
 	return r.delete(Eq{"id": id})
 }
 
-func (r mediaFileRepository) DeleteByPath(path string) error {
-	filtered, err := r.FindByPath(path)
-	if err != nil {
-		return err
-	}
-	if len(filtered) == 0 {
-		return nil
-	}
-	ids := make([]string, len(filtered))
-	for i, mf := range filtered {
-		ids[i] = mf.ID
-	}
-	log.Debug(r.ctx, "Deleting mediafiles by path", "path", path, "totalDeleted", len(ids))
-	del := Delete(r.tableName).Where(Eq{"id": ids})
-	_, err = r.executeSQL(del)
-	return err
+// DeleteByPath delete from the DB all mediafiles that are direct children of path
+func (r mediaFileRepository) DeleteByPath(path string) (int64, error) {
+	path = filepath.Clean(path)
+	del := Delete(r.tableName).
+		Where(And{pathStartsWith(path),
+			Eq{fmt.Sprintf("substr(path, %d) glob '*%s*'", len(path)+2, string(os.PathSeparator)): 0}})
+	log.Debug(r.ctx, "Deleting mediafiles by path", "path", path)
+	return r.executeSQL(del)
 }
 
 func (r mediaFileRepository) Search(q string, offset int, size int) (model.MediaFiles, error) {
@@ -155,8 +174,20 @@ func (r mediaFileRepository) EntityName() string {
 }
 
 func (r mediaFileRepository) NewInstance() interface{} {
-	return model.MediaFile{}
+	return &model.MediaFile{}
+}
+
+func (r mediaFileRepository) Save(entity interface{}) (string, error) {
+	mf := entity.(*model.MediaFile)
+	err := r.Put(mf)
+	return mf.ID, err
+}
+
+func (r mediaFileRepository) Update(entity interface{}, cols ...string) error {
+	mf := entity.(*model.MediaFile)
+	return r.Put(mf)
 }
 
 var _ model.MediaFileRepository = (*mediaFileRepository)(nil)
 var _ model.ResourceRepository = (*mediaFileRepository)(nil)
+var _ rest.Persistable = (*mediaFileRepository)(nil)

persistence/mediafile_repository_test.go

@@ -51,10 +51,29 @@ var _ = Describe("MediaRepository", func() {
 		Expect(mr.FindByAlbum("67")).To(Equal(model.MediaFiles{}))
 	})
 
-	It("finds tracks by path", func() {
-		Expect(mr.FindByPath(P("/beatles/1/sgt"))).To(Equal(model.MediaFiles{
-			songDayInALife,
-		}))
+	It("finds tracks by path when using wildcards chars", func() {
+		Expect(mr.Put(&model.MediaFile{ID: "7001", Path: P("/Find:By'Path/_/123.mp3")})).To(BeNil())
+		Expect(mr.Put(&model.MediaFile{ID: "7002", Path: P("/Find:By'Path/1/123.mp3")})).To(BeNil())
+
+		found, err := mr.FindAllByPath(P("/Find:By'Path/_/"))
+		Expect(err).To(BeNil())
+		Expect(found).To(HaveLen(1))
+		Expect(found[0].ID).To(Equal("7001"))
+	})
+
+	It("finds tracks by path case sensitively", func() {
+		Expect(mr.Put(&model.MediaFile{ID: "7003", Path: P("/Casesensitive/file1.mp3")})).To(BeNil())
+		Expect(mr.Put(&model.MediaFile{ID: "7004", Path: P("/casesensitive/file2.mp3")})).To(BeNil())
+
+		found, err := mr.FindAllByPath(P("/Casesensitive"))
+		Expect(err).To(BeNil())
+		Expect(found).To(HaveLen(1))
+		Expect(found[0].ID).To(Equal("7003"))
+
+		found, err = mr.FindAllByPath(P("/casesensitive/"))
+		Expect(err).To(BeNil())
+		Expect(found).To(HaveLen(1))
+		Expect(found[0].ID).To(Equal("7004"))
 	})
 
 	It("returns starred tracks", func() {
@@ -80,12 +99,18 @@ var _ = Describe("MediaRepository", func() {
 		id2 := "2222"
 		Expect(mr.Put(&model.MediaFile{ID: id2, Path: P("/abc/123/" + id2 + ".mp3")})).To(BeNil())
 		id3 := "3333"
-		Expect(mr.Put(&model.MediaFile{ID: id3, Path: P("/abc/" + id3 + ".mp3")})).To(BeNil())
+		Expect(mr.Put(&model.MediaFile{ID: id3, Path: P("/ab_/" + id3 + ".mp3")})).To(BeNil())
+		id4 := "4444"
+		Expect(mr.Put(&model.MediaFile{ID: id4, Path: P("/abc/" + id4 + ".mp3")})).To(BeNil())
+		id5 := "5555"
+		Expect(mr.Put(&model.MediaFile{ID: id5, Path: P("/Ab_/" + id5 + ".mp3")})).To(BeNil())
 
-		Expect(mr.DeleteByPath(P("/abc"))).To(BeNil())
+		Expect(mr.DeleteByPath(P("/ab_"))).To(Equal(int64(1)))
 
 		Expect(mr.Get(id1)).ToNot(BeNil())
 		Expect(mr.Get(id2)).ToNot(BeNil())
+		Expect(mr.Get(id4)).ToNot(BeNil())
+		Expect(mr.Get(id5)).ToNot(BeNil())
 		_, err := mr.Get(id3)
 		Expect(err).To(MatchError(model.ErrNotFound))
 	})
@@ -101,7 +126,7 @@ var _ = Describe("MediaRepository", func() {
 			Expect(err).To(BeNil())
 
 			Expect(mf.PlayDate.Unix()).To(Equal(playDate.Unix()))
-			Expect(mf.PlayCount).To(Equal(1))
+			Expect(mf.PlayCount).To(Equal(int64(1)))
 		})
 
 		It("increments play count on newly starred items", func() {
@@ -115,7 +140,7 @@ var _ = Describe("MediaRepository", func() {
 			Expect(err).To(BeNil())
 
 			Expect(mf.PlayDate.Unix()).To(Equal(playDate.Unix()))
-			Expect(mf.PlayCount).To(Equal(1))
+			Expect(mf.PlayCount).To(Equal(int64(1)))
 		})
 	})
 })

persistence/persistence.go

@@ -108,23 +108,36 @@ func (s *SQLStore) WithTx(block func(tx model.DataStore) error) error {
 }
 
 func (s *SQLStore) GC(ctx context.Context) error {
-	err := s.Album(ctx).PurgeEmpty()
+	err := s.Album(ctx).(*albumRepository).purgeEmpty()
 	if err != nil {
+		log.Error(ctx, "Error removing empty albums", err)
 		return err
 	}
-	err = s.Artist(ctx).PurgeEmpty()
+	err = s.Artist(ctx).(*artistRepository).purgeEmpty()
 	if err != nil {
+		log.Error(ctx, "Error removing empty artists", err)
 		return err
 	}
 	err = s.MediaFile(ctx).(*mediaFileRepository).cleanAnnotations()
 	if err != nil {
+		log.Error(ctx, "Error removing orphan mediafile annotations", err)
 		return err
 	}
 	err = s.Album(ctx).(*albumRepository).cleanAnnotations()
 	if err != nil {
+		log.Error(ctx, "Error removing orphan album annotations", err)
 		return err
 	}
-	return s.Artist(ctx).(*artistRepository).cleanAnnotations()
+	err = s.Artist(ctx).(*artistRepository).cleanAnnotations()
+	if err != nil {
+		log.Error(ctx, "Error removing orphan artist annotations", err)
+		return err
+	}
+	err = s.Playlist(ctx).(*playlistRepository).removeOrphans()
+	if err != nil {
+		log.Error(ctx, "Error tidying up playlists", err)
+	}
+	return err
 }
 
 func (s *SQLStore) getOrmer() orm.Ormer {

persistence/persistence_suite_test.go

@@ -21,7 +21,7 @@ func TestPersistence(t *testing.T) {
 	tests.Init(t, true)
 
 	//os.Remove("./test-123.db")
-	//conf.Server.Path = "./test-123.db"
+	//conf.Server.DbPath = "./test-123.db"
 	conf.Server.DbPath = "file::memory:?cache=shared"
 	_ = orm.RegisterDataBase("default", db.Driver, conf.Server.DbPath)
 	db.EnsureLatestVersion()
@@ -40,9 +40,9 @@ var (
 )
 
 var (
-	albumSgtPeppers    = model.Album{ID: "101", Name: "Sgt Peppers", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "1", CoverArtPath: P("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967, FullText: " beatles peppers sgt the"}
-	albumAbbeyRoad     = model.Album{ID: "102", Name: "Abbey Road", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "2", CoverArtPath: P("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969, FullText: " abbey beatles road the"}
-	albumRadioactivity = model.Album{ID: "103", Name: "Radioactivity", Artist: "Kraftwerk", AlbumArtistID: "2", Genre: "Electronic", CoverArtId: "3", CoverArtPath: P("/kraft/radio/radio.mp3"), SongCount: 2, FullText: " kraftwerk radioactivity"}
+	albumSgtPeppers    = model.Album{ID: "101", Name: "Sgt Peppers", Artist: "The Beatles", OrderAlbumName: "sgt peppers", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "1", CoverArtPath: P("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967, FullText: " beatles peppers sgt the"}
+	albumAbbeyRoad     = model.Album{ID: "102", Name: "Abbey Road", Artist: "The Beatles", OrderAlbumName: "abbey road", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "2", CoverArtPath: P("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969, FullText: " abbey beatles road the"}
+	albumRadioactivity = model.Album{ID: "103", Name: "Radioactivity", Artist: "Kraftwerk", OrderAlbumName: "radioactivity", AlbumArtistID: "2", Genre: "Electronic", CoverArtId: "3", CoverArtPath: P("/kraft/radio/radio.mp3"), SongCount: 2, FullText: " kraftwerk radioactivity"}
 	testAlbums         = model.Albums{
 		albumSgtPeppers,
 		albumAbbeyRoad,
@@ -86,7 +86,7 @@ var _ = Describe("Initialize test DB", func() {
 	BeforeSuite(func() {
 		o := orm.NewOrm()
 		ctx := log.NewContext(context.TODO())
-		ctx = request.WithUser(ctx, model.User{ID: "userid"})
+		ctx = request.WithUser(ctx, model.User{ID: "userid", UserName: "userid"})
 		mr := NewMediaFileRepository(ctx, o)
 		for i := range testSongs {
 			s := testSongs[i]

persistence/player_repository.go

@@ -19,6 +19,9 @@ func NewPlayerRepository(ctx context.Context, o orm.Ormer) model.PlayerRepositor
 	r.ctx = ctx
 	r.ormer = o
 	r.tableName = "player"
+	r.filterMappings = map[string]filterFunc{
+		"name": containsFilter,
+	}
 	return r
 }
 
@@ -43,11 +46,19 @@ func (r *playerRepository) FindByName(client, userName string) (*model.Player, e
 
 func (r *playerRepository) newRestSelect(options ...model.QueryOptions) SelectBuilder {
 	s := r.newSelect(options...)
+	return s.Where(r.addRestriction())
+}
+
+func (r *playerRepository) addRestriction(sql ...Sqlizer) Sqlizer {
+	s := And{}
+	if len(sql) > 0 {
+		s = append(s, sql[0])
+	}
 	u := loggedUser(r.ctx)
 	if u.IsAdmin {
 		return s
 	}
-	return s.Where(Eq{"user_name": u.UserName})
+	return append(s, Eq{"user_name": u.UserName})
 }
 
 func (r *playerRepository) Count(options ...rest.QueryOptions) (int64, error) {
@@ -106,7 +117,8 @@ func (r *playerRepository) Update(entity interface{}, cols ...string) error {
 }
 
 func (r *playerRepository) Delete(id string) error {
-	err := r.delete(And{Eq{"id": id}, Eq{"user_name": loggedUser(r.ctx).UserName}})
+	filter := r.addRestriction(And{Eq{"id": id}})
+	err := r.delete(filter)
 	if err == model.ErrNotFound {
 		return rest.ErrNotFound
 	}

persistence/playlist_repository.go

@@ -45,13 +45,17 @@ func (r *playlistRepository) Exists(id string) (bool, error) {
 }
 
 func (r *playlistRepository) Delete(id string) error {
-	err := r.delete(And{Eq{"id": id}, r.userFilter()})
-	if err != nil {
-		return err
+	usr := loggedUser(r.ctx)
+	if !usr.IsAdmin {
+		pls, err := r.Get(id)
+		if err != nil {
+			return err
+		}
+		if pls.Owner != usr.UserName {
+			return rest.ErrPermissionDenied
+		}
 	}
-	del := Delete("playlist_tracks").Where(Eq{"playlist_id": id})
-	_, err = r.executeSQL(del)
-	return err
+	return r.delete(And{Eq{"id": id}, r.userFilter()})
 }
 
 func (r *playlistRepository) Put(p *model.Playlist) error {
@@ -77,9 +81,13 @@ func (r *playlistRepository) Put(p *model.Playlist) error {
 		return err
 	}
 	p.ID = id
-	err = r.updateTracks(id, tracks)
-	if err != nil {
-		return err
+
+	// Only update tracks if they are specified
+	if tracks != nil {
+		err = r.updateTracks(id, tracks)
+		if err != nil {
+			return err
+		}
 	}
 	return r.loadTracks(p)
 }
@@ -95,6 +103,16 @@ func (r *playlistRepository) Get(id string) (*model.Playlist, error) {
 	return &pls, err
 }
 
+func (r *playlistRepository) FindByPath(path string) (*model.Playlist, error) {
+	sel := r.newSelect().Columns("*").Where(Eq{"path": path})
+	var pls model.Playlist
+	err := r.queryOne(sel, &pls)
+	if err != nil {
+		return nil, err
+	}
+	return &pls, err
+}
+
 func (r *playlistRepository) GetAll(options ...model.QueryOptions) (model.Playlists, error) {
 	sel := r.newSelect(options...).Columns("*").Where(r.userFilter())
 	res := model.Playlists{}
@@ -158,6 +176,10 @@ func (r *playlistRepository) Save(entity interface{}) (string, error) {
 
 func (r *playlistRepository) Update(entity interface{}, cols ...string) error {
 	pls := entity.(*model.Playlist)
+	usr := loggedUser(r.ctx)
+	if !usr.IsAdmin && pls.Owner != usr.UserName {
+		return rest.ErrPermissionDenied
+	}
 	err := r.Put(pls)
 	if err == model.ErrNotFound {
 		return rest.ErrNotFound
@@ -165,6 +187,40 @@ func (r *playlistRepository) Update(entity interface{}, cols ...string) error {
 	return err
 }
 
+func (r *playlistRepository) removeOrphans() error {
+	sel := Select("playlist_tracks.playlist_id as id", "p.name").From("playlist_tracks").
+		Join("playlist p on playlist_tracks.playlist_id = p.id").
+		LeftJoin("media_file mf on playlist_tracks.media_file_id = mf.id").
+		Where(Eq{"mf.id": nil}).
+		GroupBy("playlist_tracks.playlist_id")
+
+	var pls []struct{ Id, Name string }
+	err := r.queryAll(sel, &pls)
+	if err != nil {
+		return err
+	}
+
+	for _, pl := range pls {
+		log.Debug(r.ctx, "Cleaning-up orphan tracks from playlist", "id", pl.Id, "name", pl.Name)
+		del := Delete("playlist_tracks").Where(And{
+			ConcatExpr("media_file_id not in (select id from media_file)"),
+			Eq{"playlist_id": pl.Id},
+		})
+		n, err := r.executeSQL(del)
+		if n == 0 || err != nil {
+			return err
+		}
+		log.Debug(r.ctx, "Deleted tracks, now reordering", "id", pl.Id, "name", pl.Name, "deleted", n)
+
+		// To reorganize the playlist, just add an empty list of new tracks
+		trks := r.Tracks(pl.Id)
+		if err := trks.Add(nil); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 var _ model.PlaylistRepository = (*playlistRepository)(nil)
 var _ rest.Repository = (*playlistRepository)(nil)
 var _ rest.Persistable = (*playlistRepository)(nil)

persistence/playlist_track_repository.go

@@ -1,20 +1,25 @@
 package persistence
 
 import (
+	"time"
+
 	. "github.com/Masterminds/squirrel"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
+	"github.com/deluan/navidrome/utils"
 	"github.com/deluan/rest"
 )
 
 type playlistTrackRepository struct {
 	sqlRepository
 	sqlRestful
-	playlistId string
+	playlistId   string
+	playlistRepo model.PlaylistRepository
 }
 
 func (r *playlistRepository) Tracks(playlistId string) model.PlaylistTrackRepository {
 	p := &playlistTrackRepository{}
+	p.playlistRepo = NewPlaylistRepository(r.ctx, r.ormer)
 	p.playlistId = playlistId
 	p.ctx = r.ctx
 	p.ormer = r.ormer
@@ -66,19 +71,17 @@ func (r *playlistTrackRepository) NewInstance() interface{} {
 }
 
 func (r *playlistTrackRepository) Add(mediaFileIds []string) error {
-	log.Debug(r.ctx, "Adding songs to playlist", "playlistId", r.playlistId, "mediaFileIds", mediaFileIds)
-
-	// Get all current tracks
-	all := r.newSelect().Columns("media_file_id").Where(Eq{"playlist_id": r.playlistId}).OrderBy("id")
-	var tracks model.PlaylistTracks
-	err := r.queryAll(all, &tracks)
-	if err != nil {
-		log.Error("Error querying current tracks from playlist", "playlistId", r.playlistId, err)
-		return err
+	if !r.isWritable() {
+		return rest.ErrPermissionDenied
 	}
-	ids := make([]string, len(tracks))
-	for i := range tracks {
-		ids[i] = tracks[i].MediaFileID
+
+	if len(mediaFileIds) > 0 {
+		log.Debug(r.ctx, "Adding songs to playlist", "playlistId", r.playlistId, "mediaFileIds", mediaFileIds)
+	}
+
+	ids, err := r.getTracks()
+	if err != nil {
+		return err
 	}
 
 	// Append new tracks
@@ -88,7 +91,27 @@ func (r *playlistTrackRepository) Add(mediaFileIds []string) error {
 	return r.Update(ids)
 }
 
+func (r *playlistTrackRepository) getTracks() ([]string, error) {
+	// Get all current tracks
+	all := r.newSelect().Columns("media_file_id").Where(Eq{"playlist_id": r.playlistId}).OrderBy("id")
+	var tracks model.PlaylistTracks
+	err := r.queryAll(all, &tracks)
+	if err != nil {
+		log.Error("Error querying current tracks from playlist", "playlistId", r.playlistId, err)
+		return nil, err
+	}
+	ids := make([]string, len(tracks))
+	for i := range tracks {
+		ids[i] = tracks[i].MediaFileID
+	}
+	return ids, nil
+}
+
 func (r *playlistTrackRepository) Update(mediaFileIds []string) error {
+	if !r.isWritable() {
+		return rest.ErrPermissionDenied
+	}
+
 	// Remove old tracks
 	del := Delete(r.tableName).Where(Eq{"playlist_id": r.playlistId})
 	_, err := r.executeSQL(del)
@@ -97,20 +120,10 @@ func (r *playlistTrackRepository) Update(mediaFileIds []string) error {
 	}
 
 	// Break the track list in chunks to avoid hitting SQLITE_MAX_FUNCTION_ARG limit
-	numTracks := len(mediaFileIds)
-	const chunkSize = 50
-	var chunks [][]string
-	for i := 0; i < numTracks; i += chunkSize {
-		end := i + chunkSize
-		if end > numTracks {
-			end = numTracks
-		}
-
-		chunks = append(chunks, mediaFileIds[i:end])
-	}
+	chunks := utils.BreakUpStringSlice(mediaFileIds, 50)
 
 	// Add new tracks, chunk by chunk
-	pos := 0
+	pos := 1
 	for i := range chunks {
 		ins := Insert(r.tableName).Columns("playlist_id", "media_file_id", "id")
 		for _, t := range chunks[i] {
@@ -141,12 +154,16 @@ func (r *playlistTrackRepository) updateStats() error {
 	upd := Update("playlist").
 		Set("duration", res.Duration).
 		Set("song_count", res.Count).
+		Set("updated_at", time.Now()).
 		Where(Eq{"id": r.playlistId})
 	_, err = r.executeSQL(upd)
 	return err
 }
 
 func (r *playlistTrackRepository) Delete(id string) error {
+	if !r.isWritable() {
+		return rest.ErrPermissionDenied
+	}
 	err := r.delete(And{Eq{"playlist_id": r.playlistId}, Eq{"id": id}})
 	if err != nil {
 		return err
@@ -154,4 +171,25 @@ func (r *playlistTrackRepository) Delete(id string) error {
 	return r.updateStats()
 }
 
+func (r *playlistTrackRepository) Reorder(pos int, newPos int) error {
+	if !r.isWritable() {
+		return rest.ErrPermissionDenied
+	}
+	ids, err := r.getTracks()
+	if err != nil {
+		return err
+	}
+	newOrder := utils.MoveString(ids, pos-1, newPos-1)
+	return r.Update(newOrder)
+}
+
+func (r *playlistTrackRepository) isWritable() bool {
+	usr := loggedUser(r.ctx)
+	if usr.IsAdmin {
+		return true
+	}
+	pls, err := r.playlistRepo.Get(r.playlistId)
+	return err == nil && pls.Owner == usr.UserName
+}
+
 var _ model.PlaylistTrackRepository = (*playlistTrackRepository)(nil)

persistence/sql_annotations.go

@@ -96,3 +96,12 @@ func (r sqlRepository) cleanAnnotations() error {
 	}
 	return nil
 }
+
+func (r sqlRepository) updateAnnotations(id string, m interface{}) error {
+	ans := m.(model.AnnotatedModel).GetAnnotations()
+	err := r.SetStar(ans.Starred, id)
+	if err != nil {
+		return err
+	}
+	return r.SetRating(ans.Rating, id)
+}

persistence/sql_base_repository.go

@@ -112,6 +112,8 @@ func (r sqlRepository) executeSQL(sq Sqlizer) (int64, error) {
 	return res.RowsAffected()
 }
 
+// Note: Due to a bug in the QueryRow, this method does not map any embedded structs (ex: annotations)
+// In this case, use the queryAll method and get the first item of the returned list
 func (r sqlRepository) queryOne(sq Sqlizer, response interface{}) error {
 	query, args, err := sq.ToSql()
 	if err != nil {
@@ -169,7 +171,10 @@ func (r sqlRepository) put(id string, m interface{}) (newId string, err error) {
 			return "", err
 		}
 		if count > 0 {
-			return id, nil
+			if _, ok := m.(model.AnnotatedModel); ok {
+				err = r.updateAnnotations(id, m)
+			}
+			return id, err
 		}
 	}
 	// If does not have an id OR could not update (new record with predefined id)

persistence/sql_restful.go

@@ -52,6 +52,10 @@ func startsWithFilter(field string, value interface{}) Sqlizer {
 	return Like{field: fmt.Sprintf("%s%%", value)}
 }
 
+func containsFilter(field string, value interface{}) Sqlizer {
+	return Like{field: fmt.Sprintf("%%%s%%", value)}
+}
+
 func booleanFilter(field string, value interface{}) Sqlizer {
 	v := strings.ToLower(value.(string))
 	return Eq{field: strings.ToLower(v) == "true"}

persistence/sql_search.go

@@ -9,7 +9,7 @@ import (
 	"github.com/kennygrant/sanitize"
 )
 
-var quotesRegex = regexp.MustCompile("[“”‘’'\"]")
+var quotesRegex = regexp.MustCompile("[“”‘’'\"\\[\\(\\{\\]\\)\\}]")
 
 func getFullText(text ...string) string {
 	fullText := sanitizeStrings(text...)

persistence/sql_search_test.go

@@ -26,5 +26,9 @@ var _ = Describe("sqlRepository", func() {
 		It("remove symbols", func() {
 			Expect(getFullText("Tom’s Diner ' “40” ‘A’")).To(Equal(" 40 a diner toms"))
 		})
+
+		It("remove opening brackets", func() {
+			Expect(getFullText("[Five Years]")).To(Equal(" five years"))
+		})
 	})
 })

persistence/user_repository.go

@@ -65,6 +65,13 @@ func (r *userRepository) Put(u *model.User) error {
 	return err
 }
 
+func (r *userRepository) FindFirstAdmin() (*model.User, error) {
+	sel := r.newSelect(model.QueryOptions{Sort: "updated_at", Max: 1}).Columns("*").Where(Eq{"is_admin": true})
+	var usr model.User
+	err := r.queryOne(sel, &usr)
+	return &usr, err
+}
+
 func (r *userRepository) FindByUsername(username string) (*model.User, error) {
 	username = strings.ToLower(username)
 	sel := r.newSelect().Columns("*").Where(Eq{"user_name": username})

reflex.conf

@@ -1 +1 @@
--s -r "(\.go$$|navidrome.toml|resources)" -R "(Jamstash-master|^ui|^data)" -- go run .
+-s -r "(\.go$$|navidrome.toml|resources)" -R "(Jamstash-master|^ui|^data|^db/migration)" -- go run .

resources/i18n/cs.json

@@ -17,11 +17,14 @@
                 "year": "Rok",
                 "size": "Velikost souboru",
                 "updatedAt": "Nahráno",
-                "bitRate": "Přenosová rychlost"
+                "bitRate": "Přenosová rychlost",
+                "discSubtitle": "Podtitul disku",
+                "starred": "Oblíbené"
             },
             "actions": {
                 "addToQueue": "Přehrát později",
-                "playNow": "Přehrát nyní"
+                "playNow": "Přehrát nyní",
+                "addToPlaylist": "Přidat do seznamu skladeb"
             }
         },
         "album": {
@@ -84,6 +87,23 @@
                 "defaultBitRate": "Výchozí přenosová rychlost",
                 "command": "Příkaz"
             }
+        },
+        "playlist": {
+            "name": "Seznam skladeb |||| Seznamy skladeb",
+            "fields": {
+                "name": "Název",
+                "duration": "Délka",
+                "owner": "Vlastník",
+                "public": "Veřejný",
+                "updatedAt": "Nahrán",
+                "createdAt": "Vytvořen",
+                "songCount": "Skladby",
+                "comment": "Komentář"
+            },
+            "actions": {
+                "selectPlaylist": "Přidat skladby do seznamu:",
+                "addNewPlaylist": "Vytvořit \"%{name}\""
+            }
         }
     },
     "ra": {
@@ -138,7 +158,8 @@
             "expand": "Zvětšit",
             "close": "Zavřít",
             "open_menu": "Otevřít nabídku",
-            "close_menu": "Zavřít nabídku"
+            "close_menu": "Zavřít nabídku",
+            "unselect": "Zrušit výběr"
         },
         "boolean": {
             "true": "Ano",
@@ -154,7 +175,7 @@
             "not_found": "Nenalezeno",
             "show": "%{name} #%{id}",
             "empty": "Zatím žádné %{name}",
-            "invite": "Chcete přidat?"
+            "invite": "Chcete jeden přidat?"
         },
         "input": {
             "file": {
@@ -218,7 +239,11 @@
     "message": {
         "note": "POZNÁMKA",
         "transcodingDisabled": "Měnění nastavení překódování je ve webovém prostředí vypnuto kvůli bezpečnosti. Pokud by jste chtěli změnit (upravit nebo přidat) možnosti překódování, restartujte server s možností %{config}.",
-        "transcodingEnabled": "Navidrome právě běží s možností %{config}, umožňující spouštění systémových příkazů z nastavení překódování pomocí webového rozhraní. Doporučujeme ji vypnout kvůli bezpečnosti a použít ji pouze pokud upravujete nastavení překódování."
+        "transcodingEnabled": "Navidrome právě běží s možností %{config}, umožňující spouštění systémových příkazů z nastavení překódování pomocí webového rozhraní. Doporučujeme ji vypnout kvůli bezpečnosti a použít ji pouze pokud upravujete nastavení překódování.",
+        "songsAddedToPlaylist": "1 skladba přidána na seznam skladeb ||| %{smart_count} skladeb přidáno na seznam skladeb",
+        "noPlaylistsAvailable": "Žádné nejsou dostupné",
+        "delete_user_title": "Odstranit uživatele '%{name}'",
+        "delete_user_content": "Jste si jisti že chcete odstranit tohoto uživatele a všechny jejich data (zahrujicí seznamy skladeb a nastavení)?"
     },
     "menu": {
         "library": "Knihovna",

resources/i18n/de.json

@@ -17,11 +17,14 @@
                 "year": "Jahr",
                 "size": "Dateigröße",
                 "updatedAt": "Hochgeladen um",
-                "bitRate": "Bitrate"
+                "bitRate": "Bitrate",
+                "discSubtitle": "CD Untertitel",
+                "starred": "Favorit"
             },
             "actions": {
                 "addToQueue": "Später abspielen",
-                "playNow": "Jetzt abspielen"
+                "playNow": "Jetzt abspielen",
+                "addToPlaylist": "Zur Playlist hinzufügen"
             }
         },
         "album": {
@@ -62,7 +65,7 @@
                 "updatedAt": "Aktualisiert um",
                 "name": "Name",
                 "password": "Passwort",
-                "createdAt": "Hergestellt um"
+                "createdAt": "Erstellt um"
             }
         },
         "player": {
@@ -84,6 +87,23 @@
                 "defaultBitRate": "Standardbitrate",
                 "command": "Befehl"
             }
+        },
+        "playlist": {
+            "name": "Playlist |||| Playlists",
+            "fields": {
+                "name": "Name",
+                "duration": "Dauer",
+                "owner": "Inhaber",
+                "public": "Öffentlich",
+                "updatedAt": "Aktualisiert um",
+                "createdAt": "Erstellt um",
+                "songCount": "Songanzahl",
+                "comment": "Kommentar"
+            },
+            "actions": {
+                "selectPlaylist": "Songs zur Playlist hinzufügen",
+                "addNewPlaylist": "\"%{name}\" erstellen"
+            }
         }
     },
     "ra": {
@@ -138,7 +158,8 @@
             "expand": "Expandieren",
             "close": "Schließen",
             "open_menu": "Menü öffnen",
-            "close_menu": "Menü schließen"
+            "close_menu": "Menü schließen",
+            "unselect": "Abwählen"
         },
         "boolean": {
             "true": "Ja",
@@ -154,7 +175,7 @@
             "not_found": "Nicht gefunden",
             "show": "%{name} #%{id}",
             "empty": "Noch kein %{name}.\n",
-            "invite": "Möchten du einen hinzufügen?"
+            "invite": "Möchtest du eine hinzufügen?"
         },
         "input": {
             "file": {
@@ -218,7 +239,11 @@
     "message": {
         "note": "HINWEIS",
         "transcodingDisabled": "Die Änderung der Transcodierungskonfiguration über die Web-UI ist aus Sicherheitsgründen deaktiviert. Wenn du die Transcodierungsoptionen ändern (bearbeiten oder hinzufügen) möchtest, starte den Server mit der Konfigurationsoption %{config} neu.",
-        "transcodingEnabled": "Navidrome läuft derzeit mit %{config}, wodurch es möglich ist, Systembefehle aus den Transkodierungseinstellungen über die Webschnittstelle auszuführen. Wir empfehlen, es aus Sicherheitsgründen zu deaktivieren und nur bei der Konfiguration von Transkodierungsoptionen zu aktivieren."
+        "transcodingEnabled": "Navidrome läuft derzeit mit %{config}, wodurch es möglich ist, Systembefehle aus den Transkodierungseinstellungen über die Webschnittstelle auszuführen. Wir empfehlen, es aus Sicherheitsgründen zu deaktivieren und nur bei der Konfiguration von Transkodierungsoptionen zu aktivieren.",
+        "songsAddedToPlaylist": "Einen Song zur Playlist hinzugefügt |||| %{smart_count} Songs zur Playlist hinzugefügt",
+        "noPlaylistsAvailable": "Keine Playlist verfügbar",
+        "delete_user_title": "Benutzer \"% {name}\" löschen",
+        "delete_user_content": "Möchtest du diesen Benutzer und alle seine Daten (einschließlich Playlisten und Einstellungen) wirklich löschen?"
     },
     "menu": {
         "library": "Bibliothek",

resources/i18n/fr.json

@@ -17,11 +17,14 @@
                 "year": "Année",
                 "size": "Taille",
                 "updatedAt": "Mise à jour",
-                "bitRate": "Bitrate"
+                "bitRate": "Bitrate",
+                "discSubtitle": "Sous-titre du disque",
+                "starred": "Favoris"
             },
             "actions": {
                 "addToQueue": "Ajouter à la file",
-                "playNow": "Lire"
+                "playNow": "Lire",
+                "addToPlaylist": "Ajouter à la playlist"
             }
         },
         "album": {
@@ -84,6 +87,23 @@
                 "defaultBitRate": "Bitrate  par défaut",
                 "command": "Commande"
             }
+        },
+        "playlist": {
+            "name": "Playlist |||| Playlists",
+            "fields": {
+                "name": "Nom",
+                "duration": "Durée",
+                "owner": "Propriétaire",
+                "public": "Public",
+                "updatedAt": "Mise à jour le",
+                "createdAt": "Crée le",
+                "songCount": "Titres",
+                "comment": "Commentaire"
+            },
+            "actions": {
+                "selectPlaylist": "Ajouter les pistes à la playlist",
+                "addNewPlaylist": "Créer \"%{name}\""
+            }
         }
     },
     "ra": {
@@ -138,7 +158,8 @@
             "expand": "Étendre",
             "close": "Fermer",
             "open_menu": "Ouvrir le menu",
-            "close_menu": "Fermer le menu"
+            "close_menu": "Fermer le menu",
+            "unselect": "Désélectionner"
         },
         "boolean": {
             "true": "Oui",
@@ -218,7 +239,11 @@
     "message": {
         "note": "NOTE",
         "transcodingDisabled": "Le changement de paramètres depuis l'interface web est désactivé pour des raisons de sécurité. Pour changer (éditer ou supprimer) les options de transcodage, relancer le serveur avec l'option %{config} activée.",
-        "transcodingEnabled": "Navidrome fonctionne actuellement avec %{config}, rendant possible l’exécution de commandes arbitraires depuis l'interface web. Il est recommandé de n'activer cette fonctionnalité uniquement lors de la configuration du Transcodage."
+        "transcodingEnabled": "Navidrome fonctionne actuellement avec %{config}, rendant possible l’exécution de commandes arbitraires depuis l'interface web. Il est recommandé de n'activer cette fonctionnalité uniquement lors de la configuration du Transcodage.",
+        "songsAddedToPlaylist": "Une piste ajoutée à la playlist |||| %{smart_count} pistes ajoutées à la playlist",
+        "noPlaylistsAvailable": "Aucune playlist",
+        "delete_user_title": "",
+        "delete_user_content": ""
     },
     "menu": {
         "library": "Bibliothèque",

resources/i18n/it.json

@@ -17,11 +17,14 @@
                 "year": "Anno",
                 "size": "Dimensioni",
                 "updatedAt": "Ultimo aggiornamento",
-                "bitRate": "Bitrate"
+                "bitRate": "Bitrate",
+                "discSubtitle": "Sottotitoli disco",
+                "starred": "Preferita"
             },
             "actions": {
                 "addToQueue": "Aggiungi alla coda",
-                "playNow": "Riproduci adesso"
+                "playNow": "Riproduci adesso",
+                "addToPlaylist": "Aggiungi alla playlist"
             }
         },
         "album": {
@@ -49,7 +52,8 @@
             "name": "Artista |||| Artisti",
             "fields": {
                 "name": "Nome",
-                "albumCount": "Album"
+                "albumCount": "Album",
+                "songCount": "Numero tracce"
             }
         },
         "user": {
@@ -83,6 +87,23 @@
                 "defaultBitRate": "Bitrate predefinito",
                 "command": "Comando"
             }
+        },
+        "playlist": {
+            "name": "Playlist |||| Playlist",
+            "fields": {
+                "name": "Nome",
+                "duration": "Durata",
+                "owner": "Creatore",
+                "public": "Pubblica",
+                "updatedAt": "Ultimo aggiornamento",
+                "createdAt": "Data creazione",
+                "songCount": "Tracce",
+                "comment": "Commento"
+            },
+            "actions": {
+                "selectPlaylist": "Aggiungi tracce alla playlist:",
+                "addNewPlaylist": "Aggiungi \"%{name}\""
+            }
         }
     },
     "ra": {
@@ -137,7 +158,8 @@
             "expand": "Espandi",
             "close": "Chiudi",
             "open_menu": "Apri menù",
-            "close_menu": "Chiudi menù"
+            "close_menu": "Chiudi menù",
+            "unselect": "Deseleziona"
         },
         "boolean": {
             "true": "Si",
@@ -217,7 +239,11 @@
     "message": {
         "note": "Note",
         "transcodingDisabled": "La possibilità di modificare le opzioni di transcodifica attraverso l’interfaccia web è disabilitata per ragioni di sicurezza. Se desideri cambiare (modificare o aggiungere) opzioni di transcodifica, riavvia il server con l’opzione %{config}.",
-        "transcodingEnabled": "Navidrome è al momento attivo con %{config}, rendendo possibile eseguire comandi remoti attraverso l’interfaccia web. Si raccomanda di disabilitare questa opzione per ragioni di sicurezza e di abilitarla solo per configurare le opzioni di transcodifica."
+        "transcodingEnabled": "Navidrome è al momento attivo con %{config}, rendendo possibile eseguire comandi remoti attraverso l’interfaccia web. Si raccomanda di disabilitare questa opzione per ragioni di sicurezza e di abilitarla solo per configurare le opzioni di transcodifica.",
+        "songsAddedToPlaylist": "Aggiunta una traccia alla playlist |||| Aggiunte %{smart_count} tracce alla playlist",
+        "noPlaylistsAvailable": "Nessuna playlist",
+        "delete_user_title": "Rimuovi utente '%{name}'",
+        "delete_user_content": "Sei sicuro di voler rimuovere questo utente e tutti i suoi dati, incluse playlist e impostazioni?"
     },
     "menu": {
         "library": "Libreria",

resources/i18n/pt.json

@@ -91,15 +91,19 @@
             "name": "Playlist |||| Playlists",
             "fields": {
                 "name": "Nome",
+                "comment": "Comentário",
                 "duration": "Duração",
                 "owner": "Dono",
                 "public": "Pública",
                 "updatedAt": "Últ. Atualização",
                 "createdAt": "Data de Criação ",
-                "songCount": "Músicas"
+                "songCount": "Músicas",
+                "sync": "Auto-importar",
+                "path": "Importar de"
             },
             "actions": {
-                "selectPlaylist": "Selecione a playlist:"
+                "selectPlaylist": "Selecione a playlist:",
+                "addNewPlaylist": "Criar \"%{name}\""
             }
         }
     },
@@ -152,6 +156,7 @@
             "show": "Exibir",
             "sort": "Ordenar",
             "undo": "Desfazer",
+            "unselect": "Deselecionar",
             "expand": "Expandir",
             "close": "Fechar",
             "open_menu": "Abrir menu",
@@ -236,8 +241,10 @@
         "note": "ATENÇÃO",
         "transcodingDisabled": "Por questão de segurança, esta tela de configuração está desabilitada. Se você quiser alterar estas configurações, reinicie o servidor com a opção %{config}",
         "transcodingEnabled": "Navidrome está sendo executado com a opção %{config}. Isto permite que potencialmente se execute comandos do sistema pela interface Web. É recomendado que vc mantenha esta opção desabilitada, e só a habilite quando precisar configurar opções de Conversão",
-        "discSubtitle": "%{subtitle} (disco %{number})",
-        "discWithoutSubtitle": "Disco %{number}"
+        "songsAddedToPlaylist": "Música adicionada à playlist |||| %{smart_count} músicas adicionadas à playlist",
+        "noPlaylistsAvailable": "Nenhuma playlist",
+        "delete_user_title": "Excluir usuário '%{name}'",
+        "delete_user_content": "Você tem certeza que deseja excluir o usuário e todos os seus dados (incluindo suas playlists e preferências)?"
     },
     "menu": {
         "library": "Biblioteca",

resources/i18n/tr.json

@@ -17,11 +17,14 @@
                 "year": "Yıl",
                 "size": "Dosya boyutu",
                 "updatedAt": "Yüklendiği zaman",
-                "bitRate": "Bir sayısı"
+                "bitRate": "Bir sayısı",
+                "discSubtitle": "Disk Altyazısı",
+                "starred": "Yıldızlı"
             },
             "actions": {
                 "addToQueue": "Sonra çal",
-                "playNow": "Şimdi cal"
+                "playNow": "Şimdi cal",
+                "addToPlaylist": "Çalma listesine ekle"
             }
         },
         "album": {
@@ -84,6 +87,23 @@
                 "defaultBitRate": "Varsayılan bit orani",
                 "command": "komut"
             }
+        },
+        "playlist": {
+            "name": "Çalma listesi |||| Çalma listeler",
+            "fields": {
+                "name": "Isim",
+                "duration": "Süre",
+                "owner": "Sahibi",
+                "public": "Görülebilir",
+                "updatedAt": "Güncelleme tarihi:",
+                "createdAt": "Oluşturma tarihi:",
+                "songCount": "Şarkılar",
+                "comment": "Yorum"
+            },
+            "actions": {
+                "selectPlaylist": "Bir çalma listesi seç:",
+                "addNewPlaylist": "Oluştur  \"%{name}\""
+            }
         }
     },
     "ra": {
@@ -138,7 +158,8 @@
             "expand": "Genişlettir",
             "close": "Kapat",
             "open_menu": "Menüyü aç",
-            "close_menu": "Menüyü kapat"
+            "close_menu": "Menüyü kapat",
+            "unselect": "Seçimi kaldır"
         },
         "boolean": {
             "true": "Evet",
@@ -218,7 +239,11 @@
     "message": {
         "note": "NOT",
         "transcodingDisabled": "Transcoding ayarlari  web arayüzü üzerinden değiştirilmesi güvenlik nedeniyle devre dışı bırakılmıştır.  Kod dönüştürme seçeneklerini değiştirmek (düzenlemek veya eklemek) istiyorsan, %{config} seçeneğiyle sunucuyu yeniden başlatın.",
-        "transcodingEnabled": "Navidrome şu anda %{config} ile çalışıyor, web arayüzünü kullanarak kod dönüştürme ayarlarından sistem komutlarını çalıştırmayı mümkün kılıyor. Güvenlik nedeniyle devre dışı bırakmanızı ve yalnızca Kod Dönüştürme seçeneklerini yapılandırırken etkinleştirmenizi öneririz."
+        "transcodingEnabled": "Navidrome şu anda %{config} ile çalışıyor, web arayüzünü kullanarak kod dönüştürme ayarlarından sistem komutlarını çalıştırmayı mümkün kılıyor. Güvenlik nedeniyle devre dışı bırakmanızı ve yalnızca Kod Dönüştürme seçeneklerini yapılandırırken etkinleştirmenizi öneririz.",
+        "songsAddedToPlaylist": "Çalma listesine 1 şarkı eklendi |||| Çalma listesine %{smart_count} şarkı eklendi",
+        "noPlaylistsAvailable": "Mevcut değil",
+        "delete_user_title": "'%{name}' kullanıcısını sil",
+        "delete_user_content": "Bu kullanıcıyı ve tüm verilerini (çalma listesi ve tercihleri dahil) silmek istediğinden emin misin?"
     },
     "menu": {
         "library": "Müzik kütüphanesi",

scanner/change_detector.go

@@ -1,6 +1,8 @@
 package scanner
 
 import (
+	"context"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"time"
@@ -14,22 +16,22 @@ type dirInfo struct {
 }
 type dirInfoMap map[string]dirInfo
 
-type ChangeDetector struct {
+type changeDetector struct {
 	rootFolder string
 	dirMap     dirInfoMap
 }
 
-func NewChangeDetector(rootFolder string) *ChangeDetector {
-	return &ChangeDetector{
+func newChangeDetector(rootFolder string) *changeDetector {
+	return &changeDetector{
 		rootFolder: rootFolder,
 		dirMap:     dirInfoMap{},
 	}
 }
 
-func (s *ChangeDetector) Scan(lastModifiedSince time.Time) (changed []string, deleted []string, err error) {
+func (s *changeDetector) Scan(ctx context.Context, lastModifiedSince time.Time) (changed []string, deleted []string, err error) {
 	start := time.Now()
 	newMap := make(dirInfoMap)
-	err = s.loadMap(newMap, s.rootFolder, lastModifiedSince, false)
+	err = s.loadMap(ctx, newMap, s.rootFolder, lastModifiedSince, false)
 	if err != nil {
 		return
 	}
@@ -39,34 +41,31 @@ func (s *ChangeDetector) Scan(lastModifiedSince time.Time) (changed []string, de
 	}
 	elapsed := time.Since(start)
 
-	log.Trace("Folder analysis complete\n", "total", len(newMap), "changed", len(changed), "deleted", len(deleted), "elapsed", elapsed)
+	log.Trace(ctx, "Folder analysis complete", "total", len(newMap), "changed", len(changed), "deleted", len(deleted), "elapsed", elapsed)
 	s.dirMap = newMap
 	return
 }
 
-func (s *ChangeDetector) loadDir(dirPath string) (children []string, lastUpdated time.Time, err error) {
-	dir, err := os.Open(dirPath)
-	if err != nil {
-		return
-	}
-	defer dir.Close()
+func (s *changeDetector) loadDir(ctx context.Context, dirPath string) (children []string, lastUpdated time.Time, err error) {
 	dirInfo, err := os.Stat(dirPath)
 	if err != nil {
+		log.Error(ctx, "Error stating dir", "path", dirPath, err)
 		return
 	}
 	lastUpdated = dirInfo.ModTime()
 
-	files, err := dir.Readdir(-1)
+	files, err := ioutil.ReadDir(dirPath)
 	if err != nil {
+		log.Error(ctx, "Error reading dir", "path", dirPath, err)
 		return
 	}
 	for _, f := range files {
-		isDir, err := IsDirOrSymlinkToDir(dirPath, f)
+		isDir, err := isDirOrSymlinkToDir(dirPath, f)
 		// Skip invalid symlinks
 		if err != nil {
 			continue
 		}
-		if isDir {
+		if isDir && !isDirIgnored(dirPath, f) && isDirReadable(dirPath, f) {
 			children = append(children, filepath.Join(dirPath, f.Name()))
 		} else {
 			if f.ModTime().After(lastUpdated) {
@@ -77,34 +76,14 @@ func (s *ChangeDetector) loadDir(dirPath string) (children []string, lastUpdated
 	return
 }
 
-// IsDirOrSymlinkToDir returns true if and only if the Dirent represents a file
-// system directory, or a symbolic link to a directory. Note that if the Dirent
-// is not a directory but is a symbolic link, this method will resolve by
-// sending a request to the operating system to follow the symbolic link.
-// Copied from github.com/karrick/godirwalk
-func IsDirOrSymlinkToDir(baseDir string, info os.FileInfo) (bool, error) {
-	if info.IsDir() {
-		return true, nil
-	}
-	if info.Mode()&os.ModeSymlink == 0 {
-		return false, nil
-	}
-	// Does this symlink point to a directory?
-	info, err := os.Stat(filepath.Join(baseDir, info.Name()))
-	if err != nil {
-		return false, err
-	}
-	return info.IsDir(), nil
-}
-
-func (s *ChangeDetector) loadMap(dirMap dirInfoMap, path string, since time.Time, maybe bool) error {
-	children, lastUpdated, err := s.loadDir(path)
+func (s *changeDetector) loadMap(ctx context.Context, dirMap dirInfoMap, path string, since time.Time, maybe bool) error {
+	children, lastUpdated, err := s.loadDir(ctx, path)
 	if err != nil {
 		return err
 	}
 	maybe = maybe || lastUpdated.After(since)
 	for _, c := range children {
-		err := s.loadMap(dirMap, c, since, maybe)
+		err := s.loadMap(ctx, dirMap, c, since, maybe)
 		if err != nil {
 			return err
 		}
@@ -116,7 +95,7 @@ func (s *ChangeDetector) loadMap(dirMap dirInfoMap, path string, since time.Time
 	return nil
 }
 
-func (s *ChangeDetector) getRelativePath(subFolder string) string {
+func (s *changeDetector) getRelativePath(subFolder string) string {
 	dir, _ := filepath.Rel(s.rootFolder, subFolder)
 	if dir == "" {
 		dir = "."
@@ -124,7 +103,7 @@ func (s *ChangeDetector) getRelativePath(subFolder string) string {
 	return dir
 }
 
-func (s *ChangeDetector) checkForUpdates(lastModifiedSince time.Time, newMap dirInfoMap) (changed []string, deleted []string, err error) {
+func (s *changeDetector) checkForUpdates(lastModifiedSince time.Time, newMap dirInfoMap) (changed []string, deleted []string, err error) {
 	for dir, newEntry := range newMap {
 		lastUpdated := newEntry.mdate
 		oldLastUpdated := lastModifiedSince

scanner/change_detector_test.go

@@ -1,6 +1,7 @@
 package scanner
 
 import (
+	"context"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -10,9 +11,9 @@ import (
 	. "github.com/onsi/gomega"
 )
 
-var _ = Describe("ChangeDetector", func() {
+var _ = Describe("changeDetector", func() {
 	var testFolder string
-	var scanner *ChangeDetector
+	var scanner *changeDetector
 
 	lastModifiedSince := time.Time{}
 
@@ -22,12 +23,12 @@ var _ = Describe("ChangeDetector", func() {
 		if err != nil {
 			panic(err)
 		}
-		scanner = NewChangeDetector(testFolder)
+		scanner = newChangeDetector(testFolder)
 	})
 
 	It("detects changes recursively", func() {
 		// Scan empty folder
-		changed, deleted, err := scanner.Scan(lastModifiedSince)
+		changed, deleted, err := scanner.Scan(context.TODO(), lastModifiedSince)
 		Expect(err).To(BeNil())
 		Expect(deleted).To(BeEmpty())
 		Expect(changed).To(ConsistOf("."))
@@ -38,7 +39,7 @@ var _ = Describe("ChangeDetector", func() {
 		if err != nil {
 			panic(err)
 		}
-		changed, deleted, err = scanner.Scan(lastModifiedSince)
+		changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince)
 		Expect(err).To(BeNil())
 		Expect(deleted).To(BeEmpty())
 		Expect(changed).To(ConsistOf(".", P("a")))
@@ -49,14 +50,14 @@ var _ = Describe("ChangeDetector", func() {
 		if err != nil {
 			panic(err)
 		}
-		changed, deleted, err = scanner.Scan(lastModifiedSince)
+		changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince)
 		Expect(err).To(BeNil())
 		Expect(deleted).To(BeEmpty())
 		Expect(changed).To(ConsistOf(P("a"), P("a/b"), P("a/b/c")))
 
 		// Scan with no changes
 		lastModifiedSince = nowWithDelay()
-		changed, deleted, err = scanner.Scan(lastModifiedSince)
+		changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince)
 		Expect(err).To(BeNil())
 		Expect(deleted).To(BeEmpty())
 		Expect(changed).To(BeEmpty())
@@ -67,7 +68,7 @@ var _ = Describe("ChangeDetector", func() {
 		if err != nil {
 			panic(err)
 		}
-		changed, deleted, err = scanner.Scan(lastModifiedSince)
+		changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince)
 		Expect(err).To(BeNil())
 		Expect(deleted).To(BeEmpty())
 		Expect(changed).To(ConsistOf(P("a/b")))
@@ -78,7 +79,7 @@ var _ = Describe("ChangeDetector", func() {
 		if err != nil {
 			panic(err)
 		}
-		changed, deleted, err = scanner.Scan(lastModifiedSince)
+		changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince)
 		Expect(err).To(BeNil())
 		Expect(deleted).To(BeEmpty())
 		Expect(changed).To(ConsistOf(P("a/b")))
@@ -89,15 +90,15 @@ var _ = Describe("ChangeDetector", func() {
 		if err != nil {
 			panic(err)
 		}
-		changed, deleted, err = scanner.Scan(lastModifiedSince)
+		changed, deleted, err = scanner.Scan(context.TODO(), lastModifiedSince)
 		Expect(err).To(BeNil())
 		Expect(deleted).To(ConsistOf(P("a/b/c")))
 		Expect(changed).To(ConsistOf(P("a/b")))
 
 		// Only returns changes after lastModifiedSince
 		lastModifiedSince = nowWithDelay()
-		newScanner := NewChangeDetector(testFolder)
-		changed, deleted, err = newScanner.Scan(lastModifiedSince)
+		newScanner := newChangeDetector(testFolder)
+		changed, deleted, err = newScanner.Scan(context.TODO(), lastModifiedSince)
 		Expect(err).To(BeNil())
 		Expect(deleted).To(BeEmpty())
 		Expect(changed).To(BeEmpty())
@@ -105,28 +106,40 @@ var _ = Describe("ChangeDetector", func() {
 
 		f, _ := os.Create(filepath.Join(testFolder, "a", "b", "new.txt"))
 		_ = f.Close()
-		changed, deleted, err = newScanner.Scan(lastModifiedSince)
+		changed, deleted, err = newScanner.Scan(context.TODO(), lastModifiedSince)
 		Expect(err).To(BeNil())
 		Expect(deleted).To(BeEmpty())
 		Expect(changed).To(ConsistOf(P("a/b")))
 	})
 
-	Describe("IsDirOrSymlinkToDir", func() {
+	Describe("isDirOrSymlinkToDir", func() {
 		It("returns true for normal dirs", func() {
 			dir, _ := os.Stat("tests/fixtures")
-			Expect(IsDirOrSymlinkToDir("tests", dir)).To(BeTrue())
+			Expect(isDirOrSymlinkToDir("tests", dir)).To(BeTrue())
 		})
 		It("returns true for symlinks to dirs", func() {
 			dir, _ := os.Stat("tests/fixtures/symlink2dir")
-			Expect(IsDirOrSymlinkToDir("tests/fixtures", dir)).To(BeTrue())
+			Expect(isDirOrSymlinkToDir("tests/fixtures", dir)).To(BeTrue())
 		})
 		It("returns false for files", func() {
 			dir, _ := os.Stat("tests/fixtures/test.mp3")
-			Expect(IsDirOrSymlinkToDir("tests/fixtures", dir)).To(BeFalse())
+			Expect(isDirOrSymlinkToDir("tests/fixtures", dir)).To(BeFalse())
 		})
 		It("returns false for symlinks to files", func() {
 			dir, _ := os.Stat("tests/fixtures/symlink")
-			Expect(IsDirOrSymlinkToDir("tests/fixtures", dir)).To(BeFalse())
+			Expect(isDirOrSymlinkToDir("tests/fixtures", dir)).To(BeFalse())
+		})
+	})
+
+	Describe("isDirIgnored", func() {
+		baseDir := filepath.Join("tests", "fixtures")
+		It("returns false for normal dirs", func() {
+			dir, _ := os.Stat(filepath.Join(baseDir, "empty_folder"))
+			Expect(isDirIgnored(baseDir, dir)).To(BeFalse())
+		})
+		It("returns true when folder contains .ndignore file", func() {
+			dir, _ := os.Stat(filepath.Join(baseDir, "ignored_folder"))
+			Expect(isDirIgnored(baseDir, dir)).To(BeTrue())
 		})
 	})
 })

scanner/flushable_map.go

@@ -0,0 +1,58 @@
+package scanner
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/deluan/navidrome/log"
+)
+
+const (
+	// batchSize used for albums/artists updates
+	batchSize = 5
+)
+
+type refreshCallbackFunc = func(ids ...string) error
+
+type flushableMap struct {
+	ctx       context.Context
+	flushFunc refreshCallbackFunc
+	entity    string
+	m         map[string]struct{}
+}
+
+func newFlushableMap(ctx context.Context, entity string, flushFunc refreshCallbackFunc) *flushableMap {
+	return &flushableMap{
+		ctx:       ctx,
+		flushFunc: flushFunc,
+		entity:    entity,
+		m:         map[string]struct{}{},
+	}
+}
+
+func (f *flushableMap) update(id string) error {
+	f.m[id] = struct{}{}
+	if len(f.m) >= batchSize {
+		err := f.flush()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (f *flushableMap) flush() error {
+	if len(f.m) == 0 {
+		return nil
+	}
+	var ids []string
+	for id := range f.m {
+		ids = append(ids, id)
+		delete(f.m, id)
+	}
+	if err := f.flushFunc(ids...); err != nil {
+		log.Error(f.ctx, fmt.Sprintf("Error writing %ss to the DB", f.entity), err)
+		return err
+	}
+	return nil
+}

scanner/load_tree.go

@@ -0,0 +1,116 @@
+package scanner
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/deluan/navidrome/consts"
+	"github.com/deluan/navidrome/log"
+	"github.com/deluan/navidrome/utils"
+)
+
+type (
+	dirMapValue struct {
+		modTime     time.Time
+		hasPlaylist bool
+	}
+	dirMap = map[string]dirMapValue
+)
+
+func loadDirTree(ctx context.Context, rootFolder string) (dirMap, error) {
+	newMap := make(dirMap)
+	err := loadMap(ctx, rootFolder, rootFolder, newMap)
+	if err != nil {
+		log.Error(ctx, "Error loading directory tree", err)
+	}
+	return newMap, err
+}
+
+func loadMap(ctx context.Context, rootPath string, currentFolder string, dirMap dirMap) error {
+	children, dirMapValue, err := loadDir(ctx, currentFolder)
+	if err != nil {
+		return err
+	}
+	for _, c := range children {
+		err := loadMap(ctx, rootPath, c, dirMap)
+		if err != nil {
+			return err
+		}
+	}
+
+	dir := filepath.Clean(currentFolder)
+	dirMap[dir] = dirMapValue
+
+	return nil
+}
+
+func loadDir(ctx context.Context, dirPath string) (children []string, info dirMapValue, err error) {
+	dirInfo, err := os.Stat(dirPath)
+	if err != nil {
+		log.Error(ctx, "Error stating dir", "path", dirPath, err)
+		return
+	}
+	info.modTime = dirInfo.ModTime()
+
+	files, err := ioutil.ReadDir(dirPath)
+	if err != nil {
+		log.Error(ctx, "Error reading dir", "path", dirPath, err)
+		return
+	}
+	for _, f := range files {
+		isDir, err := isDirOrSymlinkToDir(dirPath, f)
+		// Skip invalid symlinks
+		if err != nil {
+			continue
+		}
+		if isDir && !isDirIgnored(dirPath, f) && isDirReadable(dirPath, f) {
+			children = append(children, filepath.Join(dirPath, f.Name()))
+		} else {
+			if f.ModTime().After(info.modTime) {
+				info.modTime = f.ModTime()
+			}
+			info.hasPlaylist = info.hasPlaylist || utils.IsPlaylist(f.Name())
+		}
+	}
+	return
+}
+
+// isDirOrSymlinkToDir returns true if and only if the dirInfo represents a file
+// system directory, or a symbolic link to a directory. Note that if the dirInfo
+// is not a directory but is a symbolic link, this method will resolve by
+// sending a request to the operating system to follow the symbolic link.
+// Copied from github.com/karrick/godirwalk
+func isDirOrSymlinkToDir(baseDir string, dirInfo os.FileInfo) (bool, error) {
+	if dirInfo.IsDir() {
+		return true, nil
+	}
+	if dirInfo.Mode()&os.ModeSymlink == 0 {
+		return false, nil
+	}
+	// Does this symlink point to a directory?
+	dirInfo, err := os.Stat(filepath.Join(baseDir, dirInfo.Name()))
+	if err != nil {
+		return false, err
+	}
+	return dirInfo.IsDir(), nil
+}
+
+// isDirIgnored returns true if the directory represented by dirInfo contains an
+// `ignore` file (named after consts.SkipScanFile)
+func isDirIgnored(baseDir string, dirInfo os.FileInfo) bool {
+	_, err := os.Stat(filepath.Join(baseDir, dirInfo.Name(), consts.SkipScanFile))
+	return err == nil
+}
+
+// isDirReadable returns true if the directory represented by dirInfo is readable
+func isDirReadable(baseDir string, dirInfo os.FileInfo) bool {
+	path := filepath.Join(baseDir, dirInfo.Name())
+	res, err := utils.IsDirReadable(path)
+	if !res {
+		log.Debug("Warning: Skipping unreadable directory", "path", path, err)
+	}
+	return res
+}

scanner/mapping.go

@@ -0,0 +1,120 @@
+package scanner
+
+import (
+	"crypto/md5"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/deluan/navidrome/consts"
+	"github.com/deluan/navidrome/model"
+	"github.com/deluan/navidrome/utils"
+	"github.com/kennygrant/sanitize"
+)
+
+type mediaFileMapper struct {
+	rootFolder string
+}
+
+func newMediaFileMapper(rootFolder string) *mediaFileMapper {
+	return &mediaFileMapper{rootFolder: rootFolder}
+}
+
+func (s *mediaFileMapper) toMediaFile(md *Metadata) model.MediaFile {
+	mf := &model.MediaFile{}
+	mf.ID = s.trackID(md)
+	mf.Title = s.mapTrackTitle(md)
+	mf.Album = md.Album()
+	mf.AlbumID = s.albumID(md)
+	mf.Album = s.mapAlbumName(md)
+	mf.ArtistID = s.artistID(md)
+	mf.Artist = s.mapArtistName(md)
+	mf.AlbumArtistID = s.albumArtistID(md)
+	mf.AlbumArtist = s.mapAlbumArtistName(md)
+	mf.Genre = md.Genre()
+	mf.Compilation = md.Compilation()
+	mf.Year = md.Year()
+	mf.TrackNumber, _ = md.TrackNumber()
+	mf.DiscNumber, _ = md.DiscNumber()
+	mf.DiscSubtitle = md.DiscSubtitle()
+	mf.Duration = md.Duration()
+	mf.BitRate = md.BitRate()
+	mf.Path = md.FilePath()
+	mf.Suffix = md.Suffix()
+	mf.Size = md.Size()
+	mf.HasCoverArt = md.HasPicture()
+	mf.SortTitle = md.SortTitle()
+	mf.SortAlbumName = md.SortAlbum()
+	mf.SortArtistName = md.SortArtist()
+	mf.SortAlbumArtistName = md.SortAlbumArtist()
+	mf.OrderAlbumName = sanitizeFieldForSorting(mf.Album)
+	mf.OrderArtistName = sanitizeFieldForSorting(mf.Artist)
+	mf.OrderAlbumArtistName = sanitizeFieldForSorting(mf.AlbumArtist)
+
+	// TODO Get Creation time. https://github.com/djherbis/times ?
+	mf.CreatedAt = md.ModificationTime()
+	mf.UpdatedAt = md.ModificationTime()
+
+	return *mf
+}
+
+func sanitizeFieldForSorting(originalValue string) string {
+	v := utils.NoArticle(originalValue)
+	v = strings.TrimSpace(sanitize.Accents(v))
+	return utils.NoArticle(v)
+}
+
+func (s *mediaFileMapper) mapTrackTitle(md *Metadata) string {
+	if md.Title() == "" {
+		s := strings.TrimPrefix(md.FilePath(), s.rootFolder+string(os.PathSeparator))
+		e := filepath.Ext(s)
+		return strings.TrimSuffix(s, e)
+	}
+	return md.Title()
+}
+
+func (s *mediaFileMapper) mapAlbumArtistName(md *Metadata) string {
+	switch {
+	case md.Compilation():
+		return consts.VariousArtists
+	case md.AlbumArtist() != "":
+		return md.AlbumArtist()
+	case md.Artist() != "":
+		return md.Artist()
+	default:
+		return consts.UnknownArtist
+	}
+}
+
+func (s *mediaFileMapper) mapArtistName(md *Metadata) string {
+	if md.Artist() != "" {
+		return md.Artist()
+	}
+	return consts.UnknownArtist
+}
+
+func (s *mediaFileMapper) mapAlbumName(md *Metadata) string {
+	name := md.Album()
+	if name == "" {
+		return "[Unknown Album]"
+	}
+	return name
+}
+
+func (s *mediaFileMapper) trackID(md *Metadata) string {
+	return fmt.Sprintf("%x", md5.Sum([]byte(md.FilePath())))
+}
+
+func (s *mediaFileMapper) albumID(md *Metadata) string {
+	albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", s.mapAlbumArtistName(md), s.mapAlbumName(md)))
+	return fmt.Sprintf("%x", md5.Sum([]byte(albumPath)))
+}
+
+func (s *mediaFileMapper) artistID(md *Metadata) string {
+	return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapArtistName(md)))))
+}
+
+func (s *mediaFileMapper) albumArtistID(md *Metadata) string {
+	return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapAlbumArtistName(md)))))
+}

scanner/metadata_ffmpeg.go

@@ -4,11 +4,9 @@ import (
 	"bufio"
 	"errors"
 	"fmt"
-	"mime"
 	"os"
 	"os/exec"
 	"path"
-	"path/filepath"
 	"regexp"
 	"strconv"
 	"strings"
@@ -49,37 +47,7 @@ func (m *Metadata) BitRate() int                { return m.parseInt("bitrate") }
 func (m *Metadata) ModificationTime() time.Time { return m.fileInfo.ModTime() }
 func (m *Metadata) FilePath() string            { return m.filePath }
 func (m *Metadata) Suffix() string              { return m.suffix }
-func (m *Metadata) Size() int                   { return int(m.fileInfo.Size()) }
-
-func LoadAllAudioFiles(dirPath string) (map[string]os.FileInfo, error) {
-	dir, err := os.Open(dirPath)
-	if err != nil {
-		return nil, err
-	}
-	files, err := dir.Readdir(-1)
-	if err != nil {
-		return nil, err
-	}
-	audioFiles := make(map[string]os.FileInfo)
-	for _, f := range files {
-		if f.IsDir() {
-			continue
-		}
-		filePath := filepath.Join(dirPath, f.Name())
-		extension := path.Ext(filePath)
-		if !isAudioFile(extension) {
-			continue
-		}
-		fi, err := os.Stat(filePath)
-		if err != nil {
-			log.Error("Could not stat file", "filePath", filePath, err)
-		} else {
-			audioFiles[filePath] = fi
-		}
-	}
-
-	return audioFiles, nil
-}
+func (m *Metadata) Size() int64                 { return m.fileInfo.Size() }
 
 func ExtractAllMetadata(inputs []string) (map[string]*Metadata, error) {
 	args := createProbeCommand(inputs)
@@ -120,7 +88,7 @@ var (
 )
 
 func parseOutput(output string) map[string]string {
-	split := map[string]string{}
+	outputs := map[string]string{}
 	all := inputRegex.FindAllStringSubmatchIndex(output, -1)
 	for i, loc := range all {
 		// Filename is the first captured group
@@ -136,9 +104,9 @@ func parseOutput(output string) map[string]string {
 			// if this is the last match
 			info = output[initial:]
 		}
-		split[file] = info
+		outputs[file] = info
 	}
-	return split
+	return outputs
 }
 
 func extractMetadata(filePath, info string) (*Metadata, error) {
@@ -159,11 +127,6 @@ func extractMetadata(filePath, info string) (*Metadata, error) {
 	return m, nil
 }
 
-func isAudioFile(extension string) bool {
-	typ := mime.TypeByExtension(extension)
-	return strings.HasPrefix(typ, "audio/")
-}
-
 func (m *Metadata) parseInfo(info string) {
 	reader := strings.NewReader(info)
 	scanner := bufio.NewScanner(reader)

scanner/metadata_test.go

@@ -47,25 +47,6 @@ var _ = Describe("Metadata", func() {
 		})
 	})
 
-	Context("LoadAllAudioFiles", func() {
-		It("return all audiofiles from the folder", func() {
-			files, err := LoadAllAudioFiles("tests/fixtures")
-			Expect(err).ToNot(HaveOccurred())
-			Expect(files).To(HaveLen(3))
-			Expect(files).To(HaveKey("tests/fixtures/test.ogg"))
-			Expect(files).To(HaveKey("tests/fixtures/test.mp3"))
-			Expect(files).To(HaveKey("tests/fixtures/01 Invisible (RED) Edit Version.mp3"))
-		})
-		It("returns error if path does not exist", func() {
-			_, err := LoadAllAudioFiles("./INVALID/PATH")
-			Expect(err).To(HaveOccurred())
-		})
-
-		It("returns empty map if there are no audio files in path", func() {
-			Expect(LoadAllAudioFiles("tests/fixtures/empty_folder")).To(BeEmpty())
-		})
-	})
-
 	Context("extractMetadata", func() {
 		It("detects embedded cover art correctly", func() {
 			const output = `

scanner/playlist_sync.go

@@ -0,0 +1,120 @@
+package scanner
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/deluan/navidrome/log"
+	"github.com/deluan/navidrome/model"
+	"github.com/deluan/navidrome/model/request"
+	"github.com/deluan/navidrome/utils"
+)
+
+type playlistSync struct {
+	ds model.DataStore
+}
+
+func newPlaylistSync(ds model.DataStore) *playlistSync {
+	return &playlistSync{ds: ds}
+}
+
+func (s *playlistSync) processPlaylists(ctx context.Context, dir string) int {
+	count := 0
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		log.Error(ctx, "Error reading files", "dir", dir, err)
+		return count
+	}
+	for _, f := range files {
+		if !utils.IsPlaylist(f.Name()) {
+			continue
+		}
+		pls, err := s.parsePlaylist(ctx, f.Name(), dir)
+		if err != nil {
+			log.Error(ctx, "Error parsing playlist", "playlist", f.Name(), err)
+			continue
+		}
+		log.Debug("Found playlist", "name", pls.Name, "lastUpdated", pls.UpdatedAt, "path", pls.Path, "numTracks", len(pls.Tracks))
+		err = s.updatePlaylist(ctx, pls)
+		if err != nil {
+			log.Error(ctx, "Error updating playlist", "playlist", f.Name(), err)
+		}
+		count++
+	}
+	return count
+}
+
+func (s *playlistSync) parsePlaylist(ctx context.Context, playlistFile string, baseDir string) (*model.Playlist, error) {
+	playlistPath := filepath.Join(baseDir, playlistFile)
+	file, err := os.Open(playlistPath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+	info, err := os.Stat(playlistPath)
+	if err != nil {
+		return nil, err
+	}
+
+	var extension = filepath.Ext(playlistFile)
+	var name = playlistFile[0 : len(playlistFile)-len(extension)]
+
+	pls := &model.Playlist{
+		Name:      name,
+		Comment:   fmt.Sprintf("Auto-imported from '%s'", playlistFile),
+		Public:    false,
+		Path:      playlistPath,
+		Sync:      true,
+		UpdatedAt: info.ModTime(),
+	}
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		path := scanner.Text()
+		// Skip extended info
+		if strings.HasPrefix(path, "#") {
+			continue
+		}
+		if !filepath.IsAbs(path) {
+			path = filepath.Join(baseDir, path)
+		}
+		mf, err := s.ds.MediaFile(ctx).FindByPath(path)
+		if err != nil {
+			log.Warn(ctx, "Path in playlist not found", "playlist", playlistFile, "path", path, err)
+			continue
+		}
+		pls.Tracks = append(pls.Tracks, *mf)
+	}
+
+	return pls, scanner.Err()
+}
+
+func (s *playlistSync) updatePlaylist(ctx context.Context, newPls *model.Playlist) error {
+	owner, _ := request.UsernameFrom(ctx)
+
+	pls, err := s.ds.Playlist(ctx).FindByPath(newPls.Path)
+	if err != nil && err != model.ErrNotFound {
+		return err
+	}
+	if err == nil && !pls.Sync {
+		log.Debug(ctx, "Playlist already imported and not synced", "playlist", pls.Name, "path", pls.Path)
+		return nil
+	}
+
+	if err == nil {
+		log.Info(ctx, "Updating synced playlist", "playlist", pls.Name, "path", newPls.Path)
+		newPls.ID = pls.ID
+		newPls.Name = pls.Name
+		newPls.Comment = pls.Comment
+		newPls.Owner = pls.Owner
+	} else {
+		log.Info(ctx, "Adding synced playlist", "playlist", newPls.Name, "path", newPls.Path, "owner", owner)
+		newPls.Owner = owner
+	}
+	return s.ds.Playlist(ctx).Put(newPls)
+}

scanner/scanner.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/deluan/navidrome/conf"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
 )
@@ -81,10 +82,17 @@ func (s *Scanner) loadFolders() {
 	fs, _ := s.ds.MediaFolder(context.TODO()).GetAll()
 	for _, f := range fs {
 		log.Info("Configuring Media Folder", "name", f.Name, "path", f.Path)
-		s.folders[f.Path] = NewTagScanner(f.Path, s.ds)
+		s.folders[f.Path] = s.newScanner(f)
 	}
 }
 
+func (s *Scanner) newScanner(f model.MediaFolder) FolderScanner {
+	if conf.Server.DevOldScanner {
+		return NewTagScanner(f.Path, s.ds)
+	}
+	return NewTagScanner2(f.Path, s.ds)
+}
+
 type Status int
 
 type StatusInfo struct {

scanner/tag_scanner.go

@@ -2,48 +2,69 @@ package scanner
 
 import (
 	"context"
-	"crypto/md5"
-	"fmt"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
-	"github.com/deluan/navidrome/consts"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
 	"github.com/deluan/navidrome/utils"
-	"github.com/kennygrant/sanitize"
 )
 
 type TagScanner struct {
 	rootFolder string
 	ds         model.DataStore
-	detector   *ChangeDetector
+	detector   *changeDetector
+	mapper     *mediaFileMapper
+	firstRun   sync.Once
 }
 
 func NewTagScanner(rootFolder string, ds model.DataStore) *TagScanner {
 	return &TagScanner{
 		rootFolder: rootFolder,
 		ds:         ds,
-		detector:   NewChangeDetector(rootFolder),
+		detector:   newChangeDetector(rootFolder),
+		mapper:     newMediaFileMapper(rootFolder),
+		firstRun:   sync.Once{},
 	}
 }
 
+type (
+	artistMap map[string]struct{}
+	albumMap  map[string]struct{}
+
+	counters struct {
+		added   int64
+		updated int64
+		deleted int64
+	}
+)
+
+const (
+	// filesBatchSize used for extract file metadata
+	filesBatchSize = 100
+)
+
 // Scan algorithm overview:
-// For each changed: Get all files from DB that starts with the folder, scan each file:
+// For each changed folder: Get all files from DB that starts with the folder, scan each file:
 //	    if file in folder is newer, update the one in DB
 //      if file in folder does not exists in DB, add
 // 	    for each file in the DB that is not found in the folder, delete from DB
 // For each deleted folder: delete all files from DB that starts with the folder path
+// Only on first run, check if any folder under each changed folder is missing.
+//      if it is, delete everything under it
 // Create new albums/artists, update counters:
 //      collect all albumIDs and artistIDs from previous steps
 //	    refresh the collected albums and artists with the metadata from the mediafiles
 // Delete all empty albums, delete all empty Artists
 func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) error {
 	start := time.Now()
-	changed, deleted, err := s.detector.Scan(lastModifiedSince)
+	log.Trace(ctx, "Looking for changes in music folder", "folder", s.rootFolder)
+
+	changed, deleted, err := s.detector.Scan(ctx, lastModifiedSince)
 	if err != nil {
 		return err
 	}
@@ -63,85 +84,77 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) erro
 	sort.Strings(changed)
 	sort.Strings(deleted)
 
-	updatedArtists := map[string]bool{}
-	updatedAlbums := map[string]bool{}
+	updatedArtists := artistMap{}
+	updatedAlbums := albumMap{}
+	cnt := &counters{}
 
 	for _, c := range changed {
-		err := s.processChangedDir(ctx, c, updatedArtists, updatedAlbums)
+		err := s.processChangedDir(ctx, c, updatedArtists, updatedAlbums, cnt)
 		if err != nil {
 			return err
 		}
-		if len(updatedAlbums)+len(updatedArtists) > 100 {
-			err = s.refreshAlbums(ctx, updatedAlbums)
-			if err != nil {
-				return err
-			}
-			err = s.refreshArtists(ctx, updatedArtists)
-			if err != nil {
-				return err
-			}
-			updatedAlbums = map[string]bool{}
-			updatedArtists = map[string]bool{}
-		}
+		// TODO Search for playlists and import (with `sync` on)
 	}
 	for _, c := range deleted {
-		err := s.processDeletedDir(ctx, c, updatedArtists, updatedAlbums)
+		err := s.processDeletedDir(ctx, c, updatedArtists, updatedAlbums, cnt)
 		if err != nil {
 			return err
 		}
-		if len(updatedAlbums)+len(updatedArtists) > 100 {
-			err = s.refreshAlbums(ctx, updatedAlbums)
-			if err != nil {
-				return err
-			}
-			err = s.refreshArtists(ctx, updatedArtists)
-			if err != nil {
-				return err
-			}
-			updatedAlbums = map[string]bool{}
-			updatedArtists = map[string]bool{}
-		}
+		// TODO "Un-sync" all playlists synched from a deleted folder
 	}
 
-	err = s.refreshAlbums(ctx, updatedAlbums)
+	err = s.flushAlbums(ctx, updatedAlbums)
 	if err != nil {
 		return err
 	}
 
-	err = s.refreshArtists(ctx, updatedArtists)
+	err = s.flushArtists(ctx, updatedArtists)
 	if err != nil {
 		return err
 	}
 
+	s.firstRun.Do(func() {
+		s.removeDeletedFolders(context.TODO(), changed, cnt)
+	})
+
 	err = s.ds.GC(log.NewContext(context.TODO()))
-	log.Info("Finished Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start))
+	log.Info("Finished Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start),
+		"added", cnt.added, "updated", cnt.updated, "deleted", cnt.deleted)
 
 	return err
 }
 
-func (s *TagScanner) refreshAlbums(ctx context.Context, updatedAlbums map[string]bool) error {
+func (s *TagScanner) flushAlbums(ctx context.Context, updatedAlbums albumMap) error {
+	if len(updatedAlbums) == 0 {
+		return nil
+	}
 	var ids []string
 	for id := range updatedAlbums {
 		ids = append(ids, id)
+		delete(updatedAlbums, id)
 	}
 	return s.ds.Album(ctx).Refresh(ids...)
 }
 
-func (s *TagScanner) refreshArtists(ctx context.Context, updatedArtists map[string]bool) error {
+func (s *TagScanner) flushArtists(ctx context.Context, updatedArtists artistMap) error {
+	if len(updatedArtists) == 0 {
+		return nil
+	}
 	var ids []string
 	for id := range updatedArtists {
 		ids = append(ids, id)
+		delete(updatedArtists, id)
 	}
 	return s.ds.Artist(ctx).Refresh(ids...)
 }
 
-func (s *TagScanner) processChangedDir(ctx context.Context, dir string, updatedArtists map[string]bool, updatedAlbums map[string]bool) error {
+func (s *TagScanner) processChangedDir(ctx context.Context, dir string, updatedArtists artistMap, updatedAlbums albumMap, cnt *counters) error {
 	dir = filepath.Join(s.rootFolder, dir)
 	start := time.Now()
 
 	// Load folder's current tracks from DB into a map
 	currentTracks := map[string]model.MediaFile{}
-	ct, err := s.ds.MediaFile(ctx).FindByPath(dir)
+	ct, err := s.ds.MediaFile(ctx).FindAllByPath(dir)
 	if err != nil {
 		return err
 	}
@@ -165,33 +178,59 @@ func (s *TagScanner) processChangedDir(ctx context.Context, dir string, updatedA
 	var filesToUpdate []string
 	for filePath, info := range files {
 		c, ok := currentTracks[filePath]
-		if !ok || (ok && info.ModTime().After(c.UpdatedAt)) {
+		if !ok {
 			filesToUpdate = append(filesToUpdate, filePath)
+			cnt.added++
+		}
+		if ok && info.ModTime().After(c.UpdatedAt) {
+			filesToUpdate = append(filesToUpdate, filePath)
+			cnt.updated++
 		}
 		delete(currentTracks, filePath)
+
+		// Force a refresh of the album and artist, to cater for cover art files. Ideally we would only do this
+		// if there are any image file in the folder (TODO)
+		err = s.updateAlbum(ctx, c.AlbumID, updatedAlbums)
+		if err != nil {
+			return err
+		}
+		err = s.updateArtist(ctx, c.AlbumArtistID, updatedArtists)
+		if err != nil {
+			return err
+		}
 	}
 
 	numUpdatedTracks := 0
 	numPurgedTracks := 0
 
 	if len(filesToUpdate) > 0 {
-		// Load tracks Metadata from the folder
-		newTracks, err := s.loadTracks(filesToUpdate)
-		if err != nil {
-			return err
-		}
-
-		// If track from folder is newer than the one in DB, update/insert in DB and delete from the current tracks
-		log.Trace("Updating mediaFiles in DB", "dir", dir, "files", filesToUpdate, "numFiles", len(filesToUpdate))
-		for i := range newTracks {
-			n := newTracks[i]
-			err := s.ds.MediaFile(ctx).Put(&n)
-			updatedArtists[n.AlbumArtistID] = true
-			updatedAlbums[n.AlbumID] = true
-			numUpdatedTracks++
+		// Break the file list in chunks to avoid calling ffmpeg with too many parameters
+		chunks := utils.BreakUpStringSlice(filesToUpdate, filesBatchSize)
+		for _, chunk := range chunks {
+			// Load tracks Metadata from the folder
+			newTracks, err := s.loadTracks(chunk)
 			if err != nil {
 				return err
 			}
+
+			// If track from folder is newer than the one in DB, update/insert in DB
+			log.Trace("Updating mediaFiles in DB", "dir", dir, "files", chunk, "numFiles", len(chunk))
+			for i := range newTracks {
+				n := newTracks[i]
+				err := s.ds.MediaFile(ctx).Put(&n)
+				if err != nil {
+					return err
+				}
+				err = s.updateAlbum(ctx, n.AlbumID, updatedAlbums)
+				if err != nil {
+					return err
+				}
+				err = s.updateArtist(ctx, n.AlbumArtistID, updatedArtists)
+				if err != nil {
+					return err
+				}
+				numUpdatedTracks++
+			}
 		}
 	}
 
@@ -200,11 +239,18 @@ func (s *TagScanner) processChangedDir(ctx context.Context, dir string, updatedA
 		// Remaining tracks from DB that are not in the folder are deleted
 		for _, ct := range currentTracks {
 			numPurgedTracks++
-			updatedArtists[ct.AlbumArtistID] = true
-			updatedAlbums[ct.AlbumID] = true
+			err = s.updateAlbum(ctx, ct.AlbumID, updatedAlbums)
+			if err != nil {
+				return err
+			}
+			err = s.updateArtist(ctx, ct.AlbumArtistID, updatedArtists)
+			if err != nil {
+				return err
+			}
 			if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil {
 				return err
 			}
+			cnt.deleted++
 		}
 	}
 
@@ -212,21 +258,74 @@ func (s *TagScanner) processChangedDir(ctx context.Context, dir string, updatedA
 	return nil
 }
 
-func (s *TagScanner) processDeletedDir(ctx context.Context, dir string, updatedArtists map[string]bool, updatedAlbums map[string]bool) error {
+func (s *TagScanner) updateAlbum(ctx context.Context, albumId string, updatedAlbums albumMap) error {
+	updatedAlbums[albumId] = struct{}{}
+	if len(updatedAlbums) >= batchSize {
+		err := s.flushAlbums(ctx, updatedAlbums)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *TagScanner) updateArtist(ctx context.Context, artistId string, updatedArtists artistMap) error {
+	updatedArtists[artistId] = struct{}{}
+	if len(updatedArtists) >= batchSize {
+		err := s.flushArtists(ctx, updatedArtists)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *TagScanner) processDeletedDir(ctx context.Context, dir string, updatedArtists artistMap, updatedAlbums albumMap, cnt *counters) error {
 	dir = filepath.Join(s.rootFolder, dir)
 	start := time.Now()
 
-	ct, err := s.ds.MediaFile(ctx).FindByPath(dir)
+	mfs, err := s.ds.MediaFile(ctx).FindAllByPath(dir)
 	if err != nil {
 		return err
 	}
-	for _, t := range ct {
-		updatedArtists[t.AlbumArtistID] = true
-		updatedAlbums[t.AlbumID] = true
+	for _, t := range mfs {
+		err = s.updateAlbum(ctx, t.AlbumID, updatedAlbums)
+		if err != nil {
+			return err
+		}
+		err = s.updateArtist(ctx, t.AlbumArtistID, updatedArtists)
+		if err != nil {
+			return err
+		}
 	}
 
-	log.Info("Finished processing deleted folder", "dir", dir, "purged", len(ct), "elapsed", time.Since(start))
-	return s.ds.MediaFile(ctx).DeleteByPath(dir)
+	log.Info("Finished processing deleted folder", "dir", dir, "purged", len(mfs), "elapsed", time.Since(start))
+	c, err := s.ds.MediaFile(ctx).DeleteByPath(dir)
+	cnt.deleted += c
+	return err
+}
+
+func (s *TagScanner) removeDeletedFolders(ctx context.Context, changed []string, cnt *counters) {
+	for _, dir := range changed {
+		fullPath := filepath.Join(s.rootFolder, dir)
+		paths, err := s.ds.MediaFile(ctx).FindPathsRecursively(fullPath)
+		if err != nil {
+			log.Error(ctx, "Error reading paths from DB", "path", dir, err)
+			return
+		}
+
+		// If a path is unreadable, remove from the DB
+		for _, path := range paths {
+			if readable, err := utils.IsDirReadable(path); !readable {
+				log.Info(ctx, "Path unavailable. Removing tracks from DB", "path", path, err)
+				c, err := s.ds.MediaFile(ctx).DeleteByPath(path)
+				if err != nil {
+					log.Error(ctx, "Error removing MediaFiles from DB", "path", path, err)
+				}
+				cnt.deleted += c
+			}
+		}
+	}
 }
 
 func (s *TagScanner) loadTracks(filePaths []string) (model.MediaFiles, error) {
@@ -237,106 +336,37 @@ func (s *TagScanner) loadTracks(filePaths []string) (model.MediaFiles, error) {
 
 	var mfs model.MediaFiles
 	for _, md := range mds {
-		mf := s.toMediaFile(md)
+		mf := s.mapper.toMediaFile(md)
 		mfs = append(mfs, mf)
 	}
 	return mfs, nil
 }
 
-func (s *TagScanner) toMediaFile(md *Metadata) model.MediaFile {
-	mf := &model.MediaFile{}
-	mf.ID = s.trackID(md)
-	mf.Title = s.mapTrackTitle(md)
-	mf.Album = md.Album()
-	mf.AlbumID = s.albumID(md)
-	mf.Album = s.mapAlbumName(md)
-	mf.ArtistID = s.artistID(md)
-	mf.Artist = s.mapArtistName(md)
-	mf.AlbumArtistID = s.albumArtistID(md)
-	mf.AlbumArtist = s.mapAlbumArtistName(md)
-	mf.Genre = md.Genre()
-	mf.Compilation = md.Compilation()
-	mf.Year = md.Year()
-	mf.TrackNumber, _ = md.TrackNumber()
-	mf.DiscNumber, _ = md.DiscNumber()
-	mf.DiscSubtitle = md.DiscSubtitle()
-	mf.Duration = md.Duration()
-	mf.BitRate = md.BitRate()
-	mf.Path = md.FilePath()
-	mf.Suffix = md.Suffix()
-	mf.Size = md.Size()
-	mf.HasCoverArt = md.HasPicture()
-	mf.SortTitle = md.SortTitle()
-	mf.SortAlbumName = md.SortAlbum()
-	mf.SortArtistName = md.SortArtist()
-	mf.SortAlbumArtistName = md.SortAlbumArtist()
-	mf.OrderAlbumName = sanitizeFieldForSorting(mf.Album)
-	mf.OrderArtistName = sanitizeFieldForSorting(mf.Artist)
-	mf.OrderAlbumArtistName = sanitizeFieldForSorting(mf.AlbumArtist)
-
-	// TODO Get Creation time. https://github.com/djherbis/times ?
-	mf.CreatedAt = md.ModificationTime()
-	mf.UpdatedAt = md.ModificationTime()
-
-	return *mf
-}
-
-func sanitizeFieldForSorting(originalValue string) string {
-	v := utils.NoArticle(originalValue)
-	v = strings.TrimSpace(sanitize.Accents(v))
-	return utils.NoArticle(v)
-}
-
-func (s *TagScanner) mapTrackTitle(md *Metadata) string {
-	if md.Title() == "" {
-		s := strings.TrimPrefix(md.FilePath(), s.rootFolder+string(os.PathSeparator))
-		e := filepath.Ext(s)
-		return strings.TrimSuffix(s, e)
+func LoadAllAudioFiles(dirPath string) (map[string]os.FileInfo, error) {
+	dir, err := os.Open(dirPath)
+	if err != nil {
+		return nil, err
 	}
-	return md.Title()
-}
-
-func (s *TagScanner) mapAlbumArtistName(md *Metadata) string {
-	switch {
-	case md.Compilation():
-		return consts.VariousArtists
-	case md.AlbumArtist() != "":
-		return md.AlbumArtist()
-	case md.Artist() != "":
-		return md.Artist()
-	default:
-		return consts.UnknownArtist
+	files, err := dir.Readdir(-1)
+	if err != nil {
+		return nil, err
 	}
-}
-
-func (s *TagScanner) mapArtistName(md *Metadata) string {
-	if md.Artist() != "" {
-		return md.Artist()
+	audioFiles := make(map[string]os.FileInfo)
+	for _, f := range files {
+		if f.IsDir() {
+			continue
+		}
+		filePath := filepath.Join(dirPath, f.Name())
+		if !utils.IsAudioFile(filePath) {
+			continue
+		}
+		fi, err := os.Stat(filePath)
+		if err != nil {
+			log.Error("Could not stat file", "filePath", filePath, err)
+		} else {
+			audioFiles[filePath] = fi
+		}
 	}
-	return consts.UnknownArtist
-}
 
-func (s *TagScanner) mapAlbumName(md *Metadata) string {
-	name := md.Album()
-	if name == "" {
-		return "[Unknown Album]"
-	}
-	return name
-}
-
-func (s *TagScanner) trackID(md *Metadata) string {
-	return fmt.Sprintf("%x", md5.Sum([]byte(md.FilePath())))
-}
-
-func (s *TagScanner) albumID(md *Metadata) string {
-	albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", s.mapAlbumArtistName(md), s.mapAlbumName(md)))
-	return fmt.Sprintf("%x", md5.Sum([]byte(albumPath)))
-}
-
-func (s *TagScanner) artistID(md *Metadata) string {
-	return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapArtistName(md)))))
-}
-
-func (s *TagScanner) albumArtistID(md *Metadata) string {
-	return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapAlbumArtistName(md)))))
+	return audioFiles, nil
 }

scanner/tag_scanner_2.go

@@ -0,0 +1,354 @@
+package scanner
+
+import (
+	"context"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/deluan/navidrome/log"
+	"github.com/deluan/navidrome/model"
+	"github.com/deluan/navidrome/model/request"
+	"github.com/deluan/navidrome/utils"
+)
+
+type TagScanner2 struct {
+	rootFolder string
+	ds         model.DataStore
+	mapper     *mediaFileMapper
+	plsSync    *playlistSync
+	albumMap   *flushableMap
+	artistMap  *flushableMap
+	cnt        *counters
+}
+
+func NewTagScanner2(rootFolder string, ds model.DataStore) *TagScanner2 {
+	return &TagScanner2{
+		rootFolder: rootFolder,
+		mapper:     newMediaFileMapper(rootFolder),
+		plsSync:    newPlaylistSync(ds),
+		ds:         ds,
+	}
+}
+
+// Scan algorithm overview:
+// Load all directories under the music folder, with their ModTime (self or any non-dir children, whichever is newer)
+// Find changed folders (based on lastModifiedSince) and deleted folders (comparing to the DB)
+// For each deleted folder: delete all files from DB whose path starts with the delete folder path (non-recursively)
+// For each changed folder: get all files from DB whose path starts with the changed folder (non-recursively), check each file:
+//	    if file in folder is newer, update the one in DB
+//      if file in folder does not exists in DB, add it
+// 	    for each file in the DB that is not found in the folder, delete it from DB
+// Create new albums/artists, update counters:
+//      collect all albumIDs and artistIDs from previous steps
+//	    refresh the collected albums and artists with the metadata from the mediafiles
+// For each changed folder, process playlists:
+//      If the playlist is not in the DB, import it, setting sync = true
+//      If the playlist is in the DB and sync == true, import it, or else skip it
+// Delete all empty albums, delete all empty artists, clean-up playlists
+func (s *TagScanner2) Scan(ctx context.Context, lastModifiedSince time.Time) error {
+	ctx = s.withAdminUser(ctx)
+
+	start := time.Now()
+	allDirs, err := s.getDirTree(ctx)
+	if err != nil {
+		return err
+	}
+
+	changedDirs := s.getChangedDirs(ctx, allDirs, lastModifiedSince)
+	if len(changedDirs) == 0 {
+		log.Debug(ctx, "No changes found in Music Folder", "folder", s.rootFolder)
+		return nil
+	}
+	deletedDirs, _ := s.getDeletedDirs(ctx, allDirs, changedDirs)
+
+	if log.CurrentLevel() >= log.LevelTrace {
+		log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs),
+			"changed", strings.Join(changedDirs, ";"), "deleted", strings.Join(deletedDirs, ";"))
+	} else {
+		log.Info(ctx, "Folder changes detected", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs))
+	}
+
+	s.albumMap = newFlushableMap(ctx, "album", s.ds.Album(ctx).Refresh)
+	s.artistMap = newFlushableMap(ctx, "artist", s.ds.Artist(ctx).Refresh)
+	s.cnt = &counters{}
+
+	for _, dir := range deletedDirs {
+		err := s.processDeletedDir(ctx, dir)
+		if err != nil {
+			log.Error("Error removing deleted folder from DB", "path", dir, err)
+		}
+	}
+	for _, dir := range changedDirs {
+		err := s.processChangedDir(ctx, dir)
+		if err != nil {
+			log.Error("Error updating folder in the DB", "path", dir, err)
+		}
+	}
+
+	_ = s.albumMap.flush()
+	_ = s.artistMap.flush()
+
+	// Now that all mediafiles are imported/updated, search for and import playlists
+	u, _ := request.UserFrom(ctx)
+	plsCount := 0
+	for _, dir := range changedDirs {
+		info := allDirs[dir]
+		if info.hasPlaylist {
+			if !u.IsAdmin {
+				log.Warn("Playlists will not be imported, as there are no admin users yet, "+
+					"Please create an admin user first, and then update the playlists for them to be imported", "dir", dir)
+			} else {
+				plsCount = s.plsSync.processPlaylists(ctx, dir)
+			}
+		}
+	}
+
+	err = s.ds.GC(log.NewContext(ctx))
+	log.Info("Finished processing Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start),
+		"added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted, "playlistsImported", plsCount)
+
+	return err
+}
+
+func (s *TagScanner2) getDirTree(ctx context.Context) (dirMap, error) {
+	start := time.Now()
+	log.Trace(ctx, "Loading directory tree from music folder", "folder", s.rootFolder)
+	dirs, err := loadDirTree(ctx, s.rootFolder)
+	if err != nil {
+		return nil, err
+	}
+	log.Debug("Directory tree loaded", "total", len(dirs), "elapsed", time.Since(start))
+	return dirs, nil
+}
+
+func (s *TagScanner2) getChangedDirs(ctx context.Context, dirs dirMap, lastModified time.Time) []string {
+	start := time.Now()
+	log.Trace(ctx, "Checking for changed folders")
+	var changed []string
+	for d, info := range dirs {
+		if info.modTime.After(lastModified) {
+			changed = append(changed, d)
+		}
+	}
+	sort.Strings(changed)
+	log.Debug(ctx, "Finished changed folders check", "total", len(changed), "elapsed", time.Since(start))
+	return changed
+}
+
+func (s *TagScanner2) getDeletedDirs(ctx context.Context, allDirs dirMap, changedDirs []string) ([]string, error) {
+	start := time.Now()
+	log.Trace(ctx, "Checking for deleted folders")
+
+	var deleted []string
+	repo := s.ds.MediaFile(ctx)
+
+	// If rootFolder is in the list of changedDirs, optimize and only do one query to the DB
+	var foldersToCheck []string
+	if utils.StringInSlice(s.rootFolder, changedDirs) {
+		foldersToCheck = []string{s.rootFolder}
+	} else {
+		foldersToCheck = changedDirs
+	}
+
+	for _, changedDir := range foldersToCheck {
+		dirs, err := repo.FindPathsRecursively(changedDir)
+		if err != nil {
+			log.Error("Error getting subfolders from DB", "path", changedDir, err)
+			continue
+		}
+		for _, d := range dirs {
+			d := filepath.Clean(d)
+			if _, ok := allDirs[d]; !ok {
+				deleted = append(deleted, d)
+			}
+		}
+	}
+
+	sort.Strings(deleted)
+	log.Debug(ctx, "Finished deleted folders check", "total", len(deleted), "elapsed", time.Since(start))
+	return deleted, nil
+}
+
+func (s *TagScanner2) processDeletedDir(ctx context.Context, dir string) error {
+	start := time.Now()
+
+	mfs, err := s.ds.MediaFile(ctx).FindAllByPath(dir)
+	if err != nil {
+		return err
+	}
+	for _, t := range mfs {
+		err = s.albumMap.update(t.AlbumID)
+		if err != nil {
+			return err
+		}
+		err = s.artistMap.update(t.AlbumArtistID)
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Info(ctx, "Finished processing deleted folder", "path", dir, "purged", len(mfs), "elapsed", time.Since(start))
+	c, err := s.ds.MediaFile(ctx).DeleteByPath(dir)
+	s.cnt.deleted += c
+	return err
+}
+
+func (s *TagScanner2) processChangedDir(ctx context.Context, dir string) error {
+	start := time.Now()
+
+	// Load folder's current tracks from DB into a map
+	currentTracks := map[string]model.MediaFile{}
+	ct, err := s.ds.MediaFile(ctx).FindAllByPath(dir)
+	if err != nil {
+		return err
+	}
+	for _, t := range ct {
+		currentTracks[t.Path] = t
+	}
+
+	// Load tracks FileInfo from the folder
+	files, err := LoadAllAudioFiles(dir)
+	if err != nil {
+		return err
+	}
+
+	// If no files to process, return
+	if len(files)+len(currentTracks) == 0 {
+		return nil
+	}
+
+	// If track from folder is newer than the one in DB, select for update/insert in DB
+	log.Trace(ctx, "Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(files))
+	var filesToUpdate []string
+	for filePath, info := range files {
+		c, ok := currentTracks[filePath]
+		if !ok {
+			filesToUpdate = append(filesToUpdate, filePath)
+			s.cnt.added++
+		}
+		if ok && info.ModTime().After(c.UpdatedAt) {
+			filesToUpdate = append(filesToUpdate, filePath)
+			s.cnt.updated++
+		}
+
+		// Force a refresh of the album and artist, to cater for cover art files
+		err = s.albumMap.update(c.AlbumID)
+		if err != nil {
+			return err
+		}
+		err = s.artistMap.update(c.AlbumArtistID)
+		if err != nil {
+			return err
+		}
+
+		// Remove it from currentTracks (the ones found in DB). After this loop any currentTracks remaining
+		// are considered gone from the music folder and will be deleted from DB
+		delete(currentTracks, filePath)
+	}
+
+	numUpdatedTracks := 0
+	numPurgedTracks := 0
+
+	if len(filesToUpdate) > 0 {
+		numUpdatedTracks, err = s.addOrUpdateTracksInDB(ctx, dir, filesToUpdate)
+		if err != nil {
+			return err
+		}
+	}
+
+	if len(currentTracks) > 0 {
+		numPurgedTracks, err = s.deleteOrphanSongs(ctx, dir, currentTracks)
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Info(ctx, "Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks,
+		"purged", numPurgedTracks, "elapsed", time.Since(start))
+	return nil
+}
+
+func (s *TagScanner2) deleteOrphanSongs(ctx context.Context, dir string, tracksToDelete map[string]model.MediaFile) (int, error) {
+	numPurgedTracks := 0
+
+	log.Debug(ctx, "Deleting orphan tracks from DB", "dir", dir, "numTracks", len(tracksToDelete))
+	// Remaining tracks from DB that are not in the folder are deleted
+	for _, ct := range tracksToDelete {
+		numPurgedTracks++
+		err := s.albumMap.update(ct.AlbumID)
+		if err != nil {
+			return 0, err
+		}
+		err = s.artistMap.update(ct.AlbumArtistID)
+		if err != nil {
+			return 0, err
+		}
+		if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil {
+			return 0, err
+		}
+		s.cnt.deleted++
+	}
+	return numPurgedTracks, nil
+}
+
+func (s *TagScanner2) addOrUpdateTracksInDB(ctx context.Context, dir string, filesToUpdate []string) (int, error) {
+	numUpdatedTracks := 0
+
+	log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "numFiles", len(filesToUpdate))
+	// Break the file list in chunks to avoid calling ffmpeg with too many parameters
+	chunks := utils.BreakUpStringSlice(filesToUpdate, filesBatchSize)
+	for _, chunk := range chunks {
+		// Load tracks Metadata from the folder
+		newTracks, err := s.loadTracks(chunk)
+		if err != nil {
+			return 0, err
+		}
+
+		// If track from folder is newer than the one in DB, update/insert in DB
+		log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "files", chunk, "numFiles", len(chunk))
+		for i := range newTracks {
+			n := newTracks[i]
+			err := s.ds.MediaFile(ctx).Put(&n)
+			if err != nil {
+				return 0, err
+			}
+			err = s.albumMap.update(n.AlbumID)
+			if err != nil {
+				return 0, err
+			}
+			err = s.artistMap.update(n.AlbumArtistID)
+			if err != nil {
+				return 0, err
+			}
+			numUpdatedTracks++
+		}
+	}
+	return numUpdatedTracks, nil
+}
+
+func (s *TagScanner2) loadTracks(filePaths []string) (model.MediaFiles, error) {
+	mds, err := ExtractAllMetadata(filePaths)
+	if err != nil {
+		return nil, err
+	}
+
+	var mfs model.MediaFiles
+	for _, md := range mds {
+		mf := s.mapper.toMediaFile(md)
+		mfs = append(mfs, mf)
+	}
+	return mfs, nil
+}
+
+func (s *TagScanner2) withAdminUser(ctx context.Context) context.Context {
+	u, err := s.ds.User(ctx).FindFirstAdmin()
+	if err != nil {
+		log.Warn(ctx, "No admin user found!", err)
+		u = &model.User{}
+	}
+
+	ctx = request.WithUsername(ctx, u.UserName)
+	return request.WithUser(ctx, *u)
+}

scanner/tag_scanner_test.go

@@ -18,4 +18,24 @@ var _ = Describe("TagScanner", func() {
 			Expect(sanitizeFieldForSorting("The Beatles")).To(Equal("Beatles"))
 		})
 	})
+
+	Describe("LoadAllAudioFiles", func() {
+		It("return all audio files from the folder", func() {
+			files, err := LoadAllAudioFiles("tests/fixtures")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(files).To(HaveLen(3))
+			Expect(files).To(HaveKey("tests/fixtures/test.ogg"))
+			Expect(files).To(HaveKey("tests/fixtures/test.mp3"))
+			Expect(files).To(HaveKey("tests/fixtures/01 Invisible (RED) Edit Version.mp3"))
+			Expect(files).ToNot(HaveKey("tests/fixtures/playlist.m3u"))
+		})
+		It("returns error if path does not exist", func() {
+			_, err := LoadAllAudioFiles("./INVALID/PATH")
+			Expect(err).To(HaveOccurred())
+		})
+
+		It("returns empty map if there are no audio files in path", func() {
+			Expect(LoadAllAudioFiles("tests/fixtures/empty_folder")).To(BeEmpty())
+		})
+	})
 })

server/app/app.go

@@ -8,10 +8,12 @@ import (
 
 	"github.com/deluan/navidrome/assets"
 	"github.com/deluan/navidrome/conf"
-	"github.com/deluan/navidrome/engine/auth"
+	"github.com/deluan/navidrome/core/auth"
+	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
 	"github.com/deluan/rest"
 	"github.com/go-chi/chi"
+	"github.com/go-chi/httprate"
 	"github.com/go-chi/jwtauth"
 )
 
@@ -35,7 +37,18 @@ func (app *Router) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 func (app *Router) routes(path string) http.Handler {
 	r := chi.NewRouter()
 
-	r.Post("/login", Login(app.ds))
+	if conf.Server.AuthRequestLimit > 0 {
+		log.Info("Login rate limit set", "requestLimit", conf.Server.AuthRequestLimit,
+			"windowLength", conf.Server.AuthWindowLength)
+
+		rateLimiter := httprate.LimitByIP(conf.Server.AuthRequestLimit, conf.Server.AuthWindowLength)
+		r.With(rateLimiter).Post("/login", Login(app.ds))
+	} else {
+		log.Warn("Login rate limit is disabled! Consider enabling it to be protected against brute-force attacks")
+
+		r.Post("/login", Login(app.ds))
+	}
+
 	r.Post("/createAdmin", CreateAdmin(app.ds))
 
 	r.Route("/api", func(r chi.Router) {
@@ -91,7 +104,7 @@ func (app *Router) RX(r chi.Router, pathPrefix string, constructor rest.Reposito
 type restHandler = func(rest.RepositoryConstructor, ...rest.Logger) http.HandlerFunc
 
 func (app *Router) addPlaylistTrackRoute(r chi.Router) {
-	// Add a middleware to capture the playlisId
+	// Add a middleware to capture the playlistId
 	wrapper := func(f restHandler) http.HandlerFunc {
 		return func(res http.ResponseWriter, req *http.Request) {
 			c := func(ctx context.Context) rest.Repository {
@@ -109,6 +122,9 @@ func (app *Router) addPlaylistTrackRoute(r chi.Router) {
 		r.Route("/{id}", func(r chi.Router) {
 			r.Use(UrlParams)
 			r.Get("/", wrapper(rest.Get))
+			r.Put("/", func(w http.ResponseWriter, r *http.Request) {
+				reorderItem(app.ds)(w, r)
+			})
 			r.Delete("/", func(w http.ResponseWriter, r *http.Request) {
 				deleteFromPlaylist(app.ds)(w, r)
 			})

server/app/auth.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/deluan/navidrome/consts"
-	"github.com/deluan/navidrome/engine/auth"
+	"github.com/deluan/navidrome/core/auth"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
 	"github.com/deluan/navidrome/model/request"

server/app/playlists.go

@@ -4,16 +4,13 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strconv"
 
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
 	"github.com/deluan/navidrome/utils"
 )
 
-type addTracksPayload struct {
-	Ids []string `json:"ids"`
-}
-
 func deleteFromPlaylist(ds model.DataStore) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		playlistId := utils.ParamString(r, ":playlistId")
@@ -38,6 +35,10 @@ func deleteFromPlaylist(ds model.DataStore) http.HandlerFunc {
 }
 
 func addToPlaylist(ds model.DataStore) http.HandlerFunc {
+	type addTracksPayload struct {
+		Ids []string `json:"ids"`
+	}
+
 	return func(w http.ResponseWriter, r *http.Request) {
 		playlistId := utils.ParamString(r, ":playlistId")
 		tracksRepo := ds.Playlist(r.Context()).Tracks(playlistId)
@@ -60,3 +61,40 @@ func addToPlaylist(ds model.DataStore) http.HandlerFunc {
 		}
 	}
 }
+
+func reorderItem(ds model.DataStore) http.HandlerFunc {
+	type reorderPayload struct {
+		InsertBefore string `json:"insert_before"`
+	}
+
+	return func(w http.ResponseWriter, r *http.Request) {
+		playlistId := utils.ParamString(r, ":playlistId")
+		id := utils.ParamInt(r, ":id", 0)
+		if id == 0 {
+			http.Error(w, "invalid id", http.StatusBadRequest)
+			return
+		}
+		tracksRepo := ds.Playlist(r.Context()).Tracks(playlistId)
+		var payload reorderPayload
+		err := json.NewDecoder(r.Body).Decode(&payload)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		newPos, err := strconv.Atoi(payload.InsertBefore)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+		err = tracksRepo.Reorder(id, newPos)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadRequest)
+			return
+		}
+
+		_, err = w.Write([]byte(fmt.Sprintf(`{"id":"%d"}`, id)))
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		}
+	}
+}

server/app/serve_index.go

@@ -11,10 +11,12 @@ import (
 	"github.com/deluan/navidrome/consts"
 	"github.com/deluan/navidrome/log"
 	"github.com/deluan/navidrome/model"
+	"github.com/microcosm-cc/bluemonday"
 )
 
 // Injects the config in the `index.html` template
 func ServeIndex(ds model.DataStore, fs http.FileSystem) http.HandlerFunc {
+	policy := bluemonday.UGCPolicy()
 	return func(w http.ResponseWriter, r *http.Request) {
 		c, err := ds.User(r.Context()).CountAll()
 		firstTime := c == 0 && err == nil
@@ -27,10 +29,11 @@ func ServeIndex(ds model.DataStore, fs http.FileSystem) http.HandlerFunc {
 		appConfig := map[string]interface{}{
 			"version":                 consts.Version(),
 			"firstTime":               firstTime,
-			"baseURL":                 strings.TrimSuffix(conf.Server.BaseURL, "/"),
-			"loginBackgroundURL":      conf.Server.UILoginBackgroundURL,
+			"baseURL":                 policy.Sanitize(strings.TrimSuffix(conf.Server.BaseURL, "/")),
+			"loginBackgroundURL":      policy.Sanitize(conf.Server.UILoginBackgroundURL),
+			"welcomeMessage":          policy.Sanitize(conf.Server.UIWelcomeMessage),
 			"enableTranscodingConfig": conf.Server.EnableTranscodingConfig,
-			"enablePlaylists":         conf.Server.DevEnableUIPlaylists,
+			"gaTrackingId":            conf.Server.GATrackingID,
 		}
 		j, err := json.Marshal(appConfig)
 		if err != nil {
@@ -39,6 +42,7 @@ func ServeIndex(ds model.DataStore, fs http.FileSystem) http.HandlerFunc {
 			log.Trace(r, "Injecting config in index.html", "config", string(j))
 		}
 
+		log.Debug("UI configuration", "appConfig", appConfig)
 		data := map[string]interface{}{
 			"AppConfig": string(j),
 			"Version":   consts.Version(),

server/app/serve_index_test.go

@@ -81,6 +81,39 @@ var _ = Describe("ServeIndex", func() {
 		Expect(config).To(HaveKeyWithValue("loginBackgroundURL", "my_background_url"))
 	})
 
+	It("sets the welcomeMessage", func() {
+		conf.Server.UIWelcomeMessage = "Hello"
+		r := httptest.NewRequest("GET", "/index.html", nil)
+		w := httptest.NewRecorder()
+
+		ServeIndex(ds, fs)(w, r)
+
+		config := extractAppConfig(w.Body.String())
+		Expect(config).To(HaveKeyWithValue("welcomeMessage", "Hello"))
+	})
+
+	It("sets the enableTranscodingConfig", func() {
+		conf.Server.EnableTranscodingConfig = true
+		r := httptest.NewRequest("GET", "/index.html", nil)
+		w := httptest.NewRecorder()
+
+		ServeIndex(ds, fs)(w, r)
+
+		config := extractAppConfig(w.Body.String())
+		Expect(config).To(HaveKeyWithValue("enableTranscodingConfig", true))
+	})
+
+	It("sets the gaTrackingId", func() {
+		conf.Server.GATrackingID = "UA-12345"
+		r := httptest.NewRequest("GET", "/index.html", nil)
+		w := httptest.NewRecorder()
+
+		ServeIndex(ds, fs)(w, r)
+
+		config := extractAppConfig(w.Body.String())
+		Expect(config).To(HaveKeyWithValue("gaTrackingId", "UA-12345"))
+	})
+
 	It("sets the version", func() {
 		r := httptest.NewRequest("GET", "/index.html", nil)
 		w := httptest.NewRecorder()