mirror of
https://github.com/navidrome/navidrome.git
synced 2026-09-08 19:52:49 -04:00
Compare commits
51
Commits
master
...
artwork-worker
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fc55e8bf16 | ||
|
|
9ce51cf575 | ||
|
|
b172ce4296 | ||
|
|
bba0eab3a5 | ||
|
|
f614850ff0 | ||
|
|
ba2290af6d | ||
|
|
55608b2d20 | ||
|
|
7713a6d6b2 | ||
|
|
bc30ce67c6 | ||
|
|
d6434b9929 | ||
|
|
b3526c0fba | ||
|
|
3d32157403 | ||
|
|
5482784bfc | ||
|
|
6afcb93a9b | ||
|
|
67f6d8aee8 | ||
|
|
87095fab08 | ||
|
|
c57496d50d | ||
|
|
0fbbd01357 | ||
|
|
bab9b5cd3a | ||
|
|
454fd24833 | ||
|
|
c01e9b3184 | ||
|
|
1ed8ebf9b0 | ||
|
|
ad38cd1d58 | ||
|
|
25a05fd017 | ||
|
|
57c64e386a | ||
|
|
d6fc829f84 | ||
|
|
e0655dc882 | ||
|
|
608db503a7 | ||
|
|
967de74bf7 | ||
|
|
2efa697e52 | ||
|
|
1f818e7633 | ||
|
|
c04c8ee02a | ||
|
|
ebbe533c6a | ||
|
|
0147cc59b1 | ||
|
|
034cd17498 | ||
|
|
8147f7c40b | ||
|
|
bf614e66ad | ||
|
|
623b7d6a6c | ||
|
|
6f7f9c6463 | ||
|
|
8fd7ef19f3 | ||
|
|
1041e45ca7 | ||
|
|
4f835437a9 | ||
|
|
b16ef725c9 | ||
|
|
3e7685adc2 | ||
|
|
db16b3de9a | ||
|
|
b72597821a | ||
|
|
fcff9c63e7 | ||
|
|
14dd57052e | ||
|
|
f926539c04 | ||
|
|
6cce65f759 | ||
|
|
7aacb01f4f |
No files matched your search
+58
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/navidrome/navidrome/core/artwork"
|
||||
"github.com/navidrome/navidrome/db"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
@@ -88,6 +89,9 @@ func runNavidrome(ctx context.Context) {
|
||||
g.Go(startInsightsCollector(ctx))
|
||||
g.Go(scheduleDBAnalyzer(ctx))
|
||||
g.Go(startPluginManager(ctx))
|
||||
artworkWorker := CreateArtworkWorker()
|
||||
g.Go(startArtworkWorker(ctx, artworkWorker))
|
||||
g.Go(scheduleArtworkHousekeeping(ctx, artworkWorker))
|
||||
g.Go(runInitialScan(ctx))
|
||||
if conf.Server.Scanner.Enabled {
|
||||
g.Go(startScanWatcher(ctx))
|
||||
@@ -344,6 +348,60 @@ func startPlaybackServer(ctx context.Context) func() error {
|
||||
}
|
||||
}
|
||||
|
||||
// startArtworkWorker starts the background artwork acquisition worker. It always
|
||||
// runs; the queue is simply empty until something enqueues work into it.
|
||||
func startArtworkWorker(ctx context.Context, worker *artwork.Worker) func() error {
|
||||
return func() error {
|
||||
log.Info(ctx, "Starting artwork worker")
|
||||
return worker.Run(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// scheduleArtworkHousekeeping runs the startup fingerprint backfill and registers the
|
||||
// recurring stale-absent recheck and prune jobs. Scan-triggered prune lands in a later phase.
|
||||
func scheduleArtworkHousekeeping(ctx context.Context, worker *artwork.Worker) func() error {
|
||||
return func() error {
|
||||
ds := CreateDataStore()
|
||||
schedulerInstance := scheduler.GetInstance()
|
||||
|
||||
if _, err := schedulerInstance.Add(consts.ArtworkStaleAbsentRecheckSchedule, func() {
|
||||
if err := artwork.EnqueueStaleAbsentAll(ctx, ds); err != nil {
|
||||
log.Error(ctx, "Error enqueueing stale artwork rechecks", err)
|
||||
}
|
||||
}); err != nil {
|
||||
log.Error(ctx, "Error scheduling artwork stale-absent recheck", err)
|
||||
}
|
||||
|
||||
if _, err := schedulerInstance.Add(consts.ArtworkPruneSchedule, func() {
|
||||
if err := worker.RunPrune(ctx); err != nil {
|
||||
log.Error(ctx, "Error running artwork prune", err)
|
||||
}
|
||||
}); err != nil {
|
||||
log.Error(ctx, "Error scheduling artwork prune", err)
|
||||
}
|
||||
|
||||
backfilled, err := artwork.Backfill(ctx, ds)
|
||||
if err != nil {
|
||||
log.Error(ctx, "Error running artwork backfill", err)
|
||||
return nil
|
||||
}
|
||||
if !backfilled {
|
||||
return nil
|
||||
}
|
||||
log.Info(ctx, "Artwork backfill enqueued, scheduling a follow-up prune")
|
||||
timer := time.NewTimer(consts.ArtworkPostBackfillPruneDelay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
if err := worker.RunPrune(ctx); err != nil {
|
||||
log.Error(ctx, "Error running post-backfill artwork prune", err)
|
||||
}
|
||||
case <-ctx.Done():
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// startPluginManager starts the plugin manager, if configured.
|
||||
func startPluginManager(ctx context.Context) func() error {
|
||||
return func() error {
|
||||
|
||||
@@ -236,6 +236,21 @@ func GetPlaybackServer() playback.PlaybackServer {
|
||||
return playbackServer
|
||||
}
|
||||
|
||||
func CreateArtworkWorker() *artwork.Worker {
|
||||
sqlDB := db.Db()
|
||||
dataStore := persistence.New(sqlDB)
|
||||
imageStore := artwork.ProvideImageStore()
|
||||
broker := events.GetBroker()
|
||||
metricsMetrics := metrics.GetPrometheusInstance(dataStore)
|
||||
manager := plugins.GetManager(dataStore, broker, metricsMetrics)
|
||||
agentsAgents := agents.GetAgents(dataStore, manager)
|
||||
matcherMatcher := matcher.New(dataStore)
|
||||
provider := external.NewProvider(dataStore, agentsAgents, matcherMatcher)
|
||||
fFmpeg := ffmpeg.New()
|
||||
worker := artwork.NewWorker(dataStore, imageStore, provider, fFmpeg)
|
||||
return worker
|
||||
}
|
||||
|
||||
func getPluginManager() *plugins.Manager {
|
||||
sqlDB := db.Db()
|
||||
dataStore := persistence.New(sqlDB)
|
||||
|
||||
@@ -136,6 +136,12 @@ func GetPlaybackServer() playback.PlaybackServer {
|
||||
))
|
||||
}
|
||||
|
||||
func CreateArtworkWorker() *artwork.Worker {
|
||||
panic(wire.Build(
|
||||
allProviders,
|
||||
))
|
||||
}
|
||||
|
||||
func getPluginManager() *plugins.Manager {
|
||||
panic(wire.Build(
|
||||
allProviders,
|
||||
|
||||
@@ -57,6 +57,8 @@ type configOptions struct {
|
||||
ImageCacheSize string
|
||||
AlbumPlayCountMode string
|
||||
EnableArtworkPrecache bool
|
||||
ArtworkWorkerConcurrency int
|
||||
ArtworkExternalMaxRPS int
|
||||
AutoImportPlaylists bool
|
||||
DefaultPlaylistPublicVisibility bool
|
||||
PlaylistsPath string
|
||||
@@ -346,6 +348,8 @@ func Load(noConfigDump bool) {
|
||||
mapDeprecatedOption("CoverJpegQuality", "CoverArtQuality")
|
||||
mapDeprecatedOption("SimilarSongsMatchThreshold", "Matcher.FuzzyThreshold")
|
||||
mapDeprecatedOption("EnableTranscodingCancellation", "Transcoding.EnableCancellation")
|
||||
mapDeprecatedOption("DevArtworkWorkerConcurrency", "ArtworkWorkerConcurrency")
|
||||
mapDeprecatedOption("DevArtworkExternalRPS", "ArtworkExternalMaxRPS")
|
||||
|
||||
err := viper.Unmarshal(&Server, viper.DecodeHook(
|
||||
mapstructure.ComposeDecodeHookFunc(
|
||||
@@ -900,6 +904,8 @@ func setViperDefaults() {
|
||||
viper.SetDefault("devartworkthrottlebackloglimit", consts.RequestThrottleBacklogLimit)
|
||||
viper.SetDefault("devartworkthrottlebacklogtimeout", consts.RequestThrottleBacklogTimeout)
|
||||
viper.SetDefault("devartworkthrottlebuffered", true)
|
||||
viper.SetDefault("artworkworkerconcurrency", 4)
|
||||
viper.SetDefault("artworkexternalmaxrps", 2)
|
||||
viper.SetDefault("devartistinfotimetolive", consts.ArtistInfoTimeToLive)
|
||||
viper.SetDefault("devalbuminfotimetolive", consts.AlbumInfoTimeToLive)
|
||||
viper.SetDefault("devexternalscanner", true)
|
||||
|
||||
@@ -35,6 +35,10 @@ const (
|
||||
DBAnalyzeCheckSchedule = "@every 30m"
|
||||
DBAnalyzeMaxAge = 24 * time.Hour
|
||||
|
||||
ArtworkStaleAbsentRecheckSchedule = "@every 1h"
|
||||
ArtworkPruneSchedule = "@daily"
|
||||
ArtworkPostBackfillPruneDelay = 10 * time.Minute
|
||||
|
||||
// DefaultEncryptionKey This is the encryption key used if none is specified in the `PasswordEncryptionKey` option
|
||||
// Never ever change this! Or it will break all Navidrome installations that don't set the config option
|
||||
DefaultEncryptionKey = "just for obfuscation"
|
||||
|
||||
@@ -15,9 +15,24 @@ import (
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"go.uber.org/goleak"
|
||||
)
|
||||
|
||||
func TestArtwork(t *testing.T) {
|
||||
// Runs unconditionally: the two leaks below are pre-existing and out of this
|
||||
// package's control, so they're ignored by exact top-function instead.
|
||||
defer goleak.VerifyNone(t,
|
||||
goleak.IgnoreTopFunction("github.com/onsi/ginkgo/v2/internal/interrupt_handler.(*InterruptHandler).registerForInterrupts.func2"),
|
||||
// notify's own init() starts a singleton tree the moment it's imported (via
|
||||
// core/storage/local or plugins); recursive on darwin, nonrecursive on linux.
|
||||
goleak.IgnoreTopFunction("github.com/rjeczalik/notify.(*recursiveTree).dispatch"),
|
||||
goleak.IgnoreTopFunction("github.com/rjeczalik/notify.(*nonrecursiveTree).dispatch"),
|
||||
goleak.IgnoreTopFunction("github.com/rjeczalik/notify.(*nonrecursiveTree).internal"),
|
||||
// The old cache_warmer.go starts a goroutine per NewCacheWarmer call with
|
||||
// no shutdown path (dark-launch target for Phase 2, not touched here).
|
||||
goleak.IgnoreTopFunction("github.com/navidrome/navidrome/core/artwork.(*cacheWarmer).waitSignal"),
|
||||
)
|
||||
|
||||
tests.Init(t, false)
|
||||
log.SetLevel(log.LevelFatal)
|
||||
RegisterFailHandler(Fail)
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
// Package blurhash implements the blurhash encoding algorithm (https://github.com/woltapp/blurhash),
|
||||
// matching Jellyfin's parameters so clients tuned against Jellyfin see equivalent hashes.
|
||||
package blurhash
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"image"
|
||||
"image/draw"
|
||||
"math"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
xdraw "golang.org/x/image/draw"
|
||||
)
|
||||
|
||||
const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#$%*+,-.:;=?@[]^_{|}~"
|
||||
|
||||
// maxInputSize matches Jellyfin: larger inputs are slower with no visually discernible difference.
|
||||
const maxInputSize = 128
|
||||
|
||||
// Components picks x/y component counts for an image, targeting ~16 near-square tiles (Jellyfin's formula).
|
||||
func Components(width, height int) (int, int) {
|
||||
if width <= 0 || height <= 0 {
|
||||
return 0, 0
|
||||
}
|
||||
xf := math.Sqrt(16.0 * float64(width) / float64(height))
|
||||
yf := xf * float64(height) / float64(width)
|
||||
return min(int(xf)+1, 9), min(int(yf)+1, 9)
|
||||
}
|
||||
|
||||
// Encode returns the blurhash of img using xComp x yComp components.
|
||||
func Encode(img image.Image, xComp, yComp int) (string, error) {
|
||||
if xComp < 1 || xComp > 9 || yComp < 1 || yComp > 9 {
|
||||
return "", errors.New("blurhash: components must be between 1 and 9")
|
||||
}
|
||||
rgba := toRGBA(downscale(img))
|
||||
bounds := rgba.Bounds()
|
||||
w, h := bounds.Dx(), bounds.Dy()
|
||||
if w == 0 || h == 0 {
|
||||
return "", errors.New("blurhash: empty image")
|
||||
}
|
||||
|
||||
cosX := make([][]float64, xComp)
|
||||
for i := range cosX {
|
||||
cosX[i] = make([]float64, w)
|
||||
for x := range cosX[i] {
|
||||
cosX[i][x] = math.Cos(math.Pi * float64(i) * float64(x) / float64(w))
|
||||
}
|
||||
}
|
||||
cosY := make([][]float64, yComp)
|
||||
for j := range cosY {
|
||||
cosY[j] = make([]float64, h)
|
||||
for y := range cosY[j] {
|
||||
cosY[j][y] = math.Cos(math.Pi * float64(j) * float64(y) / float64(h))
|
||||
}
|
||||
}
|
||||
|
||||
lin := srgbToLinearTable()
|
||||
factors := make([][3]float64, xComp*yComp)
|
||||
for y := 0; y < h; y++ {
|
||||
row := rgba.Pix[y*rgba.Stride:]
|
||||
for x := 0; x < w; x++ {
|
||||
p := x * 4
|
||||
lr, lg, lb := lin[row[p]], lin[row[p+1]], lin[row[p+2]]
|
||||
for j := 0; j < yComp; j++ {
|
||||
for i := 0; i < xComp; i++ {
|
||||
basis := cosX[i][x] * cosY[j][y]
|
||||
f := &factors[j*xComp+i]
|
||||
f[0] += basis * lr
|
||||
f[1] += basis * lg
|
||||
f[2] += basis * lb
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for idx := range factors {
|
||||
norm := 2.0
|
||||
if idx == 0 {
|
||||
norm = 1.0
|
||||
}
|
||||
scale := norm / float64(w*h)
|
||||
factors[idx][0] *= scale
|
||||
factors[idx][1] *= scale
|
||||
factors[idx][2] *= scale
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString(Encode83((xComp-1)+(yComp-1)*9, 1))
|
||||
|
||||
ac := factors[1:]
|
||||
maxVal := 1.0
|
||||
if len(ac) > 0 {
|
||||
actualMax := 0.0
|
||||
for _, f := range ac {
|
||||
actualMax = max(actualMax, math.Abs(f[0]), math.Abs(f[1]), math.Abs(f[2]))
|
||||
}
|
||||
quantMax := int(math.Max(0, math.Min(82, math.Floor(actualMax*166-0.5))))
|
||||
maxVal = float64(quantMax+1) / 166
|
||||
sb.WriteString(Encode83(quantMax, 1))
|
||||
} else {
|
||||
sb.WriteString(Encode83(0, 1))
|
||||
}
|
||||
|
||||
dc := factors[0]
|
||||
sb.WriteString(Encode83(linearToSRGB(dc[0])<<16|linearToSRGB(dc[1])<<8|linearToSRGB(dc[2]), 4))
|
||||
for _, f := range ac {
|
||||
sb.WriteString(Encode83(quantAC(f[0], maxVal)*19*19+quantAC(f[1], maxVal)*19+quantAC(f[2], maxVal), 2))
|
||||
}
|
||||
return sb.String(), nil
|
||||
}
|
||||
|
||||
// toRGBA gives the pixel loop direct Pix access, avoiding a per-pixel allocation through the
|
||||
// image.At interface (~16k allocs per encode).
|
||||
func toRGBA(img image.Image) *image.RGBA {
|
||||
if rgba, ok := img.(*image.RGBA); ok {
|
||||
return rgba
|
||||
}
|
||||
b := img.Bounds()
|
||||
dst := image.NewRGBA(image.Rect(0, 0, b.Dx(), b.Dy()))
|
||||
draw.Draw(dst, dst.Bounds(), img, b.Min, draw.Src)
|
||||
return dst
|
||||
}
|
||||
|
||||
var srgbToLinearTable = sync.OnceValue(func() *[256]float64 {
|
||||
var t [256]float64
|
||||
for i := range t {
|
||||
t[i] = srgbToLinear(i)
|
||||
}
|
||||
return &t
|
||||
})
|
||||
|
||||
func downscale(img image.Image) image.Image {
|
||||
b := img.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
if w <= maxInputSize && h <= maxInputSize {
|
||||
return img
|
||||
}
|
||||
scale := float64(maxInputSize) / float64(max(w, h))
|
||||
dst := image.NewRGBA(image.Rect(0, 0, max(1, int(float64(w)*scale)), max(1, int(float64(h)*scale))))
|
||||
xdraw.ApproxBiLinear.Scale(dst, dst.Bounds(), img, b, draw.Src, nil)
|
||||
return dst
|
||||
}
|
||||
|
||||
func quantAC(v, maxVal float64) int {
|
||||
return int(math.Max(0, math.Min(18, math.Floor(signPow(v/maxVal, 0.5)*9+9.5))))
|
||||
}
|
||||
|
||||
func signPow(v, exp float64) float64 {
|
||||
return math.Copysign(math.Pow(math.Abs(v), exp), v)
|
||||
}
|
||||
|
||||
func srgbToLinear(v int) float64 {
|
||||
f := float64(v) / 255
|
||||
if f <= 0.04045 {
|
||||
return f / 12.92
|
||||
}
|
||||
return math.Pow((f+0.055)/1.055, 2.4)
|
||||
}
|
||||
|
||||
func linearToSRGB(v float64) int {
|
||||
v = math.Min(math.Max(0, v), 1)
|
||||
if v <= 0.0031308 {
|
||||
return int(v*12.92*255 + 0.5)
|
||||
}
|
||||
return int((1.055*math.Pow(v, 1/2.4)-0.055)*255 + 0.5)
|
||||
}
|
||||
|
||||
// Encode83 encodes value as a fixed-width, big-endian base83 string of the given length, using the
|
||||
// blurhash spec's alphabet.
|
||||
func Encode83(value, length int) string {
|
||||
b := make([]byte, length)
|
||||
for i := length - 1; i >= 0; i-- {
|
||||
b[i] = alphabet[value%83]
|
||||
value /= 83
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package blurhash_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"testing"
|
||||
|
||||
"github.com/navidrome/navidrome/core/artwork/blurhash"
|
||||
)
|
||||
|
||||
// benchImage builds a deterministic gradient so runs are comparable across revisions.
|
||||
func benchImage(size int) image.Image {
|
||||
img := image.NewNRGBA(image.Rect(0, 0, size, size))
|
||||
for y := 0; y < size; y++ {
|
||||
for x := 0; x < size; x++ {
|
||||
img.SetNRGBA(x, y, color.NRGBA{
|
||||
R: uint8(255 * x / size),
|
||||
G: uint8(255 * y / size),
|
||||
B: uint8((x + y) * 255 / (2 * size)),
|
||||
A: 255,
|
||||
})
|
||||
}
|
||||
}
|
||||
return img
|
||||
}
|
||||
|
||||
func BenchmarkEncode(b *testing.B) {
|
||||
for _, size := range []int{100, 300, 600, 900, 1200, 1500} {
|
||||
img := benchImage(size)
|
||||
x, y := blurhash.Components(size, size)
|
||||
b.Run(fmt.Sprintf("%dx%d", size, size), func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for range b.N {
|
||||
if _, err := blurhash.Encode(img, x, y); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package blurhash_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestBlurHash(t *testing.T) {
|
||||
tests.Init(t, false)
|
||||
log.SetLevel(log.LevelFatal)
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "BlurHash Suite")
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package blurhash_test
|
||||
|
||||
import (
|
||||
"image"
|
||||
"image/color"
|
||||
"strings"
|
||||
|
||||
"github.com/navidrome/navidrome/core/artwork/blurhash"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz#$%*+,-.:;=?@[]^_{|}~"
|
||||
|
||||
func decode83(s string) int {
|
||||
v := 0
|
||||
for _, c := range s {
|
||||
v = v*83 + strings.IndexRune(alphabet, c)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func solidImage(w, h int, c color.NRGBA) image.Image {
|
||||
img := image.NewNRGBA(image.Rect(0, 0, w, h))
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
img.SetNRGBA(x, y, c)
|
||||
}
|
||||
}
|
||||
return img
|
||||
}
|
||||
|
||||
func gradientImage(w, h int) image.Image {
|
||||
img := image.NewNRGBA(image.Rect(0, 0, w, h))
|
||||
for y := 0; y < h; y++ {
|
||||
for x := 0; x < w; x++ {
|
||||
img.SetNRGBA(x, y, color.NRGBA{R: uint8(255 * x / w), G: uint8(255 * y / h), B: 128, A: 255})
|
||||
}
|
||||
}
|
||||
return img
|
||||
}
|
||||
|
||||
var _ = Describe("Components", func() {
|
||||
DescribeTable("derives component counts from aspect ratio (Jellyfin formula)",
|
||||
func(w, h, expectedX, expectedY int) {
|
||||
x, y := blurhash.Components(w, h)
|
||||
Expect(x).To(Equal(expectedX))
|
||||
Expect(y).To(Equal(expectedY))
|
||||
},
|
||||
Entry("square album art", 600, 600, 5, 5),
|
||||
Entry("small square", 1, 1, 5, 5),
|
||||
Entry("landscape 16:9", 1920, 1080, 6, 4),
|
||||
Entry("portrait 9:16", 1080, 1920, 4, 6),
|
||||
Entry("extreme landscape capped at 9", 10000, 100, 9, 1),
|
||||
Entry("zero width", 0, 600, 0, 0),
|
||||
Entry("zero height", 600, 0, 0, 0),
|
||||
)
|
||||
})
|
||||
|
||||
var _ = Describe("Encode", func() {
|
||||
It("rejects out-of-range components", func() {
|
||||
_, err := blurhash.Encode(solidImage(8, 8, color.NRGBA{A: 255}), 0, 5)
|
||||
Expect(err).To(HaveOccurred())
|
||||
_, err = blurhash.Encode(solidImage(8, 8, color.NRGBA{A: 255}), 5, 10)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
|
||||
It("produces the spec-mandated length", func() {
|
||||
// 1 (size flag) + 1 (max AC) + 4 (DC) + 2 per AC component
|
||||
h, err := blurhash.Encode(solidImage(8, 8, color.NRGBA{R: 10, G: 20, B: 30, A: 255}), 4, 3)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(h).To(HaveLen(4 + 2 + 2*(4*3-1)))
|
||||
})
|
||||
|
||||
It("encodes the size flag as the first character", func() {
|
||||
h, err := blurhash.Encode(solidImage(8, 8, color.NRGBA{A: 255}), 4, 3)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(decode83(h[:1])).To(Equal((4 - 1) + (3-1)*9))
|
||||
})
|
||||
|
||||
It("stores the average color in the DC component", func() {
|
||||
h, err := blurhash.Encode(solidImage(16, 16, color.NRGBA{R: 200, G: 100, B: 50, A: 255}), 4, 3)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
dc := decode83(h[2:6])
|
||||
Expect(dc >> 16).To(BeNumerically("~", 200, 1))
|
||||
Expect((dc >> 8) & 0xFF).To(BeNumerically("~", 100, 1))
|
||||
Expect(dc & 0xFF).To(BeNumerically("~", 50, 1))
|
||||
})
|
||||
|
||||
It("is deterministic", func() {
|
||||
img := gradientImage(64, 64)
|
||||
h1, err1 := blurhash.Encode(img, 5, 5)
|
||||
h2, err2 := blurhash.Encode(img, 5, 5)
|
||||
Expect(err1).ToNot(HaveOccurred())
|
||||
Expect(err2).ToNot(HaveOccurred())
|
||||
Expect(h1).To(Equal(h2))
|
||||
})
|
||||
|
||||
It("produces different hashes for different images", func() {
|
||||
h1, _ := blurhash.Encode(solidImage(16, 16, color.NRGBA{R: 255, A: 255}), 4, 4)
|
||||
h2, _ := blurhash.Encode(gradientImage(16, 16), 4, 4)
|
||||
Expect(h1).ToNot(Equal(h2))
|
||||
})
|
||||
|
||||
It("downscales large images internally without changing the result materially", func() {
|
||||
// A 1000px solid image must encode fine and carry the same DC as its small version.
|
||||
big, err := blurhash.Encode(solidImage(1000, 1000, color.NRGBA{R: 60, G: 120, B: 180, A: 255}), 5, 5)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
small, err := blurhash.Encode(solidImage(16, 16, color.NRGBA{R: 60, G: 120, B: 180, A: 255}), 5, 5)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(big[2:6]).To(Equal(small[2:6]))
|
||||
})
|
||||
})
|
||||
@@ -177,6 +177,9 @@ func (n *noopProvider) TopSongs(context.Context, string, int) (model.MediaFiles,
|
||||
func (n *noopProvider) ArtistImage(context.Context, string) (*url.URL, error) {
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
func (n *noopProvider) ArtistImageResult(context.Context, string) (*url.URL, error) {
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
func (n *noopProvider) AlbumImage(context.Context, string) (*url.URL, error) {
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/navidrome/navidrome/core/auth"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
)
|
||||
|
||||
// FingerprintPropertyKey is the model.PropertyRepository key Backfill compares against
|
||||
// to detect artwork-affecting config changes across restarts.
|
||||
const FingerprintPropertyKey = "artwork.fingerprint"
|
||||
|
||||
// staleAbsentAge is how old an absent resolution must be before the recheck job retries it.
|
||||
const staleAbsentAge = 24 * time.Hour
|
||||
|
||||
// staleAbsentKinds are the item kinds eligible for the periodic stale-absent recheck.
|
||||
var staleAbsentKinds = []string{"ar", "al", "pl", "ra"}
|
||||
|
||||
// Fingerprint summarizes the config knobs that affect artwork resolution outcomes; a
|
||||
// change means previously resolved (or absent) state may no longer be correct.
|
||||
func Fingerprint() string {
|
||||
raw := fmt.Sprintf("%s|%s|%s|%s|%t|%t|%s",
|
||||
conf.Server.CoverArtPriority, conf.Server.ArtistArtPriority, conf.Server.ArtistImageFolder,
|
||||
conf.Server.Agents, conf.Server.EnableExternalServices, conf.Server.EnableM3UExternalAlbumArt, consts.Version)
|
||||
sum := md5.Sum([]byte(raw)) //nolint:gosec // fingerprint, not security-sensitive
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// Backfill enqueues artwork resolution for every entity when the config fingerprint changed
|
||||
// (or was never stored), artists first so those pages resolve before the larger backlog.
|
||||
func Backfill(ctx context.Context, ds model.DataStore) (bool, error) {
|
||||
ctx = auth.WithAdminUser(ctx, ds)
|
||||
current := Fingerprint()
|
||||
props := ds.Property(ctx)
|
||||
stored, err := props.DefaultGet(FingerprintPropertyKey, "")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if stored == current {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Artists first: few entities, most external-dependent, so they get queue headstart.
|
||||
kinds := []struct {
|
||||
kind string
|
||||
fetch func() ([]string, error)
|
||||
}{
|
||||
{"ar", func() ([]string, error) { return ds.Artist(ctx).GetAllIDs() }},
|
||||
{"al", func() ([]string, error) { return ds.Album(ctx).GetAllIDs() }},
|
||||
{"pl", func() ([]string, error) { return ds.Playlist(ctx).GetAllIDs() }},
|
||||
{"ra", func() ([]string, error) { return ds.Radio(ctx).GetAllIDs() }},
|
||||
}
|
||||
for _, k := range kinds {
|
||||
ids, err := k.fetch()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := enqueueBackfillKind(ctx, ds, k.kind, ids); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
if err := props.Put(FingerprintPropertyKey, current); err != nil {
|
||||
return false, err
|
||||
}
|
||||
log.Info(ctx, "Artwork: config fingerprint changed, backfill enqueued")
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func enqueueBackfillKind(ctx context.Context, ds model.DataStore, kind string, ids []string) error {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
items := make([]model.ArtworkQueueItem, len(ids))
|
||||
for i, id := range ids {
|
||||
items[i] = model.ArtworkQueueItem{
|
||||
ItemKind: kind, ItemID: id, ImageType: model.ImageTypePrimary, Priority: model.ArtworkPriorityBackfill,
|
||||
}
|
||||
}
|
||||
return ds.ArtworkQueue(ctx).Enqueue(items...)
|
||||
}
|
||||
|
||||
// EnqueueStaleAbsentAll requeues absent-state entries older than staleAbsentAge, across
|
||||
// every artwork-bearing kind, for the periodic recheck job.
|
||||
func EnqueueStaleAbsentAll(ctx context.Context, ds model.DataStore) error {
|
||||
cutoff := time.Now().Add(-staleAbsentAge)
|
||||
queue := ds.ArtworkQueue(ctx)
|
||||
for _, kind := range staleAbsentKinds {
|
||||
if _, err := queue.EnqueueStaleAbsent(kind, cutoff); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/conf/configtest"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/model/request"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
// visibilityPlaylistDS models playlist_repository's userFilter: a private playlist is only
|
||||
// visible when the ctx carries an admin, so headless work must wrap ctx with one first.
|
||||
type visibilityPlaylistDS struct {
|
||||
*tests.MockDataStore
|
||||
private model.Playlist
|
||||
tracks model.PlaylistTrackRepository
|
||||
}
|
||||
|
||||
func (v *visibilityPlaylistDS) Playlist(ctx context.Context) model.PlaylistRepository {
|
||||
repo := tests.CreateMockPlaylistRepo()
|
||||
repo.TracksRepo = v.tracks
|
||||
if u, ok := request.UserFrom(ctx); ok && u.IsAdmin {
|
||||
repo.SetData(model.Playlists{v.private})
|
||||
}
|
||||
return repo
|
||||
}
|
||||
|
||||
func adminUserRepo() *tests.MockedUserRepo {
|
||||
repo := tests.CreateMockUserRepo()
|
||||
Expect(repo.Put(&model.User{ID: "admin", UserName: "admin", IsAdmin: true})).To(Succeed())
|
||||
return repo
|
||||
}
|
||||
|
||||
// orderTrackingQueueRepo records the item kind of each Enqueue call, so tests can
|
||||
// assert phase ordering (artists-first) that same-priority timestamps can't guarantee.
|
||||
type orderTrackingQueueRepo struct {
|
||||
*tests.MockArtworkQueueRepo
|
||||
callKinds []string
|
||||
}
|
||||
|
||||
func (o *orderTrackingQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error {
|
||||
if len(items) > 0 {
|
||||
o.callKinds = append(o.callKinds, items[0].ItemKind)
|
||||
}
|
||||
return o.MockArtworkQueueRepo.Enqueue(items...)
|
||||
}
|
||||
|
||||
var _ = Describe("Housekeeping", func() {
|
||||
var (
|
||||
ctx context.Context
|
||||
ds *tests.MockDataStore
|
||||
queueRepo *orderTrackingQueueRepo
|
||||
propRepo *tests.MockedPropertyRepo
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
ctx = context.Background()
|
||||
conf.Server.CoverArtPriority = "embedded, folder"
|
||||
conf.Server.ArtistArtPriority = "artist.jpg"
|
||||
conf.Server.Agents = "spotify"
|
||||
conf.Server.EnableExternalServices = true
|
||||
|
||||
queueRepo = &orderTrackingQueueRepo{MockArtworkQueueRepo: tests.CreateMockArtworkQueueRepo()}
|
||||
propRepo = &tests.MockedPropertyRepo{}
|
||||
ds = &tests.MockDataStore{MockedArtworkQueue: queueRepo, MockedProperty: propRepo}
|
||||
})
|
||||
|
||||
seedEntities := func() {
|
||||
artistRepo := tests.CreateMockArtistRepo()
|
||||
artistRepo.SetData(model.Artists{{ID: "ar1"}, {ID: "ar2"}})
|
||||
ds.MockedArtist = artistRepo
|
||||
|
||||
albumRepo := tests.CreateMockAlbumRepo()
|
||||
albumRepo.SetData(model.Albums{{ID: "al1"}})
|
||||
ds.MockedAlbum = albumRepo
|
||||
|
||||
playlistRepo := tests.CreateMockPlaylistRepo()
|
||||
playlistRepo.SetData(model.Playlists{{ID: "pl1"}})
|
||||
ds.MockedPlaylist = playlistRepo
|
||||
|
||||
radioRepo := tests.CreateMockedRadioRepo()
|
||||
radioRepo.All = model.Radios{{ID: "ra1"}}
|
||||
ds.MockedRadio = radioRepo
|
||||
}
|
||||
|
||||
Describe("Fingerprint", func() {
|
||||
It("changes when a fingerprint-affecting config value changes", func() {
|
||||
f1 := Fingerprint()
|
||||
conf.Server.CoverArtPriority = "folder, embedded"
|
||||
f2 := Fingerprint()
|
||||
Expect(f1).NotTo(Equal(f2))
|
||||
})
|
||||
|
||||
It("changes when ArtistImageFolder changes", func() {
|
||||
conf.Server.ArtistImageFolder = "/before"
|
||||
f1 := Fingerprint()
|
||||
conf.Server.ArtistImageFolder = "/after"
|
||||
Expect(Fingerprint()).NotTo(Equal(f1))
|
||||
})
|
||||
|
||||
It("changes when EnableM3UExternalAlbumArt is toggled", func() {
|
||||
conf.Server.EnableM3UExternalAlbumArt = false
|
||||
f1 := Fingerprint()
|
||||
conf.Server.EnableM3UExternalAlbumArt = true
|
||||
Expect(Fingerprint()).NotTo(Equal(f1))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Backfill", func() {
|
||||
It("enqueues nothing and returns false when the stored fingerprint matches", func() {
|
||||
seedEntities()
|
||||
Expect(propRepo.Put(FingerprintPropertyKey, Fingerprint())).To(Succeed())
|
||||
|
||||
did, err := Backfill(ctx, ds)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(did).To(BeFalse())
|
||||
|
||||
count, err := queueRepo.Count()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(BeZero())
|
||||
})
|
||||
|
||||
It("runs the backfill when no fingerprint was ever stored", func() {
|
||||
seedEntities()
|
||||
|
||||
did, err := Backfill(ctx, ds)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(did).To(BeTrue())
|
||||
|
||||
count, err := queueRepo.Count()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(Equal(int64(5))) // 2 artists + 1 album + 1 playlist + 1 radio
|
||||
|
||||
stored, err := propRepo.Get(FingerprintPropertyKey)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(stored).To(Equal(Fingerprint()))
|
||||
})
|
||||
|
||||
It("enqueues a private playlist by resolving it under an admin context", func() {
|
||||
ds.MockedUser = adminUserRepo()
|
||||
vds := &visibilityPlaylistDS{
|
||||
MockDataStore: ds,
|
||||
private: model.Playlist{ID: "plPrivate", OwnerID: "admin"},
|
||||
tracks: &tests.MockPlaylistTrackRepo{},
|
||||
}
|
||||
|
||||
did, err := Backfill(ctx, vds)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(did).To(BeTrue())
|
||||
Expect(findQueued(queueRepo.MockArtworkQueueRepo, "pl", "plPrivate")).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("enqueues artists before albums/playlists/radios, all at Backfill priority", func() {
|
||||
seedEntities()
|
||||
Expect(propRepo.Put(FingerprintPropertyKey, "stale-fingerprint")).To(Succeed())
|
||||
|
||||
did, err := Backfill(ctx, ds)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(did).To(BeTrue())
|
||||
|
||||
Expect(queueRepo.callKinds).ToNot(BeEmpty())
|
||||
artistCallIdx := -1
|
||||
for i, k := range queueRepo.callKinds {
|
||||
if k == "ar" {
|
||||
artistCallIdx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
Expect(artistCallIdx).To(Equal(0), "artists must be the first Enqueue call")
|
||||
for i, k := range queueRepo.callKinds {
|
||||
if k != "ar" {
|
||||
Expect(i).To(BeNumerically(">", artistCallIdx))
|
||||
}
|
||||
}
|
||||
|
||||
for _, it := range queueRepo.Data {
|
||||
Expect(it.Priority).To(Equal(model.ArtworkPriorityBackfill))
|
||||
Expect(it.ItemKind).To(BeElementOf("ar", "al", "pl", "ra"))
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
Describe("EnqueueStaleAbsentAll", func() {
|
||||
var artRepo *tests.MockArtworkRepo
|
||||
|
||||
BeforeEach(func() {
|
||||
artRepo = tests.CreateMockArtworkRepo()
|
||||
ds.MockedArtwork = artRepo
|
||||
queueRepo.ItemArtworkSource = artRepo
|
||||
})
|
||||
|
||||
It("enqueues only absent entries older than the recheck window, across all kinds", func() {
|
||||
old := time.Now().Add(-48 * time.Hour)
|
||||
recent := time.Now().Add(-time.Hour)
|
||||
|
||||
artRepo.ItemData["ar-stale"] = model.ItemArtwork{ItemKind: "ar", ItemID: "ar1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}
|
||||
artRepo.ItemData["al-stale"] = model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}
|
||||
artRepo.ItemData["pl-stale"] = model.ItemArtwork{ItemKind: "pl", ItemID: "pl1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}
|
||||
artRepo.ItemData["ra-stale"] = model.ItemArtwork{ItemKind: "ra", ItemID: "ra1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old}
|
||||
// Not stale: too recent.
|
||||
artRepo.ItemData["ar-recent"] = model.ItemArtwork{ItemKind: "ar", ItemID: "ar2", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: recent}
|
||||
// Not absent: has a resolved hash.
|
||||
artRepo.ItemData["al-resolved"] = model.ItemArtwork{ItemKind: "al", ItemID: "al2", ImageType: model.ImageTypePrimary, Hash: "somehash", AttemptedAt: old}
|
||||
|
||||
err := EnqueueStaleAbsentAll(ctx, ds)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
Expect(queueRepo.Data).To(HaveLen(4))
|
||||
for _, it := range queueRepo.Data {
|
||||
Expect(it.Priority).To(Equal(model.ArtworkPriorityRecheck))
|
||||
}
|
||||
Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar1")).ToNot(BeNil())
|
||||
Expect(findQueued(queueRepo.MockArtworkQueueRepo, "al", "al1")).ToNot(BeNil())
|
||||
Expect(findQueued(queueRepo.MockArtworkQueueRepo, "pl", "pl1")).ToNot(BeNil())
|
||||
Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ra", "ra1")).ToNot(BeNil())
|
||||
Expect(findQueued(queueRepo.MockArtworkQueueRepo, "ar", "ar2")).To(BeNil())
|
||||
Expect(findQueued(queueRepo.MockArtworkQueueRepo, "al", "al2")).To(BeNil())
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,172 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
func HashImage(r io.Reader) (string, error) {
|
||||
d := xxh3.New()
|
||||
if _, err := io.Copy(d, r); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%016x", d.Sum64()), nil
|
||||
}
|
||||
|
||||
// ImageStore is the content-addressed store for artwork images that have no
|
||||
// library file backing them (external downloads, embedded extractions, generated).
|
||||
type ImageStore struct {
|
||||
root string
|
||||
}
|
||||
|
||||
func NewImageStore(rootDir string) *ImageStore {
|
||||
return &ImageStore{root: rootDir}
|
||||
}
|
||||
|
||||
// ProvideImageStore roots the store in its own subtree under the data folder, so
|
||||
// Prune's recursive sweep never reaches the per-entity upload folders next to it.
|
||||
func ProvideImageStore() *ImageStore {
|
||||
return NewImageStore(filepath.Join(conf.Server.DataFolder.String(), consts.ArtworkFolder, "store"))
|
||||
}
|
||||
|
||||
// extForMime is deliberately NOT mime.ExtensionsByType: extensions are baked into
|
||||
// content-addressed paths and re-derived on Open, so they must be stable across OSes.
|
||||
func extForMime(m string) string {
|
||||
switch m {
|
||||
case "image/jpeg":
|
||||
return ".jpg"
|
||||
case "image/png":
|
||||
return ".png"
|
||||
case "image/gif":
|
||||
return ".gif"
|
||||
case "image/webp":
|
||||
return ".webp"
|
||||
}
|
||||
return ".img"
|
||||
}
|
||||
|
||||
// validHash rejects anything but 16 lowercase hex chars: known-absent states carry "",
|
||||
// and malformed persisted hashes must never reach path sharding (slice panics, separators).
|
||||
func validHash(hash string) bool {
|
||||
if len(hash) != 16 {
|
||||
return false
|
||||
}
|
||||
for _, c := range []byte(hash) {
|
||||
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *ImageStore) path(hash, mimeType string) string {
|
||||
return filepath.Join(s.root, hash[0:2], hash[2:4], hash+extForMime(mimeType))
|
||||
}
|
||||
|
||||
func (s *ImageStore) Write(hash, mimeType string, r io.Reader) error {
|
||||
if !validHash(hash) {
|
||||
return fmt.Errorf("imagestore: invalid hash %q", hash)
|
||||
}
|
||||
dst := s.path(hash, mimeType)
|
||||
if _, err := os.Stat(dst); err == nil {
|
||||
// A touched mtime marks the file live so a concurrent prune spares it.
|
||||
now := time.Now()
|
||||
if err := os.Chtimes(dst, now, now); err == nil {
|
||||
return nil
|
||||
}
|
||||
// touch failed (file likely pruned concurrently) — fall through and write it
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp, err := os.CreateTemp(filepath.Dir(dst), "."+hash+".tmp*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
if _, err := io.Copy(tmp, r); err != nil {
|
||||
tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp.Name(), dst)
|
||||
}
|
||||
|
||||
func (s *ImageStore) Open(hash, mimeType string) (io.ReadCloser, error) {
|
||||
if !validHash(hash) {
|
||||
return nil, fmt.Errorf("imagestore: invalid hash %q", hash)
|
||||
}
|
||||
return os.Open(s.path(hash, mimeType))
|
||||
}
|
||||
|
||||
// Remove deletes the store file unless it is newer than olderThan, in which case
|
||||
// an overlapping acquisition may have just touched it and be about to commit its row.
|
||||
func (s *ImageStore) Remove(hash, mimeType string, olderThan time.Time) error {
|
||||
if !validHash(hash) {
|
||||
return fmt.Errorf("imagestore: invalid hash %q", hash)
|
||||
}
|
||||
path := s.path(hash, mimeType)
|
||||
info, err := os.Stat(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.ModTime().After(olderThan) {
|
||||
return nil
|
||||
}
|
||||
err = os.Remove(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Sweep removes store files not accepted by keep. Files modified after cutoff
|
||||
// (including temp files) are always kept: their acquisition row may not be committed yet.
|
||||
func (s *ImageStore) Sweep(cutoff time.Time, keep func(hash, ext string) bool) (int, error) {
|
||||
removed := 0
|
||||
err := filepath.WalkDir(s.root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return err
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.ModTime().After(cutoff) {
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
remove := strings.HasPrefix(name, ".") // abandoned temp file past the grace window
|
||||
if !remove {
|
||||
ext := filepath.Ext(name)
|
||||
remove = !keep(strings.TrimSuffix(name, ext), ext)
|
||||
}
|
||||
if remove {
|
||||
// #nosec G122 -- path comes from WalkDir over our own store root, no attacker-controlled symlinks
|
||||
if err := os.Remove(path); err != nil {
|
||||
return err
|
||||
}
|
||||
removed++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return removed, nil
|
||||
}
|
||||
return removed, err
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("ImageStore", func() {
|
||||
var store *ImageStore
|
||||
var root string
|
||||
|
||||
BeforeEach(func() {
|
||||
root = GinkgoT().TempDir()
|
||||
store = NewImageStore(root)
|
||||
})
|
||||
|
||||
It("hashes deterministically", func() {
|
||||
h1, err := HashImage(bytes.NewReader([]byte("some image bytes")))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
h2, _ := HashImage(bytes.NewReader([]byte("some image bytes")))
|
||||
Expect(h1).To(Equal(h2))
|
||||
Expect(h1).To(HaveLen(16))
|
||||
h3, _ := HashImage(bytes.NewReader([]byte("other bytes")))
|
||||
Expect(h3).ToNot(Equal(h1))
|
||||
})
|
||||
|
||||
It("writes sharded and reads back", func() {
|
||||
data := []byte("jpeg-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
|
||||
Expect(filepath.Join(root, h[0:2], h[2:4], h+".jpg")).To(BeAnExistingFile())
|
||||
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
defer rc.Close()
|
||||
got, _ := io.ReadAll(rc)
|
||||
Expect(got).To(Equal(data))
|
||||
})
|
||||
|
||||
It("is idempotent on duplicate writes and preserves the original content", func() {
|
||||
data := []byte("dup")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||
// A duplicate write only touches mtime; passing different bytes under the same
|
||||
// hash proves the second reader is never consumed to overwrite the file.
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader([]byte("not-dup")))).To(Succeed())
|
||||
|
||||
rc, err := store.Open(h, "image/png")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
defer rc.Close()
|
||||
got, err := io.ReadAll(rc)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got).To(Equal(data))
|
||||
})
|
||||
|
||||
It("refreshes the mtime on a duplicate write", func() {
|
||||
data := []byte("touch-me")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed())
|
||||
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||
|
||||
info, err := os.Stat(store.path(h, "image/png"))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(info.ModTime()).To(BeTemporally(">", time.Now().Add(-time.Minute)))
|
||||
})
|
||||
|
||||
It("rewrites the bytes when the existing file vanished before the liveness touch", func() {
|
||||
data := []byte("vanishing")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
for range 10 {
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||
Expect(os.Remove(store.path(h, "image/png"))).To(Succeed())
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||
|
||||
rc, err := store.Open(h, "image/png")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
Expect(got).To(Equal(data))
|
||||
}
|
||||
})
|
||||
|
||||
It("returns fs.ErrNotExist for missing images", func() {
|
||||
_, err := store.Open("beefbeefbeefbeef", "image/jpeg")
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
})
|
||||
|
||||
It("removes without error when already gone", func() {
|
||||
Expect(store.Remove("beefbeefbeefbeef", "image/jpeg", time.Now())).To(Succeed())
|
||||
})
|
||||
|
||||
It("rejects invalid hashes instead of panicking", func() {
|
||||
for _, h := range []string{"", "ab", "BEEFBEEFBEEFBEEF", "../../../../etcpw", "beefbeefbeefbee/"} {
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader([]byte("x")))).To(MatchError(ContainSubstring("invalid hash")))
|
||||
_, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).To(MatchError(ContainSubstring("invalid hash")))
|
||||
Expect(store.Remove(h, "image/jpeg", time.Now())).To(MatchError(ContainSubstring("invalid hash")))
|
||||
}
|
||||
})
|
||||
|
||||
It("spares a file newer than the cutoff, removes an aged one", func() {
|
||||
fresh := []byte("fresh")
|
||||
hf, _ := HashImage(bytes.NewReader(fresh))
|
||||
Expect(store.Write(hf, "image/jpeg", bytes.NewReader(fresh))).To(Succeed())
|
||||
|
||||
aged := []byte("aged")
|
||||
ha, _ := HashImage(bytes.NewReader(aged))
|
||||
Expect(store.Write(ha, "image/jpeg", bytes.NewReader(aged))).To(Succeed())
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(store.path(ha, "image/jpeg"), old, old)).To(Succeed())
|
||||
|
||||
cutoff := time.Now().Add(-time.Hour)
|
||||
Expect(store.Remove(hf, "image/jpeg", cutoff)).To(Succeed())
|
||||
Expect(store.Remove(ha, "image/jpeg", cutoff)).To(Succeed())
|
||||
|
||||
rc, err := store.Open(hf, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
_, err = store.Open(ha, "image/jpeg")
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
})
|
||||
|
||||
It("sweeps unknown files, keeps known ones", func() {
|
||||
d1 := []byte("keep-me")
|
||||
h1, _ := HashImage(bytes.NewReader(d1))
|
||||
Expect(store.Write(h1, "image/jpeg", bytes.NewReader(d1))).To(Succeed())
|
||||
d2 := []byte("orphan")
|
||||
h2, _ := HashImage(bytes.NewReader(d2))
|
||||
Expect(store.Write(h2, "image/jpeg", bytes.NewReader(d2))).To(Succeed())
|
||||
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(store.path(h2, "image/jpeg"), old, old)).To(Succeed())
|
||||
|
||||
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(h, _ string) bool { return h == h1 })
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(removed).To(Equal(1))
|
||||
_, err = store.Open(h2, "image/jpeg")
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
rc, err := store.Open(h1, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("sweeps a stale mime variant of a known hash, keeps the current one", func() {
|
||||
data := []byte("same-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed())
|
||||
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
|
||||
|
||||
// The recorded mime is image/jpeg, so the .png variant is obsolete.
|
||||
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(hash, ext string) bool {
|
||||
return hash == h && ext == ".jpg"
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(removed).To(Equal(1))
|
||||
_, err = store.Open(h, "image/png")
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("keeps young unknown files inside the grace window", func() {
|
||||
d := []byte("fresh-orphan")
|
||||
h, _ := HashImage(bytes.NewReader(d))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(d))).To(Succeed())
|
||||
|
||||
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string, string) bool { return false })
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(removed).To(Equal(0))
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("removes abandoned temp files past the grace window, keeps fresh ones", func() {
|
||||
oldTmp := filepath.Join(root, ".old.tmp")
|
||||
Expect(os.WriteFile(oldTmp, []byte("x"), 0600)).To(Succeed())
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(oldTmp, old, old)).To(Succeed())
|
||||
|
||||
freshTmp := filepath.Join(root, ".fresh.tmp")
|
||||
Expect(os.WriteFile(freshTmp, []byte("y"), 0600)).To(Succeed())
|
||||
|
||||
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string, string) bool { return true })
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(removed).To(Equal(1))
|
||||
Expect(oldTmp).ToNot(BeAnExistingFile())
|
||||
Expect(freshTmp).To(BeAnExistingFile())
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,246 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/draw"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/core/artwork/blurhash"
|
||||
"github.com/navidrome/navidrome/core/external"
|
||||
"github.com/navidrome/navidrome/core/ffmpeg"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
xdraw "golang.org/x/image/draw"
|
||||
)
|
||||
|
||||
// outcome tells the worker what to do with the queue row: found/absent
|
||||
// delete it, failed reschedules it via MarkFailed.
|
||||
type outcome int
|
||||
|
||||
const (
|
||||
outcomeFound outcome = iota
|
||||
// outcomeFoundStale: state was written and is served, but a higher-priority external
|
||||
// step failed, so the row must retry (via MarkFailed) to give that source another chance.
|
||||
outcomeFoundStale
|
||||
outcomeAbsent
|
||||
outcomeFailed
|
||||
)
|
||||
|
||||
// thumbnailSize is the max dimension fed to blurhash.
|
||||
const thumbnailSize = 128
|
||||
|
||||
// maxImageBytes caps a resolved image read: a user-editable ExternalImageURL could
|
||||
// point at an arbitrarily large endpoint, and 20MB is generous for any real cover.
|
||||
const maxImageBytes = 20 << 20
|
||||
|
||||
// maxImagePixels caps declared dimensions: a tiny compressed file can declare a
|
||||
// huge canvas that image.Decode would expand into gigabytes (decompression bomb).
|
||||
const maxImagePixels = 64 << 20
|
||||
|
||||
// workerDeps are the collaborators processItem needs; extGate is set by NewWorker in
|
||||
// production and nil only in tests, where resolveItem falls back to a plain passthrough.
|
||||
type workerDeps struct {
|
||||
ds model.DataStore
|
||||
store *ImageStore
|
||||
prov external.Provider
|
||||
ffmpeg ffmpeg.FFmpeg
|
||||
extGate extGateFunc
|
||||
}
|
||||
|
||||
// processItem resolves one queue item end to end: find an image, hash/decode/
|
||||
// blurhash it, place its bytes, and persist the resulting state.
|
||||
func processItem(ctx context.Context, deps *workerDeps, item model.ArtworkQueueItem) outcome {
|
||||
repo := deps.ds.Artwork(ctx)
|
||||
|
||||
res, err := resolveItem(ctx, deps.ds, deps.prov, deps.ffmpeg, item, deps.extGate)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "artwork: could not resolve item", "kind", item.ItemKind, "id", item.ItemID, err)
|
||||
return outcomeFailed
|
||||
}
|
||||
if res.reader == nil {
|
||||
if res.extError {
|
||||
// An external source errored/timed out: never settle on absent, keep serving old state.
|
||||
return outcomeFailed
|
||||
}
|
||||
return writeAbsent(ctx, repo, item)
|
||||
}
|
||||
defer res.reader.Close()
|
||||
|
||||
data, err := readCapped(res.reader)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "artwork: failed to read resolved image", "kind", item.ItemKind, "id", item.ItemID, "source", res.source, err)
|
||||
return outcomeFailed
|
||||
}
|
||||
log.Debug(ctx, "artwork: read resolved image", "kind", item.ItemKind, "id", item.ItemID, "source", res.source, "bytes", len(data))
|
||||
|
||||
hash, err := HashImage(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
log.Warn(ctx, "artwork: failed to hash image", "kind", item.ItemKind, "id", item.ItemID, err)
|
||||
return outcomeFailed
|
||||
}
|
||||
|
||||
art, err := repo.GetImage(hash)
|
||||
switch {
|
||||
case err == nil:
|
||||
// Dedup hit: identical bytes already known, reuse dims/mime/blurhash.
|
||||
case errors.Is(err, model.ErrNotFound):
|
||||
art, err = decodeArtwork(ctx, hash, data)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "artwork: failed to decode resolved image", "kind", item.ItemKind, "id", item.ItemID, err)
|
||||
return outcomeFailed
|
||||
}
|
||||
default:
|
||||
log.Warn(ctx, "artwork: failed to look up image hash", "kind", item.ItemKind, "id", item.ItemID, err)
|
||||
return outcomeFailed
|
||||
}
|
||||
art.SizeBytes = int64(len(data))
|
||||
|
||||
sourcePath, refMtime, err := placeBytes(deps.store, art, res, data)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "artwork: failed to write image store", "kind", item.ItemKind, "id", item.ItemID, err)
|
||||
return outcomeFailed
|
||||
}
|
||||
if err := repo.PutImage(art); err != nil {
|
||||
log.Warn(ctx, "artwork: failed to persist artwork image", "kind", item.ItemKind, "id", item.ItemID, err)
|
||||
return outcomeFailed
|
||||
}
|
||||
if err := repo.PutItemArtwork(&model.ItemArtwork{
|
||||
ItemKind: item.ItemKind,
|
||||
ItemID: item.ItemID,
|
||||
ImageType: item.ImageType,
|
||||
Hash: hash,
|
||||
Source: res.source,
|
||||
SourcePath: sourcePath,
|
||||
RefMtime: refMtime,
|
||||
AttemptedAt: time.Now(),
|
||||
}); err != nil {
|
||||
log.Warn(ctx, "artwork: failed to persist item artwork state", "kind", item.ItemKind, "id", item.ItemID, err)
|
||||
return outcomeFailed
|
||||
}
|
||||
if res.extError {
|
||||
return outcomeFoundStale
|
||||
}
|
||||
return outcomeFound
|
||||
}
|
||||
|
||||
// writeAbsent records a known-absent state: every local/external source answered definitively "no".
|
||||
func writeAbsent(ctx context.Context, repo model.ArtworkRepository, item model.ArtworkQueueItem) outcome {
|
||||
err := repo.PutItemArtwork(&model.ItemArtwork{
|
||||
ItemKind: item.ItemKind,
|
||||
ItemID: item.ItemID,
|
||||
ImageType: item.ImageType,
|
||||
AttemptedAt: time.Now(),
|
||||
})
|
||||
if err != nil {
|
||||
log.Warn(ctx, "artwork: failed to persist absent state", "kind", item.ItemKind, "id", item.ItemID, err)
|
||||
return outcomeFailed
|
||||
}
|
||||
return outcomeAbsent
|
||||
}
|
||||
|
||||
// readCapped reads r, rejecting anything over maxImageBytes.
|
||||
func readCapped(r io.Reader) ([]byte, error) {
|
||||
data, err := io.ReadAll(io.LimitReader(r, maxImageBytes+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(data) > maxImageBytes {
|
||||
return nil, fmt.Errorf("image exceeds size cap %d", maxImageBytes)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// decodeCapped rejects declared dimensions over maxImagePixels BEFORE the
|
||||
// full-decode allocation, then decodes.
|
||||
func decodeCapped(data []byte) (image.Image, string, error) {
|
||||
cfg, format, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("decode image config: %w", err)
|
||||
}
|
||||
if int64(cfg.Width)*int64(cfg.Height) > maxImagePixels {
|
||||
return nil, "", fmt.Errorf("image dimensions %dx%d exceed pixel cap %d", cfg.Width, cfg.Height, maxImagePixels)
|
||||
}
|
||||
img, _, err := image.Decode(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("decode image: %w", err)
|
||||
}
|
||||
return img, format, nil
|
||||
}
|
||||
|
||||
// decodeArtwork builds a new Artwork row from raw bytes: dimensions, mime and a
|
||||
// blurhash computed from a downscaled thumbnail.
|
||||
func decodeArtwork(ctx context.Context, hash string, data []byte) (*model.Artwork, error) {
|
||||
img, format, err := decodeCapped(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
thumb := makeThumbnail(img, thumbnailSize)
|
||||
xComp, yComp := blurhash.Components(thumb.Bounds().Dx(), thumb.Bounds().Dy())
|
||||
bh, err := blurhash.Encode(thumb, xComp, yComp)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "artwork: blurhash encoding failed", "hash", hash, err)
|
||||
bh = ""
|
||||
}
|
||||
|
||||
return &model.Artwork{
|
||||
Hash: hash,
|
||||
Mime: mimeForFormat(format),
|
||||
Width: img.Bounds().Dx(),
|
||||
Height: img.Bounds().Dy(),
|
||||
BlurHash: bh,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// makeThumbnail downscales img to fit within maxSize on its longest side.
|
||||
// Images within bounds are returned as-is (no upscaling).
|
||||
func makeThumbnail(img image.Image, maxSize int) image.Image {
|
||||
b := img.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
if w <= maxSize && h <= maxSize {
|
||||
return toFastScaleType(img)
|
||||
}
|
||||
scale := float64(maxSize) / float64(max(w, h))
|
||||
dst := image.NewRGBA(image.Rect(0, 0, max(1, int(float64(w)*scale)), max(1, int(float64(h)*scale))))
|
||||
xdraw.CatmullRom.Scale(dst, dst.Bounds(), toFastScaleType(img), b, draw.Src, nil)
|
||||
return dst
|
||||
}
|
||||
|
||||
// isFileBacked reports whether a resolution's bytes already live in a library/upload
|
||||
// file, so the acquisition must not duplicate them into the content-addressed store.
|
||||
func isFileBacked(source string) bool {
|
||||
return source == "folder" || source == "upload"
|
||||
}
|
||||
|
||||
// placeBytes reports the item's backing-file provenance (folder/upload: image, embedded: audio,
|
||||
// external/generated: none) and writes the bytes into the store for the non-file-backed sources.
|
||||
func placeBytes(store *ImageStore, art *model.Artwork, res resolution, data []byte) (sourcePath string, refMtime int64, err error) {
|
||||
if isFileBacked(res.source) {
|
||||
return res.sourcePath, res.refMtime, nil
|
||||
}
|
||||
if res.source == "embedded" {
|
||||
sourcePath, refMtime = res.sourcePath, res.refMtime
|
||||
}
|
||||
return sourcePath, refMtime, store.Write(art.Hash, art.Mime, bytes.NewReader(data))
|
||||
}
|
||||
|
||||
// mimeForFormat maps an image.Decode format name to its MIME type; extForMime
|
||||
// in image_store.go performs the inverse for content-addressed file paths.
|
||||
func mimeForFormat(format string) string {
|
||||
switch format {
|
||||
case "jpeg":
|
||||
return "image/jpeg"
|
||||
case "png":
|
||||
return "image/png"
|
||||
case "gif":
|
||||
return "image/gif"
|
||||
case "webp":
|
||||
return "image/webp"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"hash/crc32"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/conf/configtest"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
// pngHeaderWithDims builds just a PNG signature + IHDR chunk declaring w×h. DecodeConfig
|
||||
// reads the header without touching pixel data, so the body can be omitted entirely.
|
||||
func pngHeaderWithDims(w, h uint32) []byte {
|
||||
ihdr := make([]byte, 13)
|
||||
binary.BigEndian.PutUint32(ihdr[0:], w)
|
||||
binary.BigEndian.PutUint32(ihdr[4:], h)
|
||||
ihdr[8] = 8 // bit depth
|
||||
ihdr[9] = 2 // color type: truecolor
|
||||
chunk := append([]byte("IHDR"), ihdr...)
|
||||
out := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}
|
||||
out = binary.BigEndian.AppendUint32(out, uint32(len(ihdr)))
|
||||
out = append(out, chunk...)
|
||||
return binary.BigEndian.AppendUint32(out, crc32.ChecksumIEEE(chunk))
|
||||
}
|
||||
|
||||
var _ = Describe("processItem", func() {
|
||||
var (
|
||||
ctx context.Context
|
||||
ds *tests.MockDataStore
|
||||
folderRepo *fakeFolderRepo
|
||||
libRepo *tests.MockLibraryRepo
|
||||
ffm *tests.MockFFmpeg
|
||||
prov *fakeExternalProvider
|
||||
store *ImageStore
|
||||
artRepo *tests.MockArtworkRepo
|
||||
repoRoot string
|
||||
deps *workerDeps
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
ctx = context.Background()
|
||||
var err error
|
||||
repoRoot, err = os.Getwd()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
folderRepo = &fakeFolderRepo{}
|
||||
libRepo = &tests.MockLibraryRepo{}
|
||||
libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}})
|
||||
ffm = tests.NewMockFFmpeg("")
|
||||
prov = &fakeExternalProvider{}
|
||||
artRepo = tests.CreateMockArtworkRepo()
|
||||
ds = &tests.MockDataStore{
|
||||
MockedFolder: folderRepo,
|
||||
MockedLibrary: libRepo,
|
||||
MockedArtwork: artRepo,
|
||||
}
|
||||
ds.MockedAlbum = tests.CreateMockAlbumRepo()
|
||||
store = NewImageStore(GinkgoT().TempDir())
|
||||
deps = &workerDeps{ds: ds, store: store, prov: prov, ffmpeg: ffm}
|
||||
|
||||
conf.Server.CoverArtPriority = "cover.jpg, embedded"
|
||||
})
|
||||
|
||||
It("found-folder: persists state from a folder image, writes no store file, keeps sourcePath/refMtime", func() {
|
||||
folderRepo.result = []model.Folder{{
|
||||
Path: "tests/fixtures/artist/an-album",
|
||||
ImageFiles: []string{"cover.jpg"},
|
||||
}}
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "al1", Name: "Album", FolderIDs: []string{"f1"}},
|
||||
})
|
||||
|
||||
out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al1"})
|
||||
Expect(out).To(Equal(outcomeFound))
|
||||
|
||||
ia, err := artRepo.GetItemArtwork("al", "al1", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ia.Hash).ToNot(BeEmpty())
|
||||
Expect(ia.Source).To(Equal("folder"))
|
||||
Expect(filepath.ToSlash(ia.SourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/cover.jpg"))
|
||||
Expect(ia.RefMtime).To(BeNumerically(">", 0))
|
||||
|
||||
art, err := artRepo.GetImage(ia.Hash)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
_, err = store.Open(ia.Hash, art.Mime)
|
||||
Expect(os.IsNotExist(err)).To(BeTrue(), "folder-backed art must not be duplicated into the store")
|
||||
})
|
||||
|
||||
It("found-embedded: writes a store file and computes a non-empty blurhash from a real fixture", func() {
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "al2", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}},
|
||||
})
|
||||
folderRepo.result = nil
|
||||
|
||||
out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al2"})
|
||||
Expect(out).To(Equal(outcomeFound))
|
||||
|
||||
ia, err := artRepo.GetItemArtwork("al", "al2", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ia.Source).To(Equal("embedded"))
|
||||
Expect(filepath.ToSlash(ia.SourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/test.mp3"))
|
||||
|
||||
art, err := artRepo.GetImage(ia.Hash)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(art.BlurHash).ToNot(BeEmpty())
|
||||
|
||||
rc, err := store.Open(ia.Hash, art.Mime)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("absent: no local source and no external error persists a known-absent state", func() {
|
||||
folderRepo.result = nil
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "al3", Name: "Album"},
|
||||
})
|
||||
|
||||
out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al3"})
|
||||
Expect(out).To(Equal(outcomeAbsent))
|
||||
|
||||
ia, err := artRepo.GetItemArtwork("al", "al3", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ia.Hash).To(BeEmpty())
|
||||
Expect(ia.Source).To(BeEmpty())
|
||||
Expect(ia.AttemptedAt).To(BeTemporally("~", time.Now(), time.Second))
|
||||
})
|
||||
|
||||
It("failed-on-extError: leaves the item's state untouched", func() {
|
||||
conf.Server.CoverArtPriority = "external"
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "al4", Name: "Album"},
|
||||
})
|
||||
prov.albumImage = func(context.Context, string) (*url.URL, error) {
|
||||
return nil, errors.New("agent timed out")
|
||||
}
|
||||
|
||||
out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al4"})
|
||||
Expect(out).To(Equal(outcomeFailed))
|
||||
|
||||
_, err := artRepo.GetItemArtwork("al", "al4", model.ImageTypePrimary)
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
})
|
||||
|
||||
It("found-stale: a fallback hit after a transient external failure persists state and returns outcomeFoundStale", func() {
|
||||
conf.Server.CoverArtPriority = "external, cover.jpg"
|
||||
folderRepo.result = []model.Folder{{
|
||||
Path: "tests/fixtures/artist/an-album",
|
||||
ImageFiles: []string{"cover.jpg"},
|
||||
}}
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "alstale", Name: "Album", FolderIDs: []string{"f1"}},
|
||||
})
|
||||
prov.albumImage = func(context.Context, string) (*url.URL, error) {
|
||||
return nil, errors.New("agent timed out")
|
||||
}
|
||||
|
||||
out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alstale"})
|
||||
Expect(out).To(Equal(outcomeFoundStale))
|
||||
|
||||
ia, err := artRepo.GetItemArtwork("al", "alstale", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ia.Hash).ToNot(BeEmpty())
|
||||
Expect(ia.Source).To(Equal("folder"))
|
||||
})
|
||||
|
||||
It("dedup: a second item with identical bytes skips decode and reuses the artwork row", func() {
|
||||
folderRepo.result = []model.Folder{{
|
||||
Path: "tests/fixtures/artist/an-album",
|
||||
ImageFiles: []string{"cover.jpg"},
|
||||
}}
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "al5", Name: "Album A", FolderIDs: []string{"f1"}},
|
||||
{ID: "al6", Name: "Album B", FolderIDs: []string{"f1"}},
|
||||
})
|
||||
|
||||
out1 := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al5"})
|
||||
Expect(out1).To(Equal(outcomeFound))
|
||||
ia1, err := artRepo.GetItemArtwork("al", "al5", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
// Poison the stored blurhash: if the second item re-decodes instead of
|
||||
// deduping on hash, this sentinel gets overwritten by a real computed value.
|
||||
poisoned := artRepo.Data[ia1.Hash]
|
||||
poisoned.BlurHash = "SENTINEL"
|
||||
artRepo.Data[ia1.Hash] = poisoned
|
||||
|
||||
out2 := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al6"})
|
||||
Expect(out2).To(Equal(outcomeFound))
|
||||
ia2, err := artRepo.GetItemArtwork("al", "al6", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ia2.Hash).To(Equal(ia1.Hash))
|
||||
|
||||
reused, err := artRepo.GetImage(ia1.Hash)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(reused.BlurHash).To(Equal("SENTINEL"))
|
||||
})
|
||||
|
||||
It("two items, two files, identical bytes: each item keeps its own provenance; the shared artwork row is written once", func() {
|
||||
// Two distinct library files with byte-identical content resolve to the same
|
||||
// hash. Provenance is per-item, so neither file's path may overwrite the other.
|
||||
libRoot := GinkgoT().TempDir()
|
||||
imgBytes, err := os.ReadFile(filepath.Join(repoRoot, "tests/fixtures/artist/an-album/cover.jpg"))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
for sub, mtime := range map[string]int64{"album-a": 1000, "album-b": 2000} {
|
||||
dir := filepath.Join(libRoot, sub)
|
||||
Expect(os.MkdirAll(dir, 0755)).To(Succeed())
|
||||
img := filepath.Join(dir, "cover.jpg")
|
||||
Expect(os.WriteFile(img, imgBytes, 0600)).To(Succeed())
|
||||
Expect(os.Chtimes(img, time.Unix(mtime, 0), time.Unix(mtime, 0))).To(Succeed())
|
||||
}
|
||||
libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(libRoot)}})
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "alA", Name: "Album A", FolderIDs: []string{"fa"}},
|
||||
{ID: "alB", Name: "Album B", FolderIDs: []string{"fb"}},
|
||||
})
|
||||
|
||||
folderRepo.result = []model.Folder{{Path: "album-a", ImageFiles: []string{"cover.jpg"}}}
|
||||
Expect(processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alA"})).To(Equal(outcomeFound))
|
||||
iaA, err := artRepo.GetItemArtwork("al", "alA", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(iaA.Source).To(Equal("folder"))
|
||||
Expect(filepath.ToSlash(iaA.SourcePath)).To(HaveSuffix("album-a/cover.jpg"))
|
||||
Expect(iaA.RefMtime).To(Equal(int64(1000)))
|
||||
|
||||
// Poison the shared row's blurhash: the second item must dedup on hash, not re-decode.
|
||||
poisoned := artRepo.Data[iaA.Hash]
|
||||
poisoned.BlurHash = "SENTINEL"
|
||||
artRepo.Data[iaA.Hash] = poisoned
|
||||
|
||||
folderRepo.result = []model.Folder{{Path: "album-b", ImageFiles: []string{"cover.jpg"}}}
|
||||
Expect(processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "alB"})).To(Equal(outcomeFound))
|
||||
iaB, err := artRepo.GetItemArtwork("al", "alB", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(iaB.Hash).To(Equal(iaA.Hash))
|
||||
Expect(filepath.ToSlash(iaB.SourcePath)).To(HaveSuffix("album-b/cover.jpg"))
|
||||
Expect(iaB.RefMtime).To(Equal(int64(2000)))
|
||||
|
||||
// The first item's provenance survives the second item processing identical bytes.
|
||||
iaAafter, err := artRepo.GetItemArtwork("al", "alA", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(filepath.ToSlash(iaAafter.SourcePath)).To(HaveSuffix("album-a/cover.jpg"))
|
||||
Expect(iaAafter.RefMtime).To(Equal(int64(1000)))
|
||||
|
||||
// One shared artwork row, and dedup preserved it untouched.
|
||||
Expect(artRepo.Data).To(HaveLen(1))
|
||||
reused, err := artRepo.GetImage(iaA.Hash)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(reused.BlurHash).To(Equal("SENTINEL"))
|
||||
})
|
||||
|
||||
It("decode failure on found bytes: fails without writing state", func() {
|
||||
tmpDir := GinkgoT().TempDir()
|
||||
conf.Server.DataFolder = conf.NewDir(tmpDir)
|
||||
Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "radio"), 0755)).To(Succeed())
|
||||
imgPath := filepath.Join(tmpDir, "artwork", "radio", "ra1_test.jpg")
|
||||
Expect(os.WriteFile(imgPath, []byte("not actually an image"), 0600)).To(Succeed())
|
||||
|
||||
radioRepo := tests.CreateMockedRadioRepo()
|
||||
radioRepo.Data = map[string]*model.Radio{"ra1": {ID: "ra1", Name: "Radio", UploadedImage: "ra1_test.jpg"}}
|
||||
ds.MockedRadio = radioRepo
|
||||
|
||||
out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "ra1"})
|
||||
Expect(out).To(Equal(outcomeFailed))
|
||||
|
||||
_, err := artRepo.GetItemArtwork("ra", "ra1", model.ImageTypePrimary)
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
})
|
||||
|
||||
It("oversized read: a resolved image larger than the cap fails without writing state", func() {
|
||||
tmpDir := GinkgoT().TempDir()
|
||||
conf.Server.DataFolder = conf.NewDir(tmpDir)
|
||||
Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "radio"), 0755)).To(Succeed())
|
||||
imgPath := filepath.Join(tmpDir, "artwork", "radio", "big_test.jpg")
|
||||
f, err := os.Create(imgPath)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(f.Truncate(maxImageBytes + 1)).To(Succeed())
|
||||
Expect(f.Close()).To(Succeed())
|
||||
|
||||
radioRepo := tests.CreateMockedRadioRepo()
|
||||
radioRepo.Data = map[string]*model.Radio{"big": {ID: "big", Name: "Radio", UploadedImage: "big_test.jpg"}}
|
||||
ds.MockedRadio = radioRepo
|
||||
|
||||
out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "big"})
|
||||
Expect(out).To(Equal(outcomeFailed))
|
||||
|
||||
_, err = artRepo.GetItemArtwork("ra", "big", model.ImageTypePrimary)
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
})
|
||||
|
||||
It("decompression bomb: rejects huge declared dimensions before the full decode", func() {
|
||||
data := pngHeaderWithDims(50000, 50000) // 2.5 gigapixels, far above the cap
|
||||
_, err := decodeArtwork(ctx, "bomb", data)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("dimensions"))
|
||||
})
|
||||
|
||||
It("store write failure: fails without writing state", func() {
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "al7", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}},
|
||||
})
|
||||
folderRepo.result = nil
|
||||
|
||||
// A store root that is a plain file makes every MkdirAll under it fail.
|
||||
blockedRoot := filepath.Join(GinkgoT().TempDir(), "not-a-dir")
|
||||
Expect(os.WriteFile(blockedRoot, []byte("x"), 0600)).To(Succeed())
|
||||
deps.store = NewImageStore(blockedRoot)
|
||||
|
||||
out := processItem(ctx, deps, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al7"})
|
||||
Expect(out).To(Equal(outcomeFailed))
|
||||
|
||||
_, err := artRepo.GetItemArtwork("al", "al7", model.ImageTypePrimary)
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,86 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
)
|
||||
|
||||
// pruneMinAge guards the window between artwork insert and item_artwork upsert.
|
||||
const pruneMinAge = time.Hour
|
||||
|
||||
func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error {
|
||||
repo := ds.Artwork(ctx)
|
||||
|
||||
purged, err := repo.PurgeDanglingItemArtwork()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if purged > 0 {
|
||||
log.Info(ctx, "Prune: purged dangling item artwork state", "count", purged)
|
||||
}
|
||||
|
||||
// Queue rows for deleted entities would otherwise retry forever (Get -> not found -> failed).
|
||||
queuePurged, err := ds.ArtworkQueue(ctx).PurgeDangling()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if queuePurged > 0 {
|
||||
log.Info(ctx, "Prune: purged dangling artwork queue rows", "count", queuePurged)
|
||||
}
|
||||
|
||||
// One grace cutoff for both the DB orphan check and the file sweep: files younger
|
||||
// than the window may belong to acquisitions whose rows aren't committed yet.
|
||||
cutoff := time.Now().Add(-pruneMinAge)
|
||||
candidates, err := repo.GetOrphanHashes(cutoff)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(candidates) > 0 {
|
||||
arts, err := repo.GetImages(candidates)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := repo.DeleteOrphans(cutoff, candidates); err != nil {
|
||||
return err
|
||||
}
|
||||
// DeleteOrphans may spare candidates reacquired since the snapshot; only remove files
|
||||
// for rows actually gone (absent from the post-delete re-read).
|
||||
survivors, err := repo.GetImages(candidates)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
removed := 0
|
||||
for _, h := range candidates {
|
||||
if _, ok := survivors[h]; ok {
|
||||
continue
|
||||
}
|
||||
// A spared fresh file is at worst a stray a later sweep reclaims;
|
||||
// Worker.RunPrune serializes prune against in-flight acquisitions.
|
||||
if err := store.Remove(h, arts[h].Mime, cutoff); err != nil {
|
||||
log.Warn(ctx, "Prune: could not remove artwork file", "hash", h, err)
|
||||
}
|
||||
removed++
|
||||
}
|
||||
log.Info(ctx, "Prune: removed orphan artwork", "count", removed)
|
||||
}
|
||||
|
||||
mimes, err := repo.GetAllMimes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
removed, err := store.Sweep(cutoff, func(hash, ext string) bool {
|
||||
// A known hash under a stale extension is a superseded mime variant — reclaim it.
|
||||
m, ok := mimes[hash]
|
||||
return ok && ext == extForMime(m)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if removed > 0 {
|
||||
log.Info(ctx, "Prune: swept stray artwork files", "count", removed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
type flakyGetArtworkRepo struct {
|
||||
*tests.MockArtworkRepo
|
||||
}
|
||||
|
||||
func (f *flakyGetArtworkRepo) GetAllMimes() (map[string]string, error) {
|
||||
return nil, errors.New("db locked")
|
||||
}
|
||||
|
||||
var _ = Describe("Prune", func() {
|
||||
var ds *tests.MockDataStore
|
||||
var store *ImageStore
|
||||
var awRepo *tests.MockArtworkRepo
|
||||
|
||||
BeforeEach(func() {
|
||||
ds = &tests.MockDataStore{}
|
||||
awRepo = ds.Artwork(context.Background()).(*tests.MockArtworkRepo)
|
||||
store = NewImageStore(GinkgoT().TempDir())
|
||||
})
|
||||
|
||||
// PutImage refreshes created_at like the SQL repo, so fixtures are aged directly.
|
||||
ageArtwork := func(h string, t time.Time) {
|
||||
a := awRepo.Data[h]
|
||||
a.CreatedAt = t
|
||||
awRepo.Data[h] = a
|
||||
}
|
||||
|
||||
It("purges dangling item_artwork state for gone entities, summed across kinds", func() {
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "gone-album", ImageType: model.ImageTypePrimary})).To(Succeed())
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "gone-artist", ImageType: model.ImageTypePrimary})).To(Succeed())
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "live-artist", ImageType: model.ImageTypePrimary})).To(Succeed())
|
||||
awRepo.ExistingIDs = map[string]map[string]bool{
|
||||
"al": {},
|
||||
"ar": {"live-artist": true},
|
||||
}
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||
|
||||
_, err := awRepo.GetItemArtwork("al", "gone-album", model.ImageTypePrimary)
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
_, err = awRepo.GetItemArtwork("ar", "gone-artist", model.ImageTypePrimary)
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
_, err = awRepo.GetItemArtwork("ar", "live-artist", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("purges dangling artwork_queue rows for gone entities", func() {
|
||||
queueRepo := tests.CreateMockArtworkQueueRepo()
|
||||
Expect(queueRepo.Enqueue(
|
||||
model.ArtworkQueueItem{ItemKind: "al", ItemID: "gone-album", ImageType: model.ImageTypePrimary},
|
||||
model.ArtworkQueueItem{ItemKind: "al", ItemID: "live-album", ImageType: model.ImageTypePrimary},
|
||||
)).To(Succeed())
|
||||
queueRepo.ExistingIDs = map[string]map[string]bool{"al": {"live-album": true}}
|
||||
ds.MockedArtworkQueue = queueRepo
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||
|
||||
Expect(findQueued(queueRepo, "al", "gone-album")).To(BeNil())
|
||||
Expect(findQueued(queueRepo, "al", "live-album")).ToNot(BeNil())
|
||||
})
|
||||
|
||||
It("deletes orphan rows and their store files, keeps referenced ones", func() {
|
||||
data := []byte("orphan-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
|
||||
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||
ageArtwork(h, old)
|
||||
awRepo.OrphanHashes = []string{h}
|
||||
|
||||
kept := []byte("kept-bytes")
|
||||
hk, _ := HashImage(bytes.NewReader(kept))
|
||||
Expect(store.Write(hk, "image/jpeg", bytes.NewReader(kept))).To(Succeed())
|
||||
Expect(awRepo.PutImage(&model.Artwork{Hash: hk, Mime: "image/jpeg"})).To(Succeed())
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||
|
||||
_, err := awRepo.GetImage(h)
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
_, err = store.Open(h, "image/jpeg")
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
rc, err := store.Open(hk, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("spares a candidate reacquired between snapshot and delete", func() {
|
||||
data := []byte("reacquired-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||
ageArtwork(h, time.Now().Add(-2*time.Hour))
|
||||
awRepo.OrphanHashes = []string{h}
|
||||
// Reacquisition: an item now references the hash the snapshot flagged as orphan.
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1",
|
||||
ImageType: model.ImageTypePrimary, Hash: h, Source: "folder"})).To(Succeed())
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||
|
||||
_, err := awRepo.GetImage(h)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("spares a candidate whose row was freshly recreated (created_at inside the grace window)", func() {
|
||||
data := []byte("fresh-reacquired-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
// Reacquisition refreshed created_at after the snapshot; still unreferenced.
|
||||
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||
awRepo.OrphanHashes = []string{h}
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||
|
||||
_, err := awRepo.GetImage(h)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("spares an orphan file freshly touched by an overlapping acquisition", func() {
|
||||
data := []byte("racing-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||
ageArtwork(h, time.Now().Add(-2*time.Hour))
|
||||
awRepo.OrphanHashes = []string{h}
|
||||
// The row is legitimately orphaned, but a concurrent acquisition just touched the
|
||||
// file's mtime (duplicate Write) and is about to commit a row referencing it.
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("sweeps store files that have no artwork row", func() {
|
||||
stray := []byte("no-row-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(stray))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(stray))).To(Succeed())
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||
|
||||
_, err := store.Open(h, "image/jpeg")
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
})
|
||||
|
||||
It("sweeps an obsolete mime variant of a reacquired hash", func() {
|
||||
data := []byte("variant-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed())
|
||||
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
|
||||
// The row records the current mime; the .png file is a superseded variant.
|
||||
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||
|
||||
_, err := store.Open(h, "image/png")
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("warns and continues past a store.Remove failure instead of aborting the loop", func() {
|
||||
tests.SkipOnWindows("uses Unix file permission bits")
|
||||
if os.Geteuid() == 0 {
|
||||
Skip("read-only dir cannot block root (e.g. tests in a container)")
|
||||
}
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
|
||||
blocked := []byte("blocked-bytes")
|
||||
hb, _ := HashImage(bytes.NewReader(blocked))
|
||||
Expect(store.Write(hb, "image/jpeg", bytes.NewReader(blocked))).To(Succeed())
|
||||
Expect(os.Chtimes(store.path(hb, "image/jpeg"), old, old)).To(Succeed())
|
||||
Expect(awRepo.PutImage(&model.Artwork{Hash: hb, Mime: "image/jpeg"})).To(Succeed())
|
||||
ageArtwork(hb, old)
|
||||
|
||||
good := []byte("good-bytes")
|
||||
hg, _ := HashImage(bytes.NewReader(good))
|
||||
Expect(store.Write(hg, "image/jpeg", bytes.NewReader(good))).To(Succeed())
|
||||
Expect(os.Chtimes(store.path(hg, "image/jpeg"), old, old)).To(Succeed())
|
||||
Expect(awRepo.PutImage(&model.Artwork{Hash: hg, Mime: "image/jpeg"})).To(Succeed())
|
||||
ageArtwork(hg, old)
|
||||
|
||||
// A read-only shard directory makes os.Remove fail (EACCES) for hb's file only.
|
||||
shardDir := filepath.Dir(store.path(hb, "image/jpeg"))
|
||||
Expect(os.Chmod(shardDir, 0500)).To(Succeed())
|
||||
DeferCleanup(func() { _ = os.Chmod(shardDir, 0755) })
|
||||
|
||||
// hb (blocked) is processed first: if store.Remove's failure aborted the loop
|
||||
// instead of warning and continuing, hg would never be reached.
|
||||
awRepo.OrphanHashes = []string{hb, hg}
|
||||
|
||||
// Prune still errors: Sweep independently revisits hb's leftover file and,
|
||||
// unlike the loop below, has no warn-and-continue fallback of its own.
|
||||
err := Prune(context.Background(), ds, store)
|
||||
Expect(err).To(HaveOccurred())
|
||||
|
||||
// hg: reached and fully pruned despite being queued after the failing hb -
|
||||
// proof the loop didn't return/break on the first Remove error.
|
||||
_, err = awRepo.GetImage(hg)
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
_, err = store.Open(hg, "image/jpeg")
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
|
||||
// hb: row still purged (DeleteOrphans doesn't depend on file removal), but the
|
||||
// file itself survives since store.Remove failed and only warned.
|
||||
_, err = awRepo.GetImage(hb)
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
rc, err := store.Open(hb, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("never sweeps files on a transient DB error", func() {
|
||||
ds.MockedArtwork = &flakyGetArtworkRepo{MockArtworkRepo: tests.CreateMockArtworkRepo()}
|
||||
|
||||
data := []byte("live-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).ToNot(Succeed())
|
||||
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,423 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/draw"
|
||||
"image/png"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Masterminds/squirrel"
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/consts"
|
||||
"github.com/navidrome/navidrome/core/external"
|
||||
"github.com/navidrome/navidrome/core/ffmpeg"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
)
|
||||
|
||||
// resolution is one attempted acquisition outcome for an entity.
|
||||
type resolution struct {
|
||||
reader io.ReadCloser // nil when no source yielded an image
|
||||
source string // model.ItemArtwork.Source value: "folder", "embedded", "external", "upload", "generated"
|
||||
sourcePath string // backing library/upload file (folder/upload: the image; embedded: the audio file); "" otherwise
|
||||
refMtime int64 // mtime of sourcePath at resolution time; 0 when no sourcePath
|
||||
// external source errored/timed out. With no reader: forces failed (never absent).
|
||||
// On a hit: a higher-priority external step failed—serve this, but retry later.
|
||||
extError bool
|
||||
}
|
||||
|
||||
// extGateFunc is an alias for the external-step wrapper the worker injects (rate
|
||||
// limiter + circuit breaker); resolveItem defaults to a plain passthrough.
|
||||
type extGateFunc = func(func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error)
|
||||
|
||||
func passthroughExtGate(f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) {
|
||||
return f()
|
||||
}
|
||||
|
||||
// resolveItem walks the kind's priority chain and returns the first hit.
|
||||
func resolveItem(ctx context.Context, ds model.DataStore, prov external.Provider, ffmpeg ffmpeg.FFmpeg, item model.ArtworkQueueItem, extGate extGateFunc) (resolution, error) {
|
||||
if extGate == nil {
|
||||
extGate = passthroughExtGate
|
||||
}
|
||||
switch item.ItemKind {
|
||||
case "al":
|
||||
return resolveAlbum(ctx, ds, prov, ffmpeg, item.ItemID, extGate)
|
||||
case "ar":
|
||||
return resolveArtist(ctx, ds, prov, ffmpeg, item.ItemID, extGate)
|
||||
case "pl":
|
||||
return resolvePlaylist(ctx, ds, prov, ffmpeg, item.ItemID, extGate)
|
||||
case "ra":
|
||||
return resolveRadio(ctx, ds, item.ItemID)
|
||||
default:
|
||||
return resolution{}, fmt.Errorf("resolveItem: kind %q is not resolvable by the worker", item.ItemKind)
|
||||
}
|
||||
}
|
||||
|
||||
// resolveAlbum ports the folder/embedded/external selection from
|
||||
// reader_album.go, walking conf.Server.CoverArtPriority.
|
||||
func resolveAlbum(ctx context.Context, ds model.DataStore, prov external.Provider, ffm ffmpeg.FFmpeg, albumID string, extGate extGateFunc) (resolution, error) {
|
||||
al, err := ds.Album(ctx).Get(albumID)
|
||||
if err != nil {
|
||||
return resolution{}, err
|
||||
}
|
||||
_, imgFiles, _, err := loadAlbumFoldersPaths(ctx, ds, *al)
|
||||
if err != nil {
|
||||
return resolution{}, err
|
||||
}
|
||||
lib, err := loadLibraryView(ctx, ds, al.LibraryID)
|
||||
if err != nil {
|
||||
return resolution{}, err
|
||||
}
|
||||
|
||||
var extErr bool
|
||||
for pattern := range strings.SplitSeq(strings.ToLower(conf.Server.CoverArtPriority), ",") {
|
||||
pattern = strings.TrimSpace(pattern)
|
||||
switch {
|
||||
case pattern == "embedded":
|
||||
if res, ok := resolveEmbedded(ctx, lib, ffm, al.EmbedArtPath); ok {
|
||||
res.extError = extErr
|
||||
return res, nil
|
||||
}
|
||||
case pattern == "external":
|
||||
if res, ok, isErr := resolveExternalStep(extGate, fromAlbumExternalSource(ctx, *al, prov)); ok {
|
||||
return res, nil
|
||||
} else if isErr {
|
||||
extErr = true
|
||||
}
|
||||
case len(imgFiles) > 0:
|
||||
if res, ok := resolveFolderFile(ctx, lib, imgFiles, pattern); ok {
|
||||
res.extError = extErr
|
||||
return res, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return resolution{extError: extErr}, nil
|
||||
}
|
||||
|
||||
// resolveArtist ports the upload/folder/external selection from
|
||||
// reader_artist.go: upload always wins, then conf.Server.ArtistArtPriority.
|
||||
func resolveArtist(ctx context.Context, ds model.DataStore, prov external.Provider, ffm ffmpeg.FFmpeg, artistID string, extGate extGateFunc) (resolution, error) {
|
||||
ar, err := ds.Artist(ctx).Get(artistID)
|
||||
if err != nil {
|
||||
return resolution{}, err
|
||||
}
|
||||
if res, ok := resolveLocalFile(ar.UploadedImagePath(), "upload"); ok {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Only consider albums where the artist is the sole album artist, same as reader_artist.go.
|
||||
als, err := ds.Album(ctx).GetAll(model.QueryOptions{
|
||||
Filters: squirrel.And{
|
||||
squirrel.Eq{"album_artist_id": artistID},
|
||||
squirrel.Eq{"json_array_length(participants, '$.albumartist')": 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return resolution{}, err
|
||||
}
|
||||
albumPaths, imgFiles, _, err := loadAlbumFoldersPaths(ctx, ds, als...)
|
||||
if err != nil {
|
||||
return resolution{}, err
|
||||
}
|
||||
artistFolder, _, err := loadArtistFolder(ctx, ds, als, albumPaths)
|
||||
if err != nil {
|
||||
return resolution{}, err
|
||||
}
|
||||
var lib libraryView
|
||||
if len(als) > 0 {
|
||||
lib, err = loadLibraryView(ctx, ds, als[0].LibraryID)
|
||||
if err != nil {
|
||||
return resolution{}, err
|
||||
}
|
||||
}
|
||||
|
||||
var extErr bool
|
||||
for pattern := range strings.SplitSeq(strings.ToLower(conf.Server.ArtistArtPriority), ",") {
|
||||
pattern = strings.TrimSpace(pattern)
|
||||
switch {
|
||||
case pattern == "external":
|
||||
if res, ok, isErr := resolveExternalStep(extGate, fromArtistExternalResult(ctx, *ar, prov)); ok {
|
||||
return res, nil
|
||||
} else if isErr {
|
||||
extErr = true
|
||||
}
|
||||
case pattern == "image-folder":
|
||||
if res, ok := resolveArtistImageFolder(ar); ok {
|
||||
res.extError = extErr
|
||||
return res, nil
|
||||
}
|
||||
case strings.HasPrefix(pattern, "album/"):
|
||||
if lib.FS == nil {
|
||||
continue
|
||||
}
|
||||
if res, ok := resolveFolderFile(ctx, lib, imgFiles, strings.TrimPrefix(pattern, "album/")); ok {
|
||||
res.extError = extErr
|
||||
return res, nil
|
||||
}
|
||||
default:
|
||||
if lib.FS == nil || artistFolder == "" {
|
||||
continue
|
||||
}
|
||||
if res, ok := resolveArtistFolderPattern(ctx, lib, artistFolder, pattern); ok {
|
||||
res.extError = extErr
|
||||
return res, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return resolution{extError: extErr}, nil
|
||||
}
|
||||
|
||||
// resolvePlaylist ports reader_playlist.go's chain: uploaded image, sidecar,
|
||||
// ExternalImageURL, then the generated 2x2 grid sourced through resolveAlbum.
|
||||
func resolvePlaylist(ctx context.Context, ds model.DataStore, prov external.Provider, ffm ffmpeg.FFmpeg, playlistID string, extGate extGateFunc) (resolution, error) {
|
||||
pl, err := ds.Playlist(ctx).Get(playlistID)
|
||||
if err != nil {
|
||||
return resolution{}, err
|
||||
}
|
||||
|
||||
var extErr bool
|
||||
if res, ok := resolveLocalFile(pl.UploadedImagePath(), "upload"); ok {
|
||||
return res, nil
|
||||
}
|
||||
if res, ok := resolveLocalFile(findPlaylistSidecarPath(ctx, pl.Path), "folder"); ok {
|
||||
return res, nil
|
||||
}
|
||||
if res, ok, isErr := resolveExternalStep(extGate, fromPlaylistExternalSource(ctx, *pl)); ok {
|
||||
return res, nil
|
||||
} else if isErr {
|
||||
extErr = true
|
||||
}
|
||||
|
||||
albumIDs, err := ds.Playlist(ctx).Tracks(pl.ID, false).GetAlbumIDs(model.QueryOptions{Max: 4, Sort: "random()"})
|
||||
if err != nil {
|
||||
return resolution{}, err
|
||||
}
|
||||
|
||||
var tiles []image.Image
|
||||
var tileErr error // first internal (non-external) tile failure, e.g. album deleted mid-flight
|
||||
for _, albumID := range albumIDs {
|
||||
res, err := resolveAlbum(ctx, ds, prov, ffm, albumID, extGate)
|
||||
if err != nil {
|
||||
if tileErr == nil {
|
||||
tileErr = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if res.extError {
|
||||
extErr = true
|
||||
}
|
||||
if res.reader == nil {
|
||||
continue
|
||||
}
|
||||
tile, decErr := decodeTile(res.reader)
|
||||
res.reader.Close()
|
||||
if decErr == nil {
|
||||
tiles = append(tiles, tile)
|
||||
}
|
||||
if len(tiles) == 4 {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(tiles) == 0 {
|
||||
// A tile-level failure must never resolve as a clean absent: propagate
|
||||
// internal errors, and force extError for external ones.
|
||||
if tileErr != nil {
|
||||
return resolution{}, fmt.Errorf("resolvePlaylist: sampled album art failed: %w", tileErr)
|
||||
}
|
||||
return resolution{extError: extErr}, nil
|
||||
}
|
||||
// Grow to 4 tiles by repeating what we have, mirroring reader_playlist.go's loadTiles.
|
||||
switch len(tiles) {
|
||||
case 2:
|
||||
tiles = append(tiles, tiles[1], tiles[0])
|
||||
case 3:
|
||||
tiles = append(tiles, tiles[0])
|
||||
}
|
||||
r, err := assembleTiles(tiles)
|
||||
if err != nil {
|
||||
return resolution{extError: extErr}, nil //nolint:nilerr // encode failure is a soft "no image", not a resolveItem error
|
||||
}
|
||||
return resolution{reader: r, source: "generated", extError: extErr}, nil
|
||||
}
|
||||
|
||||
// resolveRadio ports reader_radio.go: only an uploaded image, no fallback.
|
||||
func resolveRadio(ctx context.Context, ds model.DataStore, radioID string) (resolution, error) {
|
||||
r, err := ds.Radio(ctx).Get(radioID)
|
||||
if err != nil {
|
||||
return resolution{}, err
|
||||
}
|
||||
res, _ := resolveLocalFile(r.UploadedImagePath(), "upload")
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// resolveExternalStep runs an external sourceFunc through extGate, shared by
|
||||
// resolveAlbum and resolveArtist. ok reports a hit; extErr reports a
|
||||
// non-not-found error (a not-found is a definitive "no", not a failure).
|
||||
func resolveExternalStep(extGate extGateFunc, sf func() (io.ReadCloser, string, error)) (res resolution, ok bool, extErr bool) {
|
||||
r, path, err := extGate(sf)
|
||||
if r != nil {
|
||||
return resolution{reader: r, source: "external", sourcePath: path}, true, false
|
||||
}
|
||||
return resolution{}, false, err != nil && !errors.Is(err, model.ErrNotFound)
|
||||
}
|
||||
|
||||
// fromPlaylistExternalSource mirrors reader_playlist.go's ExternalImageURL step:
|
||||
// a remote URL (gated) when M3U external art is enabled, else a local file path.
|
||||
func fromPlaylistExternalSource(ctx context.Context, pl model.Playlist) sourceFunc {
|
||||
return func() (io.ReadCloser, string, error) {
|
||||
imgURL := pl.ExternalImageURL
|
||||
if imgURL == "" {
|
||||
return nil, "", nil
|
||||
}
|
||||
parsed, err := url.Parse(imgURL)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if parsed.Scheme == "http" || parsed.Scheme == "https" {
|
||||
if !conf.Server.EnableM3UExternalAlbumArt {
|
||||
return nil, "", nil
|
||||
}
|
||||
return fetchPlaylistImageURL(ctx, parsed)
|
||||
}
|
||||
// A missing/unreadable local file is a definitive miss, not a transient
|
||||
// failure to retry: swallow the open error and fall through to the grid.
|
||||
r, path, _ := fromLocalFile(imgURL)()
|
||||
return r, path, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Like sources.go's fromURL but maps 404/410 to ErrNotFound (definitive), so a stale M3U
|
||||
// cover URL falls through to the grid instead of retrying forever and tripping the breaker.
|
||||
func fetchPlaylistImageURL(ctx context.Context, imageURL *url.URL) (io.ReadCloser, string, error) {
|
||||
hc := http.Client{Timeout: 5 * time.Second}
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, imageURL.String(), nil)
|
||||
req.Header.Set("User-Agent", consts.HTTPUserAgent)
|
||||
resp, err := hc.Do(req) //nolint:gosec
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusGone {
|
||||
resp.Body.Close()
|
||||
return nil, "", model.ErrNotFound
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
return nil, "", fmt.Errorf("error retrieving artwork from %s: %s", imageURL, resp.Status)
|
||||
}
|
||||
return resp.Body, imageURL.String(), nil
|
||||
}
|
||||
|
||||
func resolveEmbedded(ctx context.Context, lib libraryView, ffm ffmpeg.FFmpeg, embedRel string) (resolution, bool) {
|
||||
if embedRel == "" {
|
||||
return resolution{}, false
|
||||
}
|
||||
abs := lib.Abs(embedRel)
|
||||
for _, sf := range []sourceFunc{fromTag(ctx, lib.FS, embedRel), fromFFmpegTag(ctx, ffm, abs)} {
|
||||
if r, _, _ := sf(); r != nil {
|
||||
return resolution{reader: r, source: "embedded", sourcePath: abs, refMtime: mtimeViaFS(lib.FS, embedRel)}, true
|
||||
}
|
||||
}
|
||||
return resolution{}, false
|
||||
}
|
||||
|
||||
func resolveFolderFile(ctx context.Context, lib libraryView, imgFiles []string, pattern string) (resolution, bool) {
|
||||
r, path, _ := fromExternalFile(ctx, lib.FS, imgFiles, pattern)()
|
||||
if r == nil {
|
||||
return resolution{}, false
|
||||
}
|
||||
return resolution{reader: r, source: "folder", sourcePath: lib.Abs(path), refMtime: mtimeViaFS(lib.FS, path)}, true
|
||||
}
|
||||
|
||||
func resolveArtistImageFolder(ar *model.Artist) (resolution, bool) {
|
||||
folder := conf.Server.ArtistImageFolder
|
||||
if folder == "" {
|
||||
return resolution{}, false
|
||||
}
|
||||
return resolveLocalFile(findImageInArtistFolder(folder, ar.MbzArtistID, ar.Name), "folder")
|
||||
}
|
||||
|
||||
func resolveArtistFolderPattern(ctx context.Context, lib libraryView, artistFolder, pattern string) (resolution, bool) {
|
||||
r, path, _ := fromArtistFolder(ctx, lib.FS, lib.absRoot, artistFolder, pattern)()
|
||||
if r == nil {
|
||||
return resolution{}, false
|
||||
}
|
||||
return resolution{reader: r, source: "folder", sourcePath: path, refMtime: mtimeOf(path)}, true
|
||||
}
|
||||
|
||||
// resolveLocalFile opens an absolute path directly (uploads, image-folder). A
|
||||
// missing or unreadable path is "no source", not an error.
|
||||
func resolveLocalFile(path, source string) (resolution, bool) {
|
||||
if path == "" {
|
||||
return resolution{}, false
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return resolution{}, false
|
||||
}
|
||||
return resolution{reader: f, source: source, sourcePath: path, refMtime: mtimeOf(path)}, true
|
||||
}
|
||||
|
||||
func mtimeOf(path string) int64 {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return info.ModTime().Unix()
|
||||
}
|
||||
|
||||
// mtimeViaFS stats through the library FS instead of a joined absolute path,
|
||||
// since library roots in tests may not be real OS paths (e.g. testfile://).
|
||||
func mtimeViaFS(fsys fs.FS, name string) int64 {
|
||||
if fsys == nil || name == "" {
|
||||
return 0
|
||||
}
|
||||
info, err := fs.Stat(fsys, name)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return info.ModTime().Unix()
|
||||
}
|
||||
|
||||
// decodeTile and assembleTiles mirror playlistArtworkReader's createTile/
|
||||
// createTiledImage, reusing the same rect/fillCenter cropping helpers.
|
||||
// decodeTile runs on every sampled album's resolved bytes before processItem's
|
||||
// own maxImageBytes/maxImagePixels guards apply, so it enforces them itself too.
|
||||
func decodeTile(r io.ReadCloser) (image.Image, error) {
|
||||
data, err := readCapped(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
img, _, err := decodeCapped(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return fillCenter(img, tileSize/2, tileSize/2), nil
|
||||
}
|
||||
|
||||
func assembleTiles(tiles []image.Image) (io.ReadCloser, error) {
|
||||
buf := new(bytes.Buffer)
|
||||
var err error
|
||||
if len(tiles) == 4 {
|
||||
rgba := image.NewRGBA(image.Rectangle{Max: image.Point{X: tileSize - 1, Y: tileSize - 1}})
|
||||
draw.Draw(rgba, rect(0), tiles[0], image.Point{}, draw.Src)
|
||||
draw.Draw(rgba, rect(1), tiles[1], image.Point{}, draw.Src)
|
||||
draw.Draw(rgba, rect(2), tiles[2], image.Point{}, draw.Src)
|
||||
draw.Draw(rgba, rect(3), tiles[3], image.Point{}, draw.Src)
|
||||
err = png.Encode(buf, rgba)
|
||||
} else {
|
||||
err = png.Encode(buf, tiles[0])
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.NopCloser(buf), nil
|
||||
}
|
||||
@@ -0,0 +1,562 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"image"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/conf/configtest"
|
||||
"github.com/navidrome/navidrome/core/external"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
// fakeExternalProvider is a minimal external.Provider stub for resolve_test.go;
|
||||
// only AlbumImage/ArtistImage are exercised by the resolvers.
|
||||
type fakeExternalProvider struct {
|
||||
external.Provider
|
||||
albumImage func(ctx context.Context, id string) (*url.URL, error)
|
||||
artistImage func(ctx context.Context, id string) (*url.URL, error)
|
||||
}
|
||||
|
||||
func (f *fakeExternalProvider) AlbumImage(ctx context.Context, id string) (*url.URL, error) {
|
||||
if f.albumImage != nil {
|
||||
return f.albumImage(ctx, id)
|
||||
}
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
|
||||
func (f *fakeExternalProvider) ArtistImage(ctx context.Context, id string) (*url.URL, error) {
|
||||
if f.artistImage != nil {
|
||||
return f.artistImage(ctx, id)
|
||||
}
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
|
||||
func (f *fakeExternalProvider) ArtistImageResult(ctx context.Context, id string) (*url.URL, error) {
|
||||
return f.ArtistImage(ctx, id)
|
||||
}
|
||||
|
||||
var _ = Describe("resolveItem", func() {
|
||||
var (
|
||||
ctx context.Context
|
||||
ds *tests.MockDataStore
|
||||
folderRepo *fakeFolderRepo
|
||||
libRepo *tests.MockLibraryRepo
|
||||
ffm *tests.MockFFmpeg
|
||||
prov *fakeExternalProvider
|
||||
repoRoot string
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
ctx = context.Background()
|
||||
var err error
|
||||
repoRoot, err = os.Getwd()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
folderRepo = &fakeFolderRepo{}
|
||||
libRepo = &tests.MockLibraryRepo{}
|
||||
libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}})
|
||||
ffm = tests.NewMockFFmpeg("")
|
||||
prov = &fakeExternalProvider{}
|
||||
ds = &tests.MockDataStore{
|
||||
MockedFolder: folderRepo,
|
||||
MockedLibrary: libRepo,
|
||||
}
|
||||
})
|
||||
|
||||
Describe("kind dispatch", func() {
|
||||
It("returns an error for kinds the worker never enqueues", func() {
|
||||
_, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "mf", ItemID: "x"}, nil)
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("album", func() {
|
||||
BeforeEach(func() {
|
||||
conf.Server.CoverArtPriority = "cover.jpg, embedded"
|
||||
ds.MockedAlbum = tests.CreateMockAlbumRepo()
|
||||
})
|
||||
|
||||
It("resolves folder art from the library FS", func() {
|
||||
folderRepo.result = []model.Folder{{
|
||||
Path: "tests/fixtures/artist/an-album",
|
||||
ImageFiles: []string{"cover.jpg"},
|
||||
}}
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "al1", Name: "Album", FolderIDs: []string{"f1"}},
|
||||
})
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al1"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).ToNot(BeNil())
|
||||
defer res.reader.Close()
|
||||
Expect(res.source).To(Equal("folder"))
|
||||
Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/cover.jpg"))
|
||||
Expect(res.refMtime).To(BeNumerically(">", 0))
|
||||
Expect(res.extError).To(BeFalse())
|
||||
})
|
||||
|
||||
It("falls back to embedded art when no folder image matches", func() {
|
||||
folderRepo.result = nil
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "al2", Name: "Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}},
|
||||
})
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al2"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).ToNot(BeNil())
|
||||
defer res.reader.Close()
|
||||
Expect(res.source).To(Equal("embedded"))
|
||||
Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/test.mp3"))
|
||||
Expect(res.refMtime).To(BeNumerically(">", 0))
|
||||
})
|
||||
|
||||
It("sets extError when the external source errors without being not-found", func() {
|
||||
conf.Server.CoverArtPriority = "external"
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "al3", Name: "Album"},
|
||||
})
|
||||
prov.albumImage = func(context.Context, string) (*url.URL, error) {
|
||||
return nil, errors.New("agent timed out")
|
||||
}
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al3"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).To(BeNil())
|
||||
Expect(res.extError).To(BeTrue())
|
||||
})
|
||||
|
||||
It("does not set extError when the external source reports not-found", func() {
|
||||
conf.Server.CoverArtPriority = "external"
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "al4", Name: "Album"},
|
||||
})
|
||||
// prov.albumImage left nil -> fakeExternalProvider returns model.ErrNotFound
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al4"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).To(BeNil())
|
||||
Expect(res.extError).To(BeFalse())
|
||||
})
|
||||
|
||||
It("carries extError onto a fallback folder hit after a transient external failure", func() {
|
||||
conf.Server.CoverArtPriority = "external, cover.jpg"
|
||||
folderRepo.result = []model.Folder{{
|
||||
Path: "tests/fixtures/artist/an-album",
|
||||
ImageFiles: []string{"cover.jpg"},
|
||||
}}
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "al6", Name: "Album", FolderIDs: []string{"f1"}},
|
||||
})
|
||||
prov.albumImage = func(context.Context, string) (*url.URL, error) {
|
||||
return nil, errors.New("agent timed out")
|
||||
}
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al6"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).ToNot(BeNil())
|
||||
defer res.reader.Close()
|
||||
Expect(res.source).To(Equal("folder"))
|
||||
Expect(res.extError).To(BeTrue())
|
||||
})
|
||||
|
||||
It("does not carry extError onto a fallback folder hit after a definitive external not-found", func() {
|
||||
conf.Server.CoverArtPriority = "external, cover.jpg"
|
||||
folderRepo.result = []model.Folder{{
|
||||
Path: "tests/fixtures/artist/an-album",
|
||||
ImageFiles: []string{"cover.jpg"},
|
||||
}}
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "al7", Name: "Album", FolderIDs: []string{"f1"}},
|
||||
})
|
||||
// prov.albumImage left nil -> fakeExternalProvider returns model.ErrNotFound
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al7"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).ToNot(BeNil())
|
||||
defer res.reader.Close()
|
||||
Expect(res.source).To(Equal("folder"))
|
||||
Expect(res.extError).To(BeFalse())
|
||||
})
|
||||
|
||||
It("routes the external step through a custom extGate", func() {
|
||||
conf.Server.CoverArtPriority = "external"
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "al5", Name: "Album"},
|
||||
})
|
||||
prov.albumImage = func(context.Context, string) (*url.URL, error) {
|
||||
return nil, errors.New("boom")
|
||||
}
|
||||
var extGateCalls int
|
||||
extGate := func(f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) {
|
||||
extGateCalls++
|
||||
return f()
|
||||
}
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "al", ItemID: "al5"}, extGate)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.extError).To(BeTrue())
|
||||
Expect(extGateCalls).To(Equal(1))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("artist", func() {
|
||||
It("resolves the uploaded image before any priority chain lookup", func() {
|
||||
tmpDir := GinkgoT().TempDir()
|
||||
conf.Server.DataFolder = conf.NewDir(tmpDir)
|
||||
Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "artist"), 0755)).To(Succeed())
|
||||
imgPath := filepath.Join(tmpDir, "artwork", "artist", "ar1_test.jpg")
|
||||
Expect(os.WriteFile(imgPath, []byte("uploaded artist image"), 0600)).To(Succeed())
|
||||
|
||||
artistRepo := tests.CreateMockArtistRepo()
|
||||
artistRepo.SetData(model.Artists{{ID: "ar1", Name: "Artist", UploadedImage: "ar1_test.jpg"}})
|
||||
ds.MockedArtist = artistRepo
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar1"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).ToNot(BeNil())
|
||||
defer res.reader.Close()
|
||||
Expect(res.source).To(Equal("upload"))
|
||||
Expect(res.sourcePath).To(Equal(imgPath))
|
||||
})
|
||||
|
||||
It("falls through to the ArtistArtPriority chain when there is no upload", func() {
|
||||
conf.Server.ArtistArtPriority = "album/artist.*"
|
||||
folderRepo.result = []model.Folder{{
|
||||
LibraryPath: testFileLibPath(repoRoot),
|
||||
Path: "tests/fixtures/artist/an-album",
|
||||
ImageFiles: []string{"artist.png"},
|
||||
}}
|
||||
artistRepo := tests.CreateMockArtistRepo()
|
||||
artistRepo.SetData(model.Artists{{ID: "ar2", Name: "Artist"}})
|
||||
ds.MockedArtist = artistRepo
|
||||
ds.MockedAlbum = tests.CreateMockAlbumRepo()
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).All = model.Albums{
|
||||
{ID: "al9", Name: "Album", LibraryID: 0, FolderIDs: []string{"f1"}},
|
||||
}
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar2"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).ToNot(BeNil())
|
||||
defer res.reader.Close()
|
||||
Expect(res.source).To(Equal("folder"))
|
||||
Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("tests/fixtures/artist/an-album/artist.png"))
|
||||
})
|
||||
|
||||
It("sets extError when the external source errors without being not-found", func() {
|
||||
conf.Server.ArtistArtPriority = "external"
|
||||
artistRepo := tests.CreateMockArtistRepo()
|
||||
artistRepo.SetData(model.Artists{{ID: "ar3", Name: "Artist"}})
|
||||
ds.MockedArtist = artistRepo
|
||||
prov.artistImage = func(context.Context, string) (*url.URL, error) {
|
||||
return nil, errors.New("agent timed out")
|
||||
}
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar3"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).To(BeNil())
|
||||
Expect(res.extError).To(BeTrue())
|
||||
})
|
||||
|
||||
It("does not set extError when the external source reports not-found", func() {
|
||||
conf.Server.ArtistArtPriority = "external"
|
||||
artistRepo := tests.CreateMockArtistRepo()
|
||||
artistRepo.SetData(model.Artists{{ID: "ar4", Name: "Artist"}})
|
||||
ds.MockedArtist = artistRepo
|
||||
// prov.artistImage left nil -> fakeExternalProvider returns model.ErrNotFound
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar4"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).To(BeNil())
|
||||
Expect(res.extError).To(BeFalse())
|
||||
})
|
||||
|
||||
It("routes the external step through a custom extGate", func() {
|
||||
conf.Server.ArtistArtPriority = "external"
|
||||
artistRepo := tests.CreateMockArtistRepo()
|
||||
artistRepo.SetData(model.Artists{{ID: "ar5", Name: "Artist"}})
|
||||
ds.MockedArtist = artistRepo
|
||||
prov.artistImage = func(context.Context, string) (*url.URL, error) {
|
||||
return nil, errors.New("boom")
|
||||
}
|
||||
var extGateCalls int
|
||||
extGate := func(f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) {
|
||||
extGateCalls++
|
||||
return f()
|
||||
}
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ar", ItemID: "ar5"}, extGate)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.extError).To(BeTrue())
|
||||
Expect(extGateCalls).To(Equal(1))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("radio", func() {
|
||||
It("yields an empty resolution when there is no uploaded image", func() {
|
||||
tmpDir := GinkgoT().TempDir()
|
||||
conf.Server.DataFolder = conf.NewDir(tmpDir)
|
||||
|
||||
radioRepo := tests.CreateMockedRadioRepo()
|
||||
radioRepo.Data = map[string]*model.Radio{"ra1": {ID: "ra1", Name: "Radio"}}
|
||||
ds.MockedRadio = radioRepo
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "ra1"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res).To(Equal(resolution{}))
|
||||
})
|
||||
|
||||
It("resolves the uploaded image when set", func() {
|
||||
tmpDir := GinkgoT().TempDir()
|
||||
conf.Server.DataFolder = conf.NewDir(tmpDir)
|
||||
Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "radio"), 0755)).To(Succeed())
|
||||
imgPath := filepath.Join(tmpDir, "artwork", "radio", "ra2_test.jpg")
|
||||
Expect(os.WriteFile(imgPath, []byte("uploaded radio image"), 0600)).To(Succeed())
|
||||
|
||||
radioRepo := tests.CreateMockedRadioRepo()
|
||||
radioRepo.Data = map[string]*model.Radio{"ra2": {ID: "ra2", Name: "Radio", UploadedImage: "ra2_test.jpg"}}
|
||||
ds.MockedRadio = radioRepo
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "ra", ItemID: "ra2"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).ToNot(BeNil())
|
||||
defer res.reader.Close()
|
||||
Expect(res.source).To(Equal("upload"))
|
||||
Expect(res.sourcePath).To(Equal(imgPath))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("playlist", func() {
|
||||
BeforeEach(func() {
|
||||
conf.Server.CoverArtPriority = "cover.jpg"
|
||||
folderRepo.result = []model.Folder{{
|
||||
Path: "tests/fixtures/artist/an-album",
|
||||
ImageFiles: []string{"cover.jpg"},
|
||||
}}
|
||||
ds.MockedAlbum = tests.CreateMockAlbumRepo()
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "t1", Name: "T1", FolderIDs: []string{"f1"}},
|
||||
{ID: "t2", Name: "T2", FolderIDs: []string{"f1"}},
|
||||
{ID: "t3", Name: "T3", FolderIDs: []string{"f1"}},
|
||||
{ID: "t4", Name: "T4", FolderIDs: []string{"f1"}},
|
||||
})
|
||||
})
|
||||
|
||||
DescribeTable("yields a generated grid from up to 4 album tiles",
|
||||
func(albumIDs []string, expectedSize int) {
|
||||
plRepo := tests.CreateMockPlaylistRepo()
|
||||
plRepo.SetData(model.Playlists{{ID: "pl1", Name: "Playlist"}})
|
||||
plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: albumIDs}
|
||||
ds.MockedPlaylist = plRepo
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl1"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).ToNot(BeNil())
|
||||
defer res.reader.Close()
|
||||
Expect(res.source).To(Equal("generated"))
|
||||
|
||||
img, format, err := image.Decode(res.reader)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(format).To(Equal("png"))
|
||||
Expect(img.Bounds().Dx()).To(Equal(expectedSize))
|
||||
Expect(img.Bounds().Dy()).To(Equal(expectedSize))
|
||||
},
|
||||
// tileSize-1: the 4-tile canvas is built as [0, tileSize-1], matching
|
||||
// reader_playlist.go's createTiledImage exactly.
|
||||
Entry("1 album -> single tile", []string{"t1"}, tileSize/2),
|
||||
Entry("2 albums -> duplicated to 4 tiles", []string{"t1", "t2"}, tileSize-1),
|
||||
Entry("3 albums -> duplicated to 4 tiles", []string{"t1", "t2", "t3"}, tileSize-1),
|
||||
Entry("4 albums -> full grid", []string{"t1", "t2", "t3", "t4"}, tileSize-1),
|
||||
)
|
||||
|
||||
It("resolves the uploaded image before the generated grid", func() {
|
||||
tmpDir := GinkgoT().TempDir()
|
||||
conf.Server.DataFolder = conf.NewDir(tmpDir)
|
||||
Expect(os.MkdirAll(filepath.Join(tmpDir, "artwork", "playlist"), 0755)).To(Succeed())
|
||||
imgPath := filepath.Join(tmpDir, "artwork", "playlist", "plu_test.jpg")
|
||||
Expect(os.WriteFile(imgPath, []byte("uploaded playlist image"), 0600)).To(Succeed())
|
||||
|
||||
plRepo := tests.CreateMockPlaylistRepo()
|
||||
plRepo.SetData(model.Playlists{{ID: "plu", Name: "Playlist", UploadedImage: "plu_test.jpg"}})
|
||||
plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1", "t2"}}
|
||||
ds.MockedPlaylist = plRepo
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "plu"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).ToNot(BeNil())
|
||||
defer res.reader.Close()
|
||||
Expect(res.source).To(Equal("upload"))
|
||||
Expect(res.sourcePath).To(Equal(imgPath))
|
||||
})
|
||||
|
||||
It("resolves a sidecar image next to the playlist file before the grid", func() {
|
||||
plDir := GinkgoT().TempDir()
|
||||
Expect(os.WriteFile(filepath.Join(plDir, "list.m3u"), []byte("#EXTM3U"), 0600)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(plDir, "list.jpg"), []byte("sidecar image"), 0600)).To(Succeed())
|
||||
|
||||
plRepo := tests.CreateMockPlaylistRepo()
|
||||
plRepo.SetData(model.Playlists{{ID: "pls", Name: "Playlist", Path: filepath.Join(plDir, "list.m3u")}})
|
||||
plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1", "t2"}}
|
||||
ds.MockedPlaylist = plRepo
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pls"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).ToNot(BeNil())
|
||||
defer res.reader.Close()
|
||||
Expect(res.source).To(Equal("folder"))
|
||||
Expect(filepath.ToSlash(res.sourcePath)).To(HaveSuffix("list.jpg"))
|
||||
})
|
||||
|
||||
It("routes ExternalImageURL through extGate and sets extError on transient failure", func() {
|
||||
conf.Server.EnableM3UExternalAlbumArt = true
|
||||
folderRepo.result = nil // no grid tiles, so the external failure is what surfaces
|
||||
|
||||
plRepo := tests.CreateMockPlaylistRepo()
|
||||
plRepo.SetData(model.Playlists{{ID: "ple", Name: "Playlist", ExternalImageURL: "http://example.com/cover.jpg"}})
|
||||
plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}}
|
||||
ds.MockedPlaylist = plRepo
|
||||
|
||||
var extGateCalls int
|
||||
extGate := func(f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) {
|
||||
extGateCalls++
|
||||
return nil, "", errors.New("network down")
|
||||
}
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "ple"}, extGate)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).To(BeNil())
|
||||
Expect(res.extError).To(BeTrue())
|
||||
Expect(extGateCalls).To(Equal(1))
|
||||
})
|
||||
|
||||
It("treats a missing local ExternalImageURL as a definitive miss, not extError", func() {
|
||||
folderRepo.result = nil // no grid tiles, so the local-file miss is what surfaces
|
||||
|
||||
plRepo := tests.CreateMockPlaylistRepo()
|
||||
plRepo.SetData(model.Playlists{{ID: "plm", Name: "Playlist", ExternalImageURL: "/nonexistent/path/cover.jpg"}})
|
||||
plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}}
|
||||
ds.MockedPlaylist = plRepo
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "plm"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).To(BeNil())
|
||||
Expect(res.extError).To(BeFalse())
|
||||
})
|
||||
|
||||
It("treats an ExternalImageURL 404 as a definitive miss and falls through to the grid", func() {
|
||||
conf.Server.EnableM3UExternalAlbumArt = true
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
plRepo := tests.CreateMockPlaylistRepo()
|
||||
plRepo.SetData(model.Playlists{{ID: "pl404", Name: "Playlist", ExternalImageURL: srv.URL}})
|
||||
plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}}
|
||||
ds.MockedPlaylist = plRepo
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl404"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).ToNot(BeNil())
|
||||
defer res.reader.Close()
|
||||
Expect(res.source).To(Equal("generated"))
|
||||
Expect(res.extError).To(BeFalse())
|
||||
})
|
||||
|
||||
It("treats an ExternalImageURL 500 as a transient failure and sets extError", func() {
|
||||
conf.Server.EnableM3UExternalAlbumArt = true
|
||||
folderRepo.result = nil // no grid tiles, so the external failure is what surfaces
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
plRepo := tests.CreateMockPlaylistRepo()
|
||||
plRepo.SetData(model.Playlists{{ID: "pl500", Name: "Playlist", ExternalImageURL: srv.URL}})
|
||||
plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}}
|
||||
ds.MockedPlaylist = plRepo
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl500"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).To(BeNil())
|
||||
Expect(res.extError).To(BeTrue())
|
||||
})
|
||||
|
||||
It("yields an empty resolution when no album has art", func() {
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "empty1", Name: "Empty"},
|
||||
})
|
||||
plRepo := tests.CreateMockPlaylistRepo()
|
||||
plRepo.SetData(model.Playlists{{ID: "pl2", Name: "Playlist"}})
|
||||
plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"empty1"}}
|
||||
ds.MockedPlaylist = plRepo
|
||||
folderRepo.result = nil
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl2"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).To(BeNil())
|
||||
Expect(res.source).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("skips a grid tile whose declared dimensions are a decompression bomb", func() {
|
||||
// End-to-end regression: a bomb-declaring tile must not break the grid.
|
||||
libRoot := GinkgoT().TempDir()
|
||||
Expect(os.MkdirAll(filepath.Join(libRoot, "bomb"), 0755)).To(Succeed())
|
||||
Expect(os.WriteFile(filepath.Join(libRoot, "bomb", "cover.jpg"), pngHeaderWithDims(50000, 50000), 0600)).To(Succeed())
|
||||
libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(libRoot)}})
|
||||
folderRepo.result = []model.Folder{{Path: "bomb", ImageFiles: []string{"cover.jpg"}}}
|
||||
|
||||
plRepo := tests.CreateMockPlaylistRepo()
|
||||
plRepo.SetData(model.Playlists{{ID: "plbomb", Name: "Playlist"}})
|
||||
plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"t1"}}
|
||||
ds.MockedPlaylist = plRepo
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "plbomb"}, nil)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(res.reader).To(BeNil())
|
||||
Expect(res.source).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("does not resolve as absent when every sampled album fails to resolve", func() {
|
||||
// "missing1"/"missing2" are not in MockAlbumRepo's data, so resolveAlbum
|
||||
// returns a genuine (non-external) error for every sampled tile.
|
||||
plRepo := tests.CreateMockPlaylistRepo()
|
||||
plRepo.SetData(model.Playlists{{ID: "pl3", Name: "Playlist"}})
|
||||
plRepo.TracksRepo = &tests.MockPlaylistTrackRepo{AlbumIDs: []string{"missing1", "missing2"}}
|
||||
ds.MockedPlaylist = plRepo
|
||||
|
||||
res, err := resolveItem(ctx, ds, prov, ffm, model.ArtworkQueueItem{ItemKind: "pl", ItemID: "pl3"}, nil)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(res).To(Equal(resolution{}))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
// decodeTile runs on every sampled album's resolved bytes before processItem's
|
||||
// own guards apply, so it must enforce the same caps independently.
|
||||
var _ = Describe("decodeTile", func() {
|
||||
It("rejects a decompression bomb before the full decode", func() {
|
||||
data := pngHeaderWithDims(50000, 50000) // 2.5 gigapixels, far above the cap
|
||||
_, err := decodeTile(io.NopCloser(bytes.NewReader(data)))
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(err.Error()).To(ContainSubstring("dimensions"))
|
||||
})
|
||||
|
||||
It("rejects a tile larger than the size cap", func() {
|
||||
data := bytes.Repeat([]byte{0}, maxImageBytes+1)
|
||||
_, err := decodeTile(io.NopCloser(bytes.NewReader(data)))
|
||||
Expect(err).To(HaveOccurred())
|
||||
})
|
||||
})
|
||||
@@ -198,6 +198,18 @@ func fromArtistExternalSource(ctx context.Context, ar model.Artist, provider ext
|
||||
}
|
||||
}
|
||||
|
||||
// fromArtistExternalResult is the worker's artist external step: via ArtistImageResult a
|
||||
// transient agent failure surfaces as an error (extError) rather than settling as absent.
|
||||
func fromArtistExternalResult(ctx context.Context, ar model.Artist, provider external.Provider) sourceFunc {
|
||||
return func() (io.ReadCloser, string, error) {
|
||||
imageUrl, err := provider.ArtistImageResult(ctx, ar.ID)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return fromURL(ctx, imageUrl)
|
||||
}
|
||||
}
|
||||
|
||||
func fromAlbumExternalSource(ctx context.Context, al model.Album, provider external.Provider) sourceFunc {
|
||||
return func() (io.ReadCloser, string, error) {
|
||||
imageUrl, err := provider.AlbumImage(ctx, al.ID)
|
||||
|
||||
@@ -8,4 +8,6 @@ var Set = wire.NewSet(
|
||||
NewArtwork,
|
||||
GetImageCache,
|
||||
NewCacheWarmer,
|
||||
NewWorker,
|
||||
ProvideImageStore,
|
||||
)
|
||||
@@ -0,0 +1,257 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"math/rand/v2"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/core/external"
|
||||
"github.com/navidrome/navidrome/core/ffmpeg"
|
||||
"github.com/navidrome/navidrome/core/auth"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
const (
|
||||
workerPollInterval = 5 * time.Second
|
||||
backoffBase = 5 * time.Minute
|
||||
backoffCap = 48 * time.Hour
|
||||
breakerThreshold = 5
|
||||
breakerProbeAfter = time.Minute
|
||||
)
|
||||
|
||||
var errBreakerOpen = errors.New("artwork: external circuit breaker open")
|
||||
|
||||
// Worker drains the artwork queue through processItem: the external step is rate-limited
|
||||
// and circuit-broken, and prune is serialized against in-flight acquisitions via pruneMu.
|
||||
type Worker struct {
|
||||
deps workerDeps
|
||||
limiter *rate.Limiter
|
||||
breaker *breaker
|
||||
pruneMu sync.RWMutex
|
||||
wake chan struct{}
|
||||
runCtx context.Context
|
||||
|
||||
mu sync.Mutex
|
||||
inFlight map[string]struct{}
|
||||
}
|
||||
|
||||
func NewWorker(ds model.DataStore, store *ImageStore, prov external.Provider, ffmpeg ffmpeg.FFmpeg) *Worker {
|
||||
rps := conf.Server.ArtworkExternalMaxRPS
|
||||
limit := rate.Inf // 0 or negative disables the external throttle
|
||||
if rps > 0 {
|
||||
limit = rate.Limit(rps)
|
||||
}
|
||||
w := &Worker{
|
||||
deps: workerDeps{ds: ds, store: store, prov: prov, ffmpeg: ffmpeg},
|
||||
limiter: rate.NewLimiter(limit, max(1, rps)),
|
||||
breaker: newBreaker(),
|
||||
wake: make(chan struct{}, 1),
|
||||
runCtx: context.Background(),
|
||||
inFlight: map[string]struct{}{},
|
||||
}
|
||||
w.deps.extGate = w.gate
|
||||
return w
|
||||
}
|
||||
|
||||
// Run blocks draining the queue until ctx is cancelled. It exits cleanly with no
|
||||
// leaked goroutines: each drain waits for its batch before the loop can return.
|
||||
func (w *Worker) Run(ctx context.Context) error {
|
||||
w.runCtx = ctx
|
||||
concurrency := max(1, conf.Server.ArtworkWorkerConcurrency)
|
||||
ticker := time.NewTicker(workerPollInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
n, err := w.drain(ctx, concurrency)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
log.Warn(ctx, "artwork: worker drain failed", err)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
if n > 0 {
|
||||
continue // keep draining while the queue has ready work
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
case <-w.wake:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Bump enqueues an item at the highest priority and wakes the drain loop. It is
|
||||
// non-blocking: a wake already pending is enough.
|
||||
func (w *Worker) Bump(kind, id string) {
|
||||
item := model.ArtworkQueueItem{
|
||||
ItemKind: kind,
|
||||
ItemID: id,
|
||||
ImageType: model.ImageTypePrimary,
|
||||
Priority: model.ArtworkPriorityBump,
|
||||
}
|
||||
if err := w.deps.ds.ArtworkQueue(context.Background()).Enqueue(item); err != nil {
|
||||
log.Warn("artwork: could not bump queue item", "kind", kind, "id", id, err)
|
||||
return
|
||||
}
|
||||
select {
|
||||
case w.wake <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// RunPrune runs Prune under the worker's write lock, so no acquisition can place
|
||||
// a file while orphans are being reclaimed. This is the only sanctioned prune path.
|
||||
func (w *Worker) RunPrune(ctx context.Context) error {
|
||||
w.pruneMu.Lock()
|
||||
defer w.pruneMu.Unlock()
|
||||
return Prune(ctx, w.deps.ds, w.deps.store)
|
||||
}
|
||||
|
||||
func (w *Worker) drain(ctx context.Context, concurrency int) (int, error) {
|
||||
// Resolved per drain, not once in Run: the worker starts at boot, possibly before any
|
||||
// admin exists, so a late-created admin is picked up on the next poll (private playlists).
|
||||
ctx = auth.WithAdminUser(ctx, w.deps.ds)
|
||||
batch, err := w.deps.ds.ArtworkQueue(ctx).DequeueBatch(2 * concurrency)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
items := w.claim(batch)
|
||||
if len(items) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
sem := make(chan struct{}, concurrency)
|
||||
var wg sync.WaitGroup
|
||||
for _, item := range items {
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(it model.ArtworkQueueItem) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
defer w.release(it)
|
||||
w.process(ctx, it)
|
||||
}(item)
|
||||
}
|
||||
wg.Wait()
|
||||
return len(items), nil
|
||||
}
|
||||
|
||||
func (w *Worker) process(ctx context.Context, item model.ArtworkQueueItem) {
|
||||
if item.ImageType == "" {
|
||||
item.ImageType = model.ImageTypePrimary
|
||||
}
|
||||
w.pruneMu.RLock()
|
||||
out := processItem(ctx, &w.deps, item)
|
||||
w.pruneMu.RUnlock()
|
||||
|
||||
queue := w.deps.ds.ArtworkQueue(ctx)
|
||||
switch out {
|
||||
case outcomeFound, outcomeAbsent:
|
||||
// DeleteIfUnchanged, not Delete: a scan that re-enqueued this row mid-flight reset
|
||||
// its retry_at, so the row survives here and the next drain re-resolves it.
|
||||
if err := queue.DeleteIfUnchanged(item.ItemKind, item.ItemID, item.ImageType, item.RetryAt); err != nil {
|
||||
log.Warn(ctx, "artwork: could not delete processed queue item", "kind", item.ItemKind, "id", item.ItemID, err)
|
||||
}
|
||||
case outcomeFoundStale, outcomeFailed:
|
||||
// MarkFailedIfUnchanged, not MarkFailed: a scan that re-enqueued this row mid-flight reset
|
||||
// retry_at, so stale backoff must not stomp its fresh, immediate eligibility.
|
||||
retryAt := time.Now().Add(backoff(item.Attempts))
|
||||
if err := queue.MarkFailedIfUnchanged(item.ItemKind, item.ItemID, item.ImageType, item.RetryAt, retryAt); err != nil {
|
||||
log.Warn(ctx, "artwork: could not reschedule failed queue item", "kind", item.ItemKind, "id", item.ItemID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// claim reserves items not already in flight, so a row appearing twice within a single
|
||||
// batch is processed once.
|
||||
func (w *Worker) claim(batch []model.ArtworkQueueItem) []model.ArtworkQueueItem {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
var out []model.ArtworkQueueItem
|
||||
for _, it := range batch {
|
||||
k := queueKey(it)
|
||||
if _, busy := w.inFlight[k]; busy {
|
||||
continue
|
||||
}
|
||||
w.inFlight[k] = struct{}{}
|
||||
out = append(out, it)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (w *Worker) release(it model.ArtworkQueueItem) {
|
||||
w.mu.Lock()
|
||||
delete(w.inFlight, queueKey(it))
|
||||
w.mu.Unlock()
|
||||
}
|
||||
|
||||
func queueKey(it model.ArtworkQueueItem) string {
|
||||
return it.ItemKind + "|" + it.ItemID + "|" + it.ImageType
|
||||
}
|
||||
|
||||
// gate wraps the external step with the rate limiter and circuit breaker, matching
|
||||
// extGateFunc so it can be injected via workerDeps.extGate.
|
||||
func (w *Worker) gate(f func() (io.ReadCloser, string, error)) (io.ReadCloser, string, error) {
|
||||
if !w.breaker.allow() {
|
||||
return nil, "", errBreakerOpen
|
||||
}
|
||||
if err := w.limiter.Wait(w.runCtx); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
r, path, err := f()
|
||||
w.breaker.record(err)
|
||||
return r, path, err
|
||||
}
|
||||
|
||||
// backoffFor returns min(5m×4^n, 48h) scaled by (1+jitter), with jitter in [-0.2, 0.2].
|
||||
func backoffFor(attempts int, jitter float64) time.Duration {
|
||||
d := math.Min(float64(backoffBase)*math.Pow(4, float64(attempts)), float64(backoffCap))
|
||||
return time.Duration(d * (1 + jitter))
|
||||
}
|
||||
|
||||
func backoff(attempts int) time.Duration {
|
||||
return backoffFor(attempts, rand.Float64()*0.4-0.2) //nolint:gosec // retry jitter, not security-sensitive
|
||||
}
|
||||
|
||||
// breaker opens after breakerThreshold consecutive external errors and admits a
|
||||
// single probe once breakerProbeAfter has elapsed; a success re-closes it.
|
||||
type breaker struct {
|
||||
mu sync.Mutex
|
||||
failures int
|
||||
openedAt time.Time
|
||||
}
|
||||
|
||||
func newBreaker() *breaker { return &breaker{} }
|
||||
|
||||
func (b *breaker) allow() bool {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.failures < breakerThreshold {
|
||||
return true
|
||||
}
|
||||
if time.Since(b.openedAt) >= breakerProbeAfter {
|
||||
b.openedAt = time.Now() // start a fresh probe window so only one caller passes
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (b *breaker) record(err error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
// A not-found is a definitive answer, not a fault; only real errors trip the breaker.
|
||||
if err == nil || errors.Is(err, model.ErrNotFound) {
|
||||
b.failures = 0
|
||||
return
|
||||
}
|
||||
b.failures++
|
||||
if b.failures == breakerThreshold {
|
||||
b.openedAt = time.Now()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/conf/configtest"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
// soakCycles is deliberately >2000: this is a leak regression guard, not a
|
||||
// performance benchmark, so it favors a stable signal over raw speed.
|
||||
const soakCycles = 2200
|
||||
|
||||
var _ = Describe("Worker soak", func() {
|
||||
// Runs processItem over many cycles across a mix of sources, asserting
|
||||
// goroutines/heap plateau instead of growing unbounded (a leak guard). Skipped under -short.
|
||||
It("does not leak goroutines, heap, or fds over many acquisition cycles", func() {
|
||||
if testing.Short() {
|
||||
Skip("skipping soak test in short mode")
|
||||
}
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
|
||||
repoRoot, err := os.Getwd()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
libRepo := &tests.MockLibraryRepo{}
|
||||
libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}})
|
||||
folderRepo := &fakeFolderRepo{result: []model.Folder{{
|
||||
Path: "tests/fixtures/artist/an-album",
|
||||
ImageFiles: []string{"cover.jpg"},
|
||||
}}}
|
||||
ffm := tests.NewMockFFmpeg("")
|
||||
prov := &fakeExternalProvider{}
|
||||
artRepo := tests.CreateMockArtworkRepo()
|
||||
albumRepo := tests.CreateMockAlbumRepo()
|
||||
albumRepo.SetData(model.Albums{
|
||||
{ID: "al-folder", Name: "Folder Album", FolderIDs: []string{"f1"}},
|
||||
{ID: "al-embed", Name: "Embedded Album", EmbedArtPath: "tests/fixtures/artist/an-album/test.mp3", FolderIDs: []string{"f1"}},
|
||||
})
|
||||
ds := &tests.MockDataStore{
|
||||
MockedFolder: folderRepo,
|
||||
MockedLibrary: libRepo,
|
||||
MockedArtwork: artRepo,
|
||||
MockedAlbum: albumRepo,
|
||||
}
|
||||
store := NewImageStore(GinkgoT().TempDir())
|
||||
deps := &workerDeps{ds: ds, store: store, prov: prov, ffmpeg: ffm}
|
||||
conf.Server.CoverArtPriority = "cover.jpg, embedded"
|
||||
|
||||
// Dangling refs (al/ra ids the repos don't know about) mirror an entity
|
||||
// deleted after being enqueued; ds.Radio auto-provisions an empty mock repo.
|
||||
items := []model.ArtworkQueueItem{
|
||||
{ItemKind: "al", ItemID: "al-folder"},
|
||||
{ItemKind: "al", ItemID: "al-embed"},
|
||||
{ItemKind: "al", ItemID: "al-does-not-exist"},
|
||||
{ItemKind: "ra", ItemID: "ra-does-not-exist"},
|
||||
}
|
||||
|
||||
fdCount := func() int {
|
||||
if runtime.GOOS != "linux" {
|
||||
return -1
|
||||
}
|
||||
entries, err := os.ReadDir("/proc/self/fd")
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
return len(entries)
|
||||
}
|
||||
|
||||
settleGoroutines := func() int {
|
||||
// Background goroutines (GC workers, etc.) can take a moment to wind down;
|
||||
// poll for two consecutive equal samples instead of trusting a single one.
|
||||
prev := -1
|
||||
for range 100 {
|
||||
runtime.GC()
|
||||
n := runtime.NumGoroutine()
|
||||
if n == prev {
|
||||
return n
|
||||
}
|
||||
prev = n
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
return prev
|
||||
}
|
||||
|
||||
baselineGoroutines := settleGoroutines()
|
||||
baselineFDs := fdCount()
|
||||
|
||||
var heapAt10Pct uint64
|
||||
start := time.Now()
|
||||
for i := range soakCycles {
|
||||
it := items[i%len(items)]
|
||||
out := processItem(context.Background(), deps, it)
|
||||
|
||||
// "Serve-adjacent" read-back: exercise the Phase 2 surfaces a caller would
|
||||
// use after acquisition, not the old serving pipeline.
|
||||
if out == outcomeFound {
|
||||
ia, err := artRepo.GetItemArtwork(it.ItemKind, it.ItemID, model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred(), "cycle %d: GetItemArtwork", i)
|
||||
art, err := artRepo.GetImage(ia.Hash)
|
||||
Expect(err).ToNot(HaveOccurred(), "cycle %d: GetImage", i)
|
||||
rc, err := store.Open(ia.Hash, art.Mime)
|
||||
switch {
|
||||
case err == nil:
|
||||
_, _ = io.Copy(io.Discard, rc)
|
||||
rc.Close()
|
||||
case os.IsNotExist(err):
|
||||
// Folder-backed art has no store file; that's expected.
|
||||
default:
|
||||
Expect(err).ToNot(HaveOccurred(), "cycle %d: store.Open", i)
|
||||
}
|
||||
}
|
||||
|
||||
if i == soakCycles/10 {
|
||||
runtime.GC()
|
||||
var ms runtime.MemStats
|
||||
runtime.ReadMemStats(&ms)
|
||||
heapAt10Pct = ms.HeapAlloc
|
||||
}
|
||||
}
|
||||
elapsed := time.Since(start)
|
||||
|
||||
finalGoroutines := settleGoroutines()
|
||||
finalFDs := fdCount()
|
||||
|
||||
runtime.GC()
|
||||
var ms runtime.MemStats
|
||||
runtime.ReadMemStats(&ms)
|
||||
|
||||
GinkgoWriter.Printf("soak: cycles=%d elapsed=%s goroutines(baseline=%d final=%d) heap(10%%-mark=%d final=%d) fds(baseline=%d final=%d)\n",
|
||||
soakCycles, elapsed, baselineGoroutines, finalGoroutines, heapAt10Pct, ms.HeapAlloc, baselineFDs, finalFDs)
|
||||
|
||||
Expect(finalGoroutines).To(BeNumerically("<=", baselineGoroutines), "goroutine count grew: baseline=%d final=%d", baselineGoroutines, finalGoroutines)
|
||||
if heapAt10Pct > 0 {
|
||||
Expect(ms.HeapAlloc).To(BeNumerically("<=", 2*heapAt10Pct), "heap did not plateau: 10%%-mark=%d final=%d (final > 2x 10%%-mark)", heapAt10Pct, ms.HeapAlloc)
|
||||
}
|
||||
if runtime.GOOS == "linux" && baselineFDs >= 0 {
|
||||
Expect(finalFDs).To(BeNumerically("<=", baselineFDs), "fd count grew: baseline=%d final=%d", baselineFDs, finalFDs)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,362 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/conf/configtest"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"go.uber.org/goleak"
|
||||
)
|
||||
|
||||
// reenqueueOnDequeue simulates a concurrent scan Enqueue between DequeueBatch and the
|
||||
// worker's delete by bumping retry_at, so a DeleteIfUnchanged on the dequeued value no-ops.
|
||||
type reenqueueOnDequeue struct {
|
||||
*tests.MockArtworkQueueRepo
|
||||
done bool
|
||||
}
|
||||
|
||||
func (r *reenqueueOnDequeue) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) {
|
||||
items, err := r.MockArtworkQueueRepo.DequeueBatch(n)
|
||||
if !r.done && len(items) > 0 {
|
||||
r.done = true
|
||||
for k, it := range r.Data {
|
||||
if it.ItemKind == items[0].ItemKind && it.ItemID == items[0].ItemID {
|
||||
it.RetryAt = items[0].RetryAt.Add(time.Minute)
|
||||
r.Data[k] = it
|
||||
}
|
||||
}
|
||||
}
|
||||
return items, err
|
||||
}
|
||||
|
||||
func findQueued(q *tests.MockArtworkQueueRepo, kind, id string) *model.ArtworkQueueItem {
|
||||
for _, it := range q.Data {
|
||||
if it.ItemKind == kind && it.ItemID == id {
|
||||
return &it
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ = Describe("Worker", func() {
|
||||
var (
|
||||
ctx context.Context
|
||||
ds *tests.MockDataStore
|
||||
folderRepo *fakeFolderRepo
|
||||
libRepo *tests.MockLibraryRepo
|
||||
ffm *tests.MockFFmpeg
|
||||
prov *fakeExternalProvider
|
||||
store *ImageStore
|
||||
artRepo *tests.MockArtworkRepo
|
||||
queueRepo *tests.MockArtworkQueueRepo
|
||||
repoRoot string
|
||||
w *Worker
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
ctx = context.Background()
|
||||
var err error
|
||||
repoRoot, err = os.Getwd()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
folderRepo = &fakeFolderRepo{}
|
||||
libRepo = &tests.MockLibraryRepo{}
|
||||
libRepo.SetData(model.Libraries{{ID: 0, Path: testFileLibPath(repoRoot)}})
|
||||
ffm = tests.NewMockFFmpeg("")
|
||||
prov = &fakeExternalProvider{}
|
||||
artRepo = tests.CreateMockArtworkRepo()
|
||||
queueRepo = tests.CreateMockArtworkQueueRepo()
|
||||
ds = &tests.MockDataStore{
|
||||
MockedFolder: folderRepo,
|
||||
MockedLibrary: libRepo,
|
||||
MockedArtwork: artRepo,
|
||||
MockedArtworkQueue: queueRepo,
|
||||
}
|
||||
ds.MockedAlbum = tests.CreateMockAlbumRepo()
|
||||
store = NewImageStore(GinkgoT().TempDir())
|
||||
conf.Server.CoverArtPriority = "cover.jpg, embedded"
|
||||
conf.Server.ArtworkExternalMaxRPS = 1000 // keep the limiter out of the way of behavior tests
|
||||
w = NewWorker(ds, store, prov, ffm)
|
||||
})
|
||||
|
||||
Describe("drain", func() {
|
||||
It("processes a seeded queue item and removes it from the queue", func() {
|
||||
folderRepo.result = []model.Folder{{
|
||||
Path: "tests/fixtures/artist/an-album",
|
||||
ImageFiles: []string{"cover.jpg"},
|
||||
}}
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "al1", Name: "Album", FolderIDs: []string{"f1"}},
|
||||
})
|
||||
Expect(queueRepo.Enqueue(model.ArtworkQueueItem{
|
||||
ItemKind: "al", ItemID: "al1", Priority: model.ArtworkPriorityScan,
|
||||
})).To(Succeed())
|
||||
|
||||
n, err := w.drain(ctx, 2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(n).To(Equal(1))
|
||||
|
||||
ia, err := artRepo.GetItemArtwork("al", "al1", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ia.Source).To(Equal("folder"))
|
||||
|
||||
count, err := queueRepo.Count()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(count).To(BeZero(), "a found item must be deleted from the queue")
|
||||
})
|
||||
|
||||
It("reschedules a failed item via MarkFailed with a backed-off retry_at", func() {
|
||||
conf.Server.CoverArtPriority = "external"
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "al4", Name: "Album"}})
|
||||
prov.albumImage = func(context.Context, string) (*url.URL, error) {
|
||||
return nil, errors.New("agent timed out")
|
||||
}
|
||||
Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "al", ItemID: "al4"})).To(Succeed())
|
||||
|
||||
n, err := w.drain(ctx, 2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(n).To(Equal(1))
|
||||
|
||||
it := findQueued(queueRepo, "al", "al4")
|
||||
Expect(it).ToNot(BeNil())
|
||||
Expect(it.Attempts).To(Equal(1))
|
||||
Expect(it.RetryAt).To(BeTemporally(">", time.Now()))
|
||||
|
||||
_, err = artRepo.GetItemArtwork("al", "al4", model.ImageTypePrimary)
|
||||
Expect(err).To(MatchError(model.ErrNotFound), "a timeout must never settle on absent")
|
||||
})
|
||||
|
||||
It("reschedules a found-stale item via MarkFailed while keeping its served state", func() {
|
||||
conf.Server.CoverArtPriority = "external, cover.jpg"
|
||||
folderRepo.result = []model.Folder{{
|
||||
Path: "tests/fixtures/artist/an-album",
|
||||
ImageFiles: []string{"cover.jpg"},
|
||||
}}
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "alstale", Name: "Album", FolderIDs: []string{"f1"}},
|
||||
})
|
||||
prov.albumImage = func(context.Context, string) (*url.URL, error) {
|
||||
return nil, errors.New("agent timed out")
|
||||
}
|
||||
Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "al", ItemID: "alstale"})).To(Succeed())
|
||||
|
||||
n, err := w.drain(ctx, 2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(n).To(Equal(1))
|
||||
|
||||
it := findQueued(queueRepo, "al", "alstale")
|
||||
Expect(it).ToNot(BeNil(), "a found-stale row must survive for a higher-priority retry")
|
||||
Expect(it.Attempts).To(Equal(1))
|
||||
Expect(it.RetryAt).To(BeTemporally(">", time.Now()))
|
||||
|
||||
ia, err := artRepo.GetItemArtwork("al", "alstale", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ia.Source).To(Equal("folder"), "the fallback art is served meanwhile")
|
||||
})
|
||||
|
||||
It("keeps a row re-enqueued between dequeue and delete", func() {
|
||||
folderRepo.result = []model.Folder{{
|
||||
Path: "tests/fixtures/artist/an-album",
|
||||
ImageFiles: []string{"cover.jpg"},
|
||||
}}
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{
|
||||
{ID: "al7", Name: "Album", FolderIDs: []string{"f1"}},
|
||||
})
|
||||
racing := &reenqueueOnDequeue{MockArtworkQueueRepo: queueRepo}
|
||||
ds.MockedArtworkQueue = racing
|
||||
w = NewWorker(ds, store, prov, ffm)
|
||||
Expect(queueRepo.Enqueue(model.ArtworkQueueItem{
|
||||
ItemKind: "al", ItemID: "al7", Priority: model.ArtworkPriorityScan,
|
||||
})).To(Succeed())
|
||||
|
||||
n, err := w.drain(ctx, 1)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(n).To(Equal(1))
|
||||
|
||||
// The concurrent re-enqueue changed retry_at, so the found-path delete was a no-op.
|
||||
Expect(findQueued(queueRepo, "al", "al7")).ToNot(BeNil())
|
||||
ia, err := artRepo.GetItemArtwork("al", "al7", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ia.Source).To(Equal("folder"))
|
||||
})
|
||||
|
||||
It("keeps a fresh re-enqueue ahead of a stale failure backoff", func() {
|
||||
conf.Server.CoverArtPriority = "external"
|
||||
ds.MockedAlbum.(*tests.MockAlbumRepo).SetData(model.Albums{{ID: "al8", Name: "Album"}})
|
||||
prov.albumImage = func(context.Context, string) (*url.URL, error) {
|
||||
return nil, errors.New("agent timed out")
|
||||
}
|
||||
racing := &reenqueueOnDequeue{MockArtworkQueueRepo: queueRepo}
|
||||
ds.MockedArtworkQueue = racing
|
||||
w = NewWorker(ds, store, prov, ffm)
|
||||
Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "al", ItemID: "al8"})).To(Succeed())
|
||||
dequeued := findQueued(queueRepo, "al", "al8").RetryAt
|
||||
|
||||
n, err := w.drain(ctx, 1)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(n).To(Equal(1))
|
||||
|
||||
// The concurrent re-enqueue reset retry_at; the failure path must not stomp it
|
||||
// with stale backoff nor bump attempts, so the row stays immediately eligible.
|
||||
it := findQueued(queueRepo, "al", "al8")
|
||||
Expect(it).ToNot(BeNil())
|
||||
Expect(it.Attempts).To(BeZero())
|
||||
Expect(it.RetryAt).To(BeTemporally("==", dequeued.Add(time.Minute)))
|
||||
})
|
||||
|
||||
It("resolves a private playlist under an admin context instead of failing forever", func() {
|
||||
ds.MockedUser = adminUserRepo()
|
||||
vds := &visibilityPlaylistDS{
|
||||
MockDataStore: ds,
|
||||
private: model.Playlist{ID: "plPriv", OwnerID: "admin"},
|
||||
tracks: &tests.MockPlaylistTrackRepo{},
|
||||
}
|
||||
w = NewWorker(vds, store, prov, ffm)
|
||||
Expect(queueRepo.Enqueue(model.ArtworkQueueItem{ItemKind: "pl", ItemID: "plPriv"})).To(Succeed())
|
||||
|
||||
n, err := w.drain(ctx, 1)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(n).To(Equal(1))
|
||||
|
||||
// Resolved as absent (no art) and removed — not stuck failing on ErrNotFound forever.
|
||||
Expect(findQueued(queueRepo, "pl", "plPriv")).To(BeNil())
|
||||
ia, err := artRepo.GetItemArtwork("pl", "plPriv", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ia.Hash).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("returns zero when the queue is empty", func() {
|
||||
n, err := w.drain(ctx, 2)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(n).To(BeZero())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Bump", func() {
|
||||
It("enqueues at Bump priority and wakes the loop", func() {
|
||||
w.Bump("al", "al9")
|
||||
it := findQueued(queueRepo, "al", "al9")
|
||||
Expect(it).ToNot(BeNil())
|
||||
Expect(it.Priority).To(Equal(model.ArtworkPriorityBump))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("gate/breaker", func() {
|
||||
It("opens after 5 consecutive external errors and short-circuits the step", func() {
|
||||
var calls int
|
||||
failing := func() (io.ReadCloser, string, error) {
|
||||
calls++
|
||||
return nil, "", errors.New("boom")
|
||||
}
|
||||
for range 5 {
|
||||
_, _, err := w.gate(failing)
|
||||
Expect(err).To(HaveOccurred())
|
||||
}
|
||||
Expect(calls).To(Equal(5))
|
||||
|
||||
_, _, err := w.gate(failing)
|
||||
Expect(err).To(HaveOccurred())
|
||||
Expect(calls).To(Equal(5), "an open breaker must not call the external step")
|
||||
})
|
||||
|
||||
It("resets the failure count on a successful call", func() {
|
||||
failing := func() (io.ReadCloser, string, error) { return nil, "", errors.New("boom") }
|
||||
ok := func() (io.ReadCloser, string, error) { return io.NopCloser(nil), "p", nil }
|
||||
for range 4 {
|
||||
_, _, _ = w.gate(failing)
|
||||
}
|
||||
_, _, err := w.gate(ok)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
var calls int
|
||||
counting := func() (io.ReadCloser, string, error) {
|
||||
calls++
|
||||
return nil, "", errors.New("boom")
|
||||
}
|
||||
for range 5 {
|
||||
_, _, _ = w.gate(counting)
|
||||
}
|
||||
Expect(calls).To(Equal(5), "the breaker should have re-closed after the success")
|
||||
})
|
||||
})
|
||||
|
||||
Describe("RunPrune", func() {
|
||||
It("runs a prune under the worker mutex", func() {
|
||||
Expect(w.RunPrune(ctx)).To(Succeed())
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Run", func() {
|
||||
It("exits cleanly when the context is cancelled", func() {
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- w.Run(runCtx) }()
|
||||
|
||||
cancel()
|
||||
Eventually(done, time.Second).Should(Receive(BeNil()))
|
||||
})
|
||||
|
||||
It("does not leak goroutines after Run exits", func() {
|
||||
DeferCleanup(configtest.SetupConfig())
|
||||
|
||||
ignore := goleak.IgnoreCurrent()
|
||||
DeferCleanup(func() { goleak.VerifyNone(GinkgoT(), ignore) })
|
||||
|
||||
localDS := &tests.MockDataStore{MockedArtworkQueue: tests.CreateMockArtworkQueueRepo()}
|
||||
lw := NewWorker(localDS, NewImageStore(GinkgoT().TempDir()), &fakeExternalProvider{}, tests.NewMockFFmpeg(""))
|
||||
|
||||
runCtx, cancel := context.WithCancel(ctx)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- lw.Run(runCtx) }()
|
||||
|
||||
time.Sleep(20 * time.Millisecond) // let the loop settle on the idle select
|
||||
cancel()
|
||||
Eventually(done, 2*time.Second).Should(Receive(BeNil()))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("backoff", func() {
|
||||
It("returns the expected schedule with no jitter", func() {
|
||||
for _, c := range []struct {
|
||||
attempts int
|
||||
want time.Duration
|
||||
}{
|
||||
{0, 5 * time.Minute},
|
||||
{1, 20 * time.Minute},
|
||||
{2, 80 * time.Minute},
|
||||
{3, 320 * time.Minute},
|
||||
{4, 1280 * time.Minute},
|
||||
{5, 48 * time.Hour},
|
||||
{6, 48 * time.Hour},
|
||||
} {
|
||||
Expect(backoffFor(c.attempts, 0)).To(Equal(c.want), "attempt %d", c.attempts)
|
||||
}
|
||||
})
|
||||
|
||||
It("applies jitter proportionally", func() {
|
||||
base := backoffFor(2, 0)
|
||||
Expect(backoffFor(2, 0.2)).To(Equal(time.Duration(float64(base) * 1.2)))
|
||||
Expect(backoffFor(2, -0.2)).To(Equal(time.Duration(float64(base) * 0.8)))
|
||||
})
|
||||
|
||||
It("keeps random jitter within +/-20%", func() {
|
||||
lo := time.Duration(float64(320*time.Minute) * 0.8)
|
||||
hi := time.Duration(float64(320*time.Minute) * 1.2)
|
||||
for range 200 {
|
||||
d := backoff(3)
|
||||
Expect(d).To(BeNumerically(">=", lo))
|
||||
Expect(d).To(BeNumerically("<=", hi))
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,39 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
// Drives the real breaker state machine with the fake clock. Plain test: testing/synctest
|
||||
// needs a *testing.T, which Ginkgo doesn't give.
|
||||
func TestArtworkBreakerHalfOpen(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
g := NewWithT(t)
|
||||
b := newBreaker()
|
||||
|
||||
for range breakerThreshold {
|
||||
b.record(errors.New("boom"))
|
||||
}
|
||||
g.Expect(b.allow()).To(BeFalse(), "breaker opens after consecutive errors")
|
||||
|
||||
time.Sleep(breakerProbeAfter - time.Nanosecond)
|
||||
g.Expect(b.allow()).To(BeFalse(), "still open before the probe interval")
|
||||
|
||||
time.Sleep(time.Nanosecond)
|
||||
g.Expect(b.allow()).To(BeTrue(), "half-open: one probe is granted")
|
||||
g.Expect(b.allow()).To(BeFalse(), "only a single probe per interval")
|
||||
|
||||
b.record(errors.New("boom")) // probe fails -> stay open
|
||||
time.Sleep(breakerProbeAfter)
|
||||
g.Expect(b.allow()).To(BeTrue(), "another probe after the next interval")
|
||||
|
||||
b.record(nil) // probe succeeds -> close
|
||||
g.Expect(b.allow()).To(BeTrue(), "closed breaker admits freely")
|
||||
g.Expect(b.allow()).To(BeTrue())
|
||||
})
|
||||
}
|
||||
Vendored
+34
-8
@@ -36,6 +36,8 @@ type Provider interface {
|
||||
SimilarSongs(ctx context.Context, id string, count int) (model.MediaFiles, error)
|
||||
TopSongs(ctx context.Context, artist string, count int) (model.MediaFiles, error)
|
||||
ArtistImage(ctx context.Context, id string) (*url.URL, error)
|
||||
// ArtistImageResult is like ArtistImage but reports a transient agent failure as a real error, not ErrNotFound.
|
||||
ArtistImageResult(ctx context.Context, id string) (*url.URL, error)
|
||||
AlbumImage(ctx context.Context, id string) (*url.URL, error)
|
||||
}
|
||||
|
||||
@@ -258,7 +260,7 @@ func (e *provider) populateArtistInfo(ctx context.Context, artist auxArtist) (au
|
||||
// Call all registered agents and collect information
|
||||
g := errgroup.Group{}
|
||||
g.SetLimit(2)
|
||||
g.Go(func() error { e.callGetImage(ctx, e.ag, &artist); return nil })
|
||||
g.Go(func() error { _ = e.callGetImage(ctx, e.ag, &artist); return nil })
|
||||
g.Go(func() error { e.callGetBiography(ctx, e.ag, &artist); return nil })
|
||||
g.Go(func() error { e.callGetURL(ctx, e.ag, &artist); return nil })
|
||||
g.Go(func() error { e.callGetSimilarArtists(ctx, e.ag, &artist, maxSimilarArtists, true); return nil })
|
||||
@@ -371,18 +373,35 @@ func (e *provider) similarSongsFallback(ctx context.Context, id string, count in
|
||||
}
|
||||
|
||||
func (e *provider) ArtistImage(ctx context.Context, id string) (*url.URL, error) {
|
||||
u, _, err := e.artistImage(ctx, id)
|
||||
return u, err
|
||||
}
|
||||
|
||||
// ArtistImageResult is like ArtistImage but surfaces a transient agent failure as the
|
||||
// real error, so an agent outage is not mistaken for a definitive no-image (ErrNotFound).
|
||||
func (e *provider) ArtistImageResult(ctx context.Context, id string) (*url.URL, error) {
|
||||
u, agentErr, err := e.artistImage(ctx, id)
|
||||
if agentErr != nil && errors.Is(err, model.ErrNotFound) {
|
||||
return nil, agentErr
|
||||
}
|
||||
return u, err
|
||||
}
|
||||
|
||||
// artistImage returns the agent error (agentErr) separately from the caller-facing err,
|
||||
// so ArtistImageResult can tell "agent errored" apart from "definitively no image".
|
||||
func (e *provider) artistImage(ctx context.Context, id string) (u *url.URL, agentErr error, err error) {
|
||||
artist, err := e.getArtist(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
imageUrl := artist.ArtistImageUrl()
|
||||
if imageUrl == "" {
|
||||
// No cached URL — must fetch from external source synchronously
|
||||
e.callGetImage(ctx, e.ag, &artist)
|
||||
agentErr = e.callGetImage(ctx, e.ag, &artist)
|
||||
if utils.IsCtxDone(ctx) {
|
||||
log.Warn(ctx, "ArtistImage call canceled", ctx.Err())
|
||||
return nil, ctx.Err()
|
||||
return nil, agentErr, ctx.Err()
|
||||
}
|
||||
imageUrl = artist.ArtistImageUrl()
|
||||
} else {
|
||||
@@ -396,9 +415,10 @@ func (e *provider) ArtistImage(ctx context.Context, id string) (*url.URL, error)
|
||||
}
|
||||
|
||||
if imageUrl == "" {
|
||||
return nil, model.ErrNotFound
|
||||
return nil, agentErr, model.ErrNotFound
|
||||
}
|
||||
return url.Parse(imageUrl)
|
||||
u, err = url.Parse(imageUrl)
|
||||
return u, agentErr, err
|
||||
}
|
||||
|
||||
func (e *provider) AlbumImage(ctx context.Context, id string) (*url.URL, error) {
|
||||
@@ -519,10 +539,15 @@ func (e *provider) callGetBiography(ctx context.Context, agent agents.ArtistBiog
|
||||
artist.Biography = strings.ReplaceAll(bio, "<a ", "<a target='_blank' ")
|
||||
}
|
||||
|
||||
func (e *provider) callGetImage(ctx context.Context, agent agents.ArtistImageRetriever, artist *auxArtist) {
|
||||
// callGetImage populates artist's image URLs. A transient agent failure is
|
||||
// returned as-is; a definitive "no image" is normalized to model.ErrNotFound.
|
||||
func (e *provider) callGetImage(ctx context.Context, agent agents.ArtistImageRetriever, artist *auxArtist) error {
|
||||
images, err := agent.GetArtistImages(ctx, artist.ID, artist.Name(), artist.MbzArtistID)
|
||||
if err != nil {
|
||||
return
|
||||
if errors.Is(err, agents.ErrNotFound) {
|
||||
return model.ErrNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
sort.Slice(images, func(i, j int) bool { return images[i].Size > images[j].Size })
|
||||
|
||||
@@ -535,6 +560,7 @@ func (e *provider) callGetImage(ctx context.Context, agent agents.ArtistImageRet
|
||||
if len(images) >= 3 {
|
||||
artist.SmallImageUrl = images[2].URL
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *provider) callGetSimilarArtists(ctx context.Context, agent agents.ArtistSimilarRetriever, artist *auxArtist,
|
||||
|
||||
+43
@@ -330,6 +330,49 @@ var _ = Describe("Provider - ArtistImage", func() {
|
||||
Expect(logBuf.String()).To(ContainSubstring("Artist image info expired, enqueuing background refresh"))
|
||||
})
|
||||
|
||||
Describe("ArtistImageResult", func() {
|
||||
It("returns the real agent error on a transient failure, not ErrNotFound", func() {
|
||||
agentErr := errors.New("agent timed out")
|
||||
mockImageAgent.Mock = mock.Mock{}
|
||||
mockImageAgent.On("GetArtistImages", ctx, "artist-1", "Artist One", "").Return(nil, agentErr).Once()
|
||||
|
||||
imgURL, err := provider.ArtistImageResult(ctx, "artist-1")
|
||||
|
||||
Expect(err).To(MatchError(agentErr))
|
||||
Expect(err).ToNot(MatchError(model.ErrNotFound))
|
||||
Expect(imgURL).To(BeNil())
|
||||
})
|
||||
|
||||
It("returns ErrNotFound when the agent definitively has no image", func() {
|
||||
mockImageAgent.Mock = mock.Mock{}
|
||||
mockImageAgent.On("GetArtistImages", ctx, "artist-1", "Artist One", "").Return(nil, agents.ErrNotFound).Once()
|
||||
|
||||
imgURL, err := provider.ArtistImageResult(ctx, "artist-1")
|
||||
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
Expect(imgURL).To(BeNil())
|
||||
})
|
||||
|
||||
It("returns ErrNotFound when the agent returns no images without error", func() {
|
||||
mockImageAgent.Mock = mock.Mock{}
|
||||
mockImageAgent.On("GetArtistImages", ctx, "artist-1", "Artist One", "").Return([]agents.ExternalImage{}, nil).Once()
|
||||
|
||||
imgURL, err := provider.ArtistImageResult(ctx, "artist-1")
|
||||
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
Expect(imgURL).To(BeNil())
|
||||
})
|
||||
|
||||
It("returns the largest image URL on success", func() {
|
||||
expectedURL, _ := url.Parse("http://example.com/large.jpg")
|
||||
|
||||
imgURL, err := provider.ArtistImageResult(ctx, "artist-1")
|
||||
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(imgURL).To(Equal(expectedURL))
|
||||
})
|
||||
})
|
||||
|
||||
Context("Unicode handling in artist names", func() {
|
||||
var artistWithEnDash *model.Artist
|
||||
var expectedURL *url.URL
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE artwork (
|
||||
hash TEXT PRIMARY KEY,
|
||||
mime TEXT NOT NULL,
|
||||
width INTEGER NOT NULL DEFAULT 0,
|
||||
height INTEGER NOT NULL DEFAULT 0,
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
blur_hash TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE item_artwork (
|
||||
item_kind TEXT NOT NULL,
|
||||
item_id TEXT NOT NULL,
|
||||
image_type TEXT NOT NULL DEFAULT 'primary',
|
||||
hash TEXT NOT NULL DEFAULT '',
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
source_path TEXT NOT NULL DEFAULT '',
|
||||
ref_mtime INTEGER NOT NULL DEFAULT 0,
|
||||
attempted_at TIMESTAMP,
|
||||
updated_at TIMESTAMP,
|
||||
PRIMARY KEY (item_kind, item_id, image_type)
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX ix_item_artwork_hash ON item_artwork(hash);
|
||||
|
||||
CREATE TABLE artwork_queue (
|
||||
item_kind TEXT NOT NULL,
|
||||
item_id TEXT NOT NULL,
|
||||
image_type TEXT NOT NULL DEFAULT 'primary',
|
||||
priority INTEGER NOT NULL DEFAULT 0,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
retry_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
enqueued_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (item_kind, item_id, image_type)
|
||||
) WITHOUT ROWID;
|
||||
-- Ordered to match DequeueBatch (priority DESC, enqueued_at) so drains stop after n rows; retry_at makes it covering.
|
||||
CREATE INDEX ix_artwork_queue_drain ON artwork_queue(priority DESC, enqueued_at, retry_at);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE artwork_queue;
|
||||
DROP TABLE item_artwork;
|
||||
DROP TABLE artwork;
|
||||
@@ -57,6 +57,7 @@ require (
|
||||
github.com/tetratelabs/wazero v1.12.0
|
||||
github.com/unrolled/secure v1.17.0
|
||||
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342
|
||||
github.com/zeebo/xxh3 v1.1.0
|
||||
go.senan.xyz/taglib v0.11.1
|
||||
go.uber.org/goleak v1.3.0
|
||||
golang.org/x/image v0.44.0
|
||||
@@ -128,7 +129,6 @@ require (
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect
|
||||
github.com/valyala/fastjson v1.6.10 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
|
||||
@@ -142,6 +142,7 @@ type AlbumRepository interface {
|
||||
UpdateExternalInfo(*Album) error
|
||||
Get(id string) (*Album, error)
|
||||
GetAll(...QueryOptions) (Albums, error)
|
||||
GetAllIDs(...QueryOptions) ([]string, error)
|
||||
GetCursor(...QueryOptions) (AlbumCursor, error)
|
||||
GetYears(libraryIDs ...int) ([]int, error)
|
||||
|
||||
|
||||
@@ -89,6 +89,7 @@ type ArtistRepository interface {
|
||||
UpdateExternalInfo(a *Artist) error
|
||||
Get(id string) (*Artist, error)
|
||||
GetAll(options ...QueryOptions) (Artists, error)
|
||||
GetAllIDs(options ...QueryOptions) ([]string, error)
|
||||
GetCursor(options ...QueryOptions) (ArtistCursor, error)
|
||||
GetIndex(includeMissing bool, libraryIds []int, roles ...Role) (ArtistIndexes, error)
|
||||
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
// Artwork is one unique image, identified by the XXH3-64 hash of its bytes.
|
||||
type Artwork struct {
|
||||
Hash string `structs:"hash"`
|
||||
Mime string `structs:"mime"`
|
||||
Width int `structs:"width"`
|
||||
Height int `structs:"height"`
|
||||
SizeBytes int64 `structs:"size_bytes"`
|
||||
BlurHash string `structs:"blur_hash"`
|
||||
CreatedAt time.Time `structs:"created_at"`
|
||||
}
|
||||
|
||||
const ImageTypePrimary = "primary"
|
||||
|
||||
// ItemArtwork is an entity's resolved artwork state. Hash=="" means known absent.
|
||||
type ItemArtwork struct {
|
||||
ItemKind string `structs:"item_kind"`
|
||||
ItemID string `structs:"item_id"`
|
||||
ImageType string `structs:"image_type"`
|
||||
Hash string `structs:"hash"`
|
||||
Source string `structs:"source"`
|
||||
// SourcePath is the backing file (folder/upload: the image; embedded: the audio file); "" otherwise.
|
||||
SourcePath string `structs:"source_path"`
|
||||
// RefMtime is SourcePath's mtime at resolution; 0 when there is no SourcePath.
|
||||
RefMtime int64 `structs:"ref_mtime"`
|
||||
// attempted_at/updated_at are nullable in the schema but always set by PutItemArtwork;
|
||||
// raw inserts must set them too, since these non-pointer time.Time fields fail to scan NULL.
|
||||
AttemptedAt time.Time `structs:"attempted_at"`
|
||||
UpdatedAt time.Time `structs:"updated_at"`
|
||||
}
|
||||
|
||||
// ItemArtworkInfo is the list-hydration projection (item_artwork joined with artwork).
|
||||
type ItemArtworkInfo struct {
|
||||
ItemID string
|
||||
Hash string
|
||||
BlurHash string
|
||||
}
|
||||
|
||||
// Absent reports a known-absent artwork state (resolved, no image).
|
||||
func (i ItemArtworkInfo) Absent() bool { return i.Hash == "" }
|
||||
|
||||
type ArtworkQueueItem struct {
|
||||
ItemKind string `structs:"item_kind"`
|
||||
ItemID string `structs:"item_id"`
|
||||
ImageType string `structs:"image_type"`
|
||||
Priority int `structs:"priority"`
|
||||
Attempts int `structs:"attempts"`
|
||||
RetryAt time.Time `structs:"retry_at"`
|
||||
EnqueuedAt time.Time `structs:"enqueued_at"`
|
||||
}
|
||||
|
||||
// Queue priorities: higher drains first.
|
||||
const (
|
||||
ArtworkPriorityRecheck = 0
|
||||
ArtworkPriorityBackfill = 10
|
||||
ArtworkPriorityScan = 50
|
||||
ArtworkPriorityBump = 100
|
||||
)
|
||||
|
||||
type ArtworkRepository interface {
|
||||
// Image identity (artwork table)
|
||||
GetImage(hash string) (*Artwork, error)
|
||||
PutImage(a *Artwork) error
|
||||
GetImages(hashes []string) (map[string]Artwork, error)
|
||||
// GetOrphanHashes returns hashes referenced by no item_artwork row and older than cutoff.
|
||||
GetOrphanHashes(createdBefore time.Time) ([]string, error)
|
||||
// DeleteOrphans deletes the given hashes only if still unreferenced and older than cutoff (atomic re-check).
|
||||
DeleteOrphans(createdBefore time.Time, hashes []string) error
|
||||
// Per-item state (item_artwork table)
|
||||
GetItemArtwork(kind, id, imageType string) (*ItemArtwork, error)
|
||||
PutItemArtwork(ia *ItemArtwork) error
|
||||
DeleteForItem(kind, id string) error
|
||||
// GetInfoForItems hydrates a page: one batched query, item_artwork joined to artwork.
|
||||
GetInfoForItems(kind string, ids []string) (map[string]ItemArtworkInfo, error)
|
||||
// GetAllMimes returns hash -> current mime for every stored artwork, for sweep retention checks.
|
||||
GetAllMimes() (map[string]string, error)
|
||||
// PurgeDanglingItemArtwork removes state rows whose entity no longer exists.
|
||||
PurgeDanglingItemArtwork() (int64, error)
|
||||
}
|
||||
|
||||
type ArtworkQueueRepository interface {
|
||||
// Enqueue upserts; an existing row keeps the higher of the two priorities.
|
||||
Enqueue(items ...ArtworkQueueItem) error
|
||||
// DequeueBatch returns up to n items with retry_at <= now, priority desc, enqueued_at asc.
|
||||
DequeueBatch(n int) ([]ArtworkQueueItem, error)
|
||||
// MarkFailed increments attempts and pushes retry_at into the future.
|
||||
MarkFailed(kind, id, imageType string, retryAt time.Time) error
|
||||
// MarkFailedIfUnchanged applies the failure backoff only while retry_at still matches
|
||||
// seenRetryAt; a concurrent re-enqueue (which resets retry_at) keeps its fresh eligibility.
|
||||
MarkFailedIfUnchanged(kind, id, imageType string, seenRetryAt, retryAt time.Time) error
|
||||
Delete(kind, id, imageType string) error
|
||||
// DeleteIfUnchanged deletes the row only if its retry_at still matches retryAt, so a
|
||||
// concurrent re-enqueue (which resets retry_at) survives instead of being erased.
|
||||
DeleteIfUnchanged(kind, id, imageType string, retryAt time.Time) error
|
||||
Count() (int64, error)
|
||||
// EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff.
|
||||
EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error)
|
||||
// PurgeDangling removes queue rows whose entity no longer exists.
|
||||
PurgeDangling() (int64, error)
|
||||
}
|
||||
@@ -40,6 +40,8 @@ type DataStore interface {
|
||||
ScrobbleBuffer(ctx context.Context) ScrobbleBufferRepository
|
||||
Scrobble(ctx context.Context) ScrobbleRepository
|
||||
Plugin(ctx context.Context) PluginRepository
|
||||
Artwork(ctx context.Context) ArtworkRepository
|
||||
ArtworkQueue(ctx context.Context) ArtworkQueueRepository
|
||||
|
||||
Resource(ctx context.Context, model any) ResourceRepository
|
||||
|
||||
|
||||
@@ -143,6 +143,7 @@ type PlaylistRepository interface {
|
||||
Get(id string) (*Playlist, error)
|
||||
GetWithTracks(id string, refreshSmartPlaylist, includeMissing bool) (*Playlist, error)
|
||||
GetAll(options ...QueryOptions) (Playlists, error)
|
||||
GetAllIDs(options ...QueryOptions) ([]string, error)
|
||||
GetCursor(options ...QueryOptions) (PlaylistCursor, error)
|
||||
FindByPath(path string) (*Playlist, error)
|
||||
Delete(id string) error
|
||||
|
||||
@@ -32,5 +32,6 @@ type RadioRepository interface {
|
||||
Delete(id string) error
|
||||
Get(id string) (*Radio, error)
|
||||
GetAll(options ...QueryOptions) (Radios, error)
|
||||
GetAllIDs(options ...QueryOptions) ([]string, error)
|
||||
Put(u *Radio, colsToUpdate ...string) error
|
||||
}
|
||||
@@ -254,6 +254,15 @@ func (r *albumRepository) GetAll(options ...model.QueryOptions) (model.Albums, e
|
||||
return res.toModels(), nil
|
||||
}
|
||||
|
||||
// GetAllIDs returns just the album IDs for the same row set as GetAll, skipping the
|
||||
// heavy column projection and JSON post-processing. Used by bulk enumeration (artwork backfill).
|
||||
func (r *albumRepository) GetAllIDs(options ...model.QueryOptions) ([]string, error) {
|
||||
sq := r.applyLibraryFilter(r.newSelect(options...).Columns("album.id"))
|
||||
ids := []string{}
|
||||
err := r.queryAllSlice(sq, &ids)
|
||||
return ids, err
|
||||
}
|
||||
|
||||
func (r *albumRepository) GetCursor(options ...model.QueryOptions) (model.AlbumCursor, error) {
|
||||
sq := r.selectAlbum(options...)
|
||||
cursor, err := queryWithStableResults[dbAlbum](r.sqlRepository, sq)
|
||||
|
||||
@@ -84,6 +84,21 @@ var _ = Describe("AlbumRepository", func() {
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GetAllIDs", func() {
|
||||
It("returns the same id set as GetAll", func() {
|
||||
want, err := albumRepo.GetAll()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(want).ToNot(BeEmpty())
|
||||
wantIDs := make([]string, 0, len(want))
|
||||
for _, a := range want {
|
||||
wantIDs = append(wantIDs, a.ID)
|
||||
}
|
||||
ids, err := albumRepo.GetAllIDs()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ids).To(ConsistOf(wantIDs))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GetAll", func() {
|
||||
var GetAll = func(opts ...model.QueryOptions) (model.Albums, error) {
|
||||
albums, err := albumRepo.GetAll(opts...)
|
||||
|
||||
@@ -264,6 +264,15 @@ func (r *artistRepository) GetAll(options ...model.QueryOptions) (model.Artists,
|
||||
return res, err
|
||||
}
|
||||
|
||||
// GetAllIDs returns just the artist IDs for the same row set as GetAll, skipping the
|
||||
// heavy stats/annotation columns and JSON post-processing. Used by bulk enumeration (artwork backfill).
|
||||
func (r *artistRepository) GetAllIDs(options ...model.QueryOptions) ([]string, error) {
|
||||
sq := r.applyLibraryFilterToArtistQuery(r.newSelect(options...).Columns("artist.id")).GroupBy("artist.id")
|
||||
ids := []string{}
|
||||
err := r.queryAllSlice(sq, &ids)
|
||||
return ids, err
|
||||
}
|
||||
|
||||
func (r *artistRepository) GetCursor(options ...model.QueryOptions) (model.ArtistCursor, error) {
|
||||
sel := r.selectArtist(options...)
|
||||
cursor, err := queryWithStableResults[dbArtist](r.sqlRepository, sel)
|
||||
|
||||
@@ -284,6 +284,21 @@ var _ = Describe("ArtistRepository", func() {
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GetAllIDs", func() {
|
||||
It("returns the same id set as GetAll", func() {
|
||||
want, err := repo.GetAll()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(want).ToNot(BeEmpty())
|
||||
wantIDs := make([]string, 0, len(want))
|
||||
for _, a := range want {
|
||||
wantIDs = append(wantIDs, a.ID)
|
||||
}
|
||||
ids, err := repo.GetAllIDs()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ids).To(ConsistOf(wantIDs))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Basic Operations", func() {
|
||||
Describe("Count", func() {
|
||||
It("returns the number of artists in the DB", func() {
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
. "github.com/Masterminds/squirrel"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/pocketbase/dbx"
|
||||
)
|
||||
|
||||
// enqueueChunkSize keeps each multi-row insert under SQLite's bind-variable limit (7 cols -> 700 vars).
|
||||
const enqueueChunkSize = 100
|
||||
|
||||
type artworkQueueRepository struct {
|
||||
sqlRepository
|
||||
}
|
||||
|
||||
func NewArtworkQueueRepository(ctx context.Context, db dbx.Builder) model.ArtworkQueueRepository {
|
||||
r := &artworkQueueRepository{}
|
||||
r.ctx = ctx
|
||||
r.db = db
|
||||
r.tableName = "artwork_queue"
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *artworkQueueRepository) Enqueue(items ...model.ArtworkQueueItem) error {
|
||||
now := time.Now()
|
||||
for chunk := range slices.Chunk(items, enqueueChunkSize) {
|
||||
ins := Insert(r.tableName).Columns("item_kind", "item_id", "image_type", "priority", "attempts", "retry_at", "enqueued_at")
|
||||
for _, it := range chunk {
|
||||
if it.ImageType == "" {
|
||||
it.ImageType = model.ImageTypePrimary
|
||||
}
|
||||
ins = ins.Values(it.ItemKind, it.ItemID, it.ImageType, it.Priority, 0, now, now)
|
||||
}
|
||||
ins = ins.Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET
|
||||
priority = MAX(priority, excluded.priority), retry_at = excluded.retry_at`)
|
||||
if _, err := r.executeSQL(ins); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *artworkQueueRepository) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) {
|
||||
sel := Select("*").From(r.tableName).
|
||||
Where(LtOrEq{"retry_at": time.Now()}).
|
||||
OrderBy("priority DESC", "enqueued_at ASC").
|
||||
Limit(uint64(n))
|
||||
var res []model.ArtworkQueueItem
|
||||
err := r.queryAll(sel, &res)
|
||||
return res, err
|
||||
}
|
||||
|
||||
func (r *artworkQueueRepository) MarkFailed(kind, id, imageType string, retryAt time.Time) error {
|
||||
upd := Update(r.tableName).
|
||||
Set("attempts", Expr("attempts + 1")).
|
||||
Set("retry_at", retryAt).
|
||||
Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType})
|
||||
c, err := r.executeSQL(upd)
|
||||
if err == nil && c == 0 {
|
||||
return model.ErrNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// MarkFailedIfUnchanged applies the backoff only while retry_at still equals seenRetryAt;
|
||||
// a concurrent Enqueue resets retry_at, so its fresh eligibility survives untouched.
|
||||
func (r *artworkQueueRepository) MarkFailedIfUnchanged(kind, id, imageType string, seenRetryAt, retryAt time.Time) error {
|
||||
upd := Update(r.tableName).
|
||||
Set("attempts", Expr("attempts + 1")).
|
||||
Set("retry_at", retryAt).
|
||||
Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType, "retry_at": seenRetryAt})
|
||||
_, err := r.executeSQL(upd)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *artworkQueueRepository) Delete(kind, id, imageType string) error {
|
||||
return r.delete(Eq{"item_kind": kind, "item_id": id, "image_type": imageType})
|
||||
}
|
||||
|
||||
// DeleteIfUnchanged deletes the row only while its retry_at still equals the dequeued
|
||||
// value; a concurrent Enqueue resets retry_at, so the row survives to be re-resolved.
|
||||
func (r *artworkQueueRepository) DeleteIfUnchanged(kind, id, imageType string, retryAt time.Time) error {
|
||||
return r.delete(Eq{"item_kind": kind, "item_id": id, "image_type": imageType, "retry_at": retryAt})
|
||||
}
|
||||
|
||||
// PurgeDangling removes queue rows whose entity no longer exists, per kind.
|
||||
func (r *artworkQueueRepository) PurgeDangling() (int64, error) {
|
||||
return purgeDangling(r.executeSQL, r.tableName)
|
||||
}
|
||||
|
||||
func (r *artworkQueueRepository) Count() (int64, error) {
|
||||
var res struct{ Count int64 }
|
||||
err := r.queryOne(Select("count(*) as count").From(r.tableName), &res)
|
||||
return res.Count, err
|
||||
}
|
||||
|
||||
func (r *artworkQueueRepository) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) {
|
||||
now := time.Now()
|
||||
// DO NOTHING is deliberate: rechecks must not bump priority/retry_at of already-queued items.
|
||||
ins := Expr(`INSERT INTO `+r.tableName+` (item_kind, item_id, image_type, priority, attempts, retry_at, enqueued_at)
|
||||
SELECT item_kind, item_id, image_type, ?, 0, ?, ?
|
||||
FROM `+itemArtworkTable+` WHERE item_kind = ? AND hash = '' AND attempted_at < ?
|
||||
ON CONFLICT (item_kind, item_id, image_type) DO NOTHING`,
|
||||
model.ArtworkPriorityRecheck, now, now, kind, attemptedBefore)
|
||||
return r.executeSQL(ins)
|
||||
}
|
||||
|
||||
var _ model.ArtworkQueueRepository = (*artworkQueueRepository)(nil)
|
||||
@@ -0,0 +1,160 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/model"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("ArtworkQueueRepository", func() {
|
||||
var repo model.ArtworkQueueRepository
|
||||
|
||||
item := func(kind, id string, prio int) model.ArtworkQueueItem {
|
||||
return model.ArtworkQueueItem{ItemKind: kind, ItemID: id,
|
||||
ImageType: model.ImageTypePrimary, Priority: prio}
|
||||
}
|
||||
|
||||
BeforeEach(func() {
|
||||
clearArtworkTables()
|
||||
repo = NewArtworkQueueRepository(context.Background(), GetDBXBuilder())
|
||||
})
|
||||
|
||||
It("enqueues and dequeues by priority then FIFO", func() {
|
||||
Expect(repo.Enqueue(item("al", "low", model.ArtworkPriorityBackfill))).To(Succeed())
|
||||
Expect(repo.Enqueue(item("ar", "high", model.ArtworkPriorityBump))).To(Succeed())
|
||||
|
||||
got, err := repo.DequeueBatch(10)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got).To(HaveLen(2))
|
||||
Expect(got[0].ItemID).To(Equal("high"))
|
||||
})
|
||||
|
||||
It("keeps the higher priority on duplicate enqueue", func() {
|
||||
Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBump))).To(Succeed())
|
||||
Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBackfill))).To(Succeed())
|
||||
got, _ := repo.DequeueBatch(10)
|
||||
Expect(got).To(HaveLen(1))
|
||||
Expect(got[0].Priority).To(Equal(model.ArtworkPriorityBump))
|
||||
})
|
||||
|
||||
It("hides failed items until retry_at", func() {
|
||||
Expect(repo.Enqueue(item("al", "f1", model.ArtworkPriorityScan))).To(Succeed())
|
||||
Expect(repo.MarkFailed("al", "f1", model.ImageTypePrimary, time.Now().Add(time.Hour))).To(Succeed())
|
||||
|
||||
got, err := repo.DequeueBatch(10)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got).To(BeEmpty())
|
||||
|
||||
Expect(repo.MarkFailed("al", "f1", model.ImageTypePrimary, time.Now().Add(-time.Minute))).To(Succeed())
|
||||
got, _ = repo.DequeueBatch(10)
|
||||
Expect(got).To(HaveLen(1))
|
||||
Expect(got[0].Attempts).To(Equal(2))
|
||||
})
|
||||
|
||||
It("MarkFailedIfUnchanged applies backoff only while retry_at is unchanged", func() {
|
||||
Expect(repo.Enqueue(item("al", "m1", model.ArtworkPriorityScan))).To(Succeed())
|
||||
// Anchor retry_at in the past (attempts -> 1) so it can never collide with the re-enqueue's now.
|
||||
Expect(repo.MarkFailed("al", "m1", model.ImageTypePrimary, time.Now().Add(-time.Hour))).To(Succeed())
|
||||
got, err := repo.DequeueBatch(10)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got).To(HaveLen(1))
|
||||
original := got[0].RetryAt
|
||||
|
||||
// A concurrent scan re-enqueues, resetting retry_at to now.
|
||||
Expect(repo.Enqueue(item("al", "m1", model.ArtworkPriorityScan))).To(Succeed())
|
||||
|
||||
// Failing with the stale retry_at is a no-op: the re-enqueued row keeps its fresh state.
|
||||
future := time.Now().Add(48 * time.Hour)
|
||||
Expect(repo.MarkFailedIfUnchanged("al", "m1", model.ImageTypePrimary, original, future)).To(Succeed())
|
||||
got, _ = repo.DequeueBatch(10)
|
||||
Expect(got).To(HaveLen(1), "the fresh re-enqueue stays immediately eligible")
|
||||
Expect(got[0].Attempts).To(Equal(1), "the stale failure must not bump attempts")
|
||||
current := got[0].RetryAt
|
||||
|
||||
// Failing with the current retry_at applies the backoff and bumps attempts.
|
||||
Expect(repo.MarkFailedIfUnchanged("al", "m1", model.ImageTypePrimary, current, future)).To(Succeed())
|
||||
got, _ = repo.DequeueBatch(10)
|
||||
Expect(got).To(BeEmpty(), "backed-off row is hidden until the future retry_at")
|
||||
all, _ := repo.Count()
|
||||
Expect(all).To(Equal(int64(1)))
|
||||
})
|
||||
|
||||
It("deletes on completion and counts", func() {
|
||||
Expect(repo.Enqueue(item("al", "c1", 0))).To(Succeed())
|
||||
n, _ := repo.Count()
|
||||
Expect(n).To(Equal(int64(1)))
|
||||
Expect(repo.Delete("al", "c1", model.ImageTypePrimary)).To(Succeed())
|
||||
n, _ = repo.Count()
|
||||
Expect(n).To(BeZero())
|
||||
})
|
||||
|
||||
It("DeleteIfUnchanged deletes only while retry_at is unchanged", func() {
|
||||
Expect(repo.Enqueue(item("al", "d1", model.ArtworkPriorityScan))).To(Succeed())
|
||||
// Anchor retry_at in the past so it can never collide with the re-enqueue's now.
|
||||
Expect(repo.MarkFailed("al", "d1", model.ImageTypePrimary, time.Now().Add(-time.Hour))).To(Succeed())
|
||||
got, err := repo.DequeueBatch(10)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got).To(HaveLen(1))
|
||||
original := got[0].RetryAt
|
||||
|
||||
// A concurrent scan re-enqueues, resetting retry_at to now.
|
||||
Expect(repo.Enqueue(item("al", "d1", model.ArtworkPriorityScan))).To(Succeed())
|
||||
|
||||
// Deleting with the stale retry_at is a no-op: the re-enqueued row survives.
|
||||
Expect(repo.DeleteIfUnchanged("al", "d1", model.ImageTypePrimary, original)).To(Succeed())
|
||||
n, _ := repo.Count()
|
||||
Expect(n).To(Equal(int64(1)))
|
||||
|
||||
// Deleting with the current retry_at removes it.
|
||||
got, _ = repo.DequeueBatch(10)
|
||||
Expect(got).To(HaveLen(1))
|
||||
Expect(repo.DeleteIfUnchanged("al", "d1", model.ImageTypePrimary, got[0].RetryAt)).To(Succeed())
|
||||
n, _ = repo.Count()
|
||||
Expect(n).To(BeZero())
|
||||
})
|
||||
|
||||
It("purges queue rows whose entity no longer exists, per kind", func() {
|
||||
Expect(repo.Enqueue(
|
||||
item("al", albumSgtPeppers.ID, model.ArtworkPriorityScan),
|
||||
item("al", "no-such-album", model.ArtworkPriorityScan),
|
||||
item("ar", artistKraftwerk.ID, model.ArtworkPriorityScan),
|
||||
item("ar", "no-such-artist", model.ArtworkPriorityScan),
|
||||
item("pl", plsBest.ID, model.ArtworkPriorityScan),
|
||||
item("pl", "no-such-playlist", model.ArtworkPriorityScan),
|
||||
item("ra", radioWithHomePage.ID, model.ArtworkPriorityScan),
|
||||
item("ra", "no-such-radio", model.ArtworkPriorityScan),
|
||||
)).To(Succeed())
|
||||
|
||||
purged, err := repo.PurgeDangling()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(purged).To(Equal(int64(4)))
|
||||
|
||||
got, _ := repo.DequeueBatch(100)
|
||||
ids := make([]string, 0, len(got))
|
||||
for _, it := range got {
|
||||
ids = append(ids, it.ItemID)
|
||||
}
|
||||
Expect(ids).To(ConsistOf(albumSgtPeppers.ID, artistKraftwerk.ID, plsBest.ID, radioWithHomePage.ID))
|
||||
})
|
||||
|
||||
It("enqueues stale absent states for recheck", func() {
|
||||
awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder())
|
||||
old := time.Now().Add(-48 * time.Hour)
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "stale1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old})).To(Succeed())
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed())
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old})).To(Succeed())
|
||||
|
||||
n, err := repo.EnqueueStaleAbsent("ar", time.Now().Add(-24*time.Hour))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(n).To(Equal(int64(1)))
|
||||
|
||||
items, err := repo.DequeueBatch(10)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(items).To(HaveLen(1))
|
||||
Expect(items[0].ItemID).To(Equal("stale1"))
|
||||
Expect(items[0].Priority).To(Equal(model.ArtworkPriorityRecheck))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,208 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
. "github.com/Masterminds/squirrel"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/pocketbase/dbx"
|
||||
)
|
||||
|
||||
const (
|
||||
itemArtworkTable = "item_artwork"
|
||||
artworkBatchSize = 200
|
||||
)
|
||||
|
||||
type itemArtworkSQL struct {
|
||||
sqlRepository
|
||||
}
|
||||
|
||||
type artworkRepository struct {
|
||||
sqlRepository
|
||||
items itemArtworkSQL
|
||||
}
|
||||
|
||||
func NewArtworkRepository(ctx context.Context, db dbx.Builder) model.ArtworkRepository {
|
||||
r := &artworkRepository{}
|
||||
r.ctx = ctx
|
||||
r.db = db
|
||||
r.tableName = "artwork"
|
||||
r.items.ctx = ctx
|
||||
r.items.db = db
|
||||
r.items.tableName = itemArtworkTable
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *artworkRepository) GetImage(hash string) (*model.Artwork, error) {
|
||||
sel := Select("*").From(r.tableName).Where(Eq{"hash": hash})
|
||||
var res model.Artwork
|
||||
if err := r.queryOne(sel, &res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
func (r *artworkRepository) PutImage(a *model.Artwork) error {
|
||||
// created_at is the last-acquisition-write time the prune grace window keys on.
|
||||
a.CreatedAt = time.Now()
|
||||
values, err := toSQLArgs(*a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// created_at=excluded.created_at: reacquiring an orphan must reset the prune grace window.
|
||||
ins := Insert(r.tableName).SetMap(values).Suffix(`ON CONFLICT (hash) DO UPDATE SET mime=excluded.mime, width=excluded.width,
|
||||
height=excluded.height, size_bytes=excluded.size_bytes, blur_hash=excluded.blur_hash, created_at=excluded.created_at`)
|
||||
_, err = r.executeSQL(ins)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *artworkRepository) GetImages(hashes []string) (map[string]model.Artwork, error) {
|
||||
res := map[string]model.Artwork{}
|
||||
for chunk := range slices.Chunk(hashes, artworkBatchSize) {
|
||||
sel := Select("*").From(r.tableName).Where(Eq{"hash": chunk})
|
||||
var all []model.Artwork
|
||||
if err := r.queryAll(sel, &all); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, a := range all {
|
||||
res[a.Hash] = a
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (r *artworkRepository) GetAllMimes() (map[string]string, error) {
|
||||
sel := Select("hash", "mime").From(r.tableName)
|
||||
var rows []struct {
|
||||
Hash string
|
||||
Mime string
|
||||
}
|
||||
if err := r.queryAll(sel, &rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := make(map[string]string, len(rows))
|
||||
for _, row := range rows {
|
||||
res[row.Hash] = row.Mime
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (r *artworkRepository) GetOrphanHashes(createdBefore time.Time) ([]string, error) {
|
||||
sel := Select("hash").From(r.tableName).
|
||||
Where(And{
|
||||
Lt{"created_at": createdBefore},
|
||||
Expr("hash NOT IN (SELECT hash FROM " + itemArtworkTable + " WHERE hash <> '')"),
|
||||
})
|
||||
var hashes []string
|
||||
err := r.queryAllSlice(sel, &hashes)
|
||||
return hashes, err
|
||||
}
|
||||
|
||||
func (r *artworkRepository) DeleteOrphans(createdBefore time.Time, hashes []string) error {
|
||||
for chunk := range slices.Chunk(hashes, artworkBatchSize) {
|
||||
del := Delete(r.tableName).Where(And{
|
||||
Eq{"hash": chunk},
|
||||
Lt{"created_at": createdBefore},
|
||||
Expr("hash NOT IN (SELECT hash FROM " + itemArtworkTable + " WHERE hash <> '')"),
|
||||
})
|
||||
if _, err := r.executeSQL(del); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// danglingItemArtworkKinds maps item_kind prefixes to the table that owns the entity.
|
||||
var danglingItemArtworkKinds = map[string]string{
|
||||
"al": "album",
|
||||
"ar": "artist",
|
||||
"pl": "playlist",
|
||||
"ra": "radio",
|
||||
}
|
||||
|
||||
// purgeDangling deletes rows in table whose owning entity is gone, one statement per kind.
|
||||
func purgeDangling(execute func(Sqlizer) (int64, error), table string) (int64, error) {
|
||||
var total int64
|
||||
for kind, entityTable := range danglingItemArtworkKinds {
|
||||
del := Delete(table).Where(And{
|
||||
Eq{"item_kind": kind},
|
||||
Expr("item_id NOT IN (SELECT id FROM " + entityTable + ")"),
|
||||
})
|
||||
c, err := execute(del)
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
total += c
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func (r *artworkRepository) PurgeDanglingItemArtwork() (int64, error) {
|
||||
return purgeDangling(r.items.executeSQL, itemArtworkTable)
|
||||
}
|
||||
|
||||
func (r *artworkRepository) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) {
|
||||
sel := Select("*").From(itemArtworkTable).
|
||||
Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType})
|
||||
var res model.ItemArtwork
|
||||
if err := r.items.queryOne(sel, &res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
func (r *artworkRepository) PutItemArtwork(ia *model.ItemArtwork) error {
|
||||
if ia.ImageType == "" {
|
||||
ia.ImageType = model.ImageTypePrimary
|
||||
}
|
||||
ia.UpdatedAt = time.Now()
|
||||
// PutItemArtwork records the outcome of an attempt, so an unset attempted_at is now.
|
||||
if ia.AttemptedAt.IsZero() {
|
||||
ia.AttemptedAt = ia.UpdatedAt
|
||||
}
|
||||
values, err := toSQLArgs(*ia)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ins := Insert(itemArtworkTable).SetMap(values).Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET
|
||||
hash=excluded.hash, source=excluded.source, source_path=excluded.source_path, ref_mtime=excluded.ref_mtime,
|
||||
attempted_at=excluded.attempted_at, updated_at=excluded.updated_at`)
|
||||
_, err = r.items.executeSQL(ins)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *artworkRepository) DeleteForItem(kind, id string) error {
|
||||
return r.items.delete(Eq{"item_kind": kind, "item_id": id})
|
||||
}
|
||||
|
||||
func (r *artworkRepository) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) {
|
||||
res := map[string]model.ItemArtworkInfo{}
|
||||
for chunk := range slices.Chunk(ids, artworkBatchSize) {
|
||||
sel := Select("ia.item_id", "ia.hash", "COALESCE(a.blur_hash, '') as blur_hash").
|
||||
From(itemArtworkTable + " ia").
|
||||
LeftJoin("artwork a ON a.hash = ia.hash").
|
||||
Where(And{
|
||||
Eq{"ia.item_kind": kind},
|
||||
Eq{"ia.image_type": model.ImageTypePrimary},
|
||||
Eq{"ia.item_id": chunk},
|
||||
})
|
||||
var rows []struct {
|
||||
ItemID string
|
||||
Hash string
|
||||
BlurHash string
|
||||
}
|
||||
if err := r.items.queryAll(sel, &rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, row := range rows {
|
||||
res[row.ItemID] = model.ItemArtworkInfo{
|
||||
ItemID: row.ItemID, Hash: row.Hash, BlurHash: row.BlurHash,
|
||||
}
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
var _ model.ArtworkRepository = (*artworkRepository)(nil)
|
||||
@@ -0,0 +1,233 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/model"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/pocketbase/dbx"
|
||||
)
|
||||
|
||||
// clearArtworkTables resets the shared test DB's artwork tables so specs don't leak state.
|
||||
func clearArtworkTables() {
|
||||
db := GetDBXBuilder()
|
||||
for _, t := range []string{"artwork_queue", "item_artwork", "artwork"} {
|
||||
_, err := db.NewQuery("DELETE FROM " + t).Execute()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
}
|
||||
|
||||
var _ = Describe("ArtworkRepository", func() {
|
||||
var repo model.ArtworkRepository
|
||||
|
||||
BeforeEach(func() {
|
||||
clearArtworkTables()
|
||||
repo = NewArtworkRepository(context.Background(), GetDBXBuilder())
|
||||
})
|
||||
|
||||
Context("image identity", func() {
|
||||
It("stores and retrieves an artwork by hash", func() {
|
||||
a := &model.Artwork{Hash: "abc123", Mime: "image/jpeg", Width: 500, Height: 500, SizeBytes: 1234, BlurHash: "LKO2?U%2Tw=w"}
|
||||
Expect(repo.PutImage(a)).To(Succeed())
|
||||
|
||||
got, err := repo.GetImage("abc123")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got.Mime).To(Equal("image/jpeg"))
|
||||
Expect(got.BlurHash).To(Equal("LKO2?U%2Tw=w"))
|
||||
Expect(got.CreatedAt).ToNot(BeZero())
|
||||
})
|
||||
|
||||
It("is idempotent on Put (upsert by hash)", func() {
|
||||
a := &model.Artwork{Hash: "dup1", Mime: "image/png"}
|
||||
Expect(repo.PutImage(a)).To(Succeed())
|
||||
a.BlurHash = "XYZ"
|
||||
Expect(repo.PutImage(a)).To(Succeed())
|
||||
got, _ := repo.GetImage("dup1")
|
||||
Expect(got.BlurHash).To(Equal("XYZ"))
|
||||
})
|
||||
|
||||
It("refreshes created_at when reacquiring an existing hash", func() {
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "reacq", Mime: "image/jpeg"})).To(Succeed())
|
||||
_, err := GetDBXBuilder().NewQuery("UPDATE artwork SET created_at={:t} WHERE hash='reacq'").
|
||||
Bind(dbx.Params{"t": "2000-01-01 00:00:00"}).Execute()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "reacq", Mime: "image/png"})).To(Succeed())
|
||||
|
||||
got, err := repo.GetImage("reacq")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got.CreatedAt).To(BeTemporally(">", time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)))
|
||||
})
|
||||
|
||||
It("returns ErrNotFound for a missing hash", func() {
|
||||
_, err := repo.GetImage("nope")
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
})
|
||||
|
||||
It("fetches a batch", func() {
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "b1", Mime: "image/jpeg"})).To(Succeed())
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "b2", Mime: "image/png"})).To(Succeed())
|
||||
got, err := repo.GetImages([]string{"b1", "b2", "missing"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got).To(HaveLen(2))
|
||||
Expect(got["b2"].Mime).To(Equal("image/png"))
|
||||
})
|
||||
|
||||
It("returns every stored hash with its current mime", func() {
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "all1", Mime: "image/jpeg"})).To(Succeed())
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "all2", Mime: "image/png"})).To(Succeed())
|
||||
mimes, err := repo.GetAllMimes()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mimes).To(HaveKeyWithValue("all1", "image/jpeg"))
|
||||
Expect(mimes).To(HaveKeyWithValue("all2", "image/png"))
|
||||
})
|
||||
|
||||
It("finds orphans older than cutoff, honoring item_artwork references", func() {
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "orph1", Mime: "image/jpeg"})).To(Succeed())
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "ref1", Mime: "image/jpeg"})).To(Succeed())
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", ImageType: model.ImageTypePrimary, Hash: "ref1", Source: "folder"})).To(Succeed())
|
||||
|
||||
orphans, err := repo.GetOrphanHashes(time.Now().Add(time.Minute))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(orphans).To(ContainElement("orph1"))
|
||||
Expect(orphans).ToNot(ContainElement("ref1"))
|
||||
|
||||
orphans, err = repo.GetOrphanHashes(time.Now().Add(-time.Hour))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(orphans).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("deletes only unreferenced hashes older than the cutoff", func() {
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "d1", Mime: "image/jpeg"})).To(Succeed())
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "dref", Mime: "image/jpeg"})).To(Succeed())
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1",
|
||||
ImageType: model.ImageTypePrimary, Hash: "dref", Source: "folder"})).To(Succeed())
|
||||
|
||||
Expect(repo.DeleteOrphans(time.Now().Add(time.Minute), []string{"d1", "dref"})).To(Succeed())
|
||||
|
||||
_, err := repo.GetImage("d1")
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
_, err = repo.GetImage("dref")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("spares an unreferenced hash younger than the cutoff", func() {
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "young", Mime: "image/jpeg"})).To(Succeed())
|
||||
Expect(repo.DeleteOrphans(time.Now().Add(-time.Hour), []string{"young"})).To(Succeed())
|
||||
_, err := repo.GetImage("young")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("fetches a batch larger than the SQL variable limit", func() {
|
||||
hashes := make([]string, 0, 250)
|
||||
for i := range 250 {
|
||||
h := fmt.Sprintf("big%03d", i)
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||
hashes = append(hashes, h)
|
||||
}
|
||||
hashes = append(hashes, "absent1", "absent2")
|
||||
|
||||
got, err := repo.GetImages(hashes)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got).To(HaveLen(250))
|
||||
})
|
||||
})
|
||||
|
||||
Context("dangling state cleanup", func() {
|
||||
It("purges item_artwork rows per kind whose entity no longer exists, summing counts", func() {
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: albumSgtPeppers.ID, ImageType: model.ImageTypePrimary, Hash: "keepAl"})).To(Succeed())
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "no-such-album", ImageType: model.ImageTypePrimary, Hash: "danglingAl"})).To(Succeed())
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: artistKraftwerk.ID, ImageType: model.ImageTypePrimary, Hash: "keepAr"})).To(Succeed())
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "no-such-artist", ImageType: model.ImageTypePrimary, Hash: "danglingAr"})).To(Succeed())
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "pl", ItemID: plsBest.ID, ImageType: model.ImageTypePrimary, Hash: "keepPl"})).To(Succeed())
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "pl", ItemID: "no-such-playlist", ImageType: model.ImageTypePrimary, Hash: "danglingPl"})).To(Succeed())
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ra", ItemID: radioWithHomePage.ID, ImageType: model.ImageTypePrimary, Hash: "keepRa"})).To(Succeed())
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ra", ItemID: "no-such-radio", ImageType: model.ImageTypePrimary, Hash: "danglingRa"})).To(Succeed())
|
||||
|
||||
purged, err := repo.PurgeDanglingItemArtwork()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(purged).To(Equal(int64(4)))
|
||||
|
||||
for _, kept := range []model.ItemArtwork{
|
||||
{ItemKind: "al", ItemID: albumSgtPeppers.ID},
|
||||
{ItemKind: "ar", ItemID: artistKraftwerk.ID},
|
||||
{ItemKind: "pl", ItemID: plsBest.ID},
|
||||
{ItemKind: "ra", ItemID: radioWithHomePage.ID},
|
||||
} {
|
||||
_, err := repo.GetItemArtwork(kept.ItemKind, kept.ItemID, model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
for _, gone := range []model.ItemArtwork{
|
||||
{ItemKind: "al", ItemID: "no-such-album"},
|
||||
{ItemKind: "ar", ItemID: "no-such-artist"},
|
||||
{ItemKind: "pl", ItemID: "no-such-playlist"},
|
||||
{ItemKind: "ra", ItemID: "no-such-radio"},
|
||||
} {
|
||||
_, err := repo.GetItemArtwork(gone.ItemKind, gone.ItemID, model.ImageTypePrimary)
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
Context("item state", func() {
|
||||
It("upserts and reads state, including per-item provenance", func() {
|
||||
ia := &model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary,
|
||||
Hash: "h1", Source: "folder", SourcePath: "/music/a/cover.jpg", RefMtime: 111, AttemptedAt: time.Now()}
|
||||
Expect(repo.PutItemArtwork(ia)).To(Succeed())
|
||||
ia.Source = "embedded"
|
||||
ia.SourcePath = "/music/a/track.mp3"
|
||||
ia.RefMtime = 222
|
||||
Expect(repo.PutItemArtwork(ia)).To(Succeed())
|
||||
|
||||
got, err := repo.GetItemArtwork("al", "al1", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got.Source).To(Equal("embedded"))
|
||||
Expect(got.SourcePath).To(Equal("/music/a/track.mp3"))
|
||||
Expect(got.RefMtime).To(Equal(int64(222)))
|
||||
Expect(got.UpdatedAt).ToNot(BeZero())
|
||||
})
|
||||
|
||||
It("defaults attempted_at to now when unset", func() {
|
||||
before := time.Now().Add(-time.Second)
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "noattempt",
|
||||
ImageType: model.ImageTypePrimary, Hash: ""})).To(Succeed())
|
||||
got, err := repo.GetItemArtwork("ar", "noattempt", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got.AttemptedAt).To(BeTemporally(">", before))
|
||||
})
|
||||
|
||||
It("represents known-absent as empty hash", func() {
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "ar1",
|
||||
ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed())
|
||||
got, err := repo.GetItemArtwork("ar", "ar1", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got.Hash).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("hydrates a page in one batch, including blurhash and absence", func() {
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "h9", Mime: "image/jpeg", BlurHash: "BH9"})).To(Succeed())
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x1", ImageType: model.ImageTypePrimary, Hash: "h9", Source: "folder"})).To(Succeed())
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x2", ImageType: model.ImageTypePrimary, Hash: "", Source: ""})).To(Succeed())
|
||||
|
||||
info, err := repo.GetInfoForItems("al", []string{"x1", "x2", "x3"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(info).To(HaveLen(2))
|
||||
Expect(info["x1"].Hash).To(Equal("h9"))
|
||||
Expect(info["x1"].BlurHash).To(Equal("BH9"))
|
||||
Expect(info["x1"].Absent()).To(BeFalse())
|
||||
Expect(info["x2"].Absent()).To(BeTrue())
|
||||
_, unresolved := info["x3"]
|
||||
Expect(unresolved).To(BeFalse())
|
||||
})
|
||||
|
||||
It("deletes all rows for an item", func() {
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "pl", ItemID: "p1", ImageType: model.ImageTypePrimary, Hash: "h1"})).To(Succeed())
|
||||
Expect(repo.DeleteForItem("pl", "p1")).To(Succeed())
|
||||
_, err := repo.GetItemArtwork("pl", "p1", model.ImageTypePrimary)
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -97,6 +97,14 @@ func (s *SQLStore) Plugin(ctx context.Context) model.PluginRepository {
|
||||
return NewPluginRepository(ctx, s.getDBXBuilder())
|
||||
}
|
||||
|
||||
func (s *SQLStore) Artwork(ctx context.Context) model.ArtworkRepository {
|
||||
return NewArtworkRepository(ctx, s.getDBXBuilder())
|
||||
}
|
||||
|
||||
func (s *SQLStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository {
|
||||
return NewArtworkQueueRepository(ctx, s.getDBXBuilder())
|
||||
}
|
||||
|
||||
func (s *SQLStore) Resource(ctx context.Context, m any) model.ResourceRepository {
|
||||
switch m.(type) {
|
||||
case model.User:
|
||||
|
||||
@@ -189,6 +189,16 @@ func (r *playlistRepository) GetAll(options ...model.QueryOptions) (model.Playli
|
||||
return playlists, err
|
||||
}
|
||||
|
||||
// GetAllIDs returns just the playlist IDs for the same row set as GetAll (honoring userFilter),
|
||||
// skipping the owner-name join columns and annotation. Used by bulk enumeration (artwork backfill).
|
||||
func (r *playlistRepository) GetAllIDs(options ...model.QueryOptions) ([]string, error) {
|
||||
sq := r.newSelect(options...).Columns("playlist.id").
|
||||
Join("user on user.id = owner_id").Where(r.userFilter())
|
||||
ids := []string{}
|
||||
err := r.queryAllSlice(sq, &ids)
|
||||
return ids, err
|
||||
}
|
||||
|
||||
func (r *playlistRepository) GetCursor(options ...model.QueryOptions) (model.PlaylistCursor, error) {
|
||||
// Same userFilter as GetAll: a cursor must not widen visibility beyond public/owned playlists.
|
||||
sel := r.selectPlaylist(options...).Where(r.userFilter())
|
||||
|
||||
@@ -37,6 +37,21 @@ var _ = Describe("PlaylistRepository", func() {
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GetAllIDs", func() {
|
||||
It("returns the same id set as GetAll", func() {
|
||||
want, err := repo.GetAll()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(want).ToNot(BeEmpty())
|
||||
wantIDs := make([]string, 0, len(want))
|
||||
for _, p := range want {
|
||||
wantIDs = append(wantIDs, p.ID)
|
||||
}
|
||||
ids, err := repo.GetAllIDs()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(ids).To(ConsistOf(wantIDs))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Exists", func() {
|
||||
It("returns true for an existing playlist", func() {
|
||||
Expect(repo.Exists(plsCool.ID)).To(BeTrue())
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
. "github.com/Masterminds/squirrel"
|
||||
"github.com/deluan/rest"
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/model/id"
|
||||
"github.com/pocketbase/dbx"
|
||||
@@ -58,6 +59,14 @@ func (r *radioRepository) GetAll(options ...model.QueryOptions) (model.Radios, e
|
||||
return res, err
|
||||
}
|
||||
|
||||
// GetAllIDs returns just the radio IDs. Used by bulk enumeration (artwork backfill).
|
||||
func (r *radioRepository) GetAllIDs(options ...model.QueryOptions) ([]string, error) {
|
||||
sel := r.newSelect(options...).Columns("id")
|
||||
ids := []string{}
|
||||
err := r.queryAllSlice(sel, &ids)
|
||||
return ids, err
|
||||
}
|
||||
|
||||
func (r *radioRepository) Put(radio *model.Radio, colsToUpdate ...string) error {
|
||||
if !r.isPermitted() {
|
||||
return rest.ErrPermissionDenied
|
||||
@@ -72,7 +81,16 @@ func (r *radioRepository) Put(radio *model.Radio, colsToUpdate ...string) error
|
||||
colsToUpdate = append(colsToUpdate, "UpdatedAt")
|
||||
}
|
||||
_, err := r.put(radio.ID, radio, colsToUpdate...)
|
||||
return err
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Enqueue artwork resolution for the created/updated radio. Never fails the save.
|
||||
item := model.ArtworkQueueItem{ItemKind: "ra", ItemID: radio.ID, ImageType: model.ImageTypePrimary,
|
||||
Priority: model.ArtworkPriorityScan}
|
||||
if err := NewArtworkQueueRepository(r.ctx, r.db).Enqueue(item); err != nil {
|
||||
log.Warn(r.ctx, "could not enqueue radio artwork", "id", radio.ID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *radioRepository) Count(options ...rest.QueryOptions) (int64, error) {
|
||||
|
||||
@@ -78,6 +78,21 @@ var _ = Describe("RadioRepository", func() {
|
||||
})
|
||||
})
|
||||
|
||||
Describe("GetAllIDs", func() {
|
||||
It("returns the same id set as GetAll", func() {
|
||||
want, err := repo.GetAll()
|
||||
Expect(err).To(BeNil())
|
||||
Expect(want).ToNot(BeEmpty())
|
||||
wantIDs := make([]string, 0, len(want))
|
||||
for _, r := range want {
|
||||
wantIDs = append(wantIDs, r.ID)
|
||||
}
|
||||
ids, err := repo.GetAllIDs()
|
||||
Expect(err).To(BeNil())
|
||||
Expect(ids).To(ConsistOf(wantIDs))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("Put", func() {
|
||||
It("successfully updates item", func() {
|
||||
err := repo.Put(&model.Radio{
|
||||
@@ -107,6 +122,27 @@ var _ = Describe("RadioRepository", func() {
|
||||
Expect(err).To(BeNil())
|
||||
Expect(all[2].StreamUrl).To(Equal("https://example.com:4533/app"))
|
||||
})
|
||||
|
||||
It("enqueues artwork resolution for the saved radio", func() {
|
||||
err := repo.Put(&model.Radio{
|
||||
Name: "Artwork radio",
|
||||
StreamUrl: "https://example.com:4533/artwork",
|
||||
})
|
||||
Expect(err).To(BeNil())
|
||||
|
||||
all, err := repo.GetAll()
|
||||
Expect(err).To(BeNil())
|
||||
created := all[len(all)-1]
|
||||
|
||||
queueRepo := NewArtworkQueueRepository(context.Background(), GetDBXBuilder())
|
||||
queued, err := queueRepo.DequeueBatch(1000)
|
||||
Expect(err).To(BeNil())
|
||||
Expect(queued).To(ContainElement(SatisfyAll(
|
||||
HaveField("ItemKind", "ra"),
|
||||
HaveField("ItemID", created.ID),
|
||||
HaveField("Priority", model.ArtworkPriorityScan),
|
||||
)))
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -332,6 +332,8 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error)
|
||||
|
||||
// Collect artwork IDs to pre-cache after the transaction commits
|
||||
var artworkIDs []model.ArtworkID
|
||||
// Collect artwork queue items for changed albums/artists, enqueued in the same transaction
|
||||
var queueItems []model.ArtworkQueueItem
|
||||
|
||||
err := p.ds.WithTx(func(tx model.DataStore) error {
|
||||
// Instantiate all repositories just once per folder
|
||||
@@ -372,6 +374,10 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error)
|
||||
}
|
||||
if entry.artists[i].Name != consts.UnknownArtist && entry.artists[i].Name != consts.VariousArtists {
|
||||
artworkIDs = append(artworkIDs, entry.artists[i].CoverArtID())
|
||||
queueItems = append(queueItems, model.ArtworkQueueItem{
|
||||
ItemKind: "ar", ItemID: entry.artists[i].ID, ImageType: model.ImageTypePrimary,
|
||||
Priority: model.ArtworkPriorityScan,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -384,6 +390,10 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error)
|
||||
}
|
||||
if entry.albums[i].Name != consts.UnknownAlbum {
|
||||
artworkIDs = append(artworkIDs, entry.albums[i].CoverArtID())
|
||||
queueItems = append(queueItems, model.ArtworkQueueItem{
|
||||
ItemKind: "al", ItemID: entry.albums[i].ID, ImageType: model.ImageTypePrimary,
|
||||
Priority: model.ArtworkPriorityScan,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -415,6 +425,13 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Enqueue artwork resolution for changed albums/artists. Never fails the scan.
|
||||
if len(queueItems) > 0 {
|
||||
if err := tx.ArtworkQueue(p.ctx).Enqueue(queueItems...); err != nil {
|
||||
log.Warn(p.ctx, "Scanner: could not enqueue artwork resolution", "folder", entry.path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, "scanner: persist changes")
|
||||
if err != nil {
|
||||
|
||||
@@ -149,6 +149,11 @@ func (p *phasePlaylists) processPlaylistsInFolder(folder *model.Folder) (*model.
|
||||
log.Debug("Scanner: Imported playlist", "name", pls.Name, "lastUpdated", pls.UpdatedAt, "path", pls.Path, "numTracks", len(pls.Tracks), "elapsed", time.Since(started))
|
||||
}
|
||||
p.cw.PreCache(pls.CoverArtID())
|
||||
item := model.ArtworkQueueItem{ItemKind: "pl", ItemID: pls.ID, ImageType: model.ImageTypePrimary,
|
||||
Priority: model.ArtworkPriorityScan}
|
||||
if err := p.ds.ArtworkQueue(p.ctx).Enqueue(item); err != nil {
|
||||
log.Warn(p.ctx, "Scanner: could not enqueue playlist artwork", "id", pls.ID, err)
|
||||
}
|
||||
p.refreshed.Add(1)
|
||||
}
|
||||
return folder, nil
|
||||
|
||||
@@ -193,6 +193,29 @@ var _ = Describe("phasePlaylists", func() {
|
||||
Expect(phase.refreshed.Load()).To(Equal(uint32(2)))
|
||||
})
|
||||
|
||||
It("enqueues artwork resolution for the imported playlist", func() {
|
||||
libPath := GinkgoT().TempDir()
|
||||
folder := &model.Folder{LibraryPath: libPath, Path: "path/to", Name: "folder"}
|
||||
_ = os.MkdirAll(folder.AbsolutePath(), 0755)
|
||||
|
||||
file1 := filepath.Join(folder.AbsolutePath(), "playlist1.m3u")
|
||||
_ = os.WriteFile(file1, []byte{}, 0600)
|
||||
|
||||
pls.On("ImportFromFolder", mock.Anything, folder, "playlist1.m3u").
|
||||
Return(&model.Playlist{ID: "pl1"}, nil)
|
||||
|
||||
_, err := phase.processPlaylistsInFolder(folder)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
queued, err := ds.ArtworkQueue(ctx).DequeueBatch(10)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(queued).To(ContainElement(SatisfyAll(
|
||||
HaveField("ItemKind", "pl"),
|
||||
HaveField("ItemID", "pl1"),
|
||||
HaveField("Priority", model.ArtworkPriorityScan),
|
||||
)))
|
||||
})
|
||||
|
||||
It("reports an error if there is an error reading files", func() {
|
||||
tests.SkipOnWindows("relies on Unix /etc filesystem")
|
||||
progress := make(chan *ProgressInfo)
|
||||
|
||||
@@ -152,6 +152,29 @@ var _ = Describe("Scanner", Ordered, func() {
|
||||
HaveField("SongCount", Equal(4)),
|
||||
))
|
||||
})
|
||||
It("should enqueue artwork resolution for the scanned albums and artists", func() {
|
||||
Expect(runScanner(ctx, true)).To(Succeed())
|
||||
|
||||
albums, _ := ds.Album(ctx).GetAll()
|
||||
artists, _ := ds.Artist(ctx).GetAll(model.QueryOptions{Filters: squirrel.NotEq{"name": consts.UnknownArtist}})
|
||||
queued, err := ds.ArtworkQueue(ctx).DequeueBatch(1000)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
for _, al := range albums {
|
||||
Expect(queued).To(ContainElement(SatisfyAll(
|
||||
HaveField("ItemKind", "al"),
|
||||
HaveField("ItemID", al.ID),
|
||||
HaveField("Priority", model.ArtworkPriorityScan),
|
||||
)))
|
||||
}
|
||||
for _, ar := range artists {
|
||||
Expect(queued).To(ContainElement(SatisfyAll(
|
||||
HaveField("ItemKind", "ar"),
|
||||
HaveField("ItemID", ar.ID),
|
||||
HaveField("Priority", model.ArtworkPriorityScan),
|
||||
)))
|
||||
}
|
||||
})
|
||||
})
|
||||
When("a file was changed", func() {
|
||||
It("should update the media_file", func() {
|
||||
|
||||
@@ -361,6 +361,10 @@ func (n noopProvider) ArtistImage(context.Context, string) (*url.URL, error) {
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
|
||||
func (n noopProvider) ArtistImageResult(context.Context, string) (*url.URL, error) {
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
|
||||
func (n noopProvider) AlbumImage(context.Context, string) (*url.URL, error) {
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
|
||||
@@ -76,6 +76,18 @@ func (m *MockAlbumRepo) GetAll(qo ...model.QueryOptions) (model.Albums, error) {
|
||||
return m.All, nil
|
||||
}
|
||||
|
||||
func (m *MockAlbumRepo) GetAllIDs(qo ...model.QueryOptions) ([]string, error) {
|
||||
all, err := m.GetAll(qo...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids := make([]string, len(all))
|
||||
for i, a := range all {
|
||||
ids[i] = a.ID
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (m *MockAlbumRepo) GetCursor(qo ...model.QueryOptions) (model.AlbumCursor, error) {
|
||||
res, err := m.GetAll(qo...)
|
||||
if err != nil {
|
||||
|
||||
@@ -113,6 +113,18 @@ func (m *MockArtistRepo) GetAll(options ...model.QueryOptions) (model.Artists, e
|
||||
return allArtists, nil
|
||||
}
|
||||
|
||||
func (m *MockArtistRepo) GetAllIDs(options ...model.QueryOptions) ([]string, error) {
|
||||
all, err := m.GetAll(options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids := make([]string, len(all))
|
||||
for i, a := range all {
|
||||
ids[i] = a.ID
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (m *MockArtistRepo) GetCursor(options ...model.QueryOptions) (model.ArtistCursor, error) {
|
||||
res, err := m.GetAll(options...)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/model"
|
||||
)
|
||||
|
||||
type MockArtworkQueueRepo struct {
|
||||
model.ArtworkQueueRepository
|
||||
Data map[string]model.ArtworkQueueItem // keyed by iaKey(kind, id, imageType)
|
||||
Err error
|
||||
// ItemArtworkSource, when set, backs EnqueueStaleAbsent with real item_artwork state.
|
||||
ItemArtworkSource *MockArtworkRepo
|
||||
// ExistingIDs, keyed by item_kind, backs PurgeDangling; a nil per-kind map keeps that kind.
|
||||
ExistingIDs map[string]map[string]bool
|
||||
}
|
||||
|
||||
func CreateMockArtworkQueueRepo() *MockArtworkQueueRepo {
|
||||
return &MockArtworkQueueRepo{Data: map[string]model.ArtworkQueueItem{}}
|
||||
}
|
||||
|
||||
func (m *MockArtworkQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
now := time.Now()
|
||||
for _, it := range items {
|
||||
if it.ImageType == "" {
|
||||
it.ImageType = model.ImageTypePrimary
|
||||
}
|
||||
k := iaKey(it.ItemKind, it.ItemID, it.ImageType)
|
||||
// Mirror the SQL: retry_at/enqueued_at are server-set, never taken from the caller.
|
||||
if prev, ok := m.Data[k]; ok {
|
||||
prev.Priority = max(prev.Priority, it.Priority)
|
||||
prev.RetryAt = now
|
||||
m.Data[k] = prev
|
||||
continue
|
||||
}
|
||||
it.Attempts = 0
|
||||
it.RetryAt = now
|
||||
it.EnqueuedAt = now
|
||||
m.Data[k] = it
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkQueueRepo) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) {
|
||||
if m.Err != nil {
|
||||
return nil, m.Err
|
||||
}
|
||||
var res []model.ArtworkQueueItem
|
||||
now := time.Now()
|
||||
for _, it := range m.Data {
|
||||
if !it.RetryAt.After(now) {
|
||||
res = append(res, it)
|
||||
}
|
||||
}
|
||||
sort.Slice(res, func(i, j int) bool {
|
||||
if res[i].Priority != res[j].Priority {
|
||||
return res[i].Priority > res[j].Priority
|
||||
}
|
||||
return res[i].EnqueuedAt.Before(res[j].EnqueuedAt)
|
||||
})
|
||||
if len(res) > n {
|
||||
res = res[:n]
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkQueueRepo) MarkFailed(kind, id, imageType string, retryAt time.Time) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
k := iaKey(kind, id, imageType)
|
||||
it, ok := m.Data[k]
|
||||
if !ok {
|
||||
return model.ErrNotFound
|
||||
}
|
||||
it.Attempts++
|
||||
it.RetryAt = retryAt
|
||||
m.Data[k] = it
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkQueueRepo) MarkFailedIfUnchanged(kind, id, imageType string, seenRetryAt, retryAt time.Time) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
k := iaKey(kind, id, imageType)
|
||||
if it, ok := m.Data[k]; ok && it.RetryAt.Equal(seenRetryAt) {
|
||||
it.Attempts++
|
||||
it.RetryAt = retryAt
|
||||
m.Data[k] = it
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkQueueRepo) Delete(kind, id, imageType string) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
delete(m.Data, iaKey(kind, id, imageType))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkQueueRepo) DeleteIfUnchanged(kind, id, imageType string, retryAt time.Time) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
k := iaKey(kind, id, imageType)
|
||||
if it, ok := m.Data[k]; ok && it.RetryAt.Equal(retryAt) {
|
||||
delete(m.Data, k)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkQueueRepo) PurgeDangling() (int64, error) {
|
||||
if m.Err != nil {
|
||||
return 0, m.Err
|
||||
}
|
||||
var purged int64
|
||||
for k, it := range m.Data {
|
||||
existing := m.ExistingIDs[it.ItemKind]
|
||||
if existing == nil {
|
||||
continue
|
||||
}
|
||||
if !existing[it.ItemID] {
|
||||
delete(m.Data, k)
|
||||
purged++
|
||||
}
|
||||
}
|
||||
return purged, nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkQueueRepo) Count() (int64, error) {
|
||||
if m.Err != nil {
|
||||
return 0, m.Err
|
||||
}
|
||||
return int64(len(m.Data)), nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkQueueRepo) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) {
|
||||
if m.Err != nil || m.ItemArtworkSource == nil {
|
||||
return 0, m.Err
|
||||
}
|
||||
now := time.Now()
|
||||
var inserted int64
|
||||
for _, ia := range m.ItemArtworkSource.ItemData {
|
||||
if ia.ItemKind != kind || ia.Hash != "" || !ia.AttemptedAt.Before(attemptedBefore) {
|
||||
continue
|
||||
}
|
||||
k := iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)
|
||||
if _, ok := m.Data[k]; ok { // DO NOTHING: never touch existing queue rows
|
||||
continue
|
||||
}
|
||||
m.Data[k] = model.ArtworkQueueItem{
|
||||
ItemKind: ia.ItemKind,
|
||||
ItemID: ia.ItemID,
|
||||
ImageType: ia.ImageType,
|
||||
Priority: model.ArtworkPriorityRecheck,
|
||||
RetryAt: now,
|
||||
EnqueuedAt: now,
|
||||
}
|
||||
inserted++
|
||||
}
|
||||
return inserted, nil
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/model"
|
||||
)
|
||||
|
||||
type MockArtworkRepo struct {
|
||||
model.ArtworkRepository
|
||||
Data map[string]model.Artwork
|
||||
ItemData map[string]model.ItemArtwork // keyed by iaKey(kind, id, imageType)
|
||||
OrphanHashes []string
|
||||
Err error
|
||||
// ExistingIDs, keyed by item_kind, backs PurgeDanglingItemArtwork; nil map keeps everything.
|
||||
ExistingIDs map[string]map[string]bool
|
||||
}
|
||||
|
||||
func CreateMockArtworkRepo() *MockArtworkRepo {
|
||||
return &MockArtworkRepo{Data: map[string]model.Artwork{}, ItemData: map[string]model.ItemArtwork{}}
|
||||
}
|
||||
|
||||
func iaKey(kind, id, imageType string) string { return kind + "|" + id + "|" + imageType }
|
||||
|
||||
func (m *MockArtworkRepo) GetImage(hash string) (*model.Artwork, error) {
|
||||
if m.Err != nil {
|
||||
return nil, m.Err
|
||||
}
|
||||
if a, ok := m.Data[hash]; ok {
|
||||
return &a, nil
|
||||
}
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) PutImage(a *model.Artwork) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
// Mirrors the SQL repository: every upsert refreshes created_at. Age fixtures via Data directly.
|
||||
a.CreatedAt = time.Now()
|
||||
m.Data[a.Hash] = *a
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) GetImages(hashes []string) (map[string]model.Artwork, error) {
|
||||
if m.Err != nil {
|
||||
return nil, m.Err
|
||||
}
|
||||
res := map[string]model.Artwork{}
|
||||
for _, h := range hashes {
|
||||
if a, ok := m.Data[h]; ok {
|
||||
res[h] = a
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) GetOrphanHashes(createdBefore time.Time) ([]string, error) {
|
||||
return m.OrphanHashes, m.Err
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) GetAllMimes() (map[string]string, error) {
|
||||
if m.Err != nil {
|
||||
return nil, m.Err
|
||||
}
|
||||
mimes := make(map[string]string, len(m.Data))
|
||||
for h, a := range m.Data {
|
||||
mimes[h] = a.Mime
|
||||
}
|
||||
return mimes, nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) PurgeDanglingItemArtwork() (int64, error) {
|
||||
if m.Err != nil {
|
||||
return 0, m.Err
|
||||
}
|
||||
var purged int64
|
||||
for k, ia := range m.ItemData {
|
||||
// A nil per-kind map means that kind isn't tracked by the test, so keep it.
|
||||
existing := m.ExistingIDs[ia.ItemKind]
|
||||
if existing == nil {
|
||||
continue
|
||||
}
|
||||
if !existing[ia.ItemID] {
|
||||
delete(m.ItemData, k)
|
||||
purged++
|
||||
}
|
||||
}
|
||||
return purged, nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) DeleteOrphans(createdBefore time.Time, hashes []string) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
// Mirror the SQL re-check: only unreferenced rows older than the cutoff are deleted.
|
||||
for _, h := range hashes {
|
||||
if m.referenced(h) {
|
||||
continue
|
||||
}
|
||||
if a, ok := m.Data[h]; ok && !a.CreatedAt.Before(createdBefore) {
|
||||
continue
|
||||
}
|
||||
delete(m.Data, h)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) referenced(hash string) bool {
|
||||
for _, ia := range m.ItemData {
|
||||
if ia.Hash == hash {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) {
|
||||
if m.Err != nil {
|
||||
return nil, m.Err
|
||||
}
|
||||
if ia, ok := m.ItemData[iaKey(kind, id, imageType)]; ok {
|
||||
return &ia, nil
|
||||
}
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) PutItemArtwork(ia *model.ItemArtwork) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
if ia.ImageType == "" {
|
||||
ia.ImageType = model.ImageTypePrimary
|
||||
}
|
||||
ia.UpdatedAt = time.Now()
|
||||
if ia.AttemptedAt.IsZero() {
|
||||
ia.AttemptedAt = ia.UpdatedAt
|
||||
}
|
||||
m.ItemData[iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)] = *ia
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) DeleteForItem(kind, id string) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
for k, ia := range m.ItemData {
|
||||
if ia.ItemKind == kind && ia.ItemID == id {
|
||||
delete(m.ItemData, k)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) {
|
||||
if m.Err != nil {
|
||||
return nil, m.Err
|
||||
}
|
||||
res := map[string]model.ItemArtworkInfo{}
|
||||
for _, id := range ids {
|
||||
if ia, ok := m.ItemData[iaKey(kind, id, model.ImageTypePrimary)]; ok {
|
||||
info := model.ItemArtworkInfo{ItemID: id, Hash: ia.Hash}
|
||||
if a, ok := m.Data[ia.Hash]; ok {
|
||||
info.BlurHash = a.BlurHash
|
||||
}
|
||||
res[id] = info
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
@@ -28,6 +28,8 @@ type MockDataStore struct {
|
||||
MockedScrobble model.ScrobbleRepository
|
||||
MockedRadio model.RadioRepository
|
||||
MockedPlugin model.PluginRepository
|
||||
MockedArtwork model.ArtworkRepository
|
||||
MockedArtworkQueue model.ArtworkQueueRepository
|
||||
scrobbleBufferMu sync.Mutex
|
||||
repoMu sync.Mutex
|
||||
|
||||
@@ -247,6 +249,32 @@ func (db *MockDataStore) Plugin(ctx context.Context) model.PluginRepository {
|
||||
return db.MockedPlugin
|
||||
}
|
||||
|
||||
func (db *MockDataStore) Artwork(ctx context.Context) model.ArtworkRepository {
|
||||
if db.MockedArtwork != nil {
|
||||
return db.MockedArtwork
|
||||
}
|
||||
if db.RealDS != nil {
|
||||
return db.RealDS.Artwork(ctx)
|
||||
}
|
||||
db.MockedArtwork = CreateMockArtworkRepo()
|
||||
return db.MockedArtwork
|
||||
}
|
||||
|
||||
func (db *MockDataStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository {
|
||||
if db.MockedArtworkQueue != nil {
|
||||
return db.MockedArtworkQueue
|
||||
}
|
||||
if db.RealDS != nil {
|
||||
return db.RealDS.ArtworkQueue(ctx)
|
||||
}
|
||||
q := CreateMockArtworkQueueRepo()
|
||||
if aw, ok := db.Artwork(ctx).(*MockArtworkRepo); ok {
|
||||
q.ItemArtworkSource = aw
|
||||
}
|
||||
db.MockedArtworkQueue = q
|
||||
return db.MockedArtworkQueue
|
||||
}
|
||||
|
||||
func (db *MockDataStore) WithTx(block func(tx model.DataStore) error, label ...string) error {
|
||||
return block(db)
|
||||
}
|
||||
|
||||
@@ -52,6 +52,18 @@ func (m *MockPlaylistRepo) GetAll(options ...model.QueryOptions) (model.Playlist
|
||||
return m.All, nil
|
||||
}
|
||||
|
||||
func (m *MockPlaylistRepo) GetAllIDs(options ...model.QueryOptions) ([]string, error) {
|
||||
all, err := m.GetAll(options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids := make([]string, len(all))
|
||||
for i, p := range all {
|
||||
ids[i] = p.ID
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (m *MockPlaylistRepo) GetCursor(options ...model.QueryOptions) (model.PlaylistCursor, error) {
|
||||
res, err := m.GetAll(options...)
|
||||
if err != nil {
|
||||
|
||||
@@ -14,6 +14,7 @@ type MockPlaylistTrackRepo struct {
|
||||
Reordered bool
|
||||
AddCount int
|
||||
Err error
|
||||
AlbumIDs []string // stubbed result for GetAlbumIDs, ignoring options
|
||||
}
|
||||
|
||||
func (m *MockPlaylistTrackRepo) SetData(tracks model.PlaylistTracks) {
|
||||
@@ -66,6 +67,13 @@ func (m *MockPlaylistTrackRepo) GetCursor(options ...model.QueryOptions) (model.
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *MockPlaylistTrackRepo) GetAlbumIDs(...model.QueryOptions) ([]string, error) {
|
||||
if m.Err != nil {
|
||||
return nil, m.Err
|
||||
}
|
||||
return m.AlbumIDs, nil
|
||||
}
|
||||
|
||||
func (m *MockPlaylistTrackRepo) GetMediaFileIDs(options ...model.QueryOptions) ([]string, error) {
|
||||
if m.Err != nil {
|
||||
return nil, m.Err
|
||||
|
||||
@@ -73,6 +73,18 @@ func (m *MockedRadioRepo) GetAll(qo ...model.QueryOptions) (model.Radios, error)
|
||||
return m.All, nil
|
||||
}
|
||||
|
||||
func (m *MockedRadioRepo) GetAllIDs(qo ...model.QueryOptions) ([]string, error) {
|
||||
all, err := m.GetAll(qo...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids := make([]string, len(all))
|
||||
for i, r := range all {
|
||||
ids[i] = r.ID
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (m *MockedRadioRepo) Put(radio *model.Radio, _ ...string) error {
|
||||
if m.Err {
|
||||
return errors.New("error")
|
||||
|
||||
Reference in new issue
Block a user