Compare commits

...
Author SHA1 Message Date
Deluan 1f818e7633 fix(artwork): store backing-file provenance per item, not per hash 2026-07-22 15:52:54 -04:00
Deluan c04c8ee02a fix(artwork): mock PutImage refreshes created_at like the SQL repository
Prune specs now age fixtures directly instead of seeding stale timestamps through the upsert.
2026-07-22 00:58:22 -04:00
Deluan ebbe533c6a fix(artwork): reject malformed hashes in ImageStore operations
Known-absent states carry an empty hash and malformed persisted hashes could panic path sharding or inject separators; Write/Open/Remove now return an error for anything but 16 lowercase hex chars.
2026-07-22 00:51:13 -04:00
Deluan 0147cc59b1 fix(artwork): honor the orphan cutoff in the repository mock
The mock's DeleteOrphans now applies createdBefore like the SQL implementation, and a new spec covers a freshly reacquired row surviving prune.
2026-07-22 00:42:42 -04:00
Deluan 034cd17498 fix(artwork): index artwork_queue in dequeue order
The previous leading retry_at range column forced a temp B-tree sort of the whole eligible set on every DequeueBatch; ordering the index by (priority DESC, enqueued_at) lets scans stop after the batch size.
2026-07-22 00:36:22 -04:00
Deluan 8147f7c40b fix(artwork): rewrite vanished duplicates and sweep stale mime variants
Write falls through to a real write when the liveness touch fails, and sweep retention now matches the recorded mime's extension so obsolete variants are reclaimed.
2026-07-22 00:28:51 -04:00
Deluan bf614e66ad fix(artwork): guard orphan file removal with the prune grace window
Duplicate ImageStore writes refresh the file mtime and Remove skips files newer than the cutoff, so overlapping acquisitions cannot lose their store files to a concurrent prune.
2026-07-22 00:19:45 -04:00
Deluan 623b7d6a6c fix(artwork): atomic orphan deletion and timestamp semantics from review
DeleteOrphans re-checks age+references at delete time, PutItemArtwork defaults attempted_at, queue mock timestamps mirror SQL.
2026-07-22 00:11:15 -04:00
Deluan 6f7f9c6463 fix(artwork): address review findings on prune/sweep races and mock fidelity
Sweep now honors an mtime grace window (in-flight acquisitions and temp files), reacquired orphans reset the prune grace window, and the queue mock implements real stale-absent semantics.
2026-07-21 23:57:58 -04:00
Deluan 8fd7ef19f3 refactor(artwork): apply simplify-pass cleanups
Internal item_artwork sqlRepository helper, toSQLArgs upserts, batched queue enqueue, EnqueueStaleAbsent moved to queue repo, snapshot-based prune sweep, mock/real semantics aligned.
2026-07-21 23:37:05 -04:00
Deluan 1041e45ca7 fix(artwork): chunk unbounded IN clauses and restore interface docs 2026-07-21 23:24:42 -04:00
Deluan 4f835437a9 refactor(artwork): merge item artwork state into ArtworkRepository 2026-07-21 23:14:01 -04:00
Deluan b16ef725c9 refactor(artwork): fold originals package into core/artwork as ImageStore 2026-07-21 23:09:31 -04:00
Deluan 3e7685adc2 fix(artwork): never sweep files on transient DB errors during prune 2026-07-21 23:02:47 -04:00
Deluan db16b3de9a feat(artwork): add artwork prune (orphan cleanup) 2026-07-21 22:58:20 -04:00
Deluan b72597821a feat(artwork): add content-addressed originals store 2026-07-21 22:53:00 -04:00
Deluan fcff9c63e7 feat(artwork): implement artwork_queue repository 2026-07-21 22:45:08 -04:00
Deluan 14dd57052e feat(artwork): implement item_artwork repository with batched hydration 2026-07-21 22:45:03 -04:00
Deluan f926539c04 feat(artwork): implement artwork repository 2026-07-21 22:44:24 -04:00
Deluan 6cce65f759 feat(artwork): add artwork models, repository interfaces and mocks 2026-07-21 22:35:34 -04:00
Deluan 7aacb01f4f feat(artwork): add artwork, item_artwork and artwork_queue tables 2026-07-21 22:31:03 -04:00
16 changed files with 1592 additions and 1 deletions

No files matched your search

+164
View File
@@ -0,0 +1,164 @@
package artwork
import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"time"
"github.com/zeebo/xxh3"
)
func HashImage(r io.Reader) (string, error) {
d := xxh3.New()
if _, err := io.Copy(d, r); err != nil {
return "", err
}
return fmt.Sprintf("%016x", d.Sum64()), nil
}
// ImageStore is the content-addressed store for artwork images that have no
// library file backing them (external downloads, embedded extractions, generated).
type ImageStore struct {
root string
}
func NewImageStore(rootDir string) *ImageStore {
return &ImageStore{root: rootDir}
}
// extForMime is deliberately NOT mime.ExtensionsByType: extensions are baked into
// content-addressed paths and re-derived on Open, so they must be stable across OSes.
func extForMime(m string) string {
switch m {
case "image/jpeg":
return ".jpg"
case "image/png":
return ".png"
case "image/gif":
return ".gif"
case "image/webp":
return ".webp"
}
return ".img"
}
// validHash rejects anything but 16 lowercase hex chars: known-absent states carry "",
// and malformed persisted hashes must never reach path sharding (slice panics, separators).
func validHash(hash string) bool {
if len(hash) != 16 {
return false
}
for _, c := range []byte(hash) {
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
return false
}
}
return true
}
func (s *ImageStore) path(hash, mimeType string) string {
return filepath.Join(s.root, hash[0:2], hash[2:4], hash+extForMime(mimeType))
}
func (s *ImageStore) Write(hash, mimeType string, r io.Reader) error {
if !validHash(hash) {
return fmt.Errorf("imagestore: invalid hash %q", hash)
}
dst := s.path(hash, mimeType)
if _, err := os.Stat(dst); err == nil {
// A touched mtime marks the file live so a concurrent prune spares it.
now := time.Now()
if err := os.Chtimes(dst, now, now); err == nil {
return nil
}
// touch failed (file likely pruned concurrently) — fall through and write it
}
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
return err
}
tmp, err := os.CreateTemp(filepath.Dir(dst), "."+hash+".tmp*")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
if _, err := io.Copy(tmp, r); err != nil {
tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
return os.Rename(tmp.Name(), dst)
}
func (s *ImageStore) Open(hash, mimeType string) (io.ReadCloser, error) {
if !validHash(hash) {
return nil, fmt.Errorf("imagestore: invalid hash %q", hash)
}
return os.Open(s.path(hash, mimeType))
}
// Remove deletes the store file unless it is newer than olderThan, in which case
// an overlapping acquisition may have just touched it and be about to commit its row.
func (s *ImageStore) Remove(hash, mimeType string, olderThan time.Time) error {
if !validHash(hash) {
return fmt.Errorf("imagestore: invalid hash %q", hash)
}
path := s.path(hash, mimeType)
info, err := os.Stat(path)
if errors.Is(err, fs.ErrNotExist) {
return nil
}
if err != nil {
return err
}
if info.ModTime().After(olderThan) {
return nil
}
err = os.Remove(path)
if errors.Is(err, fs.ErrNotExist) {
return nil
}
return err
}
// Sweep removes store files not accepted by keep. Files modified after cutoff
// (including temp files) are always kept: their acquisition row may not be committed yet.
func (s *ImageStore) Sweep(cutoff time.Time, keep func(hash, ext string) bool) (int, error) {
removed := 0
err := filepath.WalkDir(s.root, func(path string, d fs.DirEntry, err error) error {
if err != nil || d.IsDir() {
return err
}
info, err := d.Info()
if err != nil {
return err
}
if info.ModTime().After(cutoff) {
return nil
}
name := d.Name()
remove := strings.HasPrefix(name, ".") // abandoned temp file past the grace window
if !remove {
ext := filepath.Ext(name)
remove = !keep(strings.TrimSuffix(name, ext), ext)
}
if remove {
// #nosec G122 -- path comes from WalkDir over our own store root, no attacker-controlled symlinks
if err := os.Remove(path); err != nil {
return err
}
removed++
}
return nil
})
if errors.Is(err, fs.ErrNotExist) {
return removed, nil
}
return removed, err
}
+195
View File
@@ -0,0 +1,195 @@
package artwork
import (
"bytes"
"io"
"os"
"path/filepath"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("ImageStore", func() {
var store *ImageStore
var root string
BeforeEach(func() {
root = GinkgoT().TempDir()
store = NewImageStore(root)
})
It("hashes deterministically", func() {
h1, err := HashImage(bytes.NewReader([]byte("some image bytes")))
Expect(err).ToNot(HaveOccurred())
h2, _ := HashImage(bytes.NewReader([]byte("some image bytes")))
Expect(h1).To(Equal(h2))
Expect(h1).To(HaveLen(16))
h3, _ := HashImage(bytes.NewReader([]byte("other bytes")))
Expect(h3).ToNot(Equal(h1))
})
It("writes sharded and reads back", func() {
data := []byte("jpeg-bytes")
h, _ := HashImage(bytes.NewReader(data))
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
Expect(filepath.Join(root, h[0:2], h[2:4], h+".jpg")).To(BeAnExistingFile())
rc, err := store.Open(h, "image/jpeg")
Expect(err).ToNot(HaveOccurred())
defer rc.Close()
got, _ := io.ReadAll(rc)
Expect(got).To(Equal(data))
})
It("is idempotent on duplicate writes", func() {
data := []byte("dup")
h, _ := HashImage(bytes.NewReader(data))
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
})
It("refreshes the mtime on a duplicate write", func() {
data := []byte("touch-me")
h, _ := HashImage(bytes.NewReader(data))
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
old := time.Now().Add(-2 * time.Hour)
Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed())
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
info, err := os.Stat(store.path(h, "image/png"))
Expect(err).ToNot(HaveOccurred())
Expect(info.ModTime()).To(BeTemporally(">", time.Now().Add(-time.Minute)))
})
It("rewrites the bytes when the existing file vanished before the liveness touch", func() {
data := []byte("vanishing")
h, _ := HashImage(bytes.NewReader(data))
for range 10 {
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
Expect(os.Remove(store.path(h, "image/png"))).To(Succeed())
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
rc, err := store.Open(h, "image/png")
Expect(err).ToNot(HaveOccurred())
got, _ := io.ReadAll(rc)
rc.Close()
Expect(got).To(Equal(data))
}
})
It("returns fs.ErrNotExist for missing images", func() {
_, err := store.Open("beefbeefbeefbeef", "image/jpeg")
Expect(os.IsNotExist(err)).To(BeTrue())
})
It("removes without error when already gone", func() {
Expect(store.Remove("beefbeefbeefbeef", "image/jpeg", time.Now())).To(Succeed())
})
It("rejects invalid hashes instead of panicking", func() {
for _, h := range []string{"", "ab", "BEEFBEEFBEEFBEEF", "../../../../etcpw", "beefbeefbeefbee/"} {
Expect(store.Write(h, "image/jpeg", bytes.NewReader([]byte("x")))).To(MatchError(ContainSubstring("invalid hash")))
_, err := store.Open(h, "image/jpeg")
Expect(err).To(MatchError(ContainSubstring("invalid hash")))
Expect(store.Remove(h, "image/jpeg", time.Now())).To(MatchError(ContainSubstring("invalid hash")))
}
})
It("spares a file newer than the cutoff, removes an aged one", func() {
fresh := []byte("fresh")
hf, _ := HashImage(bytes.NewReader(fresh))
Expect(store.Write(hf, "image/jpeg", bytes.NewReader(fresh))).To(Succeed())
aged := []byte("aged")
ha, _ := HashImage(bytes.NewReader(aged))
Expect(store.Write(ha, "image/jpeg", bytes.NewReader(aged))).To(Succeed())
old := time.Now().Add(-2 * time.Hour)
Expect(os.Chtimes(store.path(ha, "image/jpeg"), old, old)).To(Succeed())
cutoff := time.Now().Add(-time.Hour)
Expect(store.Remove(hf, "image/jpeg", cutoff)).To(Succeed())
Expect(store.Remove(ha, "image/jpeg", cutoff)).To(Succeed())
rc, err := store.Open(hf, "image/jpeg")
Expect(err).ToNot(HaveOccurred())
rc.Close()
_, err = store.Open(ha, "image/jpeg")
Expect(os.IsNotExist(err)).To(BeTrue())
})
It("sweeps unknown files, keeps known ones", func() {
d1 := []byte("keep-me")
h1, _ := HashImage(bytes.NewReader(d1))
Expect(store.Write(h1, "image/jpeg", bytes.NewReader(d1))).To(Succeed())
d2 := []byte("orphan")
h2, _ := HashImage(bytes.NewReader(d2))
Expect(store.Write(h2, "image/jpeg", bytes.NewReader(d2))).To(Succeed())
old := time.Now().Add(-2 * time.Hour)
Expect(os.Chtimes(store.path(h2, "image/jpeg"), old, old)).To(Succeed())
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(h, _ string) bool { return h == h1 })
Expect(err).ToNot(HaveOccurred())
Expect(removed).To(Equal(1))
_, err = store.Open(h2, "image/jpeg")
Expect(os.IsNotExist(err)).To(BeTrue())
rc, err := store.Open(h1, "image/jpeg")
Expect(err).ToNot(HaveOccurred())
rc.Close()
})
It("sweeps a stale mime variant of a known hash, keeps the current one", func() {
data := []byte("same-bytes")
h, _ := HashImage(bytes.NewReader(data))
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
old := time.Now().Add(-2 * time.Hour)
Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed())
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
// The recorded mime is image/jpeg, so the .png variant is obsolete.
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(hash, ext string) bool {
return hash == h && ext == ".jpg"
})
Expect(err).ToNot(HaveOccurred())
Expect(removed).To(Equal(1))
_, err = store.Open(h, "image/png")
Expect(os.IsNotExist(err)).To(BeTrue())
rc, err := store.Open(h, "image/jpeg")
Expect(err).ToNot(HaveOccurred())
rc.Close()
})
It("keeps young unknown files inside the grace window", func() {
d := []byte("fresh-orphan")
h, _ := HashImage(bytes.NewReader(d))
Expect(store.Write(h, "image/jpeg", bytes.NewReader(d))).To(Succeed())
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string, string) bool { return false })
Expect(err).ToNot(HaveOccurred())
Expect(removed).To(Equal(0))
rc, err := store.Open(h, "image/jpeg")
Expect(err).ToNot(HaveOccurred())
rc.Close()
})
It("removes abandoned temp files past the grace window, keeps fresh ones", func() {
oldTmp := filepath.Join(root, ".old.tmp")
Expect(os.WriteFile(oldTmp, []byte("x"), 0600)).To(Succeed())
old := time.Now().Add(-2 * time.Hour)
Expect(os.Chtimes(oldTmp, old, old)).To(Succeed())
freshTmp := filepath.Join(root, ".fresh.tmp")
Expect(os.WriteFile(freshTmp, []byte("y"), 0600)).To(Succeed())
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string, string) bool { return true })
Expect(err).ToNot(HaveOccurred())
Expect(removed).To(Equal(1))
Expect(oldTmp).ToNot(BeAnExistingFile())
Expect(freshTmp).To(BeAnExistingFile())
})
})
+68
View File
@@ -0,0 +1,68 @@
package artwork
import (
"context"
"time"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
)
// pruneMinAge guards the window between artwork insert and item_artwork upsert.
const pruneMinAge = time.Hour
func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error {
repo := ds.Artwork(ctx)
// One grace cutoff for both the DB orphan check and the file sweep: files younger
// than the window may belong to acquisitions whose rows aren't committed yet.
cutoff := time.Now().Add(-pruneMinAge)
candidates, err := repo.GetOrphanHashes(cutoff)
if err != nil {
return err
}
if len(candidates) > 0 {
arts, err := repo.GetImages(candidates)
if err != nil {
return err
}
if err := repo.DeleteOrphans(cutoff, candidates); err != nil {
return err
}
// DeleteOrphans may spare candidates reacquired since the snapshot; only remove files
// for rows actually gone (absent from the post-delete re-read).
survivors, err := repo.GetImages(candidates)
if err != nil {
return err
}
removed := 0
for _, h := range candidates {
if _, ok := survivors[h]; ok {
continue
}
// A spared fresh file is at worst a stray a later sweep reclaims; full
// worker/prune mutual exclusion is Phase 2's concern.
if err := store.Remove(h, arts[h].Mime, cutoff); err != nil {
log.Warn(ctx, "Prune: could not remove artwork file", "hash", h, err)
}
removed++
}
log.Info(ctx, "Prune: removed orphan artwork", "count", removed)
}
mimes, err := repo.GetAllMimes()
if err != nil {
return err
}
removed, err := store.Sweep(cutoff, func(hash, ext string) bool {
// A known hash under a stale extension is a superseded mime variant — reclaim it.
m, ok := mimes[hash]
return ok && ext == extForMime(m)
})
if err != nil {
return err
}
if removed > 0 {
log.Info(ctx, "Prune: swept stray artwork files", "count", removed)
}
return nil
}
+168
View File
@@ -0,0 +1,168 @@
package artwork
import (
"bytes"
"context"
"errors"
"os"
"time"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/tests"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
type flakyGetArtworkRepo struct {
*tests.MockArtworkRepo
}
func (f *flakyGetArtworkRepo) GetAllMimes() (map[string]string, error) {
return nil, errors.New("db locked")
}
var _ = Describe("Prune", func() {
var ds *tests.MockDataStore
var store *ImageStore
var awRepo *tests.MockArtworkRepo
BeforeEach(func() {
ds = &tests.MockDataStore{}
awRepo = ds.Artwork(context.Background()).(*tests.MockArtworkRepo)
store = NewImageStore(GinkgoT().TempDir())
})
// PutImage refreshes created_at like the SQL repo, so fixtures are aged directly.
ageArtwork := func(h string, t time.Time) {
a := awRepo.Data[h]
a.CreatedAt = t
awRepo.Data[h] = a
}
It("deletes orphan rows and their store files, keeps referenced ones", func() {
data := []byte("orphan-bytes")
h, _ := HashImage(bytes.NewReader(data))
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
old := time.Now().Add(-2 * time.Hour)
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
ageArtwork(h, old)
awRepo.OrphanHashes = []string{h}
kept := []byte("kept-bytes")
hk, _ := HashImage(bytes.NewReader(kept))
Expect(store.Write(hk, "image/jpeg", bytes.NewReader(kept))).To(Succeed())
Expect(awRepo.PutImage(&model.Artwork{Hash: hk, Mime: "image/jpeg"})).To(Succeed())
Expect(Prune(context.Background(), ds, store)).To(Succeed())
_, err := awRepo.GetImage(h)
Expect(err).To(MatchError(model.ErrNotFound))
_, err = store.Open(h, "image/jpeg")
Expect(os.IsNotExist(err)).To(BeTrue())
rc, err := store.Open(hk, "image/jpeg")
Expect(err).ToNot(HaveOccurred())
rc.Close()
})
It("spares a candidate reacquired between snapshot and delete", func() {
data := []byte("reacquired-bytes")
h, _ := HashImage(bytes.NewReader(data))
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
ageArtwork(h, time.Now().Add(-2*time.Hour))
awRepo.OrphanHashes = []string{h}
// Reacquisition: an item now references the hash the snapshot flagged as orphan.
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1",
ImageType: model.ImageTypePrimary, Hash: h, Source: "folder"})).To(Succeed())
Expect(Prune(context.Background(), ds, store)).To(Succeed())
_, err := awRepo.GetImage(h)
Expect(err).ToNot(HaveOccurred())
rc, err := store.Open(h, "image/jpeg")
Expect(err).ToNot(HaveOccurred())
rc.Close()
})
It("spares a candidate whose row was freshly recreated (created_at inside the grace window)", func() {
data := []byte("fresh-reacquired-bytes")
h, _ := HashImage(bytes.NewReader(data))
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
// Reacquisition refreshed created_at after the snapshot; still unreferenced.
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
awRepo.OrphanHashes = []string{h}
Expect(Prune(context.Background(), ds, store)).To(Succeed())
_, err := awRepo.GetImage(h)
Expect(err).ToNot(HaveOccurred())
rc, err := store.Open(h, "image/jpeg")
Expect(err).ToNot(HaveOccurred())
rc.Close()
})
It("spares an orphan file freshly touched by an overlapping acquisition", func() {
data := []byte("racing-bytes")
h, _ := HashImage(bytes.NewReader(data))
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
ageArtwork(h, time.Now().Add(-2*time.Hour))
awRepo.OrphanHashes = []string{h}
// The row is legitimately orphaned, but a concurrent acquisition just touched the
// file's mtime (duplicate Write) and is about to commit a row referencing it.
Expect(Prune(context.Background(), ds, store)).To(Succeed())
rc, err := store.Open(h, "image/jpeg")
Expect(err).ToNot(HaveOccurred())
rc.Close()
})
It("sweeps store files that have no artwork row", func() {
stray := []byte("no-row-bytes")
h, _ := HashImage(bytes.NewReader(stray))
Expect(store.Write(h, "image/jpeg", bytes.NewReader(stray))).To(Succeed())
old := time.Now().Add(-2 * time.Hour)
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
Expect(Prune(context.Background(), ds, store)).To(Succeed())
_, err := store.Open(h, "image/jpeg")
Expect(os.IsNotExist(err)).To(BeTrue())
})
It("sweeps an obsolete mime variant of a reacquired hash", func() {
data := []byte("variant-bytes")
h, _ := HashImage(bytes.NewReader(data))
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
old := time.Now().Add(-2 * time.Hour)
Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed())
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
// The row records the current mime; the .png file is a superseded variant.
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
Expect(Prune(context.Background(), ds, store)).To(Succeed())
_, err := store.Open(h, "image/png")
Expect(os.IsNotExist(err)).To(BeTrue())
rc, err := store.Open(h, "image/jpeg")
Expect(err).ToNot(HaveOccurred())
rc.Close()
})
It("never sweeps files on a transient DB error", func() {
ds.MockedArtwork = &flakyGetArtworkRepo{MockArtworkRepo: tests.CreateMockArtworkRepo()}
data := []byte("live-bytes")
h, _ := HashImage(bytes.NewReader(data))
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
Expect(Prune(context.Background(), ds, store)).ToNot(Succeed())
rc, err := store.Open(h, "image/jpeg")
Expect(err).ToNot(HaveOccurred())
rc.Close()
})
})
@@ -0,0 +1,42 @@
-- +goose Up
CREATE TABLE artwork (
hash TEXT PRIMARY KEY,
mime TEXT NOT NULL,
width INTEGER NOT NULL DEFAULT 0,
height INTEGER NOT NULL DEFAULT 0,
size_bytes INTEGER NOT NULL DEFAULT 0,
blur_hash TEXT NOT NULL DEFAULT '',
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE item_artwork (
item_kind TEXT NOT NULL,
item_id TEXT NOT NULL,
image_type TEXT NOT NULL DEFAULT 'primary',
hash TEXT NOT NULL DEFAULT '',
source TEXT NOT NULL DEFAULT '',
source_path TEXT NOT NULL DEFAULT '',
ref_mtime INTEGER NOT NULL DEFAULT 0,
attempted_at TIMESTAMP,
updated_at TIMESTAMP,
PRIMARY KEY (item_kind, item_id, image_type)
) WITHOUT ROWID;
CREATE INDEX ix_item_artwork_hash ON item_artwork(hash);
CREATE TABLE artwork_queue (
item_kind TEXT NOT NULL,
item_id TEXT NOT NULL,
image_type TEXT NOT NULL DEFAULT 'primary',
priority INTEGER NOT NULL DEFAULT 0,
attempts INTEGER NOT NULL DEFAULT 0,
retry_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
enqueued_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (item_kind, item_id, image_type)
) WITHOUT ROWID;
-- Ordered to match DequeueBatch (priority DESC, enqueued_at) so drains stop after n rows; retry_at makes it covering.
CREATE INDEX ix_artwork_queue_drain ON artwork_queue(priority DESC, enqueued_at, retry_at);
-- +goose Down
DROP TABLE artwork_queue;
DROP TABLE item_artwork;
DROP TABLE artwork;
+1 -1
View File
@@ -57,6 +57,7 @@ require (
github.com/tetratelabs/wazero v1.12.0
github.com/unrolled/secure v1.17.0
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342
github.com/zeebo/xxh3 v1.1.0
go.senan.xyz/taglib v0.11.1
go.uber.org/goleak v1.3.0
golang.org/x/image v0.44.0
@@ -128,7 +129,6 @@ require (
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect
github.com/valyala/fastjson v1.6.10 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
+93
View File
@@ -0,0 +1,93 @@
package model
import "time"
// Artwork is one unique image, identified by the XXH3-64 hash of its bytes.
type Artwork struct {
Hash string `structs:"hash"`
Mime string `structs:"mime"`
Width int `structs:"width"`
Height int `structs:"height"`
SizeBytes int64 `structs:"size_bytes"`
BlurHash string `structs:"blur_hash"`
CreatedAt time.Time `structs:"created_at"`
}
const ImageTypePrimary = "primary"
// ItemArtwork is an entity's resolved artwork state. Hash=="" means known absent.
type ItemArtwork struct {
ItemKind string `structs:"item_kind"`
ItemID string `structs:"item_id"`
ImageType string `structs:"image_type"`
Hash string `structs:"hash"`
Source string `structs:"source"`
// SourcePath is the backing file (folder/upload: the image; embedded: the audio file); "" otherwise.
SourcePath string `structs:"source_path"`
// RefMtime is SourcePath's mtime at resolution; 0 when there is no SourcePath.
RefMtime int64 `structs:"ref_mtime"`
// attempted_at/updated_at are nullable in the schema but always set by PutItemArtwork;
// raw inserts must set them too, since these non-pointer time.Time fields fail to scan NULL.
AttemptedAt time.Time `structs:"attempted_at"`
UpdatedAt time.Time `structs:"updated_at"`
}
// ItemArtworkInfo is the list-hydration projection (item_artwork joined with artwork).
type ItemArtworkInfo struct {
ItemID string
Hash string
BlurHash string
}
// Absent reports a known-absent artwork state (resolved, no image).
func (i ItemArtworkInfo) Absent() bool { return i.Hash == "" }
type ArtworkQueueItem struct {
ItemKind string `structs:"item_kind"`
ItemID string `structs:"item_id"`
ImageType string `structs:"image_type"`
Priority int `structs:"priority"`
Attempts int `structs:"attempts"`
RetryAt time.Time `structs:"retry_at"`
EnqueuedAt time.Time `structs:"enqueued_at"`
}
// Queue priorities: higher drains first.
const (
ArtworkPriorityRecheck = 0
ArtworkPriorityBackfill = 10
ArtworkPriorityScan = 50
ArtworkPriorityBump = 100
)
type ArtworkRepository interface {
// Image identity (artwork table)
GetImage(hash string) (*Artwork, error)
PutImage(a *Artwork) error
GetImages(hashes []string) (map[string]Artwork, error)
// GetOrphanHashes returns hashes referenced by no item_artwork row and older than cutoff.
GetOrphanHashes(createdBefore time.Time) ([]string, error)
// DeleteOrphans deletes the given hashes only if still unreferenced and older than cutoff (atomic re-check).
DeleteOrphans(createdBefore time.Time, hashes []string) error
// Per-item state (item_artwork table)
GetItemArtwork(kind, id, imageType string) (*ItemArtwork, error)
PutItemArtwork(ia *ItemArtwork) error
DeleteForItem(kind, id string) error
// GetInfoForItems hydrates a page: one batched query, item_artwork joined to artwork.
GetInfoForItems(kind string, ids []string) (map[string]ItemArtworkInfo, error)
// GetAllMimes returns hash -> current mime for every stored artwork, for sweep retention checks.
GetAllMimes() (map[string]string, error)
}
type ArtworkQueueRepository interface {
// Enqueue upserts; an existing row keeps the higher of the two priorities.
Enqueue(items ...ArtworkQueueItem) error
// DequeueBatch returns up to n items with retry_at <= now, priority desc, enqueued_at asc.
DequeueBatch(n int) ([]ArtworkQueueItem, error)
// MarkFailed increments attempts and pushes retry_at into the future.
MarkFailed(kind, id, imageType string, retryAt time.Time) error
Delete(kind, id, imageType string) error
Count() (int64, error)
// EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff.
EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error)
}
+2
View File
@@ -40,6 +40,8 @@ type DataStore interface {
ScrobbleBuffer(ctx context.Context) ScrobbleBufferRepository
Scrobble(ctx context.Context) ScrobbleRepository
Plugin(ctx context.Context) PluginRepository
Artwork(ctx context.Context) ArtworkRepository
ArtworkQueue(ctx context.Context) ArtworkQueueRepository
Resource(ctx context.Context, model any) ResourceRepository
+90
View File
@@ -0,0 +1,90 @@
package persistence
import (
"context"
"slices"
"time"
. "github.com/Masterminds/squirrel"
"github.com/navidrome/navidrome/model"
"github.com/pocketbase/dbx"
)
// enqueueChunkSize keeps each multi-row insert under SQLite's bind-variable limit (7 cols -> 700 vars).
const enqueueChunkSize = 100
type artworkQueueRepository struct {
sqlRepository
}
func NewArtworkQueueRepository(ctx context.Context, db dbx.Builder) model.ArtworkQueueRepository {
r := &artworkQueueRepository{}
r.ctx = ctx
r.db = db
r.tableName = "artwork_queue"
return r
}
func (r *artworkQueueRepository) Enqueue(items ...model.ArtworkQueueItem) error {
now := time.Now()
for chunk := range slices.Chunk(items, enqueueChunkSize) {
ins := Insert(r.tableName).Columns("item_kind", "item_id", "image_type", "priority", "attempts", "retry_at", "enqueued_at")
for _, it := range chunk {
if it.ImageType == "" {
it.ImageType = model.ImageTypePrimary
}
ins = ins.Values(it.ItemKind, it.ItemID, it.ImageType, it.Priority, 0, now, now)
}
ins = ins.Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET
priority = MAX(priority, excluded.priority), retry_at = excluded.retry_at`)
if _, err := r.executeSQL(ins); err != nil {
return err
}
}
return nil
}
func (r *artworkQueueRepository) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) {
sel := Select("*").From(r.tableName).
Where(LtOrEq{"retry_at": time.Now()}).
OrderBy("priority DESC", "enqueued_at ASC").
Limit(uint64(n))
var res []model.ArtworkQueueItem
err := r.queryAll(sel, &res)
return res, err
}
func (r *artworkQueueRepository) MarkFailed(kind, id, imageType string, retryAt time.Time) error {
upd := Update(r.tableName).
Set("attempts", Expr("attempts + 1")).
Set("retry_at", retryAt).
Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType})
c, err := r.executeSQL(upd)
if err == nil && c == 0 {
return model.ErrNotFound
}
return err
}
func (r *artworkQueueRepository) Delete(kind, id, imageType string) error {
return r.delete(Eq{"item_kind": kind, "item_id": id, "image_type": imageType})
}
func (r *artworkQueueRepository) Count() (int64, error) {
var res struct{ Count int64 }
err := r.queryOne(Select("count(*) as count").From(r.tableName), &res)
return res.Count, err
}
func (r *artworkQueueRepository) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) {
now := time.Now()
// DO NOTHING is deliberate: rechecks must not bump priority/retry_at of already-queued items.
ins := Expr(`INSERT INTO `+r.tableName+` (item_kind, item_id, image_type, priority, attempts, retry_at, enqueued_at)
SELECT item_kind, item_id, image_type, ?, 0, ?, ?
FROM `+itemArtworkTable+` WHERE item_kind = ? AND hash = '' AND attempted_at < ?
ON CONFLICT (item_kind, item_id, image_type) DO NOTHING`,
model.ArtworkPriorityRecheck, now, now, kind, attemptedBefore)
return r.executeSQL(ins)
}
var _ model.ArtworkQueueRepository = (*artworkQueueRepository)(nil)
@@ -0,0 +1,83 @@
package persistence
import (
"context"
"time"
"github.com/navidrome/navidrome/model"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("ArtworkQueueRepository", func() {
var repo model.ArtworkQueueRepository
item := func(kind, id string, prio int) model.ArtworkQueueItem {
return model.ArtworkQueueItem{ItemKind: kind, ItemID: id,
ImageType: model.ImageTypePrimary, Priority: prio}
}
BeforeEach(func() {
clearArtworkTables()
repo = NewArtworkQueueRepository(context.Background(), GetDBXBuilder())
})
It("enqueues and dequeues by priority then FIFO", func() {
Expect(repo.Enqueue(item("al", "low", model.ArtworkPriorityBackfill))).To(Succeed())
Expect(repo.Enqueue(item("ar", "high", model.ArtworkPriorityBump))).To(Succeed())
got, err := repo.DequeueBatch(10)
Expect(err).ToNot(HaveOccurred())
Expect(got).To(HaveLen(2))
Expect(got[0].ItemID).To(Equal("high"))
})
It("keeps the higher priority on duplicate enqueue", func() {
Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBump))).To(Succeed())
Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBackfill))).To(Succeed())
got, _ := repo.DequeueBatch(10)
Expect(got).To(HaveLen(1))
Expect(got[0].Priority).To(Equal(model.ArtworkPriorityBump))
})
It("hides failed items until retry_at", func() {
Expect(repo.Enqueue(item("al", "f1", model.ArtworkPriorityScan))).To(Succeed())
Expect(repo.MarkFailed("al", "f1", model.ImageTypePrimary, time.Now().Add(time.Hour))).To(Succeed())
got, err := repo.DequeueBatch(10)
Expect(err).ToNot(HaveOccurred())
Expect(got).To(BeEmpty())
Expect(repo.MarkFailed("al", "f1", model.ImageTypePrimary, time.Now().Add(-time.Minute))).To(Succeed())
got, _ = repo.DequeueBatch(10)
Expect(got).To(HaveLen(1))
Expect(got[0].Attempts).To(Equal(2))
})
It("deletes on completion and counts", func() {
Expect(repo.Enqueue(item("al", "c1", 0))).To(Succeed())
n, _ := repo.Count()
Expect(n).To(Equal(int64(1)))
Expect(repo.Delete("al", "c1", model.ImageTypePrimary)).To(Succeed())
n, _ = repo.Count()
Expect(n).To(BeZero())
})
It("enqueues stale absent states for recheck", func() {
awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder())
old := time.Now().Add(-48 * time.Hour)
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "stale1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old})).To(Succeed())
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed())
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old})).To(Succeed())
n, err := repo.EnqueueStaleAbsent("ar", time.Now().Add(-24*time.Hour))
Expect(err).ToNot(HaveOccurred())
Expect(n).To(Equal(int64(1)))
items, err := repo.DequeueBatch(10)
Expect(err).ToNot(HaveOccurred())
Expect(items).To(HaveLen(1))
Expect(items[0].ItemID).To(Equal("stale1"))
Expect(items[0].Priority).To(Equal(model.ArtworkPriorityRecheck))
})
})
+179
View File
@@ -0,0 +1,179 @@
package persistence
import (
"context"
"slices"
"time"
. "github.com/Masterminds/squirrel"
"github.com/navidrome/navidrome/model"
"github.com/pocketbase/dbx"
)
const (
itemArtworkTable = "item_artwork"
artworkBatchSize = 200
)
type itemArtworkSQL struct {
sqlRepository
}
type artworkRepository struct {
sqlRepository
items itemArtworkSQL
}
func NewArtworkRepository(ctx context.Context, db dbx.Builder) model.ArtworkRepository {
r := &artworkRepository{}
r.ctx = ctx
r.db = db
r.tableName = "artwork"
r.items.ctx = ctx
r.items.db = db
r.items.tableName = itemArtworkTable
return r
}
func (r *artworkRepository) GetImage(hash string) (*model.Artwork, error) {
sel := Select("*").From(r.tableName).Where(Eq{"hash": hash})
var res model.Artwork
if err := r.queryOne(sel, &res); err != nil {
return nil, err
}
return &res, nil
}
func (r *artworkRepository) PutImage(a *model.Artwork) error {
// created_at is the last-acquisition-write time the prune grace window keys on.
a.CreatedAt = time.Now()
values, err := toSQLArgs(*a)
if err != nil {
return err
}
// created_at=excluded.created_at: reacquiring an orphan must reset the prune grace window.
ins := Insert(r.tableName).SetMap(values).Suffix(`ON CONFLICT (hash) DO UPDATE SET mime=excluded.mime, width=excluded.width,
height=excluded.height, size_bytes=excluded.size_bytes, blur_hash=excluded.blur_hash, created_at=excluded.created_at`)
_, err = r.executeSQL(ins)
return err
}
func (r *artworkRepository) GetImages(hashes []string) (map[string]model.Artwork, error) {
res := map[string]model.Artwork{}
for chunk := range slices.Chunk(hashes, artworkBatchSize) {
sel := Select("*").From(r.tableName).Where(Eq{"hash": chunk})
var all []model.Artwork
if err := r.queryAll(sel, &all); err != nil {
return nil, err
}
for _, a := range all {
res[a.Hash] = a
}
}
return res, nil
}
func (r *artworkRepository) GetAllMimes() (map[string]string, error) {
sel := Select("hash", "mime").From(r.tableName)
var rows []struct {
Hash string
Mime string
}
if err := r.queryAll(sel, &rows); err != nil {
return nil, err
}
res := make(map[string]string, len(rows))
for _, row := range rows {
res[row.Hash] = row.Mime
}
return res, nil
}
func (r *artworkRepository) GetOrphanHashes(createdBefore time.Time) ([]string, error) {
sel := Select("hash").From(r.tableName).
Where(And{
Lt{"created_at": createdBefore},
Expr("hash NOT IN (SELECT hash FROM " + itemArtworkTable + " WHERE hash <> '')"),
})
var hashes []string
err := r.queryAllSlice(sel, &hashes)
return hashes, err
}
func (r *artworkRepository) DeleteOrphans(createdBefore time.Time, hashes []string) error {
for chunk := range slices.Chunk(hashes, artworkBatchSize) {
del := Delete(r.tableName).Where(And{
Eq{"hash": chunk},
Lt{"created_at": createdBefore},
Expr("hash NOT IN (SELECT hash FROM " + itemArtworkTable + " WHERE hash <> '')"),
})
if _, err := r.executeSQL(del); err != nil {
return err
}
}
return nil
}
func (r *artworkRepository) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) {
sel := Select("*").From(itemArtworkTable).
Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType})
var res model.ItemArtwork
if err := r.items.queryOne(sel, &res); err != nil {
return nil, err
}
return &res, nil
}
func (r *artworkRepository) PutItemArtwork(ia *model.ItemArtwork) error {
if ia.ImageType == "" {
ia.ImageType = model.ImageTypePrimary
}
ia.UpdatedAt = time.Now()
// PutItemArtwork records the outcome of an attempt, so an unset attempted_at is now.
if ia.AttemptedAt.IsZero() {
ia.AttemptedAt = ia.UpdatedAt
}
values, err := toSQLArgs(*ia)
if err != nil {
return err
}
ins := Insert(itemArtworkTable).SetMap(values).Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET
hash=excluded.hash, source=excluded.source, source_path=excluded.source_path, ref_mtime=excluded.ref_mtime,
attempted_at=excluded.attempted_at, updated_at=excluded.updated_at`)
_, err = r.items.executeSQL(ins)
return err
}
func (r *artworkRepository) DeleteForItem(kind, id string) error {
return r.items.delete(Eq{"item_kind": kind, "item_id": id})
}
func (r *artworkRepository) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) {
res := map[string]model.ItemArtworkInfo{}
for chunk := range slices.Chunk(ids, artworkBatchSize) {
sel := Select("ia.item_id", "ia.hash", "COALESCE(a.blur_hash, '') as blur_hash").
From(itemArtworkTable + " ia").
LeftJoin("artwork a ON a.hash = ia.hash").
Where(And{
Eq{"ia.item_kind": kind},
Eq{"ia.image_type": model.ImageTypePrimary},
Eq{"ia.item_id": chunk},
})
var rows []struct {
ItemID string
Hash string
BlurHash string
}
if err := r.items.queryAll(sel, &rows); err != nil {
return nil, err
}
for _, row := range rows {
res[row.ItemID] = model.ItemArtworkInfo{
ItemID: row.ItemID, Hash: row.Hash, BlurHash: row.BlurHash,
}
}
}
return res, nil
}
var _ model.ArtworkRepository = (*artworkRepository)(nil)
+197
View File
@@ -0,0 +1,197 @@
package persistence
import (
"context"
"fmt"
"time"
"github.com/navidrome/navidrome/model"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/pocketbase/dbx"
)
// clearArtworkTables resets the shared test DB's artwork tables so specs don't leak state.
func clearArtworkTables() {
db := GetDBXBuilder()
for _, t := range []string{"artwork_queue", "item_artwork", "artwork"} {
_, err := db.NewQuery("DELETE FROM " + t).Execute()
Expect(err).ToNot(HaveOccurred())
}
}
var _ = Describe("ArtworkRepository", func() {
var repo model.ArtworkRepository
BeforeEach(func() {
clearArtworkTables()
repo = NewArtworkRepository(context.Background(), GetDBXBuilder())
})
Context("image identity", func() {
It("stores and retrieves an artwork by hash", func() {
a := &model.Artwork{Hash: "abc123", Mime: "image/jpeg", Width: 500, Height: 500, SizeBytes: 1234, BlurHash: "LKO2?U%2Tw=w"}
Expect(repo.PutImage(a)).To(Succeed())
got, err := repo.GetImage("abc123")
Expect(err).ToNot(HaveOccurred())
Expect(got.Mime).To(Equal("image/jpeg"))
Expect(got.BlurHash).To(Equal("LKO2?U%2Tw=w"))
Expect(got.CreatedAt).ToNot(BeZero())
})
It("is idempotent on Put (upsert by hash)", func() {
a := &model.Artwork{Hash: "dup1", Mime: "image/png"}
Expect(repo.PutImage(a)).To(Succeed())
a.BlurHash = "XYZ"
Expect(repo.PutImage(a)).To(Succeed())
got, _ := repo.GetImage("dup1")
Expect(got.BlurHash).To(Equal("XYZ"))
})
It("refreshes created_at when reacquiring an existing hash", func() {
Expect(repo.PutImage(&model.Artwork{Hash: "reacq", Mime: "image/jpeg"})).To(Succeed())
_, err := GetDBXBuilder().NewQuery("UPDATE artwork SET created_at={:t} WHERE hash='reacq'").
Bind(dbx.Params{"t": "2000-01-01 00:00:00"}).Execute()
Expect(err).ToNot(HaveOccurred())
Expect(repo.PutImage(&model.Artwork{Hash: "reacq", Mime: "image/png"})).To(Succeed())
got, err := repo.GetImage("reacq")
Expect(err).ToNot(HaveOccurred())
Expect(got.CreatedAt).To(BeTemporally(">", time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)))
})
It("returns ErrNotFound for a missing hash", func() {
_, err := repo.GetImage("nope")
Expect(err).To(MatchError(model.ErrNotFound))
})
It("fetches a batch", func() {
Expect(repo.PutImage(&model.Artwork{Hash: "b1", Mime: "image/jpeg"})).To(Succeed())
Expect(repo.PutImage(&model.Artwork{Hash: "b2", Mime: "image/png"})).To(Succeed())
got, err := repo.GetImages([]string{"b1", "b2", "missing"})
Expect(err).ToNot(HaveOccurred())
Expect(got).To(HaveLen(2))
Expect(got["b2"].Mime).To(Equal("image/png"))
})
It("returns every stored hash with its current mime", func() {
Expect(repo.PutImage(&model.Artwork{Hash: "all1", Mime: "image/jpeg"})).To(Succeed())
Expect(repo.PutImage(&model.Artwork{Hash: "all2", Mime: "image/png"})).To(Succeed())
mimes, err := repo.GetAllMimes()
Expect(err).ToNot(HaveOccurred())
Expect(mimes).To(HaveKeyWithValue("all1", "image/jpeg"))
Expect(mimes).To(HaveKeyWithValue("all2", "image/png"))
})
It("finds orphans older than cutoff, honoring item_artwork references", func() {
Expect(repo.PutImage(&model.Artwork{Hash: "orph1", Mime: "image/jpeg"})).To(Succeed())
Expect(repo.PutImage(&model.Artwork{Hash: "ref1", Mime: "image/jpeg"})).To(Succeed())
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", ImageType: model.ImageTypePrimary, Hash: "ref1", Source: "folder"})).To(Succeed())
orphans, err := repo.GetOrphanHashes(time.Now().Add(time.Minute))
Expect(err).ToNot(HaveOccurred())
Expect(orphans).To(ContainElement("orph1"))
Expect(orphans).ToNot(ContainElement("ref1"))
orphans, err = repo.GetOrphanHashes(time.Now().Add(-time.Hour))
Expect(err).ToNot(HaveOccurred())
Expect(orphans).To(BeEmpty())
})
It("deletes only unreferenced hashes older than the cutoff", func() {
Expect(repo.PutImage(&model.Artwork{Hash: "d1", Mime: "image/jpeg"})).To(Succeed())
Expect(repo.PutImage(&model.Artwork{Hash: "dref", Mime: "image/jpeg"})).To(Succeed())
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1",
ImageType: model.ImageTypePrimary, Hash: "dref", Source: "folder"})).To(Succeed())
Expect(repo.DeleteOrphans(time.Now().Add(time.Minute), []string{"d1", "dref"})).To(Succeed())
_, err := repo.GetImage("d1")
Expect(err).To(MatchError(model.ErrNotFound))
_, err = repo.GetImage("dref")
Expect(err).ToNot(HaveOccurred())
})
It("spares an unreferenced hash younger than the cutoff", func() {
Expect(repo.PutImage(&model.Artwork{Hash: "young", Mime: "image/jpeg"})).To(Succeed())
Expect(repo.DeleteOrphans(time.Now().Add(-time.Hour), []string{"young"})).To(Succeed())
_, err := repo.GetImage("young")
Expect(err).ToNot(HaveOccurred())
})
It("fetches a batch larger than the SQL variable limit", func() {
hashes := make([]string, 0, 250)
for i := range 250 {
h := fmt.Sprintf("big%03d", i)
Expect(repo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
hashes = append(hashes, h)
}
hashes = append(hashes, "absent1", "absent2")
got, err := repo.GetImages(hashes)
Expect(err).ToNot(HaveOccurred())
Expect(got).To(HaveLen(250))
})
})
Context("item state", func() {
It("upserts and reads state, including per-item provenance", func() {
ia := &model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary,
Hash: "h1", Source: "folder", SourcePath: "/music/a/cover.jpg", RefMtime: 111, AttemptedAt: time.Now()}
Expect(repo.PutItemArtwork(ia)).To(Succeed())
ia.Source = "embedded"
ia.SourcePath = "/music/a/track.mp3"
ia.RefMtime = 222
Expect(repo.PutItemArtwork(ia)).To(Succeed())
got, err := repo.GetItemArtwork("al", "al1", model.ImageTypePrimary)
Expect(err).ToNot(HaveOccurred())
Expect(got.Source).To(Equal("embedded"))
Expect(got.SourcePath).To(Equal("/music/a/track.mp3"))
Expect(got.RefMtime).To(Equal(int64(222)))
Expect(got.UpdatedAt).ToNot(BeZero())
})
It("defaults attempted_at to now when unset", func() {
before := time.Now().Add(-time.Second)
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "noattempt",
ImageType: model.ImageTypePrimary, Hash: ""})).To(Succeed())
got, err := repo.GetItemArtwork("ar", "noattempt", model.ImageTypePrimary)
Expect(err).ToNot(HaveOccurred())
Expect(got.AttemptedAt).To(BeTemporally(">", before))
})
It("represents known-absent as empty hash", func() {
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "ar1",
ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed())
got, err := repo.GetItemArtwork("ar", "ar1", model.ImageTypePrimary)
Expect(err).ToNot(HaveOccurred())
Expect(got.Hash).To(BeEmpty())
})
It("hydrates a page in one batch, including blurhash and absence", func() {
Expect(repo.PutImage(&model.Artwork{Hash: "h9", Mime: "image/jpeg", BlurHash: "BH9"})).To(Succeed())
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x1", ImageType: model.ImageTypePrimary, Hash: "h9", Source: "folder"})).To(Succeed())
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x2", ImageType: model.ImageTypePrimary, Hash: "", Source: ""})).To(Succeed())
info, err := repo.GetInfoForItems("al", []string{"x1", "x2", "x3"})
Expect(err).ToNot(HaveOccurred())
Expect(info).To(HaveLen(2))
Expect(info["x1"].Hash).To(Equal("h9"))
Expect(info["x1"].BlurHash).To(Equal("BH9"))
Expect(info["x1"].Absent()).To(BeFalse())
Expect(info["x2"].Absent()).To(BeTrue())
_, unresolved := info["x3"]
Expect(unresolved).To(BeFalse())
})
It("deletes all rows for an item", func() {
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "pl", ItemID: "p1", ImageType: model.ImageTypePrimary, Hash: "h1"})).To(Succeed())
Expect(repo.DeleteForItem("pl", "p1")).To(Succeed())
_, err := repo.GetItemArtwork("pl", "p1", model.ImageTypePrimary)
Expect(err).To(MatchError(model.ErrNotFound))
})
})
})
+8
View File
@@ -97,6 +97,14 @@ func (s *SQLStore) Plugin(ctx context.Context) model.PluginRepository {
return NewPluginRepository(ctx, s.getDBXBuilder())
}
func (s *SQLStore) Artwork(ctx context.Context) model.ArtworkRepository {
return NewArtworkRepository(ctx, s.getDBXBuilder())
}
func (s *SQLStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository {
return NewArtworkQueueRepository(ctx, s.getDBXBuilder())
}
func (s *SQLStore) Resource(ctx context.Context, m any) model.ResourceRepository {
switch m.(type) {
case model.User:
+125
View File
@@ -0,0 +1,125 @@
package tests
import (
"sort"
"time"
"github.com/navidrome/navidrome/model"
)
type MockArtworkQueueRepo struct {
model.ArtworkQueueRepository
Data map[string]model.ArtworkQueueItem // keyed by iaKey(kind, id, imageType)
Err error
// ItemArtworkSource, when set, backs EnqueueStaleAbsent with real item_artwork state.
ItemArtworkSource *MockArtworkRepo
}
func CreateMockArtworkQueueRepo() *MockArtworkQueueRepo {
return &MockArtworkQueueRepo{Data: map[string]model.ArtworkQueueItem{}}
}
func (m *MockArtworkQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error {
if m.Err != nil {
return m.Err
}
now := time.Now()
for _, it := range items {
if it.ImageType == "" {
it.ImageType = model.ImageTypePrimary
}
k := iaKey(it.ItemKind, it.ItemID, it.ImageType)
// Mirror the SQL: retry_at/enqueued_at are server-set, never taken from the caller.
if prev, ok := m.Data[k]; ok {
prev.Priority = max(prev.Priority, it.Priority)
prev.RetryAt = now
m.Data[k] = prev
continue
}
it.Attempts = 0
it.RetryAt = now
it.EnqueuedAt = now
m.Data[k] = it
}
return nil
}
func (m *MockArtworkQueueRepo) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) {
if m.Err != nil {
return nil, m.Err
}
var res []model.ArtworkQueueItem
now := time.Now()
for _, it := range m.Data {
if !it.RetryAt.After(now) {
res = append(res, it)
}
}
sort.Slice(res, func(i, j int) bool {
if res[i].Priority != res[j].Priority {
return res[i].Priority > res[j].Priority
}
return res[i].EnqueuedAt.Before(res[j].EnqueuedAt)
})
if len(res) > n {
res = res[:n]
}
return res, nil
}
func (m *MockArtworkQueueRepo) MarkFailed(kind, id, imageType string, retryAt time.Time) error {
if m.Err != nil {
return m.Err
}
k := iaKey(kind, id, imageType)
it, ok := m.Data[k]
if !ok {
return model.ErrNotFound
}
it.Attempts++
it.RetryAt = retryAt
m.Data[k] = it
return nil
}
func (m *MockArtworkQueueRepo) Delete(kind, id, imageType string) error {
if m.Err != nil {
return m.Err
}
delete(m.Data, iaKey(kind, id, imageType))
return nil
}
func (m *MockArtworkQueueRepo) Count() (int64, error) {
if m.Err != nil {
return 0, m.Err
}
return int64(len(m.Data)), nil
}
func (m *MockArtworkQueueRepo) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) {
if m.Err != nil || m.ItemArtworkSource == nil {
return 0, m.Err
}
now := time.Now()
var inserted int64
for _, ia := range m.ItemArtworkSource.ItemData {
if ia.ItemKind != kind || ia.Hash != "" || !ia.AttemptedAt.Before(attemptedBefore) {
continue
}
k := iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)
if _, ok := m.Data[k]; ok { // DO NOTHING: never touch existing queue rows
continue
}
m.Data[k] = model.ArtworkQueueItem{
ItemKind: ia.ItemKind,
ItemID: ia.ItemID,
ImageType: ia.ImageType,
Priority: model.ArtworkPriorityRecheck,
RetryAt: now,
EnqueuedAt: now,
}
inserted++
}
return inserted, nil
}
+149
View File
@@ -0,0 +1,149 @@
package tests
import (
"time"
"github.com/navidrome/navidrome/model"
)
type MockArtworkRepo struct {
model.ArtworkRepository
Data map[string]model.Artwork
ItemData map[string]model.ItemArtwork // keyed by iaKey(kind, id, imageType)
OrphanHashes []string
Err error
}
func CreateMockArtworkRepo() *MockArtworkRepo {
return &MockArtworkRepo{Data: map[string]model.Artwork{}, ItemData: map[string]model.ItemArtwork{}}
}
func iaKey(kind, id, imageType string) string { return kind + "|" + id + "|" + imageType }
func (m *MockArtworkRepo) GetImage(hash string) (*model.Artwork, error) {
if m.Err != nil {
return nil, m.Err
}
if a, ok := m.Data[hash]; ok {
return &a, nil
}
return nil, model.ErrNotFound
}
func (m *MockArtworkRepo) PutImage(a *model.Artwork) error {
if m.Err != nil {
return m.Err
}
// Mirrors the SQL repository: every upsert refreshes created_at. Age fixtures via Data directly.
a.CreatedAt = time.Now()
m.Data[a.Hash] = *a
return nil
}
func (m *MockArtworkRepo) GetImages(hashes []string) (map[string]model.Artwork, error) {
if m.Err != nil {
return nil, m.Err
}
res := map[string]model.Artwork{}
for _, h := range hashes {
if a, ok := m.Data[h]; ok {
res[h] = a
}
}
return res, nil
}
func (m *MockArtworkRepo) GetOrphanHashes(createdBefore time.Time) ([]string, error) {
return m.OrphanHashes, m.Err
}
func (m *MockArtworkRepo) GetAllMimes() (map[string]string, error) {
if m.Err != nil {
return nil, m.Err
}
mimes := make(map[string]string, len(m.Data))
for h, a := range m.Data {
mimes[h] = a.Mime
}
return mimes, nil
}
func (m *MockArtworkRepo) DeleteOrphans(createdBefore time.Time, hashes []string) error {
if m.Err != nil {
return m.Err
}
// Mirror the SQL re-check: only unreferenced rows older than the cutoff are deleted.
for _, h := range hashes {
if m.referenced(h) {
continue
}
if a, ok := m.Data[h]; ok && !a.CreatedAt.Before(createdBefore) {
continue
}
delete(m.Data, h)
}
return nil
}
func (m *MockArtworkRepo) referenced(hash string) bool {
for _, ia := range m.ItemData {
if ia.Hash == hash {
return true
}
}
return false
}
func (m *MockArtworkRepo) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) {
if m.Err != nil {
return nil, m.Err
}
if ia, ok := m.ItemData[iaKey(kind, id, imageType)]; ok {
return &ia, nil
}
return nil, model.ErrNotFound
}
func (m *MockArtworkRepo) PutItemArtwork(ia *model.ItemArtwork) error {
if m.Err != nil {
return m.Err
}
if ia.ImageType == "" {
ia.ImageType = model.ImageTypePrimary
}
ia.UpdatedAt = time.Now()
if ia.AttemptedAt.IsZero() {
ia.AttemptedAt = ia.UpdatedAt
}
m.ItemData[iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)] = *ia
return nil
}
func (m *MockArtworkRepo) DeleteForItem(kind, id string) error {
if m.Err != nil {
return m.Err
}
for k, ia := range m.ItemData {
if ia.ItemKind == kind && ia.ItemID == id {
delete(m.ItemData, k)
}
}
return nil
}
func (m *MockArtworkRepo) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) {
if m.Err != nil {
return nil, m.Err
}
res := map[string]model.ItemArtworkInfo{}
for _, id := range ids {
if ia, ok := m.ItemData[iaKey(kind, id, model.ImageTypePrimary)]; ok {
info := model.ItemArtworkInfo{ItemID: id, Hash: ia.Hash}
if a, ok := m.Data[ia.Hash]; ok {
info.BlurHash = a.BlurHash
}
res[id] = info
}
}
return res, nil
}
+28
View File
@@ -28,6 +28,8 @@ type MockDataStore struct {
MockedScrobble model.ScrobbleRepository
MockedRadio model.RadioRepository
MockedPlugin model.PluginRepository
MockedArtwork model.ArtworkRepository
MockedArtworkQueue model.ArtworkQueueRepository
scrobbleBufferMu sync.Mutex
repoMu sync.Mutex
@@ -247,6 +249,32 @@ func (db *MockDataStore) Plugin(ctx context.Context) model.PluginRepository {
return db.MockedPlugin
}
func (db *MockDataStore) Artwork(ctx context.Context) model.ArtworkRepository {
if db.MockedArtwork != nil {
return db.MockedArtwork
}
if db.RealDS != nil {
return db.RealDS.Artwork(ctx)
}
db.MockedArtwork = CreateMockArtworkRepo()
return db.MockedArtwork
}
func (db *MockDataStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository {
if db.MockedArtworkQueue != nil {
return db.MockedArtworkQueue
}
if db.RealDS != nil {
return db.RealDS.ArtworkQueue(ctx)
}
q := CreateMockArtworkQueueRepo()
if aw, ok := db.Artwork(ctx).(*MockArtworkRepo); ok {
q.ItemArtworkSource = aw
}
db.MockedArtworkQueue = q
return db.MockedArtworkQueue
}
func (db *MockDataStore) WithTx(block func(tx model.DataStore) error, label ...string) error {
return block(db)
}