mirror of
https://github.com/navidrome/navidrome.git
synced 2026-09-08 19:52:49 -04:00
Compare commits
21
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1f818e7633 | ||
|
|
c04c8ee02a | ||
|
|
ebbe533c6a | ||
|
|
0147cc59b1 | ||
|
|
034cd17498 | ||
|
|
8147f7c40b | ||
|
|
bf614e66ad | ||
|
|
623b7d6a6c | ||
|
|
6f7f9c6463 | ||
|
|
8fd7ef19f3 | ||
|
|
1041e45ca7 | ||
|
|
4f835437a9 | ||
|
|
b16ef725c9 | ||
|
|
3e7685adc2 | ||
|
|
db16b3de9a | ||
|
|
b72597821a | ||
|
|
fcff9c63e7 | ||
|
|
14dd57052e | ||
|
|
f926539c04 | ||
|
|
6cce65f759 | ||
|
|
7aacb01f4f |
No files matched your search
@@ -0,0 +1,164 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
func HashImage(r io.Reader) (string, error) {
|
||||
d := xxh3.New()
|
||||
if _, err := io.Copy(d, r); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%016x", d.Sum64()), nil
|
||||
}
|
||||
|
||||
// ImageStore is the content-addressed store for artwork images that have no
|
||||
// library file backing them (external downloads, embedded extractions, generated).
|
||||
type ImageStore struct {
|
||||
root string
|
||||
}
|
||||
|
||||
func NewImageStore(rootDir string) *ImageStore {
|
||||
return &ImageStore{root: rootDir}
|
||||
}
|
||||
|
||||
// extForMime is deliberately NOT mime.ExtensionsByType: extensions are baked into
|
||||
// content-addressed paths and re-derived on Open, so they must be stable across OSes.
|
||||
func extForMime(m string) string {
|
||||
switch m {
|
||||
case "image/jpeg":
|
||||
return ".jpg"
|
||||
case "image/png":
|
||||
return ".png"
|
||||
case "image/gif":
|
||||
return ".gif"
|
||||
case "image/webp":
|
||||
return ".webp"
|
||||
}
|
||||
return ".img"
|
||||
}
|
||||
|
||||
// validHash rejects anything but 16 lowercase hex chars: known-absent states carry "",
|
||||
// and malformed persisted hashes must never reach path sharding (slice panics, separators).
|
||||
func validHash(hash string) bool {
|
||||
if len(hash) != 16 {
|
||||
return false
|
||||
}
|
||||
for _, c := range []byte(hash) {
|
||||
if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *ImageStore) path(hash, mimeType string) string {
|
||||
return filepath.Join(s.root, hash[0:2], hash[2:4], hash+extForMime(mimeType))
|
||||
}
|
||||
|
||||
func (s *ImageStore) Write(hash, mimeType string, r io.Reader) error {
|
||||
if !validHash(hash) {
|
||||
return fmt.Errorf("imagestore: invalid hash %q", hash)
|
||||
}
|
||||
dst := s.path(hash, mimeType)
|
||||
if _, err := os.Stat(dst); err == nil {
|
||||
// A touched mtime marks the file live so a concurrent prune spares it.
|
||||
now := time.Now()
|
||||
if err := os.Chtimes(dst, now, now); err == nil {
|
||||
return nil
|
||||
}
|
||||
// touch failed (file likely pruned concurrently) — fall through and write it
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp, err := os.CreateTemp(filepath.Dir(dst), "."+hash+".tmp*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
if _, err := io.Copy(tmp, r); err != nil {
|
||||
tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp.Name(), dst)
|
||||
}
|
||||
|
||||
func (s *ImageStore) Open(hash, mimeType string) (io.ReadCloser, error) {
|
||||
if !validHash(hash) {
|
||||
return nil, fmt.Errorf("imagestore: invalid hash %q", hash)
|
||||
}
|
||||
return os.Open(s.path(hash, mimeType))
|
||||
}
|
||||
|
||||
// Remove deletes the store file unless it is newer than olderThan, in which case
|
||||
// an overlapping acquisition may have just touched it and be about to commit its row.
|
||||
func (s *ImageStore) Remove(hash, mimeType string, olderThan time.Time) error {
|
||||
if !validHash(hash) {
|
||||
return fmt.Errorf("imagestore: invalid hash %q", hash)
|
||||
}
|
||||
path := s.path(hash, mimeType)
|
||||
info, err := os.Stat(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.ModTime().After(olderThan) {
|
||||
return nil
|
||||
}
|
||||
err = os.Remove(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Sweep removes store files not accepted by keep. Files modified after cutoff
|
||||
// (including temp files) are always kept: their acquisition row may not be committed yet.
|
||||
func (s *ImageStore) Sweep(cutoff time.Time, keep func(hash, ext string) bool) (int, error) {
|
||||
removed := 0
|
||||
err := filepath.WalkDir(s.root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return err
|
||||
}
|
||||
info, err := d.Info()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.ModTime().After(cutoff) {
|
||||
return nil
|
||||
}
|
||||
name := d.Name()
|
||||
remove := strings.HasPrefix(name, ".") // abandoned temp file past the grace window
|
||||
if !remove {
|
||||
ext := filepath.Ext(name)
|
||||
remove = !keep(strings.TrimSuffix(name, ext), ext)
|
||||
}
|
||||
if remove {
|
||||
// #nosec G122 -- path comes from WalkDir over our own store root, no attacker-controlled symlinks
|
||||
if err := os.Remove(path); err != nil {
|
||||
return err
|
||||
}
|
||||
removed++
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return removed, nil
|
||||
}
|
||||
return removed, err
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("ImageStore", func() {
|
||||
var store *ImageStore
|
||||
var root string
|
||||
|
||||
BeforeEach(func() {
|
||||
root = GinkgoT().TempDir()
|
||||
store = NewImageStore(root)
|
||||
})
|
||||
|
||||
It("hashes deterministically", func() {
|
||||
h1, err := HashImage(bytes.NewReader([]byte("some image bytes")))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
h2, _ := HashImage(bytes.NewReader([]byte("some image bytes")))
|
||||
Expect(h1).To(Equal(h2))
|
||||
Expect(h1).To(HaveLen(16))
|
||||
h3, _ := HashImage(bytes.NewReader([]byte("other bytes")))
|
||||
Expect(h3).ToNot(Equal(h1))
|
||||
})
|
||||
|
||||
It("writes sharded and reads back", func() {
|
||||
data := []byte("jpeg-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
|
||||
Expect(filepath.Join(root, h[0:2], h[2:4], h+".jpg")).To(BeAnExistingFile())
|
||||
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
defer rc.Close()
|
||||
got, _ := io.ReadAll(rc)
|
||||
Expect(got).To(Equal(data))
|
||||
})
|
||||
|
||||
It("is idempotent on duplicate writes", func() {
|
||||
data := []byte("dup")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||
})
|
||||
|
||||
It("refreshes the mtime on a duplicate write", func() {
|
||||
data := []byte("touch-me")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed())
|
||||
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||
|
||||
info, err := os.Stat(store.path(h, "image/png"))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(info.ModTime()).To(BeTemporally(">", time.Now().Add(-time.Minute)))
|
||||
})
|
||||
|
||||
It("rewrites the bytes when the existing file vanished before the liveness touch", func() {
|
||||
data := []byte("vanishing")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
for range 10 {
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||
Expect(os.Remove(store.path(h, "image/png"))).To(Succeed())
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||
|
||||
rc, err := store.Open(h, "image/png")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
got, _ := io.ReadAll(rc)
|
||||
rc.Close()
|
||||
Expect(got).To(Equal(data))
|
||||
}
|
||||
})
|
||||
|
||||
It("returns fs.ErrNotExist for missing images", func() {
|
||||
_, err := store.Open("beefbeefbeefbeef", "image/jpeg")
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
})
|
||||
|
||||
It("removes without error when already gone", func() {
|
||||
Expect(store.Remove("beefbeefbeefbeef", "image/jpeg", time.Now())).To(Succeed())
|
||||
})
|
||||
|
||||
It("rejects invalid hashes instead of panicking", func() {
|
||||
for _, h := range []string{"", "ab", "BEEFBEEFBEEFBEEF", "../../../../etcpw", "beefbeefbeefbee/"} {
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader([]byte("x")))).To(MatchError(ContainSubstring("invalid hash")))
|
||||
_, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).To(MatchError(ContainSubstring("invalid hash")))
|
||||
Expect(store.Remove(h, "image/jpeg", time.Now())).To(MatchError(ContainSubstring("invalid hash")))
|
||||
}
|
||||
})
|
||||
|
||||
It("spares a file newer than the cutoff, removes an aged one", func() {
|
||||
fresh := []byte("fresh")
|
||||
hf, _ := HashImage(bytes.NewReader(fresh))
|
||||
Expect(store.Write(hf, "image/jpeg", bytes.NewReader(fresh))).To(Succeed())
|
||||
|
||||
aged := []byte("aged")
|
||||
ha, _ := HashImage(bytes.NewReader(aged))
|
||||
Expect(store.Write(ha, "image/jpeg", bytes.NewReader(aged))).To(Succeed())
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(store.path(ha, "image/jpeg"), old, old)).To(Succeed())
|
||||
|
||||
cutoff := time.Now().Add(-time.Hour)
|
||||
Expect(store.Remove(hf, "image/jpeg", cutoff)).To(Succeed())
|
||||
Expect(store.Remove(ha, "image/jpeg", cutoff)).To(Succeed())
|
||||
|
||||
rc, err := store.Open(hf, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
_, err = store.Open(ha, "image/jpeg")
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
})
|
||||
|
||||
It("sweeps unknown files, keeps known ones", func() {
|
||||
d1 := []byte("keep-me")
|
||||
h1, _ := HashImage(bytes.NewReader(d1))
|
||||
Expect(store.Write(h1, "image/jpeg", bytes.NewReader(d1))).To(Succeed())
|
||||
d2 := []byte("orphan")
|
||||
h2, _ := HashImage(bytes.NewReader(d2))
|
||||
Expect(store.Write(h2, "image/jpeg", bytes.NewReader(d2))).To(Succeed())
|
||||
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(store.path(h2, "image/jpeg"), old, old)).To(Succeed())
|
||||
|
||||
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(h, _ string) bool { return h == h1 })
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(removed).To(Equal(1))
|
||||
_, err = store.Open(h2, "image/jpeg")
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
rc, err := store.Open(h1, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("sweeps a stale mime variant of a known hash, keeps the current one", func() {
|
||||
data := []byte("same-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed())
|
||||
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
|
||||
|
||||
// The recorded mime is image/jpeg, so the .png variant is obsolete.
|
||||
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(hash, ext string) bool {
|
||||
return hash == h && ext == ".jpg"
|
||||
})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(removed).To(Equal(1))
|
||||
_, err = store.Open(h, "image/png")
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("keeps young unknown files inside the grace window", func() {
|
||||
d := []byte("fresh-orphan")
|
||||
h, _ := HashImage(bytes.NewReader(d))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(d))).To(Succeed())
|
||||
|
||||
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string, string) bool { return false })
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(removed).To(Equal(0))
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("removes abandoned temp files past the grace window, keeps fresh ones", func() {
|
||||
oldTmp := filepath.Join(root, ".old.tmp")
|
||||
Expect(os.WriteFile(oldTmp, []byte("x"), 0600)).To(Succeed())
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(oldTmp, old, old)).To(Succeed())
|
||||
|
||||
freshTmp := filepath.Join(root, ".fresh.tmp")
|
||||
Expect(os.WriteFile(freshTmp, []byte("y"), 0600)).To(Succeed())
|
||||
|
||||
removed, err := store.Sweep(time.Now().Add(-time.Hour), func(string, string) bool { return true })
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(removed).To(Equal(1))
|
||||
Expect(oldTmp).ToNot(BeAnExistingFile())
|
||||
Expect(freshTmp).To(BeAnExistingFile())
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,68 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/log"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
)
|
||||
|
||||
// pruneMinAge guards the window between artwork insert and item_artwork upsert.
|
||||
const pruneMinAge = time.Hour
|
||||
|
||||
func Prune(ctx context.Context, ds model.DataStore, store *ImageStore) error {
|
||||
repo := ds.Artwork(ctx)
|
||||
// One grace cutoff for both the DB orphan check and the file sweep: files younger
|
||||
// than the window may belong to acquisitions whose rows aren't committed yet.
|
||||
cutoff := time.Now().Add(-pruneMinAge)
|
||||
candidates, err := repo.GetOrphanHashes(cutoff)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(candidates) > 0 {
|
||||
arts, err := repo.GetImages(candidates)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := repo.DeleteOrphans(cutoff, candidates); err != nil {
|
||||
return err
|
||||
}
|
||||
// DeleteOrphans may spare candidates reacquired since the snapshot; only remove files
|
||||
// for rows actually gone (absent from the post-delete re-read).
|
||||
survivors, err := repo.GetImages(candidates)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
removed := 0
|
||||
for _, h := range candidates {
|
||||
if _, ok := survivors[h]; ok {
|
||||
continue
|
||||
}
|
||||
// A spared fresh file is at worst a stray a later sweep reclaims; full
|
||||
// worker/prune mutual exclusion is Phase 2's concern.
|
||||
if err := store.Remove(h, arts[h].Mime, cutoff); err != nil {
|
||||
log.Warn(ctx, "Prune: could not remove artwork file", "hash", h, err)
|
||||
}
|
||||
removed++
|
||||
}
|
||||
log.Info(ctx, "Prune: removed orphan artwork", "count", removed)
|
||||
}
|
||||
|
||||
mimes, err := repo.GetAllMimes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
removed, err := store.Sweep(cutoff, func(hash, ext string) bool {
|
||||
// A known hash under a stale extension is a superseded mime variant — reclaim it.
|
||||
m, ok := mimes[hash]
|
||||
return ok && ext == extForMime(m)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if removed > 0 {
|
||||
log.Info(ctx, "Prune: swept stray artwork files", "count", removed)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package artwork
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/navidrome/navidrome/tests"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
type flakyGetArtworkRepo struct {
|
||||
*tests.MockArtworkRepo
|
||||
}
|
||||
|
||||
func (f *flakyGetArtworkRepo) GetAllMimes() (map[string]string, error) {
|
||||
return nil, errors.New("db locked")
|
||||
}
|
||||
|
||||
var _ = Describe("Prune", func() {
|
||||
var ds *tests.MockDataStore
|
||||
var store *ImageStore
|
||||
var awRepo *tests.MockArtworkRepo
|
||||
|
||||
BeforeEach(func() {
|
||||
ds = &tests.MockDataStore{}
|
||||
awRepo = ds.Artwork(context.Background()).(*tests.MockArtworkRepo)
|
||||
store = NewImageStore(GinkgoT().TempDir())
|
||||
})
|
||||
|
||||
// PutImage refreshes created_at like the SQL repo, so fixtures are aged directly.
|
||||
ageArtwork := func(h string, t time.Time) {
|
||||
a := awRepo.Data[h]
|
||||
a.CreatedAt = t
|
||||
awRepo.Data[h] = a
|
||||
}
|
||||
|
||||
It("deletes orphan rows and their store files, keeps referenced ones", func() {
|
||||
data := []byte("orphan-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
|
||||
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||
ageArtwork(h, old)
|
||||
awRepo.OrphanHashes = []string{h}
|
||||
|
||||
kept := []byte("kept-bytes")
|
||||
hk, _ := HashImage(bytes.NewReader(kept))
|
||||
Expect(store.Write(hk, "image/jpeg", bytes.NewReader(kept))).To(Succeed())
|
||||
Expect(awRepo.PutImage(&model.Artwork{Hash: hk, Mime: "image/jpeg"})).To(Succeed())
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||
|
||||
_, err := awRepo.GetImage(h)
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
_, err = store.Open(h, "image/jpeg")
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
rc, err := store.Open(hk, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("spares a candidate reacquired between snapshot and delete", func() {
|
||||
data := []byte("reacquired-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||
ageArtwork(h, time.Now().Add(-2*time.Hour))
|
||||
awRepo.OrphanHashes = []string{h}
|
||||
// Reacquisition: an item now references the hash the snapshot flagged as orphan.
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1",
|
||||
ImageType: model.ImageTypePrimary, Hash: h, Source: "folder"})).To(Succeed())
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||
|
||||
_, err := awRepo.GetImage(h)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("spares a candidate whose row was freshly recreated (created_at inside the grace window)", func() {
|
||||
data := []byte("fresh-reacquired-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
// Reacquisition refreshed created_at after the snapshot; still unreferenced.
|
||||
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||
awRepo.OrphanHashes = []string{h}
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||
|
||||
_, err := awRepo.GetImage(h)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("spares an orphan file freshly touched by an overlapping acquisition", func() {
|
||||
data := []byte("racing-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||
ageArtwork(h, time.Now().Add(-2*time.Hour))
|
||||
awRepo.OrphanHashes = []string{h}
|
||||
// The row is legitimately orphaned, but a concurrent acquisition just touched the
|
||||
// file's mtime (duplicate Write) and is about to commit a row referencing it.
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("sweeps store files that have no artwork row", func() {
|
||||
stray := []byte("no-row-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(stray))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(stray))).To(Succeed())
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||
|
||||
_, err := store.Open(h, "image/jpeg")
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
})
|
||||
|
||||
It("sweeps an obsolete mime variant of a reacquired hash", func() {
|
||||
data := []byte("variant-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/png", bytes.NewReader(data))).To(Succeed())
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
old := time.Now().Add(-2 * time.Hour)
|
||||
Expect(os.Chtimes(store.path(h, "image/png"), old, old)).To(Succeed())
|
||||
Expect(os.Chtimes(store.path(h, "image/jpeg"), old, old)).To(Succeed())
|
||||
// The row records the current mime; the .png file is a superseded variant.
|
||||
Expect(awRepo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).To(Succeed())
|
||||
|
||||
_, err := store.Open(h, "image/png")
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
|
||||
It("never sweeps files on a transient DB error", func() {
|
||||
ds.MockedArtwork = &flakyGetArtworkRepo{MockArtworkRepo: tests.CreateMockArtworkRepo()}
|
||||
|
||||
data := []byte("live-bytes")
|
||||
h, _ := HashImage(bytes.NewReader(data))
|
||||
Expect(store.Write(h, "image/jpeg", bytes.NewReader(data))).To(Succeed())
|
||||
|
||||
Expect(Prune(context.Background(), ds, store)).ToNot(Succeed())
|
||||
|
||||
rc, err := store.Open(h, "image/jpeg")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
rc.Close()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,42 @@
|
||||
-- +goose Up
|
||||
CREATE TABLE artwork (
|
||||
hash TEXT PRIMARY KEY,
|
||||
mime TEXT NOT NULL,
|
||||
width INTEGER NOT NULL DEFAULT 0,
|
||||
height INTEGER NOT NULL DEFAULT 0,
|
||||
size_bytes INTEGER NOT NULL DEFAULT 0,
|
||||
blur_hash TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE item_artwork (
|
||||
item_kind TEXT NOT NULL,
|
||||
item_id TEXT NOT NULL,
|
||||
image_type TEXT NOT NULL DEFAULT 'primary',
|
||||
hash TEXT NOT NULL DEFAULT '',
|
||||
source TEXT NOT NULL DEFAULT '',
|
||||
source_path TEXT NOT NULL DEFAULT '',
|
||||
ref_mtime INTEGER NOT NULL DEFAULT 0,
|
||||
attempted_at TIMESTAMP,
|
||||
updated_at TIMESTAMP,
|
||||
PRIMARY KEY (item_kind, item_id, image_type)
|
||||
) WITHOUT ROWID;
|
||||
CREATE INDEX ix_item_artwork_hash ON item_artwork(hash);
|
||||
|
||||
CREATE TABLE artwork_queue (
|
||||
item_kind TEXT NOT NULL,
|
||||
item_id TEXT NOT NULL,
|
||||
image_type TEXT NOT NULL DEFAULT 'primary',
|
||||
priority INTEGER NOT NULL DEFAULT 0,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
retry_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
enqueued_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (item_kind, item_id, image_type)
|
||||
) WITHOUT ROWID;
|
||||
-- Ordered to match DequeueBatch (priority DESC, enqueued_at) so drains stop after n rows; retry_at makes it covering.
|
||||
CREATE INDEX ix_artwork_queue_drain ON artwork_queue(priority DESC, enqueued_at, retry_at);
|
||||
|
||||
-- +goose Down
|
||||
DROP TABLE artwork_queue;
|
||||
DROP TABLE item_artwork;
|
||||
DROP TABLE artwork;
|
||||
@@ -57,6 +57,7 @@ require (
|
||||
github.com/tetratelabs/wazero v1.12.0
|
||||
github.com/unrolled/secure v1.17.0
|
||||
github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342
|
||||
github.com/zeebo/xxh3 v1.1.0
|
||||
go.senan.xyz/taglib v0.11.1
|
||||
go.uber.org/goleak v1.3.0
|
||||
golang.org/x/image v0.44.0
|
||||
@@ -128,7 +129,6 @@ require (
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/tetratelabs/wabin v0.0.0-20230304001439-f6f874872834 // indirect
|
||||
github.com/valyala/fastjson v1.6.10 // indirect
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.opentelemetry.io/proto/otlp v1.10.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
// Artwork is one unique image, identified by the XXH3-64 hash of its bytes.
|
||||
type Artwork struct {
|
||||
Hash string `structs:"hash"`
|
||||
Mime string `structs:"mime"`
|
||||
Width int `structs:"width"`
|
||||
Height int `structs:"height"`
|
||||
SizeBytes int64 `structs:"size_bytes"`
|
||||
BlurHash string `structs:"blur_hash"`
|
||||
CreatedAt time.Time `structs:"created_at"`
|
||||
}
|
||||
|
||||
const ImageTypePrimary = "primary"
|
||||
|
||||
// ItemArtwork is an entity's resolved artwork state. Hash=="" means known absent.
|
||||
type ItemArtwork struct {
|
||||
ItemKind string `structs:"item_kind"`
|
||||
ItemID string `structs:"item_id"`
|
||||
ImageType string `structs:"image_type"`
|
||||
Hash string `structs:"hash"`
|
||||
Source string `structs:"source"`
|
||||
// SourcePath is the backing file (folder/upload: the image; embedded: the audio file); "" otherwise.
|
||||
SourcePath string `structs:"source_path"`
|
||||
// RefMtime is SourcePath's mtime at resolution; 0 when there is no SourcePath.
|
||||
RefMtime int64 `structs:"ref_mtime"`
|
||||
// attempted_at/updated_at are nullable in the schema but always set by PutItemArtwork;
|
||||
// raw inserts must set them too, since these non-pointer time.Time fields fail to scan NULL.
|
||||
AttemptedAt time.Time `structs:"attempted_at"`
|
||||
UpdatedAt time.Time `structs:"updated_at"`
|
||||
}
|
||||
|
||||
// ItemArtworkInfo is the list-hydration projection (item_artwork joined with artwork).
|
||||
type ItemArtworkInfo struct {
|
||||
ItemID string
|
||||
Hash string
|
||||
BlurHash string
|
||||
}
|
||||
|
||||
// Absent reports a known-absent artwork state (resolved, no image).
|
||||
func (i ItemArtworkInfo) Absent() bool { return i.Hash == "" }
|
||||
|
||||
type ArtworkQueueItem struct {
|
||||
ItemKind string `structs:"item_kind"`
|
||||
ItemID string `structs:"item_id"`
|
||||
ImageType string `structs:"image_type"`
|
||||
Priority int `structs:"priority"`
|
||||
Attempts int `structs:"attempts"`
|
||||
RetryAt time.Time `structs:"retry_at"`
|
||||
EnqueuedAt time.Time `structs:"enqueued_at"`
|
||||
}
|
||||
|
||||
// Queue priorities: higher drains first.
|
||||
const (
|
||||
ArtworkPriorityRecheck = 0
|
||||
ArtworkPriorityBackfill = 10
|
||||
ArtworkPriorityScan = 50
|
||||
ArtworkPriorityBump = 100
|
||||
)
|
||||
|
||||
type ArtworkRepository interface {
|
||||
// Image identity (artwork table)
|
||||
GetImage(hash string) (*Artwork, error)
|
||||
PutImage(a *Artwork) error
|
||||
GetImages(hashes []string) (map[string]Artwork, error)
|
||||
// GetOrphanHashes returns hashes referenced by no item_artwork row and older than cutoff.
|
||||
GetOrphanHashes(createdBefore time.Time) ([]string, error)
|
||||
// DeleteOrphans deletes the given hashes only if still unreferenced and older than cutoff (atomic re-check).
|
||||
DeleteOrphans(createdBefore time.Time, hashes []string) error
|
||||
// Per-item state (item_artwork table)
|
||||
GetItemArtwork(kind, id, imageType string) (*ItemArtwork, error)
|
||||
PutItemArtwork(ia *ItemArtwork) error
|
||||
DeleteForItem(kind, id string) error
|
||||
// GetInfoForItems hydrates a page: one batched query, item_artwork joined to artwork.
|
||||
GetInfoForItems(kind string, ids []string) (map[string]ItemArtworkInfo, error)
|
||||
// GetAllMimes returns hash -> current mime for every stored artwork, for sweep retention checks.
|
||||
GetAllMimes() (map[string]string, error)
|
||||
}
|
||||
|
||||
type ArtworkQueueRepository interface {
|
||||
// Enqueue upserts; an existing row keeps the higher of the two priorities.
|
||||
Enqueue(items ...ArtworkQueueItem) error
|
||||
// DequeueBatch returns up to n items with retry_at <= now, priority desc, enqueued_at asc.
|
||||
DequeueBatch(n int) ([]ArtworkQueueItem, error)
|
||||
// MarkFailed increments attempts and pushes retry_at into the future.
|
||||
MarkFailed(kind, id, imageType string, retryAt time.Time) error
|
||||
Delete(kind, id, imageType string) error
|
||||
Count() (int64, error)
|
||||
// EnqueueStaleAbsent inserts queue rows (priority Recheck) for absent states older than cutoff.
|
||||
EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error)
|
||||
}
|
||||
@@ -40,6 +40,8 @@ type DataStore interface {
|
||||
ScrobbleBuffer(ctx context.Context) ScrobbleBufferRepository
|
||||
Scrobble(ctx context.Context) ScrobbleRepository
|
||||
Plugin(ctx context.Context) PluginRepository
|
||||
Artwork(ctx context.Context) ArtworkRepository
|
||||
ArtworkQueue(ctx context.Context) ArtworkQueueRepository
|
||||
|
||||
Resource(ctx context.Context, model any) ResourceRepository
|
||||
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
. "github.com/Masterminds/squirrel"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/pocketbase/dbx"
|
||||
)
|
||||
|
||||
// enqueueChunkSize keeps each multi-row insert under SQLite's bind-variable limit (7 cols -> 700 vars).
|
||||
const enqueueChunkSize = 100
|
||||
|
||||
type artworkQueueRepository struct {
|
||||
sqlRepository
|
||||
}
|
||||
|
||||
func NewArtworkQueueRepository(ctx context.Context, db dbx.Builder) model.ArtworkQueueRepository {
|
||||
r := &artworkQueueRepository{}
|
||||
r.ctx = ctx
|
||||
r.db = db
|
||||
r.tableName = "artwork_queue"
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *artworkQueueRepository) Enqueue(items ...model.ArtworkQueueItem) error {
|
||||
now := time.Now()
|
||||
for chunk := range slices.Chunk(items, enqueueChunkSize) {
|
||||
ins := Insert(r.tableName).Columns("item_kind", "item_id", "image_type", "priority", "attempts", "retry_at", "enqueued_at")
|
||||
for _, it := range chunk {
|
||||
if it.ImageType == "" {
|
||||
it.ImageType = model.ImageTypePrimary
|
||||
}
|
||||
ins = ins.Values(it.ItemKind, it.ItemID, it.ImageType, it.Priority, 0, now, now)
|
||||
}
|
||||
ins = ins.Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET
|
||||
priority = MAX(priority, excluded.priority), retry_at = excluded.retry_at`)
|
||||
if _, err := r.executeSQL(ins); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *artworkQueueRepository) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) {
|
||||
sel := Select("*").From(r.tableName).
|
||||
Where(LtOrEq{"retry_at": time.Now()}).
|
||||
OrderBy("priority DESC", "enqueued_at ASC").
|
||||
Limit(uint64(n))
|
||||
var res []model.ArtworkQueueItem
|
||||
err := r.queryAll(sel, &res)
|
||||
return res, err
|
||||
}
|
||||
|
||||
func (r *artworkQueueRepository) MarkFailed(kind, id, imageType string, retryAt time.Time) error {
|
||||
upd := Update(r.tableName).
|
||||
Set("attempts", Expr("attempts + 1")).
|
||||
Set("retry_at", retryAt).
|
||||
Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType})
|
||||
c, err := r.executeSQL(upd)
|
||||
if err == nil && c == 0 {
|
||||
return model.ErrNotFound
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *artworkQueueRepository) Delete(kind, id, imageType string) error {
|
||||
return r.delete(Eq{"item_kind": kind, "item_id": id, "image_type": imageType})
|
||||
}
|
||||
|
||||
func (r *artworkQueueRepository) Count() (int64, error) {
|
||||
var res struct{ Count int64 }
|
||||
err := r.queryOne(Select("count(*) as count").From(r.tableName), &res)
|
||||
return res.Count, err
|
||||
}
|
||||
|
||||
func (r *artworkQueueRepository) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) {
|
||||
now := time.Now()
|
||||
// DO NOTHING is deliberate: rechecks must not bump priority/retry_at of already-queued items.
|
||||
ins := Expr(`INSERT INTO `+r.tableName+` (item_kind, item_id, image_type, priority, attempts, retry_at, enqueued_at)
|
||||
SELECT item_kind, item_id, image_type, ?, 0, ?, ?
|
||||
FROM `+itemArtworkTable+` WHERE item_kind = ? AND hash = '' AND attempted_at < ?
|
||||
ON CONFLICT (item_kind, item_id, image_type) DO NOTHING`,
|
||||
model.ArtworkPriorityRecheck, now, now, kind, attemptedBefore)
|
||||
return r.executeSQL(ins)
|
||||
}
|
||||
|
||||
var _ model.ArtworkQueueRepository = (*artworkQueueRepository)(nil)
|
||||
@@ -0,0 +1,83 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/model"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("ArtworkQueueRepository", func() {
|
||||
var repo model.ArtworkQueueRepository
|
||||
|
||||
item := func(kind, id string, prio int) model.ArtworkQueueItem {
|
||||
return model.ArtworkQueueItem{ItemKind: kind, ItemID: id,
|
||||
ImageType: model.ImageTypePrimary, Priority: prio}
|
||||
}
|
||||
|
||||
BeforeEach(func() {
|
||||
clearArtworkTables()
|
||||
repo = NewArtworkQueueRepository(context.Background(), GetDBXBuilder())
|
||||
})
|
||||
|
||||
It("enqueues and dequeues by priority then FIFO", func() {
|
||||
Expect(repo.Enqueue(item("al", "low", model.ArtworkPriorityBackfill))).To(Succeed())
|
||||
Expect(repo.Enqueue(item("ar", "high", model.ArtworkPriorityBump))).To(Succeed())
|
||||
|
||||
got, err := repo.DequeueBatch(10)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got).To(HaveLen(2))
|
||||
Expect(got[0].ItemID).To(Equal("high"))
|
||||
})
|
||||
|
||||
It("keeps the higher priority on duplicate enqueue", func() {
|
||||
Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBump))).To(Succeed())
|
||||
Expect(repo.Enqueue(item("al", "a1", model.ArtworkPriorityBackfill))).To(Succeed())
|
||||
got, _ := repo.DequeueBatch(10)
|
||||
Expect(got).To(HaveLen(1))
|
||||
Expect(got[0].Priority).To(Equal(model.ArtworkPriorityBump))
|
||||
})
|
||||
|
||||
It("hides failed items until retry_at", func() {
|
||||
Expect(repo.Enqueue(item("al", "f1", model.ArtworkPriorityScan))).To(Succeed())
|
||||
Expect(repo.MarkFailed("al", "f1", model.ImageTypePrimary, time.Now().Add(time.Hour))).To(Succeed())
|
||||
|
||||
got, err := repo.DequeueBatch(10)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got).To(BeEmpty())
|
||||
|
||||
Expect(repo.MarkFailed("al", "f1", model.ImageTypePrimary, time.Now().Add(-time.Minute))).To(Succeed())
|
||||
got, _ = repo.DequeueBatch(10)
|
||||
Expect(got).To(HaveLen(1))
|
||||
Expect(got[0].Attempts).To(Equal(2))
|
||||
})
|
||||
|
||||
It("deletes on completion and counts", func() {
|
||||
Expect(repo.Enqueue(item("al", "c1", 0))).To(Succeed())
|
||||
n, _ := repo.Count()
|
||||
Expect(n).To(Equal(int64(1)))
|
||||
Expect(repo.Delete("al", "c1", model.ImageTypePrimary)).To(Succeed())
|
||||
n, _ = repo.Count()
|
||||
Expect(n).To(BeZero())
|
||||
})
|
||||
|
||||
It("enqueues stale absent states for recheck", func() {
|
||||
awRepo := NewArtworkRepository(context.Background(), GetDBXBuilder())
|
||||
old := time.Now().Add(-48 * time.Hour)
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "stale1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: old})).To(Succeed())
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "fresh1", ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed())
|
||||
Expect(awRepo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "found1", ImageType: model.ImageTypePrimary, Hash: "hX", AttemptedAt: old})).To(Succeed())
|
||||
|
||||
n, err := repo.EnqueueStaleAbsent("ar", time.Now().Add(-24*time.Hour))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(n).To(Equal(int64(1)))
|
||||
|
||||
items, err := repo.DequeueBatch(10)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(items).To(HaveLen(1))
|
||||
Expect(items[0].ItemID).To(Equal("stale1"))
|
||||
Expect(items[0].Priority).To(Equal(model.ArtworkPriorityRecheck))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,179 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
. "github.com/Masterminds/squirrel"
|
||||
"github.com/navidrome/navidrome/model"
|
||||
"github.com/pocketbase/dbx"
|
||||
)
|
||||
|
||||
const (
|
||||
itemArtworkTable = "item_artwork"
|
||||
artworkBatchSize = 200
|
||||
)
|
||||
|
||||
type itemArtworkSQL struct {
|
||||
sqlRepository
|
||||
}
|
||||
|
||||
type artworkRepository struct {
|
||||
sqlRepository
|
||||
items itemArtworkSQL
|
||||
}
|
||||
|
||||
func NewArtworkRepository(ctx context.Context, db dbx.Builder) model.ArtworkRepository {
|
||||
r := &artworkRepository{}
|
||||
r.ctx = ctx
|
||||
r.db = db
|
||||
r.tableName = "artwork"
|
||||
r.items.ctx = ctx
|
||||
r.items.db = db
|
||||
r.items.tableName = itemArtworkTable
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *artworkRepository) GetImage(hash string) (*model.Artwork, error) {
|
||||
sel := Select("*").From(r.tableName).Where(Eq{"hash": hash})
|
||||
var res model.Artwork
|
||||
if err := r.queryOne(sel, &res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
func (r *artworkRepository) PutImage(a *model.Artwork) error {
|
||||
// created_at is the last-acquisition-write time the prune grace window keys on.
|
||||
a.CreatedAt = time.Now()
|
||||
values, err := toSQLArgs(*a)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// created_at=excluded.created_at: reacquiring an orphan must reset the prune grace window.
|
||||
ins := Insert(r.tableName).SetMap(values).Suffix(`ON CONFLICT (hash) DO UPDATE SET mime=excluded.mime, width=excluded.width,
|
||||
height=excluded.height, size_bytes=excluded.size_bytes, blur_hash=excluded.blur_hash, created_at=excluded.created_at`)
|
||||
_, err = r.executeSQL(ins)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *artworkRepository) GetImages(hashes []string) (map[string]model.Artwork, error) {
|
||||
res := map[string]model.Artwork{}
|
||||
for chunk := range slices.Chunk(hashes, artworkBatchSize) {
|
||||
sel := Select("*").From(r.tableName).Where(Eq{"hash": chunk})
|
||||
var all []model.Artwork
|
||||
if err := r.queryAll(sel, &all); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, a := range all {
|
||||
res[a.Hash] = a
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (r *artworkRepository) GetAllMimes() (map[string]string, error) {
|
||||
sel := Select("hash", "mime").From(r.tableName)
|
||||
var rows []struct {
|
||||
Hash string
|
||||
Mime string
|
||||
}
|
||||
if err := r.queryAll(sel, &rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res := make(map[string]string, len(rows))
|
||||
for _, row := range rows {
|
||||
res[row.Hash] = row.Mime
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (r *artworkRepository) GetOrphanHashes(createdBefore time.Time) ([]string, error) {
|
||||
sel := Select("hash").From(r.tableName).
|
||||
Where(And{
|
||||
Lt{"created_at": createdBefore},
|
||||
Expr("hash NOT IN (SELECT hash FROM " + itemArtworkTable + " WHERE hash <> '')"),
|
||||
})
|
||||
var hashes []string
|
||||
err := r.queryAllSlice(sel, &hashes)
|
||||
return hashes, err
|
||||
}
|
||||
|
||||
func (r *artworkRepository) DeleteOrphans(createdBefore time.Time, hashes []string) error {
|
||||
for chunk := range slices.Chunk(hashes, artworkBatchSize) {
|
||||
del := Delete(r.tableName).Where(And{
|
||||
Eq{"hash": chunk},
|
||||
Lt{"created_at": createdBefore},
|
||||
Expr("hash NOT IN (SELECT hash FROM " + itemArtworkTable + " WHERE hash <> '')"),
|
||||
})
|
||||
if _, err := r.executeSQL(del); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *artworkRepository) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) {
|
||||
sel := Select("*").From(itemArtworkTable).
|
||||
Where(Eq{"item_kind": kind, "item_id": id, "image_type": imageType})
|
||||
var res model.ItemArtwork
|
||||
if err := r.items.queryOne(sel, &res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
func (r *artworkRepository) PutItemArtwork(ia *model.ItemArtwork) error {
|
||||
if ia.ImageType == "" {
|
||||
ia.ImageType = model.ImageTypePrimary
|
||||
}
|
||||
ia.UpdatedAt = time.Now()
|
||||
// PutItemArtwork records the outcome of an attempt, so an unset attempted_at is now.
|
||||
if ia.AttemptedAt.IsZero() {
|
||||
ia.AttemptedAt = ia.UpdatedAt
|
||||
}
|
||||
values, err := toSQLArgs(*ia)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ins := Insert(itemArtworkTable).SetMap(values).Suffix(`ON CONFLICT (item_kind, item_id, image_type) DO UPDATE SET
|
||||
hash=excluded.hash, source=excluded.source, source_path=excluded.source_path, ref_mtime=excluded.ref_mtime,
|
||||
attempted_at=excluded.attempted_at, updated_at=excluded.updated_at`)
|
||||
_, err = r.items.executeSQL(ins)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *artworkRepository) DeleteForItem(kind, id string) error {
|
||||
return r.items.delete(Eq{"item_kind": kind, "item_id": id})
|
||||
}
|
||||
|
||||
func (r *artworkRepository) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) {
|
||||
res := map[string]model.ItemArtworkInfo{}
|
||||
for chunk := range slices.Chunk(ids, artworkBatchSize) {
|
||||
sel := Select("ia.item_id", "ia.hash", "COALESCE(a.blur_hash, '') as blur_hash").
|
||||
From(itemArtworkTable + " ia").
|
||||
LeftJoin("artwork a ON a.hash = ia.hash").
|
||||
Where(And{
|
||||
Eq{"ia.item_kind": kind},
|
||||
Eq{"ia.image_type": model.ImageTypePrimary},
|
||||
Eq{"ia.item_id": chunk},
|
||||
})
|
||||
var rows []struct {
|
||||
ItemID string
|
||||
Hash string
|
||||
BlurHash string
|
||||
}
|
||||
if err := r.items.queryAll(sel, &rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, row := range rows {
|
||||
res[row.ItemID] = model.ItemArtworkInfo{
|
||||
ItemID: row.ItemID, Hash: row.Hash, BlurHash: row.BlurHash,
|
||||
}
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
var _ model.ArtworkRepository = (*artworkRepository)(nil)
|
||||
@@ -0,0 +1,197 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/model"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/pocketbase/dbx"
|
||||
)
|
||||
|
||||
// clearArtworkTables resets the shared test DB's artwork tables so specs don't leak state.
|
||||
func clearArtworkTables() {
|
||||
db := GetDBXBuilder()
|
||||
for _, t := range []string{"artwork_queue", "item_artwork", "artwork"} {
|
||||
_, err := db.NewQuery("DELETE FROM " + t).Execute()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
}
|
||||
}
|
||||
|
||||
var _ = Describe("ArtworkRepository", func() {
|
||||
var repo model.ArtworkRepository
|
||||
|
||||
BeforeEach(func() {
|
||||
clearArtworkTables()
|
||||
repo = NewArtworkRepository(context.Background(), GetDBXBuilder())
|
||||
})
|
||||
|
||||
Context("image identity", func() {
|
||||
It("stores and retrieves an artwork by hash", func() {
|
||||
a := &model.Artwork{Hash: "abc123", Mime: "image/jpeg", Width: 500, Height: 500, SizeBytes: 1234, BlurHash: "LKO2?U%2Tw=w"}
|
||||
Expect(repo.PutImage(a)).To(Succeed())
|
||||
|
||||
got, err := repo.GetImage("abc123")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got.Mime).To(Equal("image/jpeg"))
|
||||
Expect(got.BlurHash).To(Equal("LKO2?U%2Tw=w"))
|
||||
Expect(got.CreatedAt).ToNot(BeZero())
|
||||
})
|
||||
|
||||
It("is idempotent on Put (upsert by hash)", func() {
|
||||
a := &model.Artwork{Hash: "dup1", Mime: "image/png"}
|
||||
Expect(repo.PutImage(a)).To(Succeed())
|
||||
a.BlurHash = "XYZ"
|
||||
Expect(repo.PutImage(a)).To(Succeed())
|
||||
got, _ := repo.GetImage("dup1")
|
||||
Expect(got.BlurHash).To(Equal("XYZ"))
|
||||
})
|
||||
|
||||
It("refreshes created_at when reacquiring an existing hash", func() {
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "reacq", Mime: "image/jpeg"})).To(Succeed())
|
||||
_, err := GetDBXBuilder().NewQuery("UPDATE artwork SET created_at={:t} WHERE hash='reacq'").
|
||||
Bind(dbx.Params{"t": "2000-01-01 00:00:00"}).Execute()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "reacq", Mime: "image/png"})).To(Succeed())
|
||||
|
||||
got, err := repo.GetImage("reacq")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got.CreatedAt).To(BeTemporally(">", time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)))
|
||||
})
|
||||
|
||||
It("returns ErrNotFound for a missing hash", func() {
|
||||
_, err := repo.GetImage("nope")
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
})
|
||||
|
||||
It("fetches a batch", func() {
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "b1", Mime: "image/jpeg"})).To(Succeed())
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "b2", Mime: "image/png"})).To(Succeed())
|
||||
got, err := repo.GetImages([]string{"b1", "b2", "missing"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got).To(HaveLen(2))
|
||||
Expect(got["b2"].Mime).To(Equal("image/png"))
|
||||
})
|
||||
|
||||
It("returns every stored hash with its current mime", func() {
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "all1", Mime: "image/jpeg"})).To(Succeed())
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "all2", Mime: "image/png"})).To(Succeed())
|
||||
mimes, err := repo.GetAllMimes()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(mimes).To(HaveKeyWithValue("all1", "image/jpeg"))
|
||||
Expect(mimes).To(HaveKeyWithValue("all2", "image/png"))
|
||||
})
|
||||
|
||||
It("finds orphans older than cutoff, honoring item_artwork references", func() {
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "orph1", Mime: "image/jpeg"})).To(Succeed())
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "ref1", Mime: "image/jpeg"})).To(Succeed())
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1", ImageType: model.ImageTypePrimary, Hash: "ref1", Source: "folder"})).To(Succeed())
|
||||
|
||||
orphans, err := repo.GetOrphanHashes(time.Now().Add(time.Minute))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(orphans).To(ContainElement("orph1"))
|
||||
Expect(orphans).ToNot(ContainElement("ref1"))
|
||||
|
||||
orphans, err = repo.GetOrphanHashes(time.Now().Add(-time.Hour))
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(orphans).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("deletes only unreferenced hashes older than the cutoff", func() {
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "d1", Mime: "image/jpeg"})).To(Succeed())
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "dref", Mime: "image/jpeg"})).To(Succeed())
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "a1",
|
||||
ImageType: model.ImageTypePrimary, Hash: "dref", Source: "folder"})).To(Succeed())
|
||||
|
||||
Expect(repo.DeleteOrphans(time.Now().Add(time.Minute), []string{"d1", "dref"})).To(Succeed())
|
||||
|
||||
_, err := repo.GetImage("d1")
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
_, err = repo.GetImage("dref")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("spares an unreferenced hash younger than the cutoff", func() {
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "young", Mime: "image/jpeg"})).To(Succeed())
|
||||
Expect(repo.DeleteOrphans(time.Now().Add(-time.Hour), []string{"young"})).To(Succeed())
|
||||
_, err := repo.GetImage("young")
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
})
|
||||
|
||||
It("fetches a batch larger than the SQL variable limit", func() {
|
||||
hashes := make([]string, 0, 250)
|
||||
for i := range 250 {
|
||||
h := fmt.Sprintf("big%03d", i)
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: h, Mime: "image/jpeg"})).To(Succeed())
|
||||
hashes = append(hashes, h)
|
||||
}
|
||||
hashes = append(hashes, "absent1", "absent2")
|
||||
|
||||
got, err := repo.GetImages(hashes)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got).To(HaveLen(250))
|
||||
})
|
||||
})
|
||||
|
||||
Context("item state", func() {
|
||||
It("upserts and reads state, including per-item provenance", func() {
|
||||
ia := &model.ItemArtwork{ItemKind: "al", ItemID: "al1", ImageType: model.ImageTypePrimary,
|
||||
Hash: "h1", Source: "folder", SourcePath: "/music/a/cover.jpg", RefMtime: 111, AttemptedAt: time.Now()}
|
||||
Expect(repo.PutItemArtwork(ia)).To(Succeed())
|
||||
ia.Source = "embedded"
|
||||
ia.SourcePath = "/music/a/track.mp3"
|
||||
ia.RefMtime = 222
|
||||
Expect(repo.PutItemArtwork(ia)).To(Succeed())
|
||||
|
||||
got, err := repo.GetItemArtwork("al", "al1", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got.Source).To(Equal("embedded"))
|
||||
Expect(got.SourcePath).To(Equal("/music/a/track.mp3"))
|
||||
Expect(got.RefMtime).To(Equal(int64(222)))
|
||||
Expect(got.UpdatedAt).ToNot(BeZero())
|
||||
})
|
||||
|
||||
It("defaults attempted_at to now when unset", func() {
|
||||
before := time.Now().Add(-time.Second)
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "noattempt",
|
||||
ImageType: model.ImageTypePrimary, Hash: ""})).To(Succeed())
|
||||
got, err := repo.GetItemArtwork("ar", "noattempt", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got.AttemptedAt).To(BeTemporally(">", before))
|
||||
})
|
||||
|
||||
It("represents known-absent as empty hash", func() {
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "ar", ItemID: "ar1",
|
||||
ImageType: model.ImageTypePrimary, Hash: "", AttemptedAt: time.Now()})).To(Succeed())
|
||||
got, err := repo.GetItemArtwork("ar", "ar1", model.ImageTypePrimary)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(got.Hash).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("hydrates a page in one batch, including blurhash and absence", func() {
|
||||
Expect(repo.PutImage(&model.Artwork{Hash: "h9", Mime: "image/jpeg", BlurHash: "BH9"})).To(Succeed())
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x1", ImageType: model.ImageTypePrimary, Hash: "h9", Source: "folder"})).To(Succeed())
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "al", ItemID: "x2", ImageType: model.ImageTypePrimary, Hash: "", Source: ""})).To(Succeed())
|
||||
|
||||
info, err := repo.GetInfoForItems("al", []string{"x1", "x2", "x3"})
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
Expect(info).To(HaveLen(2))
|
||||
Expect(info["x1"].Hash).To(Equal("h9"))
|
||||
Expect(info["x1"].BlurHash).To(Equal("BH9"))
|
||||
Expect(info["x1"].Absent()).To(BeFalse())
|
||||
Expect(info["x2"].Absent()).To(BeTrue())
|
||||
_, unresolved := info["x3"]
|
||||
Expect(unresolved).To(BeFalse())
|
||||
})
|
||||
|
||||
It("deletes all rows for an item", func() {
|
||||
Expect(repo.PutItemArtwork(&model.ItemArtwork{ItemKind: "pl", ItemID: "p1", ImageType: model.ImageTypePrimary, Hash: "h1"})).To(Succeed())
|
||||
Expect(repo.DeleteForItem("pl", "p1")).To(Succeed())
|
||||
_, err := repo.GetItemArtwork("pl", "p1", model.ImageTypePrimary)
|
||||
Expect(err).To(MatchError(model.ErrNotFound))
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -97,6 +97,14 @@ func (s *SQLStore) Plugin(ctx context.Context) model.PluginRepository {
|
||||
return NewPluginRepository(ctx, s.getDBXBuilder())
|
||||
}
|
||||
|
||||
func (s *SQLStore) Artwork(ctx context.Context) model.ArtworkRepository {
|
||||
return NewArtworkRepository(ctx, s.getDBXBuilder())
|
||||
}
|
||||
|
||||
func (s *SQLStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository {
|
||||
return NewArtworkQueueRepository(ctx, s.getDBXBuilder())
|
||||
}
|
||||
|
||||
func (s *SQLStore) Resource(ctx context.Context, m any) model.ResourceRepository {
|
||||
switch m.(type) {
|
||||
case model.User:
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/model"
|
||||
)
|
||||
|
||||
type MockArtworkQueueRepo struct {
|
||||
model.ArtworkQueueRepository
|
||||
Data map[string]model.ArtworkQueueItem // keyed by iaKey(kind, id, imageType)
|
||||
Err error
|
||||
// ItemArtworkSource, when set, backs EnqueueStaleAbsent with real item_artwork state.
|
||||
ItemArtworkSource *MockArtworkRepo
|
||||
}
|
||||
|
||||
func CreateMockArtworkQueueRepo() *MockArtworkQueueRepo {
|
||||
return &MockArtworkQueueRepo{Data: map[string]model.ArtworkQueueItem{}}
|
||||
}
|
||||
|
||||
func (m *MockArtworkQueueRepo) Enqueue(items ...model.ArtworkQueueItem) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
now := time.Now()
|
||||
for _, it := range items {
|
||||
if it.ImageType == "" {
|
||||
it.ImageType = model.ImageTypePrimary
|
||||
}
|
||||
k := iaKey(it.ItemKind, it.ItemID, it.ImageType)
|
||||
// Mirror the SQL: retry_at/enqueued_at are server-set, never taken from the caller.
|
||||
if prev, ok := m.Data[k]; ok {
|
||||
prev.Priority = max(prev.Priority, it.Priority)
|
||||
prev.RetryAt = now
|
||||
m.Data[k] = prev
|
||||
continue
|
||||
}
|
||||
it.Attempts = 0
|
||||
it.RetryAt = now
|
||||
it.EnqueuedAt = now
|
||||
m.Data[k] = it
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkQueueRepo) DequeueBatch(n int) ([]model.ArtworkQueueItem, error) {
|
||||
if m.Err != nil {
|
||||
return nil, m.Err
|
||||
}
|
||||
var res []model.ArtworkQueueItem
|
||||
now := time.Now()
|
||||
for _, it := range m.Data {
|
||||
if !it.RetryAt.After(now) {
|
||||
res = append(res, it)
|
||||
}
|
||||
}
|
||||
sort.Slice(res, func(i, j int) bool {
|
||||
if res[i].Priority != res[j].Priority {
|
||||
return res[i].Priority > res[j].Priority
|
||||
}
|
||||
return res[i].EnqueuedAt.Before(res[j].EnqueuedAt)
|
||||
})
|
||||
if len(res) > n {
|
||||
res = res[:n]
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkQueueRepo) MarkFailed(kind, id, imageType string, retryAt time.Time) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
k := iaKey(kind, id, imageType)
|
||||
it, ok := m.Data[k]
|
||||
if !ok {
|
||||
return model.ErrNotFound
|
||||
}
|
||||
it.Attempts++
|
||||
it.RetryAt = retryAt
|
||||
m.Data[k] = it
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkQueueRepo) Delete(kind, id, imageType string) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
delete(m.Data, iaKey(kind, id, imageType))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkQueueRepo) Count() (int64, error) {
|
||||
if m.Err != nil {
|
||||
return 0, m.Err
|
||||
}
|
||||
return int64(len(m.Data)), nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkQueueRepo) EnqueueStaleAbsent(kind string, attemptedBefore time.Time) (int64, error) {
|
||||
if m.Err != nil || m.ItemArtworkSource == nil {
|
||||
return 0, m.Err
|
||||
}
|
||||
now := time.Now()
|
||||
var inserted int64
|
||||
for _, ia := range m.ItemArtworkSource.ItemData {
|
||||
if ia.ItemKind != kind || ia.Hash != "" || !ia.AttemptedAt.Before(attemptedBefore) {
|
||||
continue
|
||||
}
|
||||
k := iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)
|
||||
if _, ok := m.Data[k]; ok { // DO NOTHING: never touch existing queue rows
|
||||
continue
|
||||
}
|
||||
m.Data[k] = model.ArtworkQueueItem{
|
||||
ItemKind: ia.ItemKind,
|
||||
ItemID: ia.ItemID,
|
||||
ImageType: ia.ImageType,
|
||||
Priority: model.ArtworkPriorityRecheck,
|
||||
RetryAt: now,
|
||||
EnqueuedAt: now,
|
||||
}
|
||||
inserted++
|
||||
}
|
||||
return inserted, nil
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/navidrome/navidrome/model"
|
||||
)
|
||||
|
||||
type MockArtworkRepo struct {
|
||||
model.ArtworkRepository
|
||||
Data map[string]model.Artwork
|
||||
ItemData map[string]model.ItemArtwork // keyed by iaKey(kind, id, imageType)
|
||||
OrphanHashes []string
|
||||
Err error
|
||||
}
|
||||
|
||||
func CreateMockArtworkRepo() *MockArtworkRepo {
|
||||
return &MockArtworkRepo{Data: map[string]model.Artwork{}, ItemData: map[string]model.ItemArtwork{}}
|
||||
}
|
||||
|
||||
func iaKey(kind, id, imageType string) string { return kind + "|" + id + "|" + imageType }
|
||||
|
||||
func (m *MockArtworkRepo) GetImage(hash string) (*model.Artwork, error) {
|
||||
if m.Err != nil {
|
||||
return nil, m.Err
|
||||
}
|
||||
if a, ok := m.Data[hash]; ok {
|
||||
return &a, nil
|
||||
}
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) PutImage(a *model.Artwork) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
// Mirrors the SQL repository: every upsert refreshes created_at. Age fixtures via Data directly.
|
||||
a.CreatedAt = time.Now()
|
||||
m.Data[a.Hash] = *a
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) GetImages(hashes []string) (map[string]model.Artwork, error) {
|
||||
if m.Err != nil {
|
||||
return nil, m.Err
|
||||
}
|
||||
res := map[string]model.Artwork{}
|
||||
for _, h := range hashes {
|
||||
if a, ok := m.Data[h]; ok {
|
||||
res[h] = a
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) GetOrphanHashes(createdBefore time.Time) ([]string, error) {
|
||||
return m.OrphanHashes, m.Err
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) GetAllMimes() (map[string]string, error) {
|
||||
if m.Err != nil {
|
||||
return nil, m.Err
|
||||
}
|
||||
mimes := make(map[string]string, len(m.Data))
|
||||
for h, a := range m.Data {
|
||||
mimes[h] = a.Mime
|
||||
}
|
||||
return mimes, nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) DeleteOrphans(createdBefore time.Time, hashes []string) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
// Mirror the SQL re-check: only unreferenced rows older than the cutoff are deleted.
|
||||
for _, h := range hashes {
|
||||
if m.referenced(h) {
|
||||
continue
|
||||
}
|
||||
if a, ok := m.Data[h]; ok && !a.CreatedAt.Before(createdBefore) {
|
||||
continue
|
||||
}
|
||||
delete(m.Data, h)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) referenced(hash string) bool {
|
||||
for _, ia := range m.ItemData {
|
||||
if ia.Hash == hash {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) GetItemArtwork(kind, id, imageType string) (*model.ItemArtwork, error) {
|
||||
if m.Err != nil {
|
||||
return nil, m.Err
|
||||
}
|
||||
if ia, ok := m.ItemData[iaKey(kind, id, imageType)]; ok {
|
||||
return &ia, nil
|
||||
}
|
||||
return nil, model.ErrNotFound
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) PutItemArtwork(ia *model.ItemArtwork) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
if ia.ImageType == "" {
|
||||
ia.ImageType = model.ImageTypePrimary
|
||||
}
|
||||
ia.UpdatedAt = time.Now()
|
||||
if ia.AttemptedAt.IsZero() {
|
||||
ia.AttemptedAt = ia.UpdatedAt
|
||||
}
|
||||
m.ItemData[iaKey(ia.ItemKind, ia.ItemID, ia.ImageType)] = *ia
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) DeleteForItem(kind, id string) error {
|
||||
if m.Err != nil {
|
||||
return m.Err
|
||||
}
|
||||
for k, ia := range m.ItemData {
|
||||
if ia.ItemKind == kind && ia.ItemID == id {
|
||||
delete(m.ItemData, k)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MockArtworkRepo) GetInfoForItems(kind string, ids []string) (map[string]model.ItemArtworkInfo, error) {
|
||||
if m.Err != nil {
|
||||
return nil, m.Err
|
||||
}
|
||||
res := map[string]model.ItemArtworkInfo{}
|
||||
for _, id := range ids {
|
||||
if ia, ok := m.ItemData[iaKey(kind, id, model.ImageTypePrimary)]; ok {
|
||||
info := model.ItemArtworkInfo{ItemID: id, Hash: ia.Hash}
|
||||
if a, ok := m.Data[ia.Hash]; ok {
|
||||
info.BlurHash = a.BlurHash
|
||||
}
|
||||
res[id] = info
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
@@ -28,6 +28,8 @@ type MockDataStore struct {
|
||||
MockedScrobble model.ScrobbleRepository
|
||||
MockedRadio model.RadioRepository
|
||||
MockedPlugin model.PluginRepository
|
||||
MockedArtwork model.ArtworkRepository
|
||||
MockedArtworkQueue model.ArtworkQueueRepository
|
||||
scrobbleBufferMu sync.Mutex
|
||||
repoMu sync.Mutex
|
||||
|
||||
@@ -247,6 +249,32 @@ func (db *MockDataStore) Plugin(ctx context.Context) model.PluginRepository {
|
||||
return db.MockedPlugin
|
||||
}
|
||||
|
||||
func (db *MockDataStore) Artwork(ctx context.Context) model.ArtworkRepository {
|
||||
if db.MockedArtwork != nil {
|
||||
return db.MockedArtwork
|
||||
}
|
||||
if db.RealDS != nil {
|
||||
return db.RealDS.Artwork(ctx)
|
||||
}
|
||||
db.MockedArtwork = CreateMockArtworkRepo()
|
||||
return db.MockedArtwork
|
||||
}
|
||||
|
||||
func (db *MockDataStore) ArtworkQueue(ctx context.Context) model.ArtworkQueueRepository {
|
||||
if db.MockedArtworkQueue != nil {
|
||||
return db.MockedArtworkQueue
|
||||
}
|
||||
if db.RealDS != nil {
|
||||
return db.RealDS.ArtworkQueue(ctx)
|
||||
}
|
||||
q := CreateMockArtworkQueueRepo()
|
||||
if aw, ok := db.Artwork(ctx).(*MockArtworkRepo); ok {
|
||||
q.ItemArtworkSource = aw
|
||||
}
|
||||
db.MockedArtworkQueue = q
|
||||
return db.MockedArtworkQueue
|
||||
}
|
||||
|
||||
func (db *MockDataStore) WithTx(block func(tx model.DataStore) error, label ...string) error {
|
||||
return block(db)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user