fix(scanner): execute GetFolderUpdateInfo in batches to avoid "Expression tree is too large (maximum depth 1000)"

Signed-off-by: Deluan <deluan@navidrome.org>
This commit is contained in:
Deluan
2025-12-16 11:37:13 -05:00
parent 017676c457
commit cde5992c46

View File

@@ -95,45 +95,82 @@ func (r folderRepository) CountAll(opt ...model.QueryOptions) (int64, error) {
} }
func (r folderRepository) GetFolderUpdateInfo(lib model.Library, targetPaths ...string) (map[string]model.FolderUpdateInfo, error) { func (r folderRepository) GetFolderUpdateInfo(lib model.Library, targetPaths ...string) (map[string]model.FolderUpdateInfo, error) {
// If no specific paths, return all folders in the library
if len(targetPaths) == 0 {
return r.getFolderUpdateInfoAll(lib)
}
// Check if any path is root (return all folders)
for _, targetPath := range targetPaths {
if targetPath == "" || targetPath == "." {
return r.getFolderUpdateInfoAll(lib)
}
}
// Process paths in batches to avoid SQLite's expression tree depth limit (max 1000).
// Each path generates ~3 conditions, so batch size of 100 keeps us well under the limit.
const batchSize = 100
result := make(map[string]model.FolderUpdateInfo)
for batch := range slices.Chunk(targetPaths, batchSize) {
batchResult, err := r.getFolderUpdateInfoBatch(lib, batch)
if err != nil {
return nil, err
}
for id, info := range batchResult {
result[id] = info
}
}
return result, nil
}
// getFolderUpdateInfoAll returns update info for all non-missing folders in the library
func (r folderRepository) getFolderUpdateInfoAll(lib model.Library) (map[string]model.FolderUpdateInfo, error) {
where := And{
Eq{"library_id": lib.ID},
Eq{"missing": false},
}
return r.queryFolderUpdateInfo(where)
}
// getFolderUpdateInfoBatch returns update info for a batch of target paths and their descendants
func (r folderRepository) getFolderUpdateInfoBatch(lib model.Library, targetPaths []string) (map[string]model.FolderUpdateInfo, error) {
where := And{ where := And{
Eq{"library_id": lib.ID}, Eq{"library_id": lib.ID},
Eq{"missing": false}, Eq{"missing": false},
} }
// If specific paths are requested, include those folders and all their descendants // Collect folder IDs for exact target folders and path conditions for descendants
if len(targetPaths) > 0 { folderIDs := make([]string, 0, len(targetPaths))
// Collect folder IDs for exact target folders and path conditions for descendants pathConditions := make(Or, 0, len(targetPaths)*2)
folderIDs := make([]string, 0, len(targetPaths))
pathConditions := make(Or, 0, len(targetPaths)*2)
for _, targetPath := range targetPaths { for _, targetPath := range targetPaths {
if targetPath == "" || targetPath == "." { // Clean the path to normalize it. Paths stored in the folder table do not have leading/trailing slashes.
// Root path - include everything in this library cleanPath := strings.TrimPrefix(targetPath, string(os.PathSeparator))
pathConditions = Or{} cleanPath = filepath.Clean(cleanPath)
folderIDs = nil
break
}
// Clean the path to normalize it. Paths stored in the folder table do not have leading/trailing slashes.
cleanPath := strings.TrimPrefix(targetPath, string(os.PathSeparator))
cleanPath = filepath.Clean(cleanPath)
// Include the target folder itself by ID // Include the target folder itself by ID
folderIDs = append(folderIDs, model.FolderID(lib, cleanPath)) folderIDs = append(folderIDs, model.FolderID(lib, cleanPath))
// Include all descendants: folders whose path field equals or starts with the target path // Include all descendants: folders whose path field equals or starts with the target path
// Note: Folder.Path is the directory path, so children have path = targetPath // Note: Folder.Path is the directory path, so children have path = targetPath
pathConditions = append(pathConditions, Eq{"path": cleanPath}) pathConditions = append(pathConditions, Eq{"path": cleanPath})
pathConditions = append(pathConditions, Like{"path": cleanPath + "/%"}) pathConditions = append(pathConditions, Like{"path": cleanPath + "/%"})
}
// Combine conditions: exact folder IDs OR descendant path patterns
if len(folderIDs) > 0 {
where = append(where, Or{Eq{"id": folderIDs}, pathConditions})
} else if len(pathConditions) > 0 {
where = append(where, pathConditions)
}
} }
// Combine conditions: exact folder IDs OR descendant path patterns
if len(folderIDs) > 0 {
where = append(where, Or{Eq{"id": folderIDs}, pathConditions})
} else if len(pathConditions) > 0 {
where = append(where, pathConditions)
}
return r.queryFolderUpdateInfo(where)
}
// queryFolderUpdateInfo executes the query and returns the result map
func (r folderRepository) queryFolderUpdateInfo(where And) (map[string]model.FolderUpdateInfo, error) {
sq := r.newSelect().Columns("id", "updated_at", "hash").Where(where) sq := r.newSelect().Columns("id", "updated_at", "hash").Where(where)
var res []struct { var res []struct {
ID string ID string