diff --git a/src/server/lib/throttle.ts b/src/server/lib/throttle.ts index 62f0174..81179ed 100644 --- a/src/server/lib/throttle.ts +++ b/src/server/lib/throttle.ts @@ -1,8 +1,8 @@ import pThrottle from 'p-throttle'; const throttle = pThrottle({ - limit: 20, interval: 1000, + limit: 10, }); export default throttle; diff --git a/src/server/queue/index.ts b/src/server/queue/index.ts index 4362c82..0e16dd6 100644 --- a/src/server/queue/index.ts +++ b/src/server/queue/index.ts @@ -1,5 +1,5 @@ -export * from './subsonic/subsonic-scanner'; export * from './subsonic/subsonic-api'; +export * from './subsonic/subsonic-tasks'; export * from './jellyfin/jellyfin-api'; export * from './jellyfin/jellyfin-tasks'; export * from './scanner-queue'; diff --git a/src/server/queue/subsonic/subsonic-api.ts b/src/server/queue/subsonic/subsonic-api.ts new file mode 100644 index 0000000..59c942d --- /dev/null +++ b/src/server/queue/subsonic/subsonic-api.ts @@ -0,0 +1,129 @@ +import axios from 'axios'; +import { Server } from '../../types/types'; +import { + SSAlbumListEntry, + SSAlbumListResponse, + SSAlbumResponse, + SSAlbumsParams, + SSArtistIndex, + SSArtistInfoResponse, + SSArtistsResponse, + SSGenresResponse, + SSMusicFoldersResponse, +} from './subsonic-types'; + +const api = axios.create({ + validateStatus: (status) => status >= 200, +}); + +api.interceptors.response.use( + (res: any) => { + res.data = res.data['subsonic-response']; + return res; + }, + (err: any) => { + return Promise.reject(err); + } +); + +const getMusicFolders = async (server: Server) => { + const { data } = await api.get( + `${server.url}/rest/getMusicFolders.view?v=1.13.0&c=sonixd&f=json&${server.token}` + ); + + return data.musicFolders.musicFolder; +}; + +const getArtists = async (server: Server, musicFolderId: string) => { + const { data } = await api.get( + `${server.url}/rest/getArtists.view?v=1.13.0&c=sonixd&f=json&${server.token}`, + { params: { musicFolderId } } + ); + + const artists = (data.artists?.index || []).flatMap( + (index: SSArtistIndex) => index.artist + ); + + return artists; +}; + +const getGenres = async (server: Server) => { + const { data: genres } = await api.get( + `${server.url}/rest/getGenres.view?v=1.13.0&c=sonixd&f=json&${server.token}` + ); + + return genres; +}; + +const getAlbum = async (server: Server, id: string) => { + const { data: album } = await api.get( + `${server.url}/rest/getAlbum.view?v=1.13.0&c=sonixd&f=json&${server.token}`, + { params: { id } } + ); + + return album; +}; + +const getAlbums = async ( + server: Server, + params: SSAlbumsParams, + recursiveData: any[] = [] +) => { + const albums: any = api + .get( + `${server.url}/rest/getAlbumList2.view?v=1.13.0&c=sonixd&f=json&${server.token}`, + { params } + ) + .then((res) => { + if ( + !res.data.albumList2.album || + res.data.albumList2.album.length === 0 + ) { + // Flatten and return once there are no more albums left + return recursiveData.flatMap((album) => album); + } + + // On every iteration, push the existing combined album array and increase the offset + recursiveData.push(res.data.albumList2.album); + return getAlbums( + server, + { + musicFolderId: params.musicFolderId, + offset: (params.offset || 0) + (params.size || 0), + size: params.size, + type: 'newest', + }, + + recursiveData + ); + }) + .catch((err) => console.log(err)); + + return albums as SSAlbumListEntry[]; +}; + +const getArtistInfo = async (server: Server, id: string) => { + const { data: artistInfo } = await api.get( + `${server.url}/rest/getArtistInfo2.view?v=1.13.0&c=sonixd&f=json&${server.token}`, + { params: { id } } + ); + + return { + ...artistInfo, + artistInfo2: { + ...artistInfo.artistInfo2, + biography: artistInfo.artistInfo2.biography + .replaceAll(//gm, '') + .replace('Biography not available', ''), + }, + }; +}; + +export const subsonicApi = { + getAlbum, + getAlbums, + getArtistInfo, + getArtists, + getGenres, + getMusicFolders, +}; diff --git a/src/server/queue/subsonic/subsonic-scanner.ts b/src/server/queue/subsonic/subsonic-scanner.ts new file mode 100644 index 0000000..a85626e --- /dev/null +++ b/src/server/queue/subsonic/subsonic-scanner.ts @@ -0,0 +1,134 @@ +// import Queue from 'better-queue'; +// import { prisma } from '../../lib'; +// import { Server, ServerFolder } from '../../types/types'; +// import { +// scanAlbumDetail, +// importAlbumArtists, +// scanAlbums, +// scanGenres, +// } from './subsonic-tasks'; + +// const q = new Queue( +// async ( +// task: { +// dbId: number; +// id: string; +// server: Server; +// serverFolder: ServerFolder; +// type: 'genres' | 'albumArtists' | 'albums' | 'albumDetail'; +// }, +// cb: any +// ) => { +// await prisma.task.update({ +// data: { completed: false, inProgress: true }, +// where: { id: task.dbId }, +// }); + +// if (task.type === 'genres') { +// await scanGenres(task.server); +// } + +// if (task.type === 'albumArtists') { +// await importAlbumArtists(task.server, task.serverFolder); +// } + +// if (task.type === 'albums') { +// await scanAlbums(task.server, task.serverFolder); +// } + +// if (task.type === 'albumDetail') { +// await scanAlbumDetail(task.server, task.serverFolder, task.dbId); +// } + +// const result = await cb(null, task); +// return result; +// }, +// { +// batchSize: 1, +// concurrent: 1, +// filo: true, +// maxTimeout: 60000, +// } +// ); + +// q.on('task_finish', async (_taskId, result) => { +// await prisma.task.update({ +// data: { completed: true, inProgress: false }, +// where: { id: Number(result.dbId) }, +// }); +// }); + +// // const scannerTask = async ( +// // userId: number, +// // server: Server, +// // type: 'genres' | 'albumArtists' | 'albums' | 'album' +// // ) => { +// // const task = await prisma.task.create({ +// // data: { +// // name: `[${server.name || server.url}]: scan ${type}`, +// // completed: false, +// // inProgress: false, +// // userId, +// // }, +// // }); + +// // q.push({ id: task.id, server, type }); +// // }; + +// const fullScan = async (server: Server) => { +// const task = await prisma.task.create({ +// data: { +// completed: false, +// inProgress: false, +// name: `[${server.name || server.url}]: fullscan`, +// serverFolderId: 1, +// }, +// }); + +// const args = { +// dbId: task.id, +// id: 'fullscan', +// server, +// }; + +// if (server.serverFolder) { +// server.serverFolder +// .filter((folder) => folder.enabled) +// .forEach((folder) => { +// q.push({ +// ...args, +// serverFolder: folder, +// type: 'genres', +// }).on('finish', () => { +// q.push({ +// ...args, +// serverFolder: folder, +// type: 'albumArtists', +// }).on('finish', () => { +// q.push({ +// ...args, +// serverFolder: folder, +// type: 'albums', +// }).on('finish', () => { +// q.push({ +// ...args, +// server, +// serverFolder: folder, +// type: 'albumDetail', +// }); +// }); +// }); +// }); +// }); +// } +// }; + +// q.on('task_progress', (taskId, completed, total) => { +// console.log('taskId', taskId); +// console.log('completed', completed); +// console.log('total', total); +// }); + +// export const subsonicScanner = { +// fullScan, +// }; diff --git a/src/server/queue/subsonic/subsonic-tasks.ts b/src/server/queue/subsonic/subsonic-tasks.ts new file mode 100644 index 0000000..a9e1320 --- /dev/null +++ b/src/server/queue/subsonic/subsonic-tasks.ts @@ -0,0 +1,479 @@ +/* eslint-disable no-await-in-loop */ +import { prisma, throttle } from '../../lib'; +import { Server, ServerFolder } from '../../types/types'; +import { groupByProperty, uniqueArray } from '../../utils'; +import { q } from '../scanner-queue'; +import { subsonicApi } from './subsonic-api'; +import { SSAlbumListEntry } from './subsonic-types'; + +// const getCoverArtUrl = (server: Server, item: any, size?: number) => { +// if (!item.coverArt && !item.artistImageUrl) { +// return null; +// } + +// if ( +// !item.coverArt && +// !item.artistImageUrl?.match('2a96cbd8b46e442fc41c2b86b821562f') +// ) { +// return item.artistImageUrl; +// } + +// if (item.artistImageUrl?.match('2a96cbd8b46e442fc41c2b86b821562f')) { +// return null; +// } + +// return ( +// `${server.url}/getCoverArt.view` + +// `?id=${item.coverArt}` + +// `&v=1.13.0` + +// `&c=sonixd` + +// `${size ? `&size=${size}` : ''}` +// ); +// }; + +export const scanGenres = async ( + server: Server, + serverFolder: ServerFolder +) => { + const taskId = `[${server.name} (${serverFolder.name})] genres`; + + q.push({ + fn: async () => { + const task = await prisma.task.create({ + data: { + inProgress: true, + name: taskId, + serverFolderId: serverFolder.id, + }, + }); + + const res = await subsonicApi.getGenres(server); + + const genres = res.genres.genre.map((genre) => { + return { name: genre.value }; + }); + + const createdGenres = await prisma.genre.createMany({ + data: genres, + skipDuplicates: true, + }); + + const message = `Imported ${createdGenres.count} new genres.`; + + return { message, task }; + }, + id: taskId, + }); +}; + +export const scanAlbumArtists = async ( + server: Server, + serverFolder: ServerFolder +) => { + const taskId = `[${server.name} (${serverFolder.name})] album artists`; + + q.push({ + fn: async () => { + const task = await prisma.task.create({ + data: { + inProgress: true, + name: taskId, + serverFolderId: serverFolder.id, + }, + }); + + const artists = await subsonicApi.getArtists( + server, + serverFolder.remoteId + ); + + for (const artist of artists) { + await prisma.albumArtist.upsert({ + create: { + name: artist.name, + remoteId: artist.id, + serverFolderId: serverFolder.id, + }, + update: { + name: artist.name, + remoteId: artist.id, + serverFolderId: serverFolder.id, + }, + where: { + uniqueAlbumArtistId: { + remoteId: artist.id, + serverFolderId: serverFolder.id, + }, + }, + }); + + await prisma.artist.upsert({ + create: { + name: artist.name, + remoteId: artist.id, + serverFolderId: serverFolder.id, + }, + update: { + name: artist.name, + remoteId: artist.id, + serverFolderId: serverFolder.id, + }, + where: { + uniqueArtistId: { + remoteId: artist.id, + serverFolderId: serverFolder.id, + }, + }, + }); + } + + const message = `Scanned ${artists.length} album artists.`; + + return { message, task }; + }, + id: taskId, + }); +}; + +export const scanAlbums = async ( + server: Server, + serverFolder: ServerFolder +) => { + const taskId = `[${server.name} (${serverFolder.name})] albums`; + + q.push({ + fn: async () => { + const task = await prisma.task.create({ + data: { + inProgress: true, + name: taskId, + serverFolderId: serverFolder.id, + }, + }); + + const promises: any[] = []; + const albums = await subsonicApi.getAlbums(server, { + musicFolderId: serverFolder.id, + offset: 0, + size: 500, + type: 'newest', + }); + + const albumArtistGroups = groupByProperty(albums, 'artistId'); + + const addAlbums = async ( + a: SSAlbumListEntry[], + albumArtistRemoteId: string + ) => { + const albumArtist = await prisma.albumArtist.findUnique({ + where: { + uniqueAlbumArtistId: { + remoteId: albumArtistRemoteId, + serverFolderId: serverFolder.id, + }, + }, + }); + + if (albumArtist) { + a.forEach(async (album) => { + const imagesConnectOrCreate = album.coverArt + ? { + create: { name: 'Primary', url: album.coverArt }, + where: { + uniqueImageId: { name: 'Primary', url: album.coverArt }, + }, + } + : []; + + await prisma.album.upsert({ + create: { + albumArtistId: albumArtist.id, + images: { connectOrCreate: imagesConnectOrCreate }, + name: album.title, + remoteCreatedAt: album.created, + remoteId: album.id, + serverFolderId: serverFolder.id, + year: album.year, + }, + update: { + albumArtistId: albumArtist.id, + images: { connectOrCreate: imagesConnectOrCreate }, + name: album.title, + remoteCreatedAt: album.created, + remoteId: album.id, + serverFolderId: serverFolder.id, + year: album.year, + }, + where: { + uniqueAlbumId: { + remoteId: album.id, + serverFolderId: serverFolder.id, + }, + }, + }); + }); + } + }; + + Object.keys(albumArtistGroups).forEach((key) => { + promises.push(addAlbums(albumArtistGroups[key], key)); + }); + + await Promise.all(promises); + + const message = `Scanned ${albums.length} albums.`; + + return { message, task }; + }, + id: taskId, + }); +}; + +const throttledAlbumFetch = throttle( + async (server: Server, serverFolder: ServerFolder, album: any, i: number) => { + const albumRes = await subsonicApi.getAlbum(server, album.remoteId); + + console.log('fetch', i); + + if (albumRes) { + const songsUpsert = albumRes.album.song.map((song) => { + const genresConnectOrCreate = song.genre + ? { + create: { name: song.genre }, + where: { name: song.genre }, + } + : []; + + const imagesConnectOrCreate = song.coverArt + ? { + create: { name: 'Primary', url: song.coverArt }, + where: { uniqueImageId: { name: 'Primary', url: song.coverArt } }, + } + : []; + + const artistsConnect = song.artistId + ? { + uniqueArtistId: { + remoteId: song.artistId, + serverFolderId: serverFolder.id, + }, + } + : []; + + return { + create: { + artistName: !song.artistId ? song.artist : undefined, + artists: { connect: artistsConnect }, + bitRate: song.bitRate, + container: song.suffix, + disc: song.discNumber, + duration: song.duration, + genres: { connectOrCreate: genresConnectOrCreate }, + images: { connectOrCreate: imagesConnectOrCreate }, + name: song.title, + remoteCreatedAt: song.created, + remoteId: song.id, + serverFolderId: serverFolder.id, + track: song.track, + year: song.year, + }, + update: { + artistName: !song.artistId ? song.artist : undefined, + artists: { connect: artistsConnect }, + bitRate: song.bitRate, + container: song.suffix, + disc: song.discNumber, + duration: song.duration, + genres: { connectOrCreate: genresConnectOrCreate }, + images: { connectOrCreate: imagesConnectOrCreate }, + name: song.title, + remoteCreatedAt: song.created, + remoteId: song.id, + track: song.track, + year: song.year, + }, + where: { + uniqueSongId: { + remoteId: song.id, + serverFolderId: serverFolder.id, + }, + }, + }; + }); + + const uniqueArtistIds = albumRes.album.song + .map((song) => song.artistId) + .filter(uniqueArray); + + const artistsConnect = uniqueArtistIds.map((artistId) => { + return { + uniqueArtistId: { + remoteId: artistId!, + serverFolderId: serverFolder.id, + }, + }; + }); + + try { + await prisma.album.update({ + data: { + artists: { connect: artistsConnect }, + songs: { upsert: songsUpsert }, + }, + where: { + uniqueAlbumId: { + remoteId: albumRes.album.id, + serverFolderId: serverFolder.id, + }, + }, + }); + } catch (err) { + console.log(err); + } + } + } +); + +export const scanAlbumDetail = async ( + server: Server, + serverFolder: ServerFolder +) => { + const taskId = `[${server.name} (${serverFolder.name})] albums detail`; + + q.push({ + fn: async () => { + const task = await prisma.task.create({ + data: { + inProgress: true, + name: taskId, + serverFolderId: serverFolder.id, + }, + }); + + const promises = []; + const dbAlbums = await prisma.album.findMany({ + where: { serverFolderId: serverFolder.id }, + }); + + for (let i = 0; i < dbAlbums.length; i += 1) { + promises.push( + throttledAlbumFetch(server, serverFolder, dbAlbums[i], i) + ); + } + + await Promise.all(promises); + const message = `Scanned ${dbAlbums.length} albums.`; + + return { message, task }; + }, + id: taskId, + }); +}; + +const throttledArtistDetailFetch = throttle( + async ( + server: Server, + artistId: number, + artistRemoteId: string, + i: number + ) => { + console.log('artisdetail', i); + + const artistInfo = await subsonicApi.getArtistInfo(server, artistRemoteId); + + const externalsConnectOrCreate = []; + if (artistInfo.artistInfo2.lastFmUrl) { + externalsConnectOrCreate.push({ + create: { + name: 'Last.fm', + url: artistInfo.artistInfo2.lastFmUrl, + }, + where: { + uniqueExternalId: { + name: 'Last.fm', + url: artistInfo.artistInfo2.lastFmUrl, + }, + }, + }); + } + + if (artistInfo.artistInfo2.musicBrainzId) { + externalsConnectOrCreate.push({ + create: { + name: 'MusicBrainz', + url: `https://musicbrainz.org/artist/${artistInfo.artistInfo2.musicBrainzId}`, + }, + where: { + uniqueExternalId: { + name: 'MusicBrainz', + url: `https://musicbrainz.org/artist/${artistInfo.artistInfo2.musicBrainzId}`, + }, + }, + }); + } + + try { + await prisma.albumArtist.update({ + data: { + biography: artistInfo.artistInfo2.biography, + externals: { connectOrCreate: externalsConnectOrCreate }, + }, + where: { id: artistId }, + }); + } catch (err) { + console.log(err); + } + } +); + +export const scanAlbumArtistDetail = async ( + server: Server, + serverFolder: ServerFolder +) => { + const taskId = `[${server.name} (${serverFolder.name})] artists detail`; + + q.push({ + fn: async () => { + const task = await prisma.task.create({ + data: { + inProgress: true, + name: taskId, + serverFolderId: serverFolder.id, + }, + }); + + const promises = []; + const dbArtists = await prisma.albumArtist.findMany({ + where: { serverFolderId: serverFolder.id }, + }); + + for (let i = 0; i < dbArtists.length; i += 1) { + promises.push( + throttledArtistDetailFetch( + server, + dbArtists[i].id, + dbArtists[i].remoteId, + i + ) + ); + } + + return { task }; + }, + id: taskId, + }); +}; + +const scanAll = async (server: Server, serverFolder: ServerFolder) => { + await scanGenres(server, serverFolder); + await scanAlbumArtists(server, serverFolder); + await scanAlbumArtistDetail(server, serverFolder); + await scanAlbums(server, serverFolder); + await scanAlbumDetail(server, serverFolder); + // await scanSongs(server, serverFolder); +}; + +export const subsonicTasks = { + scanAll, + scanGenres, +}; diff --git a/src/server/queue/subsonic/subsonic-types.ts b/src/server/queue/subsonic/subsonic-types.ts new file mode 100644 index 0000000..85c7c0c --- /dev/null +++ b/src/server/queue/subsonic/subsonic-types.ts @@ -0,0 +1,139 @@ +export interface SSBaseResponse { + serverVersion?: 'string'; + status: 'string'; + type?: 'string'; + version: 'string'; +} + +export interface SSMusicFoldersResponse extends SSBaseResponse { + musicFolders: { + musicFolder: SSMusicFolder[]; + }; +} + +export interface SSGenresResponse extends SSBaseResponse { + genres: { + genre: SSGenre[]; + }; +} + +export interface SSArtistsResponse extends SSBaseResponse { + artists: { + ignoredArticles: string; + index: SSArtistIndex[]; + lastModified: number; + }; +} + +export interface SSAlbumListResponse extends SSBaseResponse { + albumList2: { + album: SSAlbumListEntry[]; + }; +} + +export interface SSAlbumResponse extends SSBaseResponse { + album: SSAlbum; +} + +export interface SSArtistInfoResponse extends SSBaseResponse { + artistInfo2: SSArtistInfo; +} + +export interface SSArtistInfo { + biography: string; + largeImageUrl?: string; + lastFmUrl?: string; + mediumImageUrl?: string; + musicBrainzId?: string; + smallImageUrl?: string; +} + +export interface SSMusicFolder { + id: number; + name: string; +} + +export interface SSGenre { + albumCount: number; + songCount: number; + value: string; +} + +export interface SSArtistIndex { + artist: SSArtistListEntry[]; + name: string; +} + +export interface SSArtistListEntry { + albumCount: string; + artistImageUrl?: string; + coverArt?: string; + id: string; + name: string; +} + +export interface SSAlbumListEntry { + album: string; + artist: string; + artistId: string; + coverArt: string; + created: string; + duration: number; + genre?: string; + id: string; + isDir: boolean; + isVideo: boolean; + name: string; + parent: string; + songCount: number; + starred?: boolean; + title: string; + userRating?: number; + year: number; +} + +export interface SSAlbum extends SSAlbumListEntry { + song: SSSong[]; +} + +export interface SSSong { + album: string; + albumId: string; + artist: string; + artistId?: string; + bitRate: number; + contentType: string; + coverArt: string; + created: string; + discNumber?: number; + duration: number; + genre: string; + id: string; + isDir: boolean; + isVideo: boolean; + parent: string; + path: string; + playCount: number; + size: number; + starred?: boolean; + suffix: string; + title: string; + track: number; + type: string; + userRating?: number; + year: number; +} + +export interface SSAlbumsParams { + fromYear?: number; + genre?: string; + musicFolderId?: number; + offset?: number; + size?: number; + toYear?: number; + type: string; +} + +export interface SSArtistsParams { + musicFolderId?: number; +}