Add subsonic scanner

src/server/lib/throttle.ts

@@ -1,8 +1,8 @@
 import pThrottle from 'p-throttle';
 
 const throttle = pThrottle({
-  limit: 20,
   interval: 1000,
+  limit: 10,
 });
 
 export default throttle;

src/server/queue/index.ts

@@ -1,5 +1,5 @@
-export * from './subsonic/subsonic-scanner';
 export * from './subsonic/subsonic-api';
+export * from './subsonic/subsonic-tasks';
 export * from './jellyfin/jellyfin-api';
 export * from './jellyfin/jellyfin-tasks';
 export * from './scanner-queue';

src/server/queue/subsonic/subsonic-api.ts

@@ -0,0 +1,129 @@
+import axios from 'axios';
+import { Server } from '../../types/types';
+import {
+  SSAlbumListEntry,
+  SSAlbumListResponse,
+  SSAlbumResponse,
+  SSAlbumsParams,
+  SSArtistIndex,
+  SSArtistInfoResponse,
+  SSArtistsResponse,
+  SSGenresResponse,
+  SSMusicFoldersResponse,
+} from './subsonic-types';
+
+const api = axios.create({
+  validateStatus: (status) => status >= 200,
+});
+
+api.interceptors.response.use(
+  (res: any) => {
+    res.data = res.data['subsonic-response'];
+    return res;
+  },
+  (err: any) => {
+    return Promise.reject(err);
+  }
+);
+
+const getMusicFolders = async (server: Server) => {
+  const { data } = await api.get<SSMusicFoldersResponse>(
+    `${server.url}/rest/getMusicFolders.view?v=1.13.0&c=sonixd&f=json&${server.token}`
+  );
+
+  return data.musicFolders.musicFolder;
+};
+
+const getArtists = async (server: Server, musicFolderId: string) => {
+  const { data } = await api.get<SSArtistsResponse>(
+    `${server.url}/rest/getArtists.view?v=1.13.0&c=sonixd&f=json&${server.token}`,
+    { params: { musicFolderId } }
+  );
+
+  const artists = (data.artists?.index || []).flatMap(
+    (index: SSArtistIndex) => index.artist
+  );
+
+  return artists;
+};
+
+const getGenres = async (server: Server) => {
+  const { data: genres } = await api.get<SSGenresResponse>(
+    `${server.url}/rest/getGenres.view?v=1.13.0&c=sonixd&f=json&${server.token}`
+  );
+
+  return genres;
+};
+
+const getAlbum = async (server: Server, id: string) => {
+  const { data: album } = await api.get<SSAlbumResponse>(
+    `${server.url}/rest/getAlbum.view?v=1.13.0&c=sonixd&f=json&${server.token}`,
+    { params: { id } }
+  );
+
+  return album;
+};
+
+const getAlbums = async (
+  server: Server,
+  params: SSAlbumsParams,
+  recursiveData: any[] = []
+) => {
+  const albums: any = api
+    .get<SSAlbumListResponse>(
+      `${server.url}/rest/getAlbumList2.view?v=1.13.0&c=sonixd&f=json&${server.token}`,
+      { params }
+    )
+    .then((res) => {
+      if (
+        !res.data.albumList2.album ||
+        res.data.albumList2.album.length === 0
+      ) {
+        // Flatten and return once there are no more albums left
+        return recursiveData.flatMap((album) => album);
+      }
+
+      // On every iteration, push the existing combined album array and increase the offset
+      recursiveData.push(res.data.albumList2.album);
+      return getAlbums(
+        server,
+        {
+          musicFolderId: params.musicFolderId,
+          offset: (params.offset || 0) + (params.size || 0),
+          size: params.size,
+          type: 'newest',
+        },
+
+        recursiveData
+      );
+    })
+    .catch((err) => console.log(err));
+
+  return albums as SSAlbumListEntry[];
+};
+
+const getArtistInfo = async (server: Server, id: string) => {
+  const { data: artistInfo } = await api.get<SSArtistInfoResponse>(
+    `${server.url}/rest/getArtistInfo2.view?v=1.13.0&c=sonixd&f=json&${server.token}`,
+    { params: { id } }
+  );
+
+  return {
+    ...artistInfo,
+    artistInfo2: {
+      ...artistInfo.artistInfo2,
+      biography: artistInfo.artistInfo2.biography
+        .replaceAll(/<a target.*<\/a>/gm, '')
+        .replace('Biography not available', ''),
+    },
+  };
+};
+
+export const subsonicApi = {
+  getAlbum,
+  getAlbums,
+  getArtistInfo,
+  getArtists,
+  getGenres,
+  getMusicFolders,
+};

src/server/queue/subsonic/subsonic-scanner.ts

@@ -0,0 +1,134 @@
+// import Queue from 'better-queue';
+// import { prisma } from '../../lib';
+// import { Server, ServerFolder } from '../../types/types';
+// import {
+//   scanAlbumDetail,
+//   importAlbumArtists,
+//   scanAlbums,
+//   scanGenres,
+// } from './subsonic-tasks';
+
+// const q = new Queue(
+//   async (
+//     task: {
+//       dbId: number;
+//       id: string;
+//       server: Server;
+//       serverFolder: ServerFolder;
+//       type: 'genres' | 'albumArtists' | 'albums' | 'albumDetail';
+//     },
+//     cb: any
+//   ) => {
+//     await prisma.task.update({
+//       data: { completed: false, inProgress: true },
+//       where: { id: task.dbId },
+//     });
+
+//     if (task.type === 'genres') {
+//       await scanGenres(task.server);
+//     }
+
+//     if (task.type === 'albumArtists') {
+//       await importAlbumArtists(task.server, task.serverFolder);
+//     }
+
+//     if (task.type === 'albums') {
+//       await scanAlbums(task.server, task.serverFolder);
+//     }
+
+//     if (task.type === 'albumDetail') {
+//       await scanAlbumDetail(task.server, task.serverFolder, task.dbId);
+//     }
+
+//     const result = await cb(null, task);
+//     return result;
+//   },
+//   {
+//     batchSize: 1,
+//     concurrent: 1,
+//     filo: true,
+//     maxTimeout: 60000,
+//   }
+// );
+
+// q.on('task_finish', async (_taskId, result) => {
+//   await prisma.task.update({
+//     data: { completed: true, inProgress: false },
+//     where: { id: Number(result.dbId) },
+//   });
+// });
+
+// // const scannerTask = async (
+// //   userId: number,
+// //   server: Server,
+// //   type: 'genres' | 'albumArtists' | 'albums' | 'album'
+// // ) => {
+// //   const task = await prisma.task.create({
+// //     data: {
+// //       name: `[${server.name || server.url}]: scan ${type}`,
+// //       completed: false,
+// //       inProgress: false,
+// //       userId,
+// //     },
+// //   });
+
+// //   q.push({ id: task.id, server, type });
+// // };
+
+// const fullScan = async (server: Server) => {
+//   const task = await prisma.task.create({
+//     data: {
+//       completed: false,
+//       inProgress: false,
+//       name: `[${server.name || server.url}]: fullscan`,
+//       serverFolderId: 1,
+//     },
+//   });
+
+//   const args = {
+//     dbId: task.id,
+//     id: 'fullscan',
+//     server,
+//   };
+
+//   if (server.serverFolder) {
+//     server.serverFolder
+//       .filter((folder) => folder.enabled)
+//       .forEach((folder) => {
+//         q.push({
+//           ...args,
+//           serverFolder: folder,
+//           type: 'genres',
+//         }).on('finish', () => {
+//           q.push({
+//             ...args,
+//             serverFolder: folder,
+//             type: 'albumArtists',
+//           }).on('finish', () => {
+//             q.push({
+//               ...args,
+//               serverFolder: folder,
+//               type: 'albums',
+//             }).on('finish', () => {
+//               q.push({
+//                 ...args,
+//                 server,
+//                 serverFolder: folder,
+//                 type: 'albumDetail',
+//               });
+//             });
+//           });
+//         });
+//       });
+//   }
+// };
+
+// q.on('task_progress', (taskId, completed, total) => {
+//   console.log('taskId', taskId);
+//   console.log('completed', completed);
+//   console.log('total', total);
+// });
+
+// export const subsonicScanner = {
+//   fullScan,
+// };

src/server/queue/subsonic/subsonic-tasks.ts

@@ -0,0 +1,479 @@
+/* eslint-disable no-await-in-loop */
+import { prisma, throttle } from '../../lib';
+import { Server, ServerFolder } from '../../types/types';
+import { groupByProperty, uniqueArray } from '../../utils';
+import { q } from '../scanner-queue';
+import { subsonicApi } from './subsonic-api';
+import { SSAlbumListEntry } from './subsonic-types';
+
+// const getCoverArtUrl = (server: Server, item: any, size?: number) => {
+//   if (!item.coverArt && !item.artistImageUrl) {
+//     return null;
+//   }
+
+//   if (
+//     !item.coverArt &&
+//     !item.artistImageUrl?.match('2a96cbd8b46e442fc41c2b86b821562f')
+//   ) {
+//     return item.artistImageUrl;
+//   }
+
+//   if (item.artistImageUrl?.match('2a96cbd8b46e442fc41c2b86b821562f')) {
+//     return null;
+//   }
+
+//   return (
+//     `${server.url}/getCoverArt.view` +
+//     `?id=${item.coverArt}` +
+//     `&v=1.13.0` +
+//     `&c=sonixd` +
+//     `${size ? `&size=${size}` : ''}`
+//   );
+// };
+
+export const scanGenres = async (
+  server: Server,
+  serverFolder: ServerFolder
+) => {
+  const taskId = `[${server.name} (${serverFolder.name})] genres`;
+
+  q.push({
+    fn: async () => {
+      const task = await prisma.task.create({
+        data: {
+          inProgress: true,
+          name: taskId,
+          serverFolderId: serverFolder.id,
+        },
+      });
+
+      const res = await subsonicApi.getGenres(server);
+
+      const genres = res.genres.genre.map((genre) => {
+        return { name: genre.value };
+      });
+
+      const createdGenres = await prisma.genre.createMany({
+        data: genres,
+        skipDuplicates: true,
+      });
+
+      const message = `Imported ${createdGenres.count} new genres.`;
+
+      return { message, task };
+    },
+    id: taskId,
+  });
+};
+
+export const scanAlbumArtists = async (
+  server: Server,
+  serverFolder: ServerFolder
+) => {
+  const taskId = `[${server.name} (${serverFolder.name})] album artists`;
+
+  q.push({
+    fn: async () => {
+      const task = await prisma.task.create({
+        data: {
+          inProgress: true,
+          name: taskId,
+          serverFolderId: serverFolder.id,
+        },
+      });
+
+      const artists = await subsonicApi.getArtists(
+        server,
+        serverFolder.remoteId
+      );
+
+      for (const artist of artists) {
+        await prisma.albumArtist.upsert({
+          create: {
+            name: artist.name,
+            remoteId: artist.id,
+            serverFolderId: serverFolder.id,
+          },
+          update: {
+            name: artist.name,
+            remoteId: artist.id,
+            serverFolderId: serverFolder.id,
+          },
+          where: {
+            uniqueAlbumArtistId: {
+              remoteId: artist.id,
+              serverFolderId: serverFolder.id,
+            },
+          },
+        });
+
+        await prisma.artist.upsert({
+          create: {
+            name: artist.name,
+            remoteId: artist.id,
+            serverFolderId: serverFolder.id,
+          },
+          update: {
+            name: artist.name,
+            remoteId: artist.id,
+            serverFolderId: serverFolder.id,
+          },
+          where: {
+            uniqueArtistId: {
+              remoteId: artist.id,
+              serverFolderId: serverFolder.id,
+            },
+          },
+        });
+      }
+
+      const message = `Scanned ${artists.length} album artists.`;
+
+      return { message, task };
+    },
+    id: taskId,
+  });
+};
+
+export const scanAlbums = async (
+  server: Server,
+  serverFolder: ServerFolder
+) => {
+  const taskId = `[${server.name} (${serverFolder.name})] albums`;
+
+  q.push({
+    fn: async () => {
+      const task = await prisma.task.create({
+        data: {
+          inProgress: true,
+          name: taskId,
+          serverFolderId: serverFolder.id,
+        },
+      });
+
+      const promises: any[] = [];
+      const albums = await subsonicApi.getAlbums(server, {
+        musicFolderId: serverFolder.id,
+        offset: 0,
+        size: 500,
+        type: 'newest',
+      });
+
+      const albumArtistGroups = groupByProperty(albums, 'artistId');
+
+      const addAlbums = async (
+        a: SSAlbumListEntry[],
+        albumArtistRemoteId: string
+      ) => {
+        const albumArtist = await prisma.albumArtist.findUnique({
+          where: {
+            uniqueAlbumArtistId: {
+              remoteId: albumArtistRemoteId,
+              serverFolderId: serverFolder.id,
+            },
+          },
+        });
+
+        if (albumArtist) {
+          a.forEach(async (album) => {
+            const imagesConnectOrCreate = album.coverArt
+              ? {
+                  create: { name: 'Primary', url: album.coverArt },
+                  where: {
+                    uniqueImageId: { name: 'Primary', url: album.coverArt },
+                  },
+                }
+              : [];
+
+            await prisma.album.upsert({
+              create: {
+                albumArtistId: albumArtist.id,
+                images: { connectOrCreate: imagesConnectOrCreate },
+                name: album.title,
+                remoteCreatedAt: album.created,
+                remoteId: album.id,
+                serverFolderId: serverFolder.id,
+                year: album.year,
+              },
+              update: {
+                albumArtistId: albumArtist.id,
+                images: { connectOrCreate: imagesConnectOrCreate },
+                name: album.title,
+                remoteCreatedAt: album.created,
+                remoteId: album.id,
+                serverFolderId: serverFolder.id,
+                year: album.year,
+              },
+              where: {
+                uniqueAlbumId: {
+                  remoteId: album.id,
+                  serverFolderId: serverFolder.id,
+                },
+              },
+            });
+          });
+        }
+      };
+
+      Object.keys(albumArtistGroups).forEach((key) => {
+        promises.push(addAlbums(albumArtistGroups[key], key));
+      });
+
+      await Promise.all(promises);
+
+      const message = `Scanned ${albums.length} albums.`;
+
+      return { message, task };
+    },
+    id: taskId,
+  });
+};
+
+const throttledAlbumFetch = throttle(
+  async (server: Server, serverFolder: ServerFolder, album: any, i: number) => {
+    const albumRes = await subsonicApi.getAlbum(server, album.remoteId);
+
+    console.log('fetch', i);
+
+    if (albumRes) {
+      const songsUpsert = albumRes.album.song.map((song) => {
+        const genresConnectOrCreate = song.genre
+          ? {
+              create: { name: song.genre },
+              where: { name: song.genre },
+            }
+          : [];
+
+        const imagesConnectOrCreate = song.coverArt
+          ? {
+              create: { name: 'Primary', url: song.coverArt },
+              where: { uniqueImageId: { name: 'Primary', url: song.coverArt } },
+            }
+          : [];
+
+        const artistsConnect = song.artistId
+          ? {
+              uniqueArtistId: {
+                remoteId: song.artistId,
+                serverFolderId: serverFolder.id,
+              },
+            }
+          : [];
+
+        return {
+          create: {
+            artistName: !song.artistId ? song.artist : undefined,
+            artists: { connect: artistsConnect },
+            bitRate: song.bitRate,
+            container: song.suffix,
+            disc: song.discNumber,
+            duration: song.duration,
+            genres: { connectOrCreate: genresConnectOrCreate },
+            images: { connectOrCreate: imagesConnectOrCreate },
+            name: song.title,
+            remoteCreatedAt: song.created,
+            remoteId: song.id,
+            serverFolderId: serverFolder.id,
+            track: song.track,
+            year: song.year,
+          },
+          update: {
+            artistName: !song.artistId ? song.artist : undefined,
+            artists: { connect: artistsConnect },
+            bitRate: song.bitRate,
+            container: song.suffix,
+            disc: song.discNumber,
+            duration: song.duration,
+            genres: { connectOrCreate: genresConnectOrCreate },
+            images: { connectOrCreate: imagesConnectOrCreate },
+            name: song.title,
+            remoteCreatedAt: song.created,
+            remoteId: song.id,
+            track: song.track,
+            year: song.year,
+          },
+          where: {
+            uniqueSongId: {
+              remoteId: song.id,
+              serverFolderId: serverFolder.id,
+            },
+          },
+        };
+      });
+
+      const uniqueArtistIds = albumRes.album.song
+        .map((song) => song.artistId)
+        .filter(uniqueArray);
+
+      const artistsConnect = uniqueArtistIds.map((artistId) => {
+        return {
+          uniqueArtistId: {
+            remoteId: artistId!,
+            serverFolderId: serverFolder.id,
+          },
+        };
+      });
+
+      try {
+        await prisma.album.update({
+          data: {
+            artists: { connect: artistsConnect },
+            songs: { upsert: songsUpsert },
+          },
+          where: {
+            uniqueAlbumId: {
+              remoteId: albumRes.album.id,
+              serverFolderId: serverFolder.id,
+            },
+          },
+        });
+      } catch (err) {
+        console.log(err);
+      }
+    }
+  }
+);
+
+export const scanAlbumDetail = async (
+  server: Server,
+  serverFolder: ServerFolder
+) => {
+  const taskId = `[${server.name} (${serverFolder.name})] albums detail`;
+
+  q.push({
+    fn: async () => {
+      const task = await prisma.task.create({
+        data: {
+          inProgress: true,
+          name: taskId,
+          serverFolderId: serverFolder.id,
+        },
+      });
+
+      const promises = [];
+      const dbAlbums = await prisma.album.findMany({
+        where: { serverFolderId: serverFolder.id },
+      });
+
+      for (let i = 0; i < dbAlbums.length; i += 1) {
+        promises.push(
+          throttledAlbumFetch(server, serverFolder, dbAlbums[i], i)
+        );
+      }
+
+      await Promise.all(promises);
+      const message = `Scanned ${dbAlbums.length} albums.`;
+
+      return { message, task };
+    },
+    id: taskId,
+  });
+};
+
+const throttledArtistDetailFetch = throttle(
+  async (
+    server: Server,
+    artistId: number,
+    artistRemoteId: string,
+    i: number
+  ) => {
+    console.log('artisdetail', i);
+
+    const artistInfo = await subsonicApi.getArtistInfo(server, artistRemoteId);
+
+    const externalsConnectOrCreate = [];
+    if (artistInfo.artistInfo2.lastFmUrl) {
+      externalsConnectOrCreate.push({
+        create: {
+          name: 'Last.fm',
+          url: artistInfo.artistInfo2.lastFmUrl,
+        },
+        where: {
+          uniqueExternalId: {
+            name: 'Last.fm',
+            url: artistInfo.artistInfo2.lastFmUrl,
+          },
+        },
+      });
+    }
+
+    if (artistInfo.artistInfo2.musicBrainzId) {
+      externalsConnectOrCreate.push({
+        create: {
+          name: 'MusicBrainz',
+          url: `https://musicbrainz.org/artist/${artistInfo.artistInfo2.musicBrainzId}`,
+        },
+        where: {
+          uniqueExternalId: {
+            name: 'MusicBrainz',
+            url: `https://musicbrainz.org/artist/${artistInfo.artistInfo2.musicBrainzId}`,
+          },
+        },
+      });
+    }
+
+    try {
+      await prisma.albumArtist.update({
+        data: {
+          biography: artistInfo.artistInfo2.biography,
+          externals: { connectOrCreate: externalsConnectOrCreate },
+        },
+        where: { id: artistId },
+      });
+    } catch (err) {
+      console.log(err);
+    }
+  }
+);
+
+export const scanAlbumArtistDetail = async (
+  server: Server,
+  serverFolder: ServerFolder
+) => {
+  const taskId = `[${server.name} (${serverFolder.name})] artists detail`;
+
+  q.push({
+    fn: async () => {
+      const task = await prisma.task.create({
+        data: {
+          inProgress: true,
+          name: taskId,
+          serverFolderId: serverFolder.id,
+        },
+      });
+
+      const promises = [];
+      const dbArtists = await prisma.albumArtist.findMany({
+        where: { serverFolderId: serverFolder.id },
+      });
+
+      for (let i = 0; i < dbArtists.length; i += 1) {
+        promises.push(
+          throttledArtistDetailFetch(
+            server,
+            dbArtists[i].id,
+            dbArtists[i].remoteId,
+            i
+          )
+        );
+      }
+
+      return { task };
+    },
+    id: taskId,
+  });
+};
+
+const scanAll = async (server: Server, serverFolder: ServerFolder) => {
+  await scanGenres(server, serverFolder);
+  await scanAlbumArtists(server, serverFolder);
+  await scanAlbumArtistDetail(server, serverFolder);
+  await scanAlbums(server, serverFolder);
+  await scanAlbumDetail(server, serverFolder);
+  // await scanSongs(server, serverFolder);
+};
+
+export const subsonicTasks = {
+  scanAll,
+  scanGenres,
+};

src/server/queue/subsonic/subsonic-types.ts

@@ -0,0 +1,139 @@
+export interface SSBaseResponse {
+  serverVersion?: 'string';
+  status: 'string';
+  type?: 'string';
+  version: 'string';
+}
+
+export interface SSMusicFoldersResponse extends SSBaseResponse {
+  musicFolders: {
+    musicFolder: SSMusicFolder[];
+  };
+}
+
+export interface SSGenresResponse extends SSBaseResponse {
+  genres: {
+    genre: SSGenre[];
+  };
+}
+
+export interface SSArtistsResponse extends SSBaseResponse {
+  artists: {
+    ignoredArticles: string;
+    index: SSArtistIndex[];
+    lastModified: number;
+  };
+}
+
+export interface SSAlbumListResponse extends SSBaseResponse {
+  albumList2: {
+    album: SSAlbumListEntry[];
+  };
+}
+
+export interface SSAlbumResponse extends SSBaseResponse {
+  album: SSAlbum;
+}
+
+export interface SSArtistInfoResponse extends SSBaseResponse {
+  artistInfo2: SSArtistInfo;
+}
+
+export interface SSArtistInfo {
+  biography: string;
+  largeImageUrl?: string;
+  lastFmUrl?: string;
+  mediumImageUrl?: string;
+  musicBrainzId?: string;
+  smallImageUrl?: string;
+}
+
+export interface SSMusicFolder {
+  id: number;
+  name: string;
+}
+
+export interface SSGenre {
+  albumCount: number;
+  songCount: number;
+  value: string;
+}
+
+export interface SSArtistIndex {
+  artist: SSArtistListEntry[];
+  name: string;
+}
+
+export interface SSArtistListEntry {
+  albumCount: string;
+  artistImageUrl?: string;
+  coverArt?: string;
+  id: string;
+  name: string;
+}
+
+export interface SSAlbumListEntry {
+  album: string;
+  artist: string;
+  artistId: string;
+  coverArt: string;
+  created: string;
+  duration: number;
+  genre?: string;
+  id: string;
+  isDir: boolean;
+  isVideo: boolean;
+  name: string;
+  parent: string;
+  songCount: number;
+  starred?: boolean;
+  title: string;
+  userRating?: number;
+  year: number;
+}
+
+export interface SSAlbum extends SSAlbumListEntry {
+  song: SSSong[];
+}
+
+export interface SSSong {
+  album: string;
+  albumId: string;
+  artist: string;
+  artistId?: string;
+  bitRate: number;
+  contentType: string;
+  coverArt: string;
+  created: string;
+  discNumber?: number;
+  duration: number;
+  genre: string;
+  id: string;
+  isDir: boolean;
+  isVideo: boolean;
+  parent: string;
+  path: string;
+  playCount: number;
+  size: number;
+  starred?: boolean;
+  suffix: string;
+  title: string;
+  track: number;
+  type: string;
+  userRating?: number;
+  year: number;
+}
+
+export interface SSAlbumsParams {
+  fromYear?: number;
+  genre?: string;
+  musicFolderId?: number;
+  offset?: number;
+  size?: number;
+  toYear?: number;
+  type: string;
+}
+
+export interface SSArtistsParams {
+  musicFolderId?: number;
+}