From 6fe959604a4700a9ca8dd82d7d77ea8a8fabf605 Mon Sep 17 00:00:00 2001 From: santiagosayshey Date: Sat, 27 Jun 2026 14:06:52 +0930 Subject: [PATCH] feat: support database sync priority per Arr instance (#694) --- docs/backend/drift.md | 27 ++ docs/backend/sync.md | 107 +++++++- src/lib/client/ui/list/DraggableCard.svelte | 63 +++-- src/lib/server/db/migrations.ts | 4 +- .../069_create_arr_sync_database_priority.ts | 37 +++ src/lib/server/db/queries/arrInstances.ts | 9 +- src/lib/server/db/queries/arrSync.ts | 58 +++++ .../server/db/queries/databaseInstances.ts | 9 +- src/lib/server/db/schema.sql | 15 ++ src/lib/server/drift/check.ts | 47 ++-- src/lib/server/drift/customFormats.ts | 77 ++++++ src/lib/server/drift/display.ts | 238 ++++++++++++++++-- src/lib/server/drift/qualityProfiles.ts | 99 ++++++++ .../notifications/definitions/arrDrift.ts | 7 +- src/lib/server/sync/qualityProfiles/syncer.ts | 171 ++++++++----- src/lib/shared/drift.ts | 10 + src/routes/arr/[id]/drift/+page.server.ts | 12 +- src/routes/arr/[id]/drift/+page.svelte | 25 +- src/routes/arr/[id]/sync/+page.server.ts | 111 +++++--- src/routes/arr/[id]/sync/+page.svelte | 11 +- .../sync/components/QualityProfiles.svelte | 187 ++++++++++---- 21 files changed, 1100 insertions(+), 224 deletions(-) create mode 100644 src/lib/server/db/migrations/069_create_arr_sync_database_priority.ts diff --git a/docs/backend/drift.md b/docs/backend/drift.md index 9fcd80bb..33ffc229 100644 --- a/docs/backend/drift.md +++ b/docs/backend/drift.md @@ -175,8 +175,10 @@ Each entity carries: - `section` and `sectionLabel` (e.g. `custom_formats` / `Custom Format`) - `state` and `stateLabel` (`missing` / `modified` / `extra`) +- optional `databaseId` / `databaseName` for database-scoped QP and CF drift - `tone` for badge color signaling - `summary` (one-line description, e.g. `3 changes detected`) +- optional duplicate drift metadata for lower-priority database duplicates - `changes[]`: per-field `DriftDisplayChange` rows with `label`, optional `detail`, and `expected` / `actual` `DriftDisplayValue`s. Values carry `text`, optional `mono`, and optional `tone`. @@ -198,6 +200,29 @@ flags, minimum score, order, and tags into friendly field rows. For media management the formatter turns media settings, naming, and quality definition changes into friendly field rows. +### Duplicate Drift Metadata + +Quality profile sync can select profiles from multiple databases. When more +than one selected database manages the same quality profile or custom format +name, priority determines the effective winner. The highest-priority database +is synced last and owns the Arr state for that name. + +Drift display entities can mark lower-priority rows with duplicate metadata: + +| Field | Purpose | +| -------------------- | -------------------------------------------------- | +| `reason` | Currently `lower_priority_duplicate` | +| `key` | Stable section/name key for the duplicated entity | +| `winnerDatabaseId` | Database id that owns the effective synced version | +| `winnerDatabaseName` | Database name shown on the drift page | + +Current behavior is intentionally conservative: duplicate drift is still shown, +counted, hashed, included in progress chips, included in tab badges, and +eligible for notifications. The metadata only exposes which rows are +lower-priority duplicates. A future settings-backed "include duplicate drift" +preference may use the same metadata to control drift page visibility, sync +progress counts, badge counts, and notifications. + Display types live in `src/lib/shared/drift.ts`. ## Arr Drift Page @@ -216,6 +241,8 @@ Layout: columns, mirroring the dev changes-page diff idiom. Each drifted entity is one row; expanding shows a `DriftFieldDiffTable` with `Field` / `Profilarr` / ` - ` columns rendering the entity's `changes[]`. +- Lower-priority duplicate drift rows are labelled with the duplicate marker + and the winning database name. They remain visible in the table. State rendering inside the entities section: diff --git a/docs/backend/sync.md b/docs/backend/sync.md index d5e26071..13001864 100644 --- a/docs/backend/sync.md +++ b/docs/backend/sync.md @@ -21,6 +21,12 @@ control. - [Processor Flow](#processor-flow) - [Status](#status) - [Section Registry](#section-registry) +- [Database Priority](#database-priority) + - [Schema](#schema) + - [Lifecycle](#lifecycle) + - [Sync Order](#sync-order) + - [UI](#ui) + - [Open Questions](#open-questions) - [Transformation](#transformation) - [Custom Formats](#custom-formats) - [Quality Profiles](#quality-profiles) @@ -104,11 +110,93 @@ logic. Each section type implements a `SectionHandler` interface: Three handlers register themselves on import via `registerSection()`: -| Section | Config scope | -| ----------------- | ----------------------------------------------------------------------- | -| `qualityProfiles` | Multi-profile selection from one database | -| `delayProfiles` | Single profile selection (or none) | -| `mediaManagement` | Three independent configs (naming, media settings, quality definitions) | +| Section | Config scope | +| ----------------- | ------------------------------------------------------------------------------------------------ | +| `qualityProfiles` | Multi-profile selection from one or more databases (see [Database Priority](#database-priority)) | +| `delayProfiles` | Single profile selection (or none) | +| `mediaManagement` | Three independent configs (naming, media settings, quality definitions) | + +## Database Priority + +Quality profile sync supports multiple databases per Arr instance. A priority +ordering determines which database's version of an entity wins when more than +one database manages the same name. Other sections (delay profiles, media +management) are unaffected because they are idempotent, single-config +selections where priority has no practical effect. + +### Schema + +A junction table stores the priority ordering per instance: + +```sql +CREATE TABLE arr_sync_database_priority ( + instance_id INTEGER NOT NULL, + database_id INTEGER NOT NULL, + priority INTEGER NOT NULL, + PRIMARY KEY (instance_id, database_id), + FOREIGN KEY (instance_id) REFERENCES arr_instances(id) ON DELETE CASCADE, + FOREIGN KEY (database_id) REFERENCES database_instances(id) ON DELETE CASCADE +); +``` + +Every (instance, database) pair has a row. Priority 1 is the highest priority. +The table is always fully populated: every Arr instance has a row for every +linked database. + +### Lifecycle + +Priority rows are created and removed automatically. The user never manually +adds or removes a database from the priority list. + +| Event | Action | +| ------------------ | ---------------------------------------------------------------- | +| Database linked | Insert a row for every Arr instance, appended at lowest priority | +| Instance created | Insert a row for every linked database, ordered by database ID | +| Database unlinked | FK cascade deletes all rows for that database | +| Instance deleted | FK cascade deletes all rows for that instance | +| User reorders (UI) | Update priority values to reflect the new drag-and-drop ordering | + +On migration, existing (instance, database) pairs are seeded using +`ROW_NUMBER() OVER (PARTITION BY instance_id ORDER BY database_id)` so every +instance gets a clean 1, 2, 3... sequence based on database creation order. + +### Sync Order + +The syncer groups quality profile selections by database, then iterates +databases in priority order from lowest to highest. The highest-priority +database syncs last, so its version of any shared entity overwrites earlier +versions via Arr's name-based matching. + +For a setup with two databases (foo at priority 1, bar at priority 2): + +1. Bar syncs first (lower priority). Its profiles and referenced CFs are + pushed to Arr. +2. Foo syncs next (higher priority). Its profiles and referenced CFs are + pushed, overwriting any names that bar already created. + +The Arr API's name-based create-or-update behavior handles the overwriting +naturally. No merge logic or conflict resolution is needed. + +CFs are pulled along with their profiles as they are today. If both databases +define a CF with the same name, the higher-priority database's version is the +one that persists after sync completes. + +### UI + +The sync page shows each database as a draggable card, ordered by priority. +Profile toggles are nested inside each database card. Users can select +profiles from multiple databases simultaneously. Drag-and-drop reordering +sets the priority, with the topmost card being highest priority. + +### Open Questions + +- **On-demand entity sync and hierarchy.** Entity sync takes an explicit + (instanceId, databaseId, entityName) and pushes from that database + regardless of priority. If a user edits an entity in a lower-priority + database and pushes, it overwrites the higher-priority version until the + next full sync restores it. This is accepted for now because the user + explicitly chose to push that edit. Revisit if this causes confusion in + practice. ## Transformation @@ -271,6 +359,15 @@ Drift currently covers custom formats, quality profiles, and the default delay profile. Media management coverage (naming, media settings, quality definitions) is planned as a follow-up. +For custom formats and quality profiles, drift is computed per-database. Each +database's expected state is built independently (no cross-database dedup), so +overlapping entities appear in every database that manages them. If a +higher-priority database overwrites a lower-priority database's entity, the +lower-priority database reports drift for that entity. This is intentional: +it gives visibility into which databases conflict and what each one expects. +Drift counts are raw sums across databases (the same entity drifted in two +databases counts as two). + After a successful sync that touched Arr, the sync handler enqueues an `arr.drift` job for the same instance so the drift status (and the per-section progress chips on the sync page) reflect the new state without waiting for the diff --git a/src/lib/client/ui/list/DraggableCard.svelte b/src/lib/client/ui/list/DraggableCard.svelte index 32d3b026..6343e485 100644 --- a/src/lib/client/ui/list/DraggableCard.svelte +++ b/src/lib/client/ui/list/DraggableCard.svelte @@ -1,13 +1,20 @@
- - {/if}
@@ -276,9 +282,22 @@ {#if column.key === 'title'}
- - {row.title} - +
+ + {row.title} + + {#if row.duplicateDrift} + + + {/if} +
+ {#if row.databaseName} + + {row.databaseName} + + {/if} {#if row.summary} {row.summary} diff --git a/src/routes/arr/[id]/sync/+page.server.ts b/src/routes/arr/[id]/sync/+page.server.ts index f6bed174..a1b51caf 100644 --- a/src/routes/arr/[id]/sync/+page.server.ts +++ b/src/routes/arr/[id]/sync/+page.server.ts @@ -15,11 +15,12 @@ import { calculateNextRun } from '$lib/server/sync/utils.ts'; import { scheduleArrSyncForInstance } from '$lib/server/jobs/init.ts'; import { enqueueJob } from '$lib/server/jobs/queueService.ts'; import { buildJobDisplayName } from '$lib/server/jobs/display.ts'; -import { buildExpectedCustomFormats } from '$drift/customFormats.ts'; -import type { CustomFormatDriftDiff } from '$drift/customFormats.ts'; +import { buildExpectedCustomFormatsPerDatabase } from '$drift/customFormats.ts'; +import type { CustomFormatDriftDiff, DatabaseCustomFormatDriftDiff } from '$drift/customFormats.ts'; import type { QualityProfileDriftDiff, - QualityProfileModifiedDiff + QualityProfileModifiedDiff, + DatabaseQualityProfileDriftDiff } from '$drift/qualityProfiles.ts'; import type { MediaManagementDriftDiff } from '$drift/mediaManagement.ts'; import { FEATURES } from '$shared/features.ts'; @@ -72,39 +73,69 @@ function isTransitiveOnlyQp(modified: QualityProfileModifiedDiff): boolean { return modified.fields.every(fieldIsMissingCustomFormat); } -function classifyQpDrift(qpDiff: QualityProfileDriftDiff | undefined) { +function extractQpDiffs(raw: unknown): QualityProfileDriftDiff[] { + if (Array.isArray(raw)) { + return raw + .filter( + (entry): entry is DatabaseQualityProfileDriftDiff => + entry != null && typeof entry === 'object' && 'diff' in entry + ) + .map((entry) => entry.diff); + } + if (raw != null && typeof raw === 'object' && ('missing' in raw || 'modified' in raw)) { + return [raw as QualityProfileDriftDiff]; + } + return []; +} + +function extractCfDiffs(raw: unknown): CustomFormatDriftDiff[] { + if (Array.isArray(raw)) { + return raw + .filter( + (entry): entry is DatabaseCustomFormatDriftDiff => + entry != null && typeof entry === 'object' && 'diff' in entry + ) + .map((entry) => entry.diff); + } + if (raw != null && typeof raw === 'object' && ('missing' in raw || 'modified' in raw)) { + return [raw as CustomFormatDriftDiff]; + } + return []; +} + +function classifyQpDrift(qpDiffs: QualityProfileDriftDiff[]) { const result = { directNames: [] as string[], transitiveNames: [] as string[], affectedQpNames: new Set() }; - if (!qpDiff) return result; - // Missing QPs (selected but not in Arr) are always direct. - for (const m of qpDiff.missing) { - result.directNames.push(m.name); - } - - for (const m of qpDiff.modified) { - if (isTransitiveOnlyQp(m)) { - result.transitiveNames.push(m.name); - result.affectedQpNames.add(m.name); - } else { + for (const qpDiff of qpDiffs) { + for (const m of qpDiff.missing) { result.directNames.push(m.name); - // A direct drift can also reference missing CFs; track that too. - if (m.fields.some(fieldIsMissingCustomFormat)) { + } + + for (const m of qpDiff.modified) { + if (isTransitiveOnlyQp(m)) { + result.transitiveNames.push(m.name); result.affectedQpNames.add(m.name); + } else { + result.directNames.push(m.name); + if (m.fields.some(fieldIsMissingCustomFormat)) { + result.affectedQpNames.add(m.name); + } } } } return result; } -function collectCfNames(cfDiff: CustomFormatDriftDiff | undefined): string[] { - if (!cfDiff) return []; +function collectCfNames(cfDiffs: CustomFormatDriftDiff[]): string[] { const names: string[] = []; - for (const m of cfDiff.missing) names.push(m.name); - for (const m of cfDiff.modified) names.push(m.name); + for (const cfDiff of cfDiffs) { + for (const m of cfDiff.missing) names.push(m.name); + for (const m of cfDiff.modified) names.push(m.name); + } return names; } @@ -149,7 +180,15 @@ async function loadDriftProgress( const qpTotal = qpSync.selections.length; const dpTotal = dpSync.databaseId && dpSync.profileName ? 1 : 0; - const cfTotal = qpTotal > 0 ? (await buildExpectedCustomFormats(instanceId, arrType)).length : 0; + + let cfTotal = 0; + if (qpTotal > 0) { + const perDb = await buildExpectedCustomFormatsPerDatabase(instanceId, arrType); + for (const { formats } of perDb.values()) { + cfTotal += formats.length; + } + } + const namingTotal = mmSync.namingDatabaseId !== null && mmSync.namingConfigName ? 1 : 0; const qualityDefinitionsTotal = mmSync.qualityDefinitionsDatabaseId !== null && mmSync.qualityDefinitionsConfigName ? 1 : 0; @@ -157,16 +196,12 @@ async function loadDriftProgress( mmSync.mediaSettingsDatabaseId !== null && mmSync.mediaSettingsConfigName ? 1 : 0; const counts = status.counts ?? {}; - const diff = status.diff as - | { - quality_profiles?: QualityProfileDriftDiff; - custom_formats?: CustomFormatDriftDiff; - media_management?: MediaManagementDriftDiff; - } - | undefined; - const qpClass = classifyQpDrift(diff?.quality_profiles); - const cfNames = collectCfNames(diff?.custom_formats); - const mm = diff?.media_management; + const diff = status.diff as Record | undefined; + const qpDiffs = extractQpDiffs(diff?.quality_profiles); + const cfDiffsList = extractCfDiffs(diff?.custom_formats); + const qpClass = classifyQpDrift(qpDiffs); + const cfNames = collectCfNames(cfDiffsList); + const mm = diff?.media_management as MediaManagementDriftDiff | undefined; const progress: DriftProgress = {}; if (qpTotal > 0) { @@ -305,6 +340,7 @@ export const load: ServerLoad = async ({ params }) => { // Load existing sync data const syncData = arrSyncQueries.getFullSyncData(id); const driftProgress = await loadDriftProgress(id, arrType); + const databasePriorities = arrSyncQueries.getDatabasePriorities(id); const { api_key: _, ...safeInstance } = instance; @@ -312,7 +348,8 @@ export const load: ServerLoad = async ({ params }) => { instance: safeInstance, databases: databasesWithProfiles, syncData, - driftProgress + driftProgress, + databasePriorities }; }; @@ -326,6 +363,7 @@ export const actions: Actions = { const instance = arrInstancesQueries.getById(id); const formData = await request.formData(); const selectionsJson = formData.get('selections') as string; + const prioritiesJson = formData.get('priorities') as string; const trigger = formData.get('trigger') as SyncTrigger; const cron = formData.get('cron') as string | null; @@ -339,6 +377,13 @@ export const actions: Actions = { nextRunAt: effectiveTrigger === 'schedule' ? calculateNextRun(effectiveCron) : null }); + const priorities: { databaseId: number; priority: number }[] = JSON.parse( + prioritiesJson || '[]' + ); + if (priorities.length > 0) { + await arrSyncQueries.saveDatabasePriorities(id, priorities); + } + await logger.info(`Quality profiles sync config saved for "${instance?.name}"`, { source: 'sync', meta: { instanceId: id, profileCount: selections.length, trigger } diff --git a/src/routes/arr/[id]/sync/+page.svelte b/src/routes/arr/[id]/sync/+page.svelte index b99c7b43..81c1dd83 100644 --- a/src/routes/arr/[id]/sync/+page.svelte +++ b/src/routes/arr/[id]/sync/+page.svelte @@ -168,6 +168,7 @@ />
-
- One Database Per Instance -
+
Database Priority

- Each Arr instance syncs from a single database. Quality profiles and their custom formats - all come from the same database. If you want to use profiles from a different database, - sync it to a separate Arr instance. + You can sync quality profiles from multiple databases. Drag to reorder databases by + priority. The topmost database has the highest priority and its profiles will overwrite + any matching profiles from lower-priority databases.

diff --git a/src/routes/arr/[id]/sync/components/QualityProfiles.svelte b/src/routes/arr/[id]/sync/components/QualityProfiles.svelte index 7037daec..a2eaf34e 100644 --- a/src/routes/arr/[id]/sync/components/QualityProfiles.svelte +++ b/src/routes/arr/[id]/sync/components/QualityProfiles.svelte @@ -1,9 +1,12 @@