mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-07-31 23:06:19 -04:00
Keep Prowlarr results from distinct indexer entries separate (#1140)
Results were deduplicated on `guid` alone. When one tracker is configured in Prowlarr as several indexer entries differing only by a server-side search filter, all of them return the same guid for the same torrent, so every entry but the first was silently discarded. Freeleech and other filter-specific releases became invisible, replaced by the unfiltered entry's copy, and which copy survived depended on query ordering rather than user intent. Include the indexer id in the dedup key so the entries stay distinct. Release.source_id is qualified the same way. It keys the release cache and becomes the download task id, so rows sharing a guid would otherwise collide and a grab would route through whichever entry cached last, defeating the point of showing them separately. The handler's task matcher still accepts a bare guid or infoUrl so tasks queued before this change still resolve. Ordering now follows the priority already configured in Prowlarr (1-50, lower preferred) rather than a new setting, since users curate that ranking there and filtered entries are typically ranked ahead of their unfiltered counterparts. The enabled-indexer list is fetched once and reused for the enrichment check, so this costs no extra round trip. Releases carry extra.indexer_priority, and the sort dropdown gains an "Indexer priority" entry, ascending, alongside the existing alphabetical "Indexer" sort; SortOption grew a default_direction for that, defaulting to desc so "Peers" is unchanged. PROWLARR_COLLAPSE_DUPLICATES (default off) optionally collapses a release back to one row, resolved by the same Prowlarr priority. Left off, every entry that carried a release keeps its own row, which is what makes filtered results visible again. Identity handling is defensive about partial payloads: a result that cannot be identified is never dropped or merged, and collapse only merges on a strong identifier (guid/downloadUrl/magnetUrl/infoUrl) because merging on title alone would discard genuinely different releases that share a name. Fixes #1137 Co-authored-by: delize <4028612+delize@users.noreply.github.com>
This commit is contained in:
@@ -1211,6 +1211,7 @@ How long to cache individual book details. Default: 600 (10 minutes). Max: 60480
|
||||
| `PROWLARR_API_KEY` | Found in Prowlarr: Settings > General > API Key | string (secret) | _none_ |
|
||||
| `PROWLARR_INDEXERS` | Select which indexers to search. 📚 = has book categories. Leave empty to search all. | string (comma-separated) | _empty list_ |
|
||||
| `PROWLARR_AUTO_EXPAND` | Automatically retry search without category filtering if no results are found | boolean | `false` |
|
||||
| `PROWLARR_COLLAPSE_DUPLICATES` | Collapse a release that several indexer entries returned down to a single row. Leave this off to see every entry that carried it, which is what makes results from filter-specific entries (freeleech and the like) visible. | boolean | `false` |
|
||||
| `PROWLARR_USE_SEED_PREFERENCES` | Apply per-indexer seed time and ratio preferences from Prowlarr when sending torrents to the download client | boolean | `false` |
|
||||
|
||||
<details>
|
||||
@@ -1263,6 +1264,15 @@ Automatically retry search without category filtering if no results are found
|
||||
- **Type:** boolean
|
||||
- **Default:** `false`
|
||||
|
||||
#### `PROWLARR_COLLAPSE_DUPLICATES`
|
||||
|
||||
**Show one row per release**
|
||||
|
||||
Collapse a release that several indexer entries returned down to a single row. Leave this off to see every entry that carried it, which is what makes results from filter-specific entries (freeleech and the like) visible.
|
||||
|
||||
- **Type:** boolean
|
||||
- **Default:** `false`
|
||||
|
||||
#### `PROWLARR_USE_SEED_PREFERENCES`
|
||||
|
||||
**Use Prowlarr seed preferences**
|
||||
|
||||
@@ -163,6 +163,7 @@ class SortOption:
|
||||
|
||||
label: str # Display label in the sort dropdown
|
||||
sort_key: str # Field to sort by on the Release object
|
||||
default_direction: Literal["asc", "desc"] = "desc" # Which way "best first" runs
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -261,7 +262,12 @@ def serialize_column_config(config: ReleaseColumnConfig) -> dict[str, Any]:
|
||||
# Include extra sort options (sort entries not tied to a column)
|
||||
if config.extra_sort_options:
|
||||
result["extra_sort_options"] = [
|
||||
{"label": opt.label, "sort_key": opt.sort_key} for opt in config.extra_sort_options
|
||||
{
|
||||
"label": opt.label,
|
||||
"sort_key": opt.sort_key,
|
||||
"default_direction": opt.default_direction,
|
||||
}
|
||||
for opt in config.extra_sort_options
|
||||
]
|
||||
|
||||
# Include action button if specified (replaces default expand search)
|
||||
|
||||
@@ -189,16 +189,24 @@ class ProwlarrClient:
|
||||
indexers = self.get_indexers(raise_on_error=raise_on_error)
|
||||
return [idx for idx in indexers if idx.get("enable", False)]
|
||||
|
||||
def get_enriched_indexer_ids(self, *, restrict_to: list[int] | None = None) -> list[int]:
|
||||
def get_enriched_indexer_ids(
|
||||
self,
|
||||
*,
|
||||
restrict_to: list[int] | None = None,
|
||||
indexers: list[dict[str, Any]] | None = None,
|
||||
) -> list[int]:
|
||||
"""Return enabled indexer IDs that benefit from extra Torznab handling.
|
||||
|
||||
Args:
|
||||
restrict_to: Optional list of candidate indexer IDs to consider.
|
||||
indexers: Optional already-fetched enabled indexer list, so callers
|
||||
that need the full records for other reasons can avoid a second
|
||||
round trip.
|
||||
|
||||
"""
|
||||
enriched_ids: list[int] = []
|
||||
|
||||
for idx in self.get_enabled_indexers_detailed():
|
||||
for idx in indexers if indexers is not None else self.get_enabled_indexers_detailed():
|
||||
idx_id_int = coerce_int_like(idx.get("id"))
|
||||
if idx_id_int is None:
|
||||
continue
|
||||
|
||||
@@ -34,6 +34,7 @@ from shelfmark.release_sources.prowlarr.api import IndexerSeedSettings, Prowlarr
|
||||
from shelfmark.release_sources.prowlarr.cache import cache_release, get_release, remove_release
|
||||
from shelfmark.release_sources.prowlarr.source import ProwlarrSource
|
||||
from shelfmark.release_sources.prowlarr.utils import (
|
||||
build_source_id,
|
||||
coerce_int_like,
|
||||
get_preferred_download_url,
|
||||
get_protocol,
|
||||
@@ -318,11 +319,25 @@ class ProwlarrHandler(ExternalClientHandler):
|
||||
|
||||
@staticmethod
|
||||
def _raw_release_matches_task(raw_release: dict[str, Any], task_id: str) -> bool:
|
||||
identities = (
|
||||
normalize_optional_text(raw_release.get("guid")),
|
||||
normalize_optional_text(raw_release.get("infoUrl")),
|
||||
)
|
||||
return any(identity == task_id for identity in identities)
|
||||
wanted = normalize_optional_text(task_id)
|
||||
if wanted is None:
|
||||
return False
|
||||
|
||||
bare = [
|
||||
identity
|
||||
for identity in (
|
||||
normalize_optional_text(raw_release.get("guid")),
|
||||
normalize_optional_text(raw_release.get("infoUrl")),
|
||||
)
|
||||
if identity is not None
|
||||
]
|
||||
|
||||
identities = [*bare, build_source_id(raw_release)]
|
||||
indexer_id = coerce_int_like(raw_release.get("indexerId"))
|
||||
if indexer_id is not None:
|
||||
identities.extend(f"{indexer_id}:{identity}" for identity in bare)
|
||||
|
||||
return wanted in identities
|
||||
|
||||
def _refresh_download_request_after_add_failure(
|
||||
self,
|
||||
|
||||
@@ -190,6 +190,17 @@ def prowlarr_config_settings() -> list[SettingsField]:
|
||||
description="Automatically retry search without category filtering if no results are found",
|
||||
show_when={"field": "PROWLARR_ENABLED", "value": True},
|
||||
),
|
||||
CheckboxField(
|
||||
key="PROWLARR_COLLAPSE_DUPLICATES",
|
||||
label="Show one row per release",
|
||||
default=False,
|
||||
description=(
|
||||
"Collapse a release that several indexer entries returned down to a single row. "
|
||||
"Leave this off to see every entry that carried it, which is what makes results "
|
||||
"from filter-specific entries (freeleech and the like) visible."
|
||||
),
|
||||
show_when={"field": "PROWLARR_ENABLED", "value": True},
|
||||
),
|
||||
CheckboxField(
|
||||
key="PROWLARR_USE_SEED_PREFERENCES",
|
||||
label="Use Prowlarr seed preferences",
|
||||
|
||||
@@ -33,6 +33,7 @@ from shelfmark.release_sources import (
|
||||
from shelfmark.release_sources.prowlarr.api import IndexerSeedSettings, ProwlarrClient
|
||||
from shelfmark.release_sources.prowlarr.cache import cache_release
|
||||
from shelfmark.release_sources.prowlarr.utils import (
|
||||
build_source_id,
|
||||
coerce_float_like,
|
||||
coerce_int_like,
|
||||
get_protocol,
|
||||
@@ -44,6 +45,9 @@ _SIZE_UNIT_BASE = 1024
|
||||
_TWO_FORMATS = 2
|
||||
_PROWLARR_SOURCE_ERRORS = (AttributeError, OSError, RuntimeError, TypeError, ValueError)
|
||||
|
||||
# Prowlarr indexer priority is 1-50 and lower is preferred; unknown sorts last.
|
||||
_UNRANKED_INDEXER_RANK = 51
|
||||
|
||||
# Errors that can surface from ProwlarrClient.get_indexer_seed_settings(). The
|
||||
# client raises requests exceptions (subclasses of OSError via IOError lineage
|
||||
# is not guaranteed), so include RequestException explicitly.
|
||||
@@ -69,6 +73,118 @@ def _coerce_indexer_id(value: object) -> int | None:
|
||||
return coerce_int_like(value)
|
||||
|
||||
|
||||
def _identity_text(value: object) -> str | None:
|
||||
"""Trimmed text for an identity field, or None when there is nothing usable."""
|
||||
if isinstance(value, str):
|
||||
return value.strip() or None
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
|
||||
def _release_identity(result: dict) -> str | None:
|
||||
"""Identify the underlying release, independent of which indexer surfaced it.
|
||||
|
||||
Strong identifiers only. Title is deliberately excluded because matching on
|
||||
it here would merge two genuinely different releases that happen to share a
|
||||
name, and every caller of this either drops or overwrites a row on a match.
|
||||
Returns None when nothing identifies the result.
|
||||
"""
|
||||
for field in ("guid", "downloadUrl", "magnetUrl", "infoUrl"):
|
||||
identity = _identity_text(result.get(field))
|
||||
if identity is not None:
|
||||
return identity
|
||||
return None
|
||||
|
||||
|
||||
def _result_dedup_key(result: dict) -> tuple[int | None, str] | None:
|
||||
"""Dedup key for a raw Prowlarr result, or None if it cannot be identified.
|
||||
|
||||
One tracker is often configured in Prowlarr as several indexer entries that
|
||||
differ only by a server-side search filter, say a "freeleech only" entry
|
||||
alongside an unfiltered one. Those entries return the same guid for the same
|
||||
torrent, so keying on the guid alone throws away the filtered entry's copy
|
||||
and with it the only signal that the release matched the filter. Including
|
||||
the indexer id keeps the entries distinct.
|
||||
|
||||
Title is an acceptable last resort here, unlike in _release_identity, because
|
||||
the indexer id is part of the key: it only ever collapses a literal repeat
|
||||
from one indexer, never two rows from different entries.
|
||||
"""
|
||||
identity = _release_identity(result) or _identity_text(result.get("title"))
|
||||
if identity is None:
|
||||
return None
|
||||
return (_coerce_indexer_id(result.get("indexerId")), identity)
|
||||
|
||||
|
||||
def _build_indexer_priority(indexers: list[dict]) -> dict[int, int]:
|
||||
"""Map indexer id to the priority configured in Prowlarr. Lower is preferred.
|
||||
|
||||
Users already rank their indexers in Prowlarr, and on trackers configured as
|
||||
several entries that ranking is usually the meaningful one: a "freeleech
|
||||
only" entry is typically given a better priority than the unfiltered entry
|
||||
beside it. Reusing it avoids asking for the same ordering a second time.
|
||||
"""
|
||||
priority: dict[int, int] = {}
|
||||
for indexer in indexers:
|
||||
indexer_id = _coerce_indexer_id(indexer.get("id"))
|
||||
if indexer_id is None:
|
||||
continue
|
||||
rank = coerce_int_like(indexer.get("priority"))
|
||||
if rank is not None:
|
||||
priority[indexer_id] = rank
|
||||
|
||||
return priority
|
||||
|
||||
|
||||
def _rank_for_indexer_id(indexer_id: object, priority: dict[int, int]) -> int:
|
||||
"""Preference rank for an indexer id. Lower wins, unknown ranks last."""
|
||||
coerced = _coerce_indexer_id(indexer_id)
|
||||
if coerced is None:
|
||||
return _UNRANKED_INDEXER_RANK
|
||||
return priority.get(coerced, _UNRANKED_INDEXER_RANK)
|
||||
|
||||
|
||||
def _indexer_rank(result: dict, priority: dict[int, int]) -> int:
|
||||
"""Preference rank of the indexer that surfaced a raw result."""
|
||||
return _rank_for_indexer_id(result.get("indexerId"), priority)
|
||||
|
||||
|
||||
def _release_indexer_rank(release: Release, priority: dict[int, int]) -> int:
|
||||
"""Preference rank of the indexer that surfaced a converted release."""
|
||||
return _rank_for_indexer_id(release.extra.get("indexer_id"), priority)
|
||||
|
||||
|
||||
def _collapse_duplicate_indexer_results(
|
||||
results: list[dict], priority: dict[int, int]
|
||||
) -> list[dict]:
|
||||
"""Reduce a release to a single row, keeping the preferred indexer entry.
|
||||
|
||||
Opt-in behaviour for users who want one row per torrent. Ties keep the
|
||||
result that was queried first, and the winner holds the loser's position so
|
||||
the overall result order stays stable.
|
||||
"""
|
||||
position_by_identity: dict[str, int] = {}
|
||||
kept: list[dict] = []
|
||||
|
||||
for result in results:
|
||||
identity = _release_identity(result)
|
||||
if identity is None:
|
||||
kept.append(result)
|
||||
continue
|
||||
|
||||
existing_position = position_by_identity.get(identity)
|
||||
if existing_position is None:
|
||||
position_by_identity[identity] = len(kept)
|
||||
kept.append(result)
|
||||
continue
|
||||
|
||||
if _indexer_rank(result, priority) < _indexer_rank(kept[existing_position], priority):
|
||||
kept[existing_position] = result
|
||||
|
||||
return kept
|
||||
|
||||
|
||||
def _parse_size(size_bytes: int | None) -> str | None:
|
||||
"""Convert bytes to human-readable size string."""
|
||||
if size_bytes is None or size_bytes <= 0:
|
||||
@@ -387,8 +503,7 @@ def _prowlarr_result_to_release(
|
||||
formats_display = _formats_display(formats)
|
||||
language_detected = _extract_mam_language(str(raw_title or ""))
|
||||
|
||||
# Build the source_id from GUID or generate from indexer + title
|
||||
source_id = result.get("guid") or f"{indexer}:{hash(raw_title)}"
|
||||
source_id = build_source_id(result)
|
||||
|
||||
# Cache the raw Prowlarr result so handler can look it up by source_id
|
||||
cache_release(source_id, result)
|
||||
@@ -612,6 +727,11 @@ class ProwlarrSource(ReleaseSource):
|
||||
],
|
||||
extra_sort_options=[
|
||||
SortOption(label="Peers", sort_key="seeders"),
|
||||
SortOption(
|
||||
label="Indexer priority",
|
||||
sort_key="extra.indexer_priority",
|
||||
default_direction="asc",
|
||||
),
|
||||
],
|
||||
grid_template="minmax(0,2fr) minmax(140px,1fr) 50px 50px 90px 80px",
|
||||
leading_cell=LeadingCellConfig(
|
||||
@@ -817,8 +937,12 @@ class ProwlarrSource(ReleaseSource):
|
||||
try:
|
||||
auto_expand_enabled = config.get("PROWLARR_AUTO_EXPAND", False)
|
||||
deadline = time.monotonic() + PROWLARR_SEARCH_TIMEOUT_SECONDS
|
||||
enabled_indexers = client.get_enabled_indexers_detailed()
|
||||
indexer_priority = _build_indexer_priority(enabled_indexers)
|
||||
# Some indexers benefit from title+author queries and extra format detection.
|
||||
enriched_indexer_ids = client.get_enriched_indexer_ids(restrict_to=indexer_ids)
|
||||
enriched_indexer_ids = client.get_enriched_indexer_ids(
|
||||
restrict_to=indexer_ids, indexers=enabled_indexers
|
||||
)
|
||||
enriched_indexer_ids_set = set(enriched_indexer_ids)
|
||||
indexer_seed_settings = (
|
||||
_fetch_indexer_seed_settings(client, indexer_ids)
|
||||
@@ -859,7 +983,7 @@ class ProwlarrSource(ReleaseSource):
|
||||
|
||||
return results
|
||||
|
||||
seen_keys: set[str] = set()
|
||||
seen_keys: set[tuple[int | None, str]] = set()
|
||||
all_results: list[dict] = []
|
||||
|
||||
for idx, variant in enumerate(variants, start=1):
|
||||
@@ -887,18 +1011,22 @@ class ProwlarrSource(ReleaseSource):
|
||||
self.last_search_type = "expanded"
|
||||
|
||||
for r in raw_results:
|
||||
key = (
|
||||
r.get("guid")
|
||||
or r.get("downloadUrl")
|
||||
or r.get("magnetUrl")
|
||||
or r.get("infoUrl")
|
||||
or f"{r.get('indexerId')}:{r.get('title')}"
|
||||
)
|
||||
if key in seen_keys:
|
||||
continue
|
||||
seen_keys.add(key)
|
||||
key = _result_dedup_key(r)
|
||||
if key is not None:
|
||||
if key in seen_keys:
|
||||
continue
|
||||
seen_keys.add(key)
|
||||
all_results.append(r)
|
||||
|
||||
if config.get("PROWLARR_COLLAPSE_DUPLICATES", False):
|
||||
before_collapse = len(all_results)
|
||||
all_results = _collapse_duplicate_indexer_results(all_results, indexer_priority)
|
||||
if len(all_results) != before_collapse:
|
||||
logger.debug(
|
||||
"Prowlarr: collapsed %s duplicate result(s) across indexer entries",
|
||||
before_collapse - len(all_results),
|
||||
)
|
||||
|
||||
results: list[Release] = []
|
||||
enriched_source_ids: set[str] = set()
|
||||
|
||||
@@ -917,13 +1045,19 @@ class ProwlarrSource(ReleaseSource):
|
||||
content_type,
|
||||
enable_format_detection=is_enriched,
|
||||
)
|
||||
if idx_id_int is not None and idx_id_int in indexer_priority:
|
||||
release.extra["indexer_priority"] = indexer_priority[idx_id_int]
|
||||
results.append(release)
|
||||
|
||||
if is_enriched:
|
||||
enriched_source_ids.add(release.source_id)
|
||||
|
||||
# Sort results: enriched indexers first, then others
|
||||
results.sort(key=lambda r: 0 if r.source_id in enriched_source_ids else 1)
|
||||
results.sort(
|
||||
key=lambda r: (
|
||||
_release_indexer_rank(r, indexer_priority),
|
||||
0 if r.source_id in enriched_source_ids else 1,
|
||||
)
|
||||
)
|
||||
|
||||
if results:
|
||||
torrent_count = sum(1 for r in results if r.protocol == ReleaseProtocol.TORRENT)
|
||||
|
||||
@@ -32,6 +32,28 @@ def coerce_int_like(value: object) -> int | None:
|
||||
return int(normalized)
|
||||
|
||||
|
||||
def build_source_id(result: dict) -> str:
|
||||
"""Build the Release.source_id for a raw Prowlarr result.
|
||||
|
||||
Qualified by the indexer id because one tracker is often configured in
|
||||
Prowlarr as several indexer entries that differ only by a server-side search
|
||||
filter, and those entries return the same guid for the same torrent. Without
|
||||
the qualifier the entries collide in the release cache and a grab routes
|
||||
through whichever entry happened to cache last.
|
||||
"""
|
||||
guid = result.get("guid")
|
||||
if guid:
|
||||
base = str(guid)
|
||||
else:
|
||||
indexer = result.get("indexer", "Unknown")
|
||||
base = f"{indexer}:{hash(result.get('title', 'Unknown'))}"
|
||||
|
||||
indexer_id = coerce_int_like(result.get("indexerId"))
|
||||
if indexer_id is None:
|
||||
return base
|
||||
return f"{indexer_id}:{base}"
|
||||
|
||||
|
||||
def coerce_float_like(value: object) -> float | None:
|
||||
"""Return a float for float-like config/API values, else None."""
|
||||
if isinstance(value, bool):
|
||||
|
||||
@@ -382,7 +382,11 @@ export const ReleaseCell = ({
|
||||
className={`flex items-center ${alignClass} gap-1.5 truncate text-xs text-gray-600 dark:text-gray-300`}
|
||||
>
|
||||
<span className={`h-2 w-2 shrink-0 rounded-full ${dotColor}`} title={protocolLabel} />
|
||||
<span className="truncate">{displayValue}</span>
|
||||
{/* Titled because one tracker can appear as several indexer entries whose
|
||||
names share a prefix, and truncation would make the rows look identical */}
|
||||
<span className="truncate" title={displayValue}>
|
||||
{displayValue}
|
||||
</span>
|
||||
{peers && <span className="shrink-0 text-gray-400 dark:text-gray-500">{peers}</span>}
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -956,7 +956,7 @@ const ReleaseModalSession = ({
|
||||
const fromExtra = (columnConfig.extra_sort_options || []).map((opt) => ({
|
||||
label: opt.label,
|
||||
sortKey: opt.sort_key,
|
||||
defaultDirection: 'desc' as const, // Extra sort options are typically numeric (e.g., peers)
|
||||
defaultDirection: opt.default_direction ?? ('desc' as const),
|
||||
}));
|
||||
return [...fromColumns, ...fromExtra];
|
||||
}, [sortableColumns, columnConfig.extra_sort_options]);
|
||||
|
||||
@@ -414,6 +414,7 @@ export interface LeadingCellConfig {
|
||||
export interface ExtraSortOption {
|
||||
label: string; // Display label in the sort dropdown
|
||||
sort_key: string; // Field to sort by on the Release object
|
||||
default_direction?: 'asc' | 'desc'; // Which way "best first" runs (defaults to desc)
|
||||
}
|
||||
|
||||
export interface SourceActionButton {
|
||||
|
||||
@@ -18,7 +18,7 @@ from shelfmark.download.clients import (
|
||||
from shelfmark.release_sources import Release, ReleaseProtocol
|
||||
from shelfmark.release_sources.prowlarr.cache import cache_release, remove_release
|
||||
from shelfmark.release_sources.prowlarr.handler import ProwlarrHandler
|
||||
from shelfmark.release_sources.prowlarr.utils import get_protocol
|
||||
from shelfmark.release_sources.prowlarr.utils import build_source_id, get_protocol
|
||||
|
||||
|
||||
class ProgressRecorder:
|
||||
@@ -1584,3 +1584,57 @@ class TestProwlarrHandlerPostProcessCleanup:
|
||||
assert args[1] == "nzbget"
|
||||
assert args[2] == "123"
|
||||
assert str(args[3]) == "delete failed"
|
||||
|
||||
|
||||
class TestRawReleaseMatchesTask:
|
||||
"""Refreshing a stale release has to find it by whichever id form the task holds."""
|
||||
|
||||
RAW = {
|
||||
"guid": "https://tracker.example/torrent/555",
|
||||
"infoUrl": "https://tracker.example/details/555",
|
||||
"indexerId": 25,
|
||||
}
|
||||
|
||||
def test_matches_the_indexer_qualified_source_id(self):
|
||||
assert ProwlarrHandler._raw_release_matches_task(
|
||||
self.RAW, "25:https://tracker.example/torrent/555"
|
||||
)
|
||||
|
||||
def test_still_matches_a_bare_guid_from_a_task_queued_before_the_change(self):
|
||||
assert ProwlarrHandler._raw_release_matches_task(
|
||||
self.RAW, "https://tracker.example/torrent/555"
|
||||
)
|
||||
|
||||
def test_still_matches_a_bare_info_url(self):
|
||||
assert ProwlarrHandler._raw_release_matches_task(
|
||||
self.RAW, "https://tracker.example/details/555"
|
||||
)
|
||||
|
||||
def test_does_not_match_the_same_guid_from_a_different_indexer_entry(self):
|
||||
assert not ProwlarrHandler._raw_release_matches_task(
|
||||
self.RAW, "10:https://tracker.example/torrent/555"
|
||||
)
|
||||
|
||||
def test_does_not_match_an_unrelated_release(self):
|
||||
assert not ProwlarrHandler._raw_release_matches_task(self.RAW, "25:something-else")
|
||||
|
||||
|
||||
class TestRawReleaseMatchesTaskEdgeCases:
|
||||
"""Matching the wrong release here grabs the wrong torrent."""
|
||||
|
||||
def test_blank_task_id_never_matches(self):
|
||||
raw = {"guid": "g", "infoUrl": "i", "indexerId": 25}
|
||||
|
||||
assert not ProwlarrHandler._raw_release_matches_task(raw, "")
|
||||
assert not ProwlarrHandler._raw_release_matches_task(raw, " ")
|
||||
assert not ProwlarrHandler._raw_release_matches_task(raw, None)
|
||||
|
||||
def test_a_release_with_no_identifiers_does_not_match_a_blank_task_id(self):
|
||||
assert not ProwlarrHandler._raw_release_matches_task({}, None)
|
||||
assert not ProwlarrHandler._raw_release_matches_task({}, "")
|
||||
|
||||
def test_matches_a_qualified_id_built_from_a_guidless_release(self):
|
||||
raw = {"indexerId": 25, "indexer": "MAM", "title": "Dune"}
|
||||
built = build_source_id(raw)
|
||||
|
||||
assert ProwlarrHandler._raw_release_matches_task(raw, built)
|
||||
|
||||
@@ -11,13 +11,21 @@ from shelfmark.metadata_providers import BookMetadata
|
||||
from shelfmark.release_sources.prowlarr.api import ProwlarrClient
|
||||
from shelfmark.release_sources.prowlarr.source import (
|
||||
ProwlarrSource,
|
||||
_build_indexer_priority,
|
||||
_collapse_duplicate_indexer_results,
|
||||
_detect_content_type_from_categories,
|
||||
_extract_format,
|
||||
_fetch_indexer_seed_settings,
|
||||
_last_known_seed_settings,
|
||||
_parse_size,
|
||||
_release_identity,
|
||||
_result_dedup_key,
|
||||
)
|
||||
from shelfmark.release_sources.prowlarr.utils import (
|
||||
build_source_id,
|
||||
get_protocol_display,
|
||||
sanitize_download_url,
|
||||
)
|
||||
from shelfmark.release_sources.prowlarr.utils import get_protocol_display, sanitize_download_url
|
||||
|
||||
|
||||
class _AvailableSource:
|
||||
@@ -265,8 +273,8 @@ class FakeTorznabClient:
|
||||
self.queries.append(query)
|
||||
return self.search_results
|
||||
|
||||
def get_enriched_indexer_ids(self, restrict_to=None):
|
||||
del restrict_to
|
||||
def get_enriched_indexer_ids(self, restrict_to=None, indexers=None):
|
||||
del restrict_to, indexers
|
||||
return []
|
||||
|
||||
def get_indexer_seed_settings(self, restrict_to=None):
|
||||
@@ -719,3 +727,416 @@ class TestFetchIndexerSeedSettingsFallback:
|
||||
raise RuntimeError("indexers unavailable")
|
||||
|
||||
assert _fetch_indexer_seed_settings(FailingClient(), None) == {}
|
||||
|
||||
|
||||
class _MultiIndexerClient:
|
||||
"""Torznab client where each indexer entry returns its own result set.
|
||||
|
||||
Models one tracker configured in Prowlarr as several entries differing by a
|
||||
server-side search filter, so the same guid comes back from more than one.
|
||||
"""
|
||||
|
||||
def __init__(self, results_by_indexer: dict[int, list[dict]], priorities=None):
|
||||
self.results_by_indexer = results_by_indexer
|
||||
self.priorities = priorities or {}
|
||||
|
||||
def get_enabled_indexers_detailed(self, *, raise_on_error=False):
|
||||
del raise_on_error
|
||||
return [
|
||||
{
|
||||
"id": indexer_id,
|
||||
"enable": True,
|
||||
"priority": self.priorities.get(indexer_id, 25),
|
||||
"capabilities": {
|
||||
"categories": [
|
||||
{"id": 7000, "subCategories": []},
|
||||
{"id": 3030, "subCategories": []},
|
||||
]
|
||||
},
|
||||
}
|
||||
for indexer_id in sorted(self.results_by_indexer)
|
||||
]
|
||||
|
||||
def torznab_search(
|
||||
self,
|
||||
*,
|
||||
indexer_id: int,
|
||||
query: str,
|
||||
categories=None,
|
||||
search_type="book",
|
||||
limit=100,
|
||||
offset=0,
|
||||
):
|
||||
del query, categories, search_type, limit, offset
|
||||
return self.results_by_indexer.get(indexer_id, [])
|
||||
|
||||
def get_enriched_indexer_ids(self, restrict_to=None, indexers=None):
|
||||
del restrict_to, indexers
|
||||
return []
|
||||
|
||||
def get_indexer_seed_settings(self, restrict_to=None):
|
||||
del restrict_to
|
||||
return {}
|
||||
|
||||
|
||||
def _mam_result(indexer_id: int, indexer: str, guid: str, *, freeleech: bool = False) -> dict:
|
||||
return {
|
||||
"guid": guid,
|
||||
"title": "Dune",
|
||||
"indexerId": indexer_id,
|
||||
"indexer": indexer,
|
||||
"protocol": "torrent",
|
||||
"size": 1048576,
|
||||
"seeders": 10,
|
||||
"leechers": 1,
|
||||
"categories": [{"id": 7020}],
|
||||
"downloadVolumeFactor": 0.0 if freeleech else 1.0,
|
||||
"infoUrl": f"https://tracker.example/{guid}",
|
||||
}
|
||||
|
||||
|
||||
class TestIndexerAwareDeduplication:
|
||||
"""One tracker as several Prowlarr entries must not collapse to one row (#1137)."""
|
||||
|
||||
ONLY_ACTIVE = 10
|
||||
FREELEECH = 25
|
||||
|
||||
def _search(self, monkeypatch, results_by_indexer, config_values=None, priorities=None):
|
||||
import shelfmark.release_sources.prowlarr.source as prowlarr_source
|
||||
from shelfmark.core.search_plan import build_release_search_plan
|
||||
|
||||
values = {"PROWLARR_INDEXERS": "", "PROWLARR_AUTO_EXPAND": False}
|
||||
values.update(config_values or {})
|
||||
monkeypatch.setattr(
|
||||
prowlarr_source.config, "get", lambda key, default=None: values.get(key, default)
|
||||
)
|
||||
|
||||
source = ProwlarrSource()
|
||||
monkeypatch.setattr(
|
||||
source, "_get_client", lambda: _MultiIndexerClient(results_by_indexer, priorities)
|
||||
)
|
||||
|
||||
book = BookMetadata(
|
||||
provider="hardcover", provider_id="123", title="Dune", authors=["Frank Herbert"]
|
||||
)
|
||||
plan = build_release_search_plan(book, languages=["en"])
|
||||
return source.search(book, plan)
|
||||
|
||||
def test_same_guid_from_two_indexer_entries_both_survive(self, monkeypatch):
|
||||
shared_guid = "https://tracker.example/torrent/555"
|
||||
releases = self._search(
|
||||
monkeypatch,
|
||||
{
|
||||
self.ONLY_ACTIVE: [_mam_result(self.ONLY_ACTIVE, "MyAnonamouse", shared_guid)],
|
||||
self.FREELEECH: [
|
||||
_mam_result(
|
||||
self.FREELEECH, "MyAnonamouse - Freeleech", shared_guid, freeleech=True
|
||||
)
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert len(releases) == 2
|
||||
assert {r.indexer for r in releases} == {"MyAnonamouse", "MyAnonamouse - Freeleech"}
|
||||
assert [r.extra["freeleech"] for r in releases].count(True) == 1
|
||||
|
||||
def test_surviving_rows_have_distinct_source_ids(self, monkeypatch):
|
||||
shared_guid = "https://tracker.example/torrent/555"
|
||||
releases = self._search(
|
||||
monkeypatch,
|
||||
{
|
||||
self.ONLY_ACTIVE: [_mam_result(self.ONLY_ACTIVE, "MyAnonamouse", shared_guid)],
|
||||
self.FREELEECH: [
|
||||
_mam_result(self.FREELEECH, "MyAnonamouse - Freeleech", shared_guid)
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
source_ids = [r.source_id for r in releases]
|
||||
assert len(source_ids) == 2
|
||||
assert len(set(source_ids)) == 2
|
||||
|
||||
def test_source_ids_resolve_to_their_own_indexer_entry(self, monkeypatch):
|
||||
from shelfmark.release_sources.prowlarr.cache import get_release
|
||||
|
||||
shared_guid = "https://tracker.example/torrent/555"
|
||||
releases = self._search(
|
||||
monkeypatch,
|
||||
{
|
||||
self.ONLY_ACTIVE: [_mam_result(self.ONLY_ACTIVE, "MyAnonamouse", shared_guid)],
|
||||
self.FREELEECH: [
|
||||
_mam_result(self.FREELEECH, "MyAnonamouse - Freeleech", shared_guid)
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
assert len(releases) == 2
|
||||
cached_indexer_ids = []
|
||||
for release in releases:
|
||||
cached = get_release(release.source_id)
|
||||
assert cached is not None
|
||||
assert cached["indexerId"] == release.extra["indexer_id"]
|
||||
cached_indexer_ids.append(cached["indexerId"])
|
||||
|
||||
assert sorted(cached_indexer_ids) == [self.ONLY_ACTIVE, self.FREELEECH]
|
||||
|
||||
def test_repeat_of_same_result_from_one_indexer_still_collapses(self, monkeypatch):
|
||||
duplicate = _mam_result(self.ONLY_ACTIVE, "MyAnonamouse", "https://tracker.example/t/1")
|
||||
releases = self._search(monkeypatch, {self.ONLY_ACTIVE: [duplicate, dict(duplicate)]})
|
||||
|
||||
assert len(releases) == 1
|
||||
|
||||
|
||||
class TestBuildSourceId:
|
||||
"""source_id keys the release cache, so it must survive shared guids."""
|
||||
|
||||
def test_same_guid_from_two_indexer_entries_gets_two_ids(self):
|
||||
guid = "https://tracker.example/torrent/555"
|
||||
|
||||
assert build_source_id({"guid": guid, "indexerId": 10}) != build_source_id(
|
||||
{"guid": guid, "indexerId": 25}
|
||||
)
|
||||
|
||||
def test_same_indexer_and_guid_is_stable(self):
|
||||
result = {"guid": "https://tracker.example/torrent/555", "indexerId": 10}
|
||||
|
||||
assert build_source_id(result) == build_source_id(dict(result))
|
||||
|
||||
def test_falls_back_to_the_bare_guid_without_an_indexer_id(self):
|
||||
guid = "https://tracker.example/torrent/555"
|
||||
|
||||
assert build_source_id({"guid": guid}) == guid
|
||||
|
||||
def test_falls_back_to_indexer_and_title_without_a_guid(self):
|
||||
source_id = build_source_id({"indexerId": 10, "indexer": "MAM", "title": "Dune"})
|
||||
|
||||
assert source_id.startswith("10:MAM:")
|
||||
|
||||
|
||||
class TestCollapseDuplicatesSetting:
|
||||
"""Opt-in one-row-per-release collapse, resolved by Prowlarr's priority."""
|
||||
|
||||
ONLY_ACTIVE = 10
|
||||
FREELEECH = 25
|
||||
|
||||
def _search_collapsed(self, monkeypatch, priorities):
|
||||
shared_guid = "https://tracker.example/torrent/555"
|
||||
return TestIndexerAwareDeduplication()._search(
|
||||
monkeypatch,
|
||||
{
|
||||
self.ONLY_ACTIVE: [_mam_result(self.ONLY_ACTIVE, "MyAnonamouse", shared_guid)],
|
||||
self.FREELEECH: [
|
||||
_mam_result(self.FREELEECH, "MyAnonamouse - Freeleech", shared_guid)
|
||||
],
|
||||
},
|
||||
config_values={"PROWLARR_COLLAPSE_DUPLICATES": True},
|
||||
priorities=priorities,
|
||||
)
|
||||
|
||||
def test_collapse_keeps_the_better_prowlarr_priority(self, monkeypatch):
|
||||
releases = self._search_collapsed(monkeypatch, {self.FREELEECH: 20, self.ONLY_ACTIVE: 24})
|
||||
|
||||
assert len(releases) == 1
|
||||
assert releases[0].indexer == "MyAnonamouse - Freeleech"
|
||||
|
||||
def test_collapse_honours_the_reverse_priority(self, monkeypatch):
|
||||
releases = self._search_collapsed(monkeypatch, {self.FREELEECH: 30, self.ONLY_ACTIVE: 24})
|
||||
|
||||
assert len(releases) == 1
|
||||
assert releases[0].indexer == "MyAnonamouse"
|
||||
|
||||
def test_equal_priority_keeps_the_first_queried(self, monkeypatch):
|
||||
releases = self._search_collapsed(monkeypatch, {self.FREELEECH: 25, self.ONLY_ACTIVE: 25})
|
||||
|
||||
assert len(releases) == 1
|
||||
assert releases[0].indexer == "MyAnonamouse"
|
||||
|
||||
def test_collapse_off_by_default_keeps_both_rows(self, monkeypatch):
|
||||
releases = TestIndexerAwareDeduplication()._search(
|
||||
monkeypatch,
|
||||
{
|
||||
self.ONLY_ACTIVE: [
|
||||
_mam_result(self.ONLY_ACTIVE, "MyAnonamouse", "https://tracker.example/t/9")
|
||||
],
|
||||
self.FREELEECH: [
|
||||
_mam_result(self.FREELEECH, "MAM - Freeleech", "https://tracker.example/t/9")
|
||||
],
|
||||
},
|
||||
priorities={self.FREELEECH: 20, self.ONLY_ACTIVE: 24},
|
||||
)
|
||||
|
||||
assert len(releases) == 2
|
||||
|
||||
|
||||
class TestBuildIndexerPriority:
|
||||
"""The priority NUMBER from Prowlarr is the rank; the id is only the key."""
|
||||
|
||||
def test_reads_the_priority_number_per_indexer(self):
|
||||
priority = _build_indexer_priority([{"id": 25, "priority": 20}, {"id": 10, "priority": 24}])
|
||||
|
||||
assert priority == {25: 20, 10: 24}
|
||||
|
||||
def test_a_lower_number_is_preferred(self):
|
||||
priority = _build_indexer_priority([{"id": 25, "priority": 20}, {"id": 10, "priority": 24}])
|
||||
|
||||
assert priority[25] < priority[10]
|
||||
|
||||
def test_coerces_string_ids_and_priorities(self):
|
||||
assert _build_indexer_priority([{"id": "25", "priority": "20"}]) == {25: 20}
|
||||
|
||||
def test_skips_records_without_a_usable_id_or_priority(self):
|
||||
assert (
|
||||
_build_indexer_priority([{"priority": 20}, {"id": "abc", "priority": 20}, {"id": 7}])
|
||||
== {}
|
||||
)
|
||||
|
||||
def test_empty_input_yields_no_preferences(self):
|
||||
assert _build_indexer_priority([]) == {}
|
||||
|
||||
|
||||
class TestDedupEdgeCases:
|
||||
"""Malformed and partial results must never silently lose a row."""
|
||||
|
||||
def test_unidentifiable_results_are_all_kept(self, monkeypatch):
|
||||
blank = {"indexerId": 10, "indexer": "MAM", "protocol": "torrent"}
|
||||
releases = TestIndexerAwareDeduplication()._search(
|
||||
monkeypatch, {10: [dict(blank), dict(blank)]}
|
||||
)
|
||||
|
||||
assert len(releases) == 2
|
||||
|
||||
def test_dedup_key_is_none_when_nothing_identifies_the_result(self):
|
||||
assert _result_dedup_key({"indexerId": 10}) is None
|
||||
assert _result_dedup_key({"guid": " ", "title": " "}) is None
|
||||
|
||||
def test_dedup_falls_back_to_title_within_one_indexer(self):
|
||||
first = {"indexerId": 10, "title": "Dune"}
|
||||
second = {"indexerId": 10, "title": "Dune"}
|
||||
assert _result_dedup_key(first) == _result_dedup_key(second)
|
||||
|
||||
def test_same_title_from_two_indexers_is_not_deduped(self):
|
||||
assert _result_dedup_key({"indexerId": 10, "title": "Dune"}) != _result_dedup_key(
|
||||
{"indexerId": 25, "title": "Dune"}
|
||||
)
|
||||
|
||||
def test_identity_ignores_whitespace_only_fields(self):
|
||||
assert _release_identity({"guid": " ", "downloadUrl": "https://x/1"}) == "https://x/1"
|
||||
|
||||
def test_identity_accepts_a_numeric_guid(self):
|
||||
assert _release_identity({"guid": 12345}) == "12345"
|
||||
|
||||
def test_identity_is_none_without_a_strong_identifier(self):
|
||||
assert _release_identity({"title": "Dune", "indexerId": 10}) is None
|
||||
|
||||
|
||||
class TestCollapseEdgeCases:
|
||||
"""Collapse discards rows, so it must only ever merge on a strong identifier."""
|
||||
|
||||
def test_unidentifiable_results_are_never_collapsed(self):
|
||||
blank = {"indexerId": 10, "title": "Dune"}
|
||||
kept = _collapse_duplicate_indexer_results([dict(blank), dict(blank)], {})
|
||||
|
||||
assert len(kept) == 2
|
||||
|
||||
def test_same_title_different_torrents_are_not_collapsed(self):
|
||||
results = [
|
||||
{"indexerId": 10, "guid": "guid-a", "title": "Dune"},
|
||||
{"indexerId": 25, "guid": "guid-b", "title": "Dune"},
|
||||
]
|
||||
|
||||
assert len(_collapse_duplicate_indexer_results(results, {})) == 2
|
||||
|
||||
def test_collapse_preserves_the_position_of_the_row_it_replaces(self):
|
||||
results = [
|
||||
{"indexerId": 10, "guid": "shared"},
|
||||
{"indexerId": 99, "guid": "other"},
|
||||
{"indexerId": 25, "guid": "shared"},
|
||||
]
|
||||
kept = _collapse_duplicate_indexer_results(results, {25: 0, 10: 1})
|
||||
|
||||
assert [r["indexerId"] for r in kept] == [25, 99]
|
||||
|
||||
def test_results_without_an_indexer_id_still_collapse_on_guid(self):
|
||||
results = [{"guid": "shared"}, {"guid": "shared"}]
|
||||
|
||||
assert len(_collapse_duplicate_indexer_results(results, {})) == 1
|
||||
|
||||
|
||||
class TestIndexerPrioritySorting:
|
||||
"""Results are listed by the indexer priority configured in Prowlarr."""
|
||||
|
||||
ONLY_ACTIVE = 10
|
||||
FREELEECH = 25
|
||||
OTHER = 99
|
||||
|
||||
def _search(self, monkeypatch, priorities):
|
||||
return TestIndexerAwareDeduplication()._search(
|
||||
monkeypatch,
|
||||
{
|
||||
self.ONLY_ACTIVE: [_mam_result(self.ONLY_ACTIVE, "MyAnonamouse", "https://t/a")],
|
||||
self.FREELEECH: [
|
||||
_mam_result(self.FREELEECH, "MyAnonamouse - Freeleech", "https://t/b")
|
||||
],
|
||||
self.OTHER: [_mam_result(self.OTHER, "Some Other Tracker", "https://t/c")],
|
||||
},
|
||||
priorities=priorities,
|
||||
)
|
||||
|
||||
def test_results_follow_the_priority_numbers(self, monkeypatch):
|
||||
releases = self._search(
|
||||
monkeypatch, {self.FREELEECH: 20, self.ONLY_ACTIVE: 24, self.OTHER: 50}
|
||||
)
|
||||
|
||||
assert [r.extra["indexer_id"] for r in releases] == [
|
||||
self.FREELEECH,
|
||||
self.ONLY_ACTIVE,
|
||||
self.OTHER,
|
||||
]
|
||||
|
||||
def test_reversing_the_numbers_reverses_the_list(self, monkeypatch):
|
||||
releases = self._search(
|
||||
monkeypatch, {self.FREELEECH: 50, self.ONLY_ACTIVE: 24, self.OTHER: 20}
|
||||
)
|
||||
|
||||
assert [r.extra["indexer_id"] for r in releases] == [
|
||||
self.OTHER,
|
||||
self.ONLY_ACTIVE,
|
||||
self.FREELEECH,
|
||||
]
|
||||
|
||||
def test_equal_priorities_keep_query_order(self, monkeypatch):
|
||||
releases = self._search(
|
||||
monkeypatch, {self.FREELEECH: 25, self.ONLY_ACTIVE: 25, self.OTHER: 25}
|
||||
)
|
||||
|
||||
assert [r.extra["indexer_id"] for r in releases] == [
|
||||
self.ONLY_ACTIVE,
|
||||
self.FREELEECH,
|
||||
self.OTHER,
|
||||
]
|
||||
|
||||
|
||||
class TestIndexerPrioritySortOption:
|
||||
"""The UI needs the priority on the row to offer an explicit sort on it."""
|
||||
|
||||
def test_releases_carry_their_prowlarr_priority(self, monkeypatch):
|
||||
releases = TestIndexerPrioritySorting()._search(monkeypatch, {10: 24, 25: 20, 99: 50})
|
||||
|
||||
by_indexer = {r.extra["indexer_id"]: r.extra.get("indexer_priority") for r in releases}
|
||||
assert by_indexer == {10: 24, 25: 20, 99: 50}
|
||||
|
||||
def test_priority_is_absent_when_prowlarr_reports_none(self, monkeypatch):
|
||||
releases = TestIndexerPrioritySorting()._search(monkeypatch, {})
|
||||
|
||||
assert all(r.extra.get("indexer_priority") == 25 for r in releases)
|
||||
|
||||
def test_sort_option_is_offered_lowest_first(self):
|
||||
from shelfmark.release_sources import serialize_column_config
|
||||
|
||||
source = ProwlarrSource()
|
||||
serialized = serialize_column_config(source.get_column_config())
|
||||
options = {o["label"]: o for o in serialized["extra_sort_options"]}
|
||||
|
||||
assert options["Indexer priority"]["sort_key"] == "extra.indexer_priority"
|
||||
assert options["Indexer priority"]["default_direction"] == "asc"
|
||||
assert options["Peers"]["default_direction"] == "desc"
|
||||
|
||||
Reference in New Issue
Block a user