mirror of
https://github.com/Cleanuparr/Cleanuparr.git
synced 2025-12-31 09:58:53 -05:00
Compare commits
62 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9463d7587f | ||
|
|
7d2bf41bec | ||
|
|
93bb8cc18d | ||
|
|
449d9e623f | ||
|
|
3a50d9be3c | ||
|
|
693f80fe6a | ||
|
|
8cfc73213a | ||
|
|
6fbae768a4 | ||
|
|
8e9d0127e0 | ||
|
|
b92d70769a | ||
|
|
75b001cf6a | ||
|
|
479ca7884e | ||
|
|
00d8910118 | ||
|
|
bd28c7ab05 | ||
|
|
720279df65 | ||
|
|
2d4ec648b8 | ||
|
|
704fdaca4a | ||
|
|
b134136e51 | ||
|
|
5ca717d7e0 | ||
|
|
7068ee5e5a | ||
|
|
9f770473e5 | ||
|
|
5fe0f5750a | ||
|
|
b8ce225ccc | ||
|
|
f21f7388b7 | ||
|
|
a1354f231a | ||
|
|
4bc1c33e81 | ||
|
|
32bcbab523 | ||
|
|
b94ae21e11 | ||
|
|
a92ebd75c2 | ||
|
|
e6d3929fc9 | ||
|
|
a68e13af35 | ||
|
|
324c3ace8f | ||
|
|
3a9d5d9085 | ||
|
|
89a6eaf0ce | ||
|
|
027c4a0f4d | ||
|
|
81990c6768 | ||
|
|
ba02aa0e49 | ||
|
|
5adbdbd920 | ||
|
|
b3b211d956 | ||
|
|
279bd6d82d | ||
|
|
5dced28228 | ||
|
|
51bdaf64e4 | ||
|
|
9c8e0ebedc | ||
|
|
e1bea8a8c8 | ||
|
|
a6d3820104 | ||
|
|
36c793a5fb | ||
|
|
aade8a91c3 | ||
|
|
3fe7c3de1a | ||
|
|
596a5aed8d | ||
|
|
19b3675701 | ||
|
|
1713d0fd1e | ||
|
|
3a95a302c0 | ||
|
|
e738ba2334 | ||
|
|
c813215f3e | ||
|
|
0f63a2d271 | ||
|
|
133c34de53 | ||
|
|
a3ca735b12 | ||
|
|
519ab6a0cd | ||
|
|
0c691a540a | ||
|
|
209f78717f | ||
|
|
a02be80ac1 | ||
|
|
8a8b906b6f |
1
.github/FUNDING.yml
vendored
Normal file
1
.github/FUNDING.yml
vendored
Normal file
@@ -0,0 +1 @@
|
||||
github: Flaminel
|
||||
12
.github/ISSUE_TEMPLATE/1-bug.yml
vendored
12
.github/ISSUE_TEMPLATE/1-bug.yml
vendored
@@ -14,8 +14,12 @@ body:
|
||||
options:
|
||||
- label: Reviewed the documentation.
|
||||
required: true
|
||||
- label: Ensured I am using ghcr.io/flmorg/cleanuperr docker repository.
|
||||
required: true
|
||||
- label: Ensured I am using the latest version.
|
||||
required: true
|
||||
- label: Enabled verbose logging.
|
||||
required: true
|
||||
- type: textarea
|
||||
id: what-happened
|
||||
attributes:
|
||||
@@ -23,14 +27,6 @@ body:
|
||||
description: If applicable, mention what you expected to happen.
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: Version
|
||||
description: What version of our software are you running?
|
||||
placeholder: e.g. 1.3.0 or latest
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: os
|
||||
attributes:
|
||||
|
||||
13
.github/ISSUE_TEMPLATE/3-help.yml
vendored
13
.github/ISSUE_TEMPLATE/3-help.yml
vendored
@@ -7,6 +7,19 @@ body:
|
||||
attributes:
|
||||
value: |
|
||||
If you are experiencing unexpected behavior, please consider submitting a bug report instead.
|
||||
- type: checkboxes
|
||||
id: init
|
||||
attributes:
|
||||
label: "Before submitting a help request, I have:"
|
||||
options:
|
||||
- label: Reviewed the documentation.
|
||||
required: true
|
||||
- label: Ensured I am using ghcr.io/flmorg/cleanuperr docker repository.
|
||||
required: true
|
||||
- label: Ensured I am using the latest version.
|
||||
required: true
|
||||
- label: Enabled verbose logging.
|
||||
required: true
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
|
||||
49
.github/workflows/docs.yml
vendored
Normal file
49
.github/workflows/docs.yml
vendored
Normal file
@@ -0,0 +1,49 @@
|
||||
name: Deploy Docusaurus to GitHub Pages
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'docs/**'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pages: write
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
environment:
|
||||
name: github-pages
|
||||
url: ${{ steps.deployment.outputs.page_url }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20.x
|
||||
cache: yarn
|
||||
cache-dependency-path: docs/yarn.lock
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: docs
|
||||
run: yarn install --frozen-lockfile
|
||||
|
||||
- name: Build Docusaurus
|
||||
working-directory: docs
|
||||
run: yarn build
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: docs/build
|
||||
retention-days: 1
|
||||
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
301
README.md
301
README.md
@@ -1,290 +1,59 @@
|
||||
# <img width="24px" src="./Logo/256.png" alt="cleanuperr"></img> cleanuperr
|
||||
_Love this project? Give it a ⭐️ and let others know!_
|
||||
|
||||
cleanuperr is a tool for automating the cleanup of unwanted or blocked files in Sonarr, Radarr, and supported download clients like qBittorrent. It removes incomplete or blocked downloads, updates queues, and enforces blacklists or whitelists to manage file selection. After removing blocked content, cleanuperr can also trigger a search to replace the deleted shows/movies.
|
||||
# <img width="24px" src="./Logo/256.png" alt="cleanuperr"></img> Cleanuperr
|
||||
|
||||
cleanuperr was created primarily to address malicious files, such as `*.lnk` or `*.zipx`, that were getting stuck in Sonarr/Radarr and required manual intervention. Some of the reddit posts that made cleanuperr come to life can be found [here](https://www.reddit.com/r/sonarr/comments/1gqnx16/psa_sonarr_downloaded_a_virus/), [here](https://www.reddit.com/r/sonarr/comments/1gqwklr/sonar_downloaded_a_mkv_file_which_looked_like_a/), [here](https://www.reddit.com/r/sonarr/comments/1gpw2wa/downloaded_waiting_to_import/) and [here](https://www.reddit.com/r/sonarr/comments/1gpi344/downloads_not_importing_no_files_found/).
|
||||
[](https://discord.gg/sWggpnmGNY)
|
||||
|
||||
The tool supports both qBittorrent's built-in exclusion features and its own blocklist-based system. Binaries for all platforms are provided, along with Docker images for easy deployment.
|
||||
Cleanuperr is a tool for automating the cleanup of unwanted or blocked files in Sonarr, Radarr, and supported download clients like qBittorrent. It removes incomplete or blocked downloads, updates queues, and enforces blacklists or whitelists to manage file selection. After removing blocked content, Cleanuperr can also trigger a search to replace the deleted shows/movies.
|
||||
|
||||
Refer to the [Environment variables](#Environment-variables) section for detailed configuration instructions and the [Setup](#Setup) section for an in-depth explanation of the cleanup process.
|
||||
Cleanuperr was created primarily to address malicious files, such as `*.lnk` or `*.zipx`, that were getting stuck in Sonarr/Radarr and required manual intervention. Some of the reddit posts that made Cleanuperr come to life can be found [here](https://www.reddit.com/r/sonarr/comments/1gqnx16/psa_sonarr_downloaded_a_virus/), [here](https://www.reddit.com/r/sonarr/comments/1gqwklr/sonar_downloaded_a_mkv_file_which_looked_like_a/), [here](https://www.reddit.com/r/sonarr/comments/1gpw2wa/downloaded_waiting_to_import/) and [here](https://www.reddit.com/r/sonarr/comments/1gpi344/downloads_not_importing_no_files_found/).
|
||||
|
||||
## Key features
|
||||
- Marks unwanted files as skip/unwanted in the download client.
|
||||
- Automatically strikes stalled or stuck downloads.
|
||||
- Removes and blocks downloads that reached the maximum number of strikes or are marked as unwanted by the download client or by cleanuperr and triggers a search for removed downloads.
|
||||
> [!IMPORTANT]
|
||||
> **Features:**
|
||||
> - Strike system to mark stalled or downloads stuck in metadata downloading.
|
||||
> - Remove and block downloads that reached a maximum number of strikes.
|
||||
> - Remove and block downloads that have a low download speed or high estimated completion time.
|
||||
> - Remove downloads blocked by qBittorrent or by Cleanuperr's **content blocker**.
|
||||
> - Trigger a search for downloads removed from the *arrs.
|
||||
> - Clean up downloads that have been seeding for a certain amount of time.
|
||||
> - Notify on strike or download removal.
|
||||
> - Ignore certain torrent hashes, categories, tags or trackers from being processed by Cleanuperr.
|
||||
|
||||
## Important note
|
||||
Cleanuperr supports both qBittorrent's built-in exclusion features and its own blocklist-based system. Binaries for all platforms are provided, along with Docker images for easy deployment.
|
||||
|
||||
Only the **latest versions** of the following apps are supported, or earlier versions that have the same API as the latest version:
|
||||
- qBittorrent
|
||||
- Deluge
|
||||
- Transmission
|
||||
- Sonarr
|
||||
- Radarr
|
||||
- Lidarr
|
||||
## Quick Start
|
||||
|
||||
This tool is actively developed and still a work in progress, so using the `latest` Docker tag may result in breaking changes. Join the Discord server if you want to reach out to me quickly (or just stay updated on new releases) so we can squash those pesky bugs together:
|
||||
> [!NOTE]
|
||||
>
|
||||
> 1. **Docker (Recommended)**
|
||||
> Pull the Docker image from `ghcr.io/flmorg/cleanuperr:latest`.
|
||||
>
|
||||
> 2. **Unraid (for Unraid users)**
|
||||
> Use the Unraid Community App.
|
||||
>
|
||||
> 3. **Manual Installation (if you're not using Docker)**
|
||||
> Go to [Windows](#windows), [Linux](#linux) or [MacOS](#macos).
|
||||
|
||||
> https://discord.gg/sWggpnmGNY
|
||||
# Docs
|
||||
|
||||
# How it works
|
||||
Docs can be found [here](https://flmorg.github.io/cleanuperr/).
|
||||
|
||||
1. **Content blocker** will:
|
||||
- Run every 5 minutes (or configured cron).
|
||||
- Process all items in the *arr queue.
|
||||
- Find the corresponding item from the download client for each queue item.
|
||||
- Mark the files that were found in the queue as **unwanted/skipped** if:
|
||||
- They **are listed in the blacklist**, or
|
||||
- They **are not included in the whitelist**.
|
||||
- If **all files** of a download **are unwanted**:
|
||||
- It will be removed from the *arr's queue and blocked.
|
||||
- It will be deleted from the download client.
|
||||
- A new search will be triggered for the *arr item.
|
||||
2. **Queue cleaner** will:
|
||||
- Run every 5 minutes (or configured cron, or right after `content blocker`).
|
||||
- Process all items in the *arr queue.
|
||||
- Check each queue item if it is **stalled (download speed is 0)**, **stuck in matadata downloading** or **failed to be imported**.
|
||||
- If it is, the item receives a **strike** and will continue to accumulate strikes every time it meets any of these conditions.
|
||||
- Check each queue item if it meets one of the following condition in the download client:
|
||||
- **Marked as completed, but 0 bytes have been downloaded** (due to files being blocked by qBittorrent or the **content blocker**).
|
||||
- All associated files of are marked as **unwanted/skipped**.
|
||||
- If the item **DOES NOT** match the above criteria, it will be skipped.
|
||||
- If the item **DOES** match the criteria or has received the **maximum number of strikes**:
|
||||
- It will be removed from the *arr's queue and blocked.
|
||||
- It will be deleted from the download client.
|
||||
- A new search will be triggered for the *arr item.
|
||||
# <img width="24px" src="./Logo/256.png" alt="Cleanuperr"> Cleanuperr <svg width="14px" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><!--!Font Awesome Free 6.7.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free Copyright 2025 Fonticons, Inc.--><path d="M376.6 84.5c11.3-13.6 9.5-33.8-4.1-45.1s-33.8-9.5-45.1 4.1L192 206 56.6 43.5C45.3 29.9 25.1 28.1 11.5 39.4S-3.9 70.9 7.4 84.5L150.3 256 7.4 427.5c-11.3 13.6-9.5 33.8 4.1 45.1s33.8 9.5 45.1-4.1L192 306 327.4 468.5c11.3 13.6 31.5 15.4 45.1 4.1s15.4-31.5 4.1-45.1L233.7 256 376.6 84.5z"/></svg> Huntarr <img width="24px" src="https://github.com/plexguide/Huntarr.io/blob/main/frontend/static/logo/256.png?raw=true" alt Huntarr></img>
|
||||
|
||||
# Setup
|
||||
Think of **Cleanuperr** as the janitor of your server; it keeps your download queue spotless, removes clutter, and blocks malicious files. Now imagine combining that with **Huntarr**, the compulsive librarian who finds missing and upgradable media to complete your collection
|
||||
|
||||
## Using qBittorrent's built-in feature (works only with qBittorrent)
|
||||
While **Huntarr** fills in the blanks and improves what you already have, **Cleanuperr** makes sure that only clean downloads get through. If you're aiming for a reliable and self-sufficient setup, **Cleanuperr** and **Huntarr** will take your automated media stack to another level.
|
||||
|
||||
1. Go to qBittorrent -> Options -> Downloads -> make sure `Excluded file names` is checked -> Paste an exclusion list that you have copied.
|
||||
- [blacklist](https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist), or
|
||||
- [permissive blacklist](https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist_permissive), or
|
||||
- create your own
|
||||
2. qBittorrent will block files from being downloaded. In the case of malicious content, **nothing is downloaded and the torrent is marked as complete**.
|
||||
3. Start **cleanuperr** with `QUEUECLEANER__ENABLED` set to `true`.
|
||||
4. The **queue cleaner** will perform a cleanup process as described in the [How it works](#how-it-works) section.
|
||||
|
||||
## Using cleanuperr's blocklist (works with all supported download clients)
|
||||
|
||||
1. Set both `QUEUECLEANER__ENABLED` and `CONTENTBLOCKER__ENABLED` to `true` in your environment variables.
|
||||
2. Configure and enable either a **blacklist** or a **whitelist** as described in the [Arr variables](#Arr-variables) section.
|
||||
3. Once configured, cleanuperr will perform the following tasks:
|
||||
- Execute the **content blocker** job, as explained in the [How it works](#how-it-works) section.
|
||||
- Execute the **queue cleaner** job, as explained in the [How it works](#how-it-works) section.
|
||||
|
||||
## Using cleanuperr just for failed *arr imports (works for Usenet users as well)
|
||||
|
||||
1. Set `QUEUECLEANER__ENABLED` to `true`.
|
||||
2. Set `QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES` to a desired value.
|
||||
3. Optionally set failed import message patterns to ignore using `QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__<NUMBER>`.
|
||||
4. Set `DOWNLOAD_CLIENT` to `none`.
|
||||
|
||||
**No other action involving a download client would work (e.g. content blocking, removing stalled downloads, excluding private trackers).**
|
||||
|
||||
## Usage
|
||||
|
||||
### Docker compose yaml
|
||||
|
||||
```
|
||||
version: "3.3"
|
||||
services:
|
||||
cleanuperr:
|
||||
volumes:
|
||||
- ./cleanuperr/logs:/var/logs
|
||||
environment:
|
||||
- LOGGING__LOGLEVEL=Information
|
||||
- LOGGING__FILE__ENABLED=false
|
||||
- LOGGING__FILE__PATH=/var/logs/
|
||||
- LOGGING__ENHANCED=true
|
||||
|
||||
- TRIGGERS__QUEUECLEANER=0 0/5 * * * ?
|
||||
- TRIGGERS__CONTENTBLOCKER=0 0/5 * * * ?
|
||||
|
||||
- QUEUECLEANER__ENABLED=true
|
||||
- QUEUECLEANER__RUNSEQUENTIALLY=true
|
||||
- QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES=5
|
||||
- QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE=false
|
||||
# - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0=title mismatch
|
||||
# - QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__1=manual import required
|
||||
- QUEUECLEANER__STALLED_MAX_STRIKES=5
|
||||
- QUEUECLEANER__STALLED_IGNORE_PRIVATE=false
|
||||
|
||||
- CONTENTBLOCKER__ENABLED=true
|
||||
- CONTENTBLOCKER__IGNORE_PRIVATE=true
|
||||
|
||||
- DOWNLOAD_CLIENT=none
|
||||
# OR
|
||||
# - DOWNLOAD_CLIENT=qBittorrent
|
||||
# - QBITTORRENT__URL=http://localhost:8080
|
||||
# - QBITTORRENT__USERNAME=user
|
||||
# - QBITTORRENT__PASSWORD=pass
|
||||
# OR
|
||||
# - DOWNLOAD_CLIENT=deluge
|
||||
# - DELUGE__URL=http://localhost:8112
|
||||
# - DELUGE__PASSWORD=testing
|
||||
# OR
|
||||
# - DOWNLOAD_CLIENT=transmission
|
||||
# - TRANSMISSION__URL=http://localhost:9091
|
||||
# - TRANSMISSION__USERNAME=test
|
||||
# - TRANSMISSION__PASSWORD=testing
|
||||
|
||||
- SONARR__ENABLED=true
|
||||
- SONARR__SEARCHTYPE=Episode
|
||||
- SONARR__BLOCK__TYPE=blacklist
|
||||
- SONARR__BLOCK__PATH=https://example.com/path/to/file.txt
|
||||
- SONARR__INSTANCES__0__URL=http://localhost:8989
|
||||
- SONARR__INSTANCES__0__APIKEY=secret1
|
||||
- SONARR__INSTANCES__1__URL=http://localhost:8990
|
||||
- SONARR__INSTANCES__1__APIKEY=secret2
|
||||
|
||||
- RADARR__ENABLED=true
|
||||
- RADARR__BLOCK__TYPE=blacklist
|
||||
- RADARR__BLOCK__PATH=https://example.com/path/to/file.txt
|
||||
- RADARR__INSTANCES__0__URL=http://localhost:7878
|
||||
- RADARR__INSTANCES__0__APIKEY=secret3
|
||||
- RADARR__INSTANCES__1__URL=http://localhost:7879
|
||||
- RADARR__INSTANCES__1__APIKEY=secret4
|
||||
|
||||
- LIDARR__ENABLED=true
|
||||
- LIDARR__BLOCK__TYPE=blacklist
|
||||
- LIDARR__BLOCK__PATH=https://example.com/path/to/file.txt
|
||||
- LIDARR__INSTANCES__0__URL=http://radarr:8686
|
||||
- LIDARR__INSTANCES__0__APIKEY=secret5
|
||||
- LIDARR__INSTANCES__1__URL=http://radarr:8687
|
||||
- LIDARR__INSTANCES__1__APIKEY=secret6
|
||||
image: ghcr.io/flmorg/cleanuperr:latest
|
||||
restart: unless-stopped
|
||||
```
|
||||
|
||||
## Environment variables
|
||||
|
||||
### General variables
|
||||
<details>
|
||||
<summary>Click here</summary>
|
||||
|
||||
| Variable | Required | Description | Default value |
|
||||
|---|---|---|---|
|
||||
| LOGGING__LOGLEVEL | No | Can be `Verbose`, `Debug`, `Information`, `Warning`, `Error` or `Fatal` | `Information` |
|
||||
| LOGGING__FILE__ENABLED | No | Enable or disable logging to file | false |
|
||||
| LOGGING__FILE__PATH | No | Directory where to save the log files | empty |
|
||||
| LOGGING__ENHANCED | No | Enhance logs whenever possible<br>A more detailed description is provided [here](variables.md#LOGGING__ENHANCED) | true |
|
||||
|||||
|
||||
| TRIGGERS__QUEUECLEANER | Yes if queue cleaner is enabled | [Quartz cron trigger](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html)<br>Can be a max of 6h interval<br>**Is ignored if `QUEUECLEANER__RUNSEQUENTIALLY=true` and `CONTENTBLOCKER__ENABLED=true`** | 0 0/5 * * * ? |
|
||||
| TRIGGERS__CONTENTBLOCKER | Yes if content blocker is enabled | [Quartz cron trigger](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html)<br>Can be a max of 6h interval | 0 0/5 * * * ? |
|
||||
|||||
|
||||
| QUEUECLEANER__ENABLED | No | Enable or disable the queue cleaner | true |
|
||||
| QUEUECLEANER__RUNSEQUENTIALLY | No | If set to true, the queue cleaner will run after the content blocker instead of running in parallel, streamlining the cleaning process | true |
|
||||
| QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES | No | After how many strikes should a failed import be removed<br>0 means never | 0 |
|
||||
| QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE | No | Whether to ignore failed imports from private trackers | false |
|
||||
| QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__0 | No | First pattern to look for when an import is failed<br>If the specified message pattern is found, the item is skipped | empty |
|
||||
| QUEUECLEANER__STALLED_MAX_STRIKES | No | After how many strikes should a stalled download be removed<br>0 means never | 0 |
|
||||
| QUEUECLEANER__STALLED_IGNORE_PRIVATE | No | Whether to ignore stalled downloads from private trackers | false |
|
||||
|||||
|
||||
| CONTENTBLOCKER__ENABLED | No | Enable or disable the content blocker | false |
|
||||
| CONTENTBLOCKER__IGNORE_PRIVATE | No | Whether to ignore downloads from private trackers | false |
|
||||
</details>
|
||||
|
||||
### Download client variables
|
||||
<details>
|
||||
<summary>Click here</summary>
|
||||
|
||||
| Variable | Required | Description | Default value |
|
||||
|---|---|---|---|
|
||||
| DOWNLOAD_CLIENT | No | Download client that is used by *arrs<br>Can be `qbittorrent`, `deluge`, `transmission` or `none` | `none` |
|
||||
| QBITTORRENT__URL | No | qBittorrent instance url | http://localhost:8112 |
|
||||
| QBITTORRENT__USERNAME | No | qBittorrent user | empty |
|
||||
| QBITTORRENT__PASSWORD | No | qBittorrent password | empty |
|
||||
|||||
|
||||
| DELUGE__URL | No | Deluge instance url | http://localhost:8080 |
|
||||
| DELUGE__PASSWORD | No | Deluge password | empty |
|
||||
|||||
|
||||
| TRANSMISSION__URL | No | Transmission instance url | http://localhost:9091 |
|
||||
| TRANSMISSION__USERNAME | No | Transmission user | empty |
|
||||
| TRANSMISSION__PASSWORD | No | Transmission password | empty |
|
||||
</details>
|
||||
|
||||
### Arr variables
|
||||
<details>
|
||||
<summary>Click here</summary>
|
||||
|
||||
| Variable | Required | Description | Default value |
|
||||
|---|---|---|---|
|
||||
| SONARR__ENABLED | No | Enable or disable Sonarr cleanup | false |
|
||||
| SONARR__BLOCK__TYPE | No | Block type<br>Can be `blacklist` or `whitelist` | `blacklist` |
|
||||
| SONARR__BLOCK__PATH | No | Path to the blocklist (local file or url)<br>Needs to be json compatible | empty |
|
||||
| SONARR__SEARCHTYPE | No | What to search for after removing a queue item<br>Can be `Episode`, `Season` or `Series` | `Episode` |
|
||||
| SONARR__INSTANCES__0__URL | No | First Sonarr instance url | http://localhost:8989 |
|
||||
| SONARR__INSTANCES__0__APIKEY | No | First Sonarr instance API key | empty |
|
||||
|||||
|
||||
| RADARR__ENABLED | No | Enable or disable Radarr cleanup | false |
|
||||
| RADARR__BLOCK__TYPE | No | Block type<br>Can be `blacklist` or `whitelist` | `blacklist` |
|
||||
| RADARR__BLOCK__PATH | No | Path to the blocklist (local file or url)<br>Needs to be json compatible | empty |
|
||||
| RADARR__INSTANCES__0__URL | No | First Radarr instance url | http://localhost:7878 |
|
||||
| RADARR__INSTANCES__0__APIKEY | No | First Radarr instance API key | empty |
|
||||
|||||
|
||||
| LIDARR__ENABLED | No | Enable or disable LIDARR cleanup | false |
|
||||
| LIDARR__BLOCK__TYPE | No | Block type<br>Can be `blacklist` or `whitelist` | `blacklist` |
|
||||
| LIDARR__BLOCK__PATH | No | Path to the blocklist (local file or url)<br>Needs to be json compatible | empty |
|
||||
| LIDARR__INSTANCES__0__URL | No | First LIDARR instance url | http://localhost:8686 |
|
||||
| LIDARR__INSTANCES__0__APIKEY | No | First LIDARR instance API key | empty |
|
||||
</details>
|
||||
|
||||
### Advanced variables
|
||||
<details>
|
||||
<summary>Click here</summary>
|
||||
|
||||
| Variable | Required | Description | Default value |
|
||||
|---|---|---|---|
|
||||
| HTTP_MAX_RETRIES | No | The number of times to retry a failed HTTP call (to *arrs, download clients etc.) | 0 |
|
||||
| HTTP_TIMEOUT | No | The number of seconds to wait before failing an HTTP call (to *arrs, download clients etc.) | 100 |
|
||||
</details>
|
||||
|
||||
#
|
||||
|
||||
### To be noted
|
||||
|
||||
1. The queue cleaner and content blocker can be enabled or disabled separately, if you want to run only one of them.
|
||||
2. Only one download client can be enabled at a time. If you have more than one download client, you should deploy multiple instances of cleanuperr.
|
||||
3. The blocklists (blacklist/whitelist) should have a single pattern on each line and supports the following:
|
||||
```
|
||||
*example // file name ends with "example"
|
||||
example* // file name starts with "example"
|
||||
*example* // file name has "example" in the name
|
||||
example // file name is exactly the word "example"
|
||||
regex:<ANY_REGEX> // regex that needs to be marked at the start of the line with "regex:"
|
||||
```
|
||||
4. Multiple Sonarr/Radarr/Lidarr instances can be specified using this format, where `<NUMBER>` starts from `0`:
|
||||
```
|
||||
SONARR__INSTANCES__<NUMBER>__URL
|
||||
SONARR__INSTANCES__<NUMBER>__APIKEY
|
||||
```
|
||||
5. Multiple failed import patterns can be specified using this format, where `<NUMBER>` starts from 0:
|
||||
```
|
||||
QUEUECLEANER__IMPORT_FAILED_IGNORE_PATTERNS__<NUMBER>
|
||||
```
|
||||
6. [This blacklist](https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist) and [this whitelist](https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/whitelist) can be used for Sonarr and Radarr, but they are not suitable for other *arrs.
|
||||
#
|
||||
|
||||
### Binaries (if you're not using Docker)
|
||||
|
||||
1. Download the binaries from [releases](https://github.com/flmorg/cleanuperr/releases).
|
||||
2. Extract them from the zip file.
|
||||
3. Edit **appsettings.json**. The paths from this json file correspond with the docker env vars, as described [above](#environment-variables).
|
||||
|
||||
### Run as a Windows Service
|
||||
|
||||
Check out this stackoverflow answer on how to do it: https://stackoverflow.com/a/15719678
|
||||
<span style="font-size:24px"> ➡️ [**Huntarr**](https://github.com/plexguide/Huntarr.io) </span>
|
||||
|
||||
# Credits
|
||||
Special thanks for inspiration go to:
|
||||
- [ThijmenGThN/swaparr](https://github.com/ThijmenGThN/swaparr)
|
||||
- [ManiMatter/decluttarr](https://github.com/ManiMatter/decluttarr)
|
||||
- [PaeyMoopy/sonarr-radarr-queue-cleaner](https://github.com/PaeyMoopy/sonarr-radarr-queue-cleaner)
|
||||
- [Sonarr](https://github.com/Sonarr/Sonarr) & [Radarr](https://github.com/Radarr/Radarr) for the logo
|
||||
- [Sonarr](https://github.com/Sonarr/Sonarr) & [Radarr](https://github.com/Radarr/Radarr)
|
||||
|
||||
# Buy me a coffee
|
||||
If I made your life just a tiny bit easier, consider buying me a coffee!
|
||||
|
||||
<a href="https://buymeacoffee.com/flaminel" target="_blank"><img src="https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png" alt="Buy Me A Coffee" style="height: 41px !important;width: 174px !important;box-shadow: 0px 3px 2px 0px rgba(190, 190, 190, 0.5) !important;-webkit-box-shadow: 0px 3px 2px 0px rgba(190, 190, 190, 0.5) !important;" ></a>
|
||||
<a href="https://buymeacoffee.com/flaminel" target="_blank"><img src="https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png" alt="Buy Me A Coffee" style="height: 41px !important;width: 174px !important;box-shadow: 0px 3px 2px 0px rgba(190, 190, 190, 0.5) !important;-webkit-box-shadow: 0px 3px 2px 0px rgba(190, 190, 190, 0.5) !important;" ></a>
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
*.apk
|
||||
*.arj
|
||||
*.bat
|
||||
*.bin
|
||||
*.bmp
|
||||
|
||||
@@ -10,50 +10,89 @@ deployment:
|
||||
repository: ghcr.io/flmorg/cleanuperr
|
||||
tag: latest
|
||||
env:
|
||||
- name: DRY_RUN
|
||||
value: "false"
|
||||
|
||||
- name: LOGGING__LOGLEVEL
|
||||
value: Information
|
||||
value: Verbose
|
||||
- name: LOGGING__FILE__ENABLED
|
||||
value: "true"
|
||||
- name: LOGGING__FILE__PATH
|
||||
value: /var/logs
|
||||
- name: LOGGING__ENHANCED
|
||||
value: "true"
|
||||
|
||||
- name: TRIGGERS__QUEUECLEANER
|
||||
value: 0 0/5 * * * ?
|
||||
- name: TRIGGERS__CONTENTBLOCKER
|
||||
value: 0 0/5 * * * ?
|
||||
|
||||
- name: QUEUECLEANER__ENABLED
|
||||
value: "true"
|
||||
- name: QUEUECLEANER__RUNSEQUENTIALLY
|
||||
value: "true"
|
||||
- name: QUEUECLEANER__IMPORT_FAILED_MAX_STRIKES
|
||||
value: "3"
|
||||
- name: QUEUECLEANER__IMPORT_FAILED_IGNORE_PRIVATE
|
||||
value: "false"
|
||||
- name: QUEUECLEANER__IMPORT_FAILED_DELETE_PRIVATE
|
||||
value: "false"
|
||||
- name: QUEUECLEANER__STALLED_MAX_STRIKES
|
||||
value: "3"
|
||||
- name: QUEUECLEANER__STALLED_IGNORE_PRIVATE
|
||||
value: "false"
|
||||
- name: QUEUECLEANER__STALLED_DELETE_PRIVATE
|
||||
value: "false"
|
||||
|
||||
- name: CONTENTBLOCKER__ENABLED
|
||||
value: "true"
|
||||
- name: CONTENTBLOCKER__BLACKLIST__ENABLED
|
||||
- name: CONTENTBLOCKER__IGNORE_PRIVATE
|
||||
value: "true"
|
||||
- name: CONTENTBLOCKER__BLACKLIST__PATH
|
||||
value: https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist
|
||||
- name: CONTENTBLOCKER__DELETE_PRIVATE
|
||||
value: "false"
|
||||
|
||||
- name: DOWNLOADCLEANER__ENABLED
|
||||
value: "false"
|
||||
|
||||
- name: DOWNLOAD_CLIENT
|
||||
value: qbittorrent
|
||||
- name: QBITTORRENT__URL
|
||||
value: http://service.qbittorrent-videos.svc.cluster.local
|
||||
|
||||
- name: SONARR__ENABLED
|
||||
value: "true"
|
||||
- name: SONARR__SEARCHTYPE
|
||||
value: Episode
|
||||
- name: SONARR__BLOCK__TYPE
|
||||
value: blacklist
|
||||
- name: SONARR__BLOCK__PATH
|
||||
value: https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist
|
||||
- name: SONARR__INSTANCES__0__URL
|
||||
value: http://service.sonarr-low-res.svc.cluster.local
|
||||
- name: SONARR__INSTANCES__1__URL
|
||||
value: http://service.sonarr-high-res.svc.cluster.local
|
||||
|
||||
- name: RADARR__ENABLED
|
||||
value: "true"
|
||||
- name: RADARR__BLOCK__TYPE
|
||||
value: blacklist
|
||||
- name: RADARR__BLOCK__PATH
|
||||
value: https://raw.githubusercontent.com/flmorg/cleanuperr/refs/heads/main/blacklist
|
||||
- name: RADARR__INSTANCES__0__URL
|
||||
value: http://service.radarr-low-res.svc.cluster.local
|
||||
- name: RADARR__INSTANCES__1__URL
|
||||
value: http://service.radarr-high-res.svc.cluster.local
|
||||
|
||||
- name: NOTIFIARR__ON_IMPORT_FAILED_STRIKE
|
||||
value: "true"
|
||||
- name: NOTIFIARR__ON_STALLED_STRIKE
|
||||
value: "true"
|
||||
- name: NOTIFIARR__ON_QUEUE_ITEM_DELETED
|
||||
value: "true"
|
||||
- name: NOTIFIARR__ON_DOWNLOAD_CLEANED
|
||||
value: "true"
|
||||
- name: NOTIFIARR__CHANNEL_ID
|
||||
value: "1340708411259748413"
|
||||
envFromSecret:
|
||||
- secretName: qbit-auth
|
||||
envs:
|
||||
@@ -73,6 +112,10 @@ deployment:
|
||||
key: RDRL_API_KEY
|
||||
- name: RADARR__INSTANCES__1__APIKEY
|
||||
key: RDRH_API_KEY
|
||||
- secretName: notifiarr-auth
|
||||
envs:
|
||||
- name: NOTIFIARR__API_KEY
|
||||
key: API_KEY
|
||||
resources:
|
||||
requests:
|
||||
cpu: 0m
|
||||
@@ -112,4 +155,8 @@ vaultSecrets:
|
||||
path: secrets/sonarr
|
||||
templates:
|
||||
SNRL_API_KEY: "{% .Secrets.low_api_key %}"
|
||||
SNRH_API_KEY: "{% .Secrets.high_api_key %}"
|
||||
SNRH_API_KEY: "{% .Secrets.high_api_key %}"
|
||||
- name: notifiarr-auth
|
||||
path: secrets/notifiarr
|
||||
templates:
|
||||
API_KEY: "{% .Secrets.passthrough_api_key %}"
|
||||
6
code/Common/Attributes/DryRunSafeguardAttribute.cs
Normal file
6
code/Common/Attributes/DryRunSafeguardAttribute.cs
Normal file
@@ -0,0 +1,6 @@
|
||||
namespace Common.Attributes;
|
||||
|
||||
[AttributeUsage(AttributeTargets.Method, Inherited = true)]
|
||||
public class DryRunSafeguardAttribute : Attribute
|
||||
{
|
||||
}
|
||||
@@ -7,7 +7,7 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="9.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="9.0.2" />
|
||||
<PackageReference Include="Serilog" Version="4.2.0" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -11,9 +11,9 @@ public abstract record ArrConfig
|
||||
public required List<ArrInstance> Instances { get; init; }
|
||||
}
|
||||
|
||||
public record Block
|
||||
public readonly record struct Block
|
||||
{
|
||||
public BlocklistType Type { get; set; }
|
||||
public BlocklistType Type { get; init; }
|
||||
|
||||
public string? Path { get; set; }
|
||||
public string? Path { get; init; }
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace Common.Configuration.ContentBlocker;
|
||||
|
||||
public sealed record ContentBlockerConfig : IJobConfig
|
||||
public sealed record ContentBlockerConfig : IJobConfig, IIgnoredDownloadsConfig
|
||||
{
|
||||
public const string SectionName = "ContentBlocker";
|
||||
|
||||
@@ -11,6 +11,12 @@ public sealed record ContentBlockerConfig : IJobConfig
|
||||
[ConfigurationKeyName("IGNORE_PRIVATE")]
|
||||
public bool IgnorePrivate { get; init; }
|
||||
|
||||
[ConfigurationKeyName("DELETE_PRIVATE")]
|
||||
public bool DeletePrivate { get; init; }
|
||||
|
||||
[ConfigurationKeyName("IGNORED_DOWNLOADS_PATH")]
|
||||
public string? IgnoredDownloadsPath { get; init; }
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
}
|
||||
|
||||
45
code/Common/Configuration/DownloadCleaner/CleanCategory.cs
Normal file
45
code/Common/Configuration/DownloadCleaner/CleanCategory.cs
Normal file
@@ -0,0 +1,45 @@
|
||||
using Common.Exceptions;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace Common.Configuration.DownloadCleaner;
|
||||
|
||||
public sealed record CleanCategory : IConfig
|
||||
{
|
||||
public required string Name { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Max ratio before removing a download.
|
||||
/// </summary>
|
||||
[ConfigurationKeyName("MAX_RATIO")]
|
||||
public required double MaxRatio { get; init; } = -1;
|
||||
|
||||
/// <summary>
|
||||
/// Min number of hours to seed before removing a download, if the ratio has been met.
|
||||
/// </summary>
|
||||
[ConfigurationKeyName("MIN_SEED_TIME")]
|
||||
public required double MinSeedTime { get; init; } = 0;
|
||||
|
||||
/// <summary>
|
||||
/// Number of hours to seed before removing a download.
|
||||
/// </summary>
|
||||
[ConfigurationKeyName("MAX_SEED_TIME")]
|
||||
public required double MaxSeedTime { get; init; } = -1;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(Name))
|
||||
{
|
||||
throw new ValidationException($"{nameof(Name)} can not be empty");
|
||||
}
|
||||
|
||||
if (MaxRatio < 0 && MaxSeedTime < 0)
|
||||
{
|
||||
throw new ValidationException($"both {nameof(MaxRatio)} and {nameof(MaxSeedTime)} are disabled");
|
||||
}
|
||||
|
||||
if (MinSeedTime < 0)
|
||||
{
|
||||
throw new ValidationException($"{nameof(MinSeedTime)} can not be negative");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
using Common.Exceptions;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace Common.Configuration.DownloadCleaner;
|
||||
|
||||
public sealed record DownloadCleanerConfig : IJobConfig, IIgnoredDownloadsConfig
|
||||
{
|
||||
public const string SectionName = "DownloadCleaner";
|
||||
|
||||
public bool Enabled { get; init; }
|
||||
|
||||
public List<CleanCategory>? Categories { get; init; }
|
||||
|
||||
[ConfigurationKeyName("DELETE_PRIVATE")]
|
||||
public bool DeletePrivate { get; init; }
|
||||
|
||||
[ConfigurationKeyName("IGNORED_DOWNLOADS_PATH")]
|
||||
public string? IgnoredDownloadsPath { get; init; }
|
||||
|
||||
[ConfigurationKeyName("UNLINKED_TARGET_CATEGORY")]
|
||||
public string UnlinkedTargetCategory { get; init; } = "cleanuperr-unlinked";
|
||||
|
||||
[ConfigurationKeyName("UNLINKED_IGNORED_ROOT_DIR")]
|
||||
public string UnlinkedIgnoredRootDir { get; init; } = string.Empty;
|
||||
|
||||
[ConfigurationKeyName("UNLINKED_CATEGORIES")]
|
||||
public List<string>? UnlinkedCategories { get; init; }
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (!Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (Categories?.Count is null or 0)
|
||||
{
|
||||
throw new ValidationException("no categories configured");
|
||||
}
|
||||
|
||||
if (Categories?.GroupBy(x => x.Name).Any(x => x.Count() > 1) is true)
|
||||
{
|
||||
throw new ValidationException("duplicated clean categories found");
|
||||
}
|
||||
|
||||
Categories?.ForEach(x => x.Validate());
|
||||
|
||||
if (string.IsNullOrEmpty(UnlinkedTargetCategory))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (UnlinkedCategories?.Count is null or 0)
|
||||
{
|
||||
throw new ValidationException("no unlinked categories configured");
|
||||
}
|
||||
|
||||
if (UnlinkedCategories.Contains(UnlinkedTargetCategory))
|
||||
{
|
||||
throw new ValidationException($"{SectionName.ToUpperInvariant()}__UNLINKED_TARGET_CATEGORY should not be present in {SectionName.ToUpperInvariant()}__UNLINKED_CATEGORIES");
|
||||
}
|
||||
|
||||
if (UnlinkedCategories.Any(string.IsNullOrEmpty))
|
||||
{
|
||||
throw new ValidationException("empty unlinked category filter found");
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(UnlinkedIgnoredRootDir) && !Directory.Exists(UnlinkedIgnoredRootDir))
|
||||
{
|
||||
throw new ValidationException($"{UnlinkedIgnoredRootDir} root directory does not exist");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,7 @@
|
||||
namespace Common.Configuration.DownloadClient;
|
||||
using Common.Exceptions;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace Common.Configuration.DownloadClient;
|
||||
|
||||
public sealed record DelugeConfig : IConfig
|
||||
{
|
||||
@@ -6,13 +9,16 @@ public sealed record DelugeConfig : IConfig
|
||||
|
||||
public Uri? Url { get; init; }
|
||||
|
||||
[ConfigurationKeyName("URL_BASE")]
|
||||
public string UrlBase { get; init; } = string.Empty;
|
||||
|
||||
public string? Password { get; init; }
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (Url is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(Url));
|
||||
throw new ValidationException($"{nameof(Url)} is empty");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,7 @@
|
||||
namespace Common.Configuration.DownloadClient;
|
||||
using Common.Exceptions;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace Common.Configuration.DownloadClient;
|
||||
|
||||
public sealed class QBitConfig : IConfig
|
||||
{
|
||||
@@ -6,6 +9,9 @@ public sealed class QBitConfig : IConfig
|
||||
|
||||
public Uri? Url { get; init; }
|
||||
|
||||
[ConfigurationKeyName("URL_BASE")]
|
||||
public string UrlBase { get; init; } = string.Empty;
|
||||
|
||||
public string? Username { get; init; }
|
||||
|
||||
public string? Password { get; init; }
|
||||
@@ -14,7 +20,7 @@ public sealed class QBitConfig : IConfig
|
||||
{
|
||||
if (Url is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(Url));
|
||||
throw new ValidationException($"{nameof(Url)} is empty");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,7 @@
|
||||
namespace Common.Configuration.DownloadClient;
|
||||
using Common.Exceptions;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace Common.Configuration.DownloadClient;
|
||||
|
||||
public record TransmissionConfig : IConfig
|
||||
{
|
||||
@@ -6,6 +9,9 @@ public record TransmissionConfig : IConfig
|
||||
|
||||
public Uri? Url { get; init; }
|
||||
|
||||
[ConfigurationKeyName("URL_BASE")]
|
||||
public string UrlBase { get; init; } = "transmission";
|
||||
|
||||
public string? Username { get; init; }
|
||||
|
||||
public string? Password { get; init; }
|
||||
@@ -14,7 +20,7 @@ public record TransmissionConfig : IConfig
|
||||
{
|
||||
if (Url is null)
|
||||
{
|
||||
throw new ArgumentNullException(nameof(Url));
|
||||
throw new ValidationException($"{nameof(Url)} is empty");
|
||||
}
|
||||
}
|
||||
}
|
||||
9
code/Common/Configuration/General/DryRunConfig.cs
Normal file
9
code/Common/Configuration/General/DryRunConfig.cs
Normal file
@@ -0,0 +1,9 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace Common.Configuration.General;
|
||||
|
||||
public sealed record DryRunConfig
|
||||
{
|
||||
[ConfigurationKeyName("DRY_RUN")]
|
||||
public bool IsDryRun { get; init; }
|
||||
}
|
||||
@@ -1,20 +1,25 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Common.Enums;
|
||||
using Common.Exceptions;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace Common.Configuration.General;
|
||||
|
||||
public class HttpConfig : IConfig
|
||||
public sealed record HttpConfig : IConfig
|
||||
{
|
||||
[ConfigurationKeyName("HTTP_MAX_RETRIES")]
|
||||
public ushort MaxRetries { get; init; }
|
||||
|
||||
[ConfigurationKeyName("HTTP_TIMEOUT")]
|
||||
public ushort Timeout { get; init; } = 100;
|
||||
|
||||
[ConfigurationKeyName("HTTP_VALIDATE_CERT")]
|
||||
public CertificateValidationType CertificateValidation { get; init; } = CertificateValidationType.Enabled;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (Timeout is 0)
|
||||
{
|
||||
throw new ArgumentException("HTTP_TIMEOUT must be greater than 0");
|
||||
throw new ValidationException("HTTP_TIMEOUT must be greater than 0");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,4 +7,6 @@ public sealed class TriggersConfig
|
||||
public required string QueueCleaner { get; init; }
|
||||
|
||||
public required string ContentBlocker { get; init; }
|
||||
|
||||
public required string DownloadCleaner { get; init; }
|
||||
}
|
||||
6
code/Common/Configuration/IIgnoredDownloadsConfig.cs
Normal file
6
code/Common/Configuration/IIgnoredDownloadsConfig.cs
Normal file
@@ -0,0 +1,6 @@
|
||||
namespace Common.Configuration;
|
||||
|
||||
public interface IIgnoredDownloadsConfig
|
||||
{
|
||||
string? IgnoredDownloadsPath { get; }
|
||||
}
|
||||
34
code/Common/Configuration/Notification/NotificationConfig.cs
Normal file
34
code/Common/Configuration/Notification/NotificationConfig.cs
Normal file
@@ -0,0 +1,34 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace Common.Configuration.Notification;
|
||||
|
||||
public abstract record NotificationConfig
|
||||
{
|
||||
[ConfigurationKeyName("ON_IMPORT_FAILED_STRIKE")]
|
||||
public bool OnImportFailedStrike { get; init; }
|
||||
|
||||
[ConfigurationKeyName("ON_STALLED_STRIKE")]
|
||||
public bool OnStalledStrike { get; init; }
|
||||
|
||||
[ConfigurationKeyName("ON_SLOW_STRIKE")]
|
||||
public bool OnSlowStrike { get; init; }
|
||||
|
||||
[ConfigurationKeyName("ON_QUEUE_ITEM_DELETED")]
|
||||
public bool OnQueueItemDeleted { get; init; }
|
||||
|
||||
[ConfigurationKeyName("ON_DOWNLOAD_CLEANED")]
|
||||
public bool OnDownloadCleaned { get; init; }
|
||||
|
||||
[ConfigurationKeyName("ON_CATEGORY_CHANGED")]
|
||||
public bool OnCategoryChanged { get; init; }
|
||||
|
||||
public bool IsEnabled =>
|
||||
OnImportFailedStrike ||
|
||||
OnStalledStrike ||
|
||||
OnSlowStrike ||
|
||||
OnQueueItemDeleted ||
|
||||
OnDownloadCleaned ||
|
||||
OnCategoryChanged;
|
||||
|
||||
public abstract bool IsValid();
|
||||
}
|
||||
@@ -1,8 +1,10 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Common.CustomDataTypes;
|
||||
using Common.Exceptions;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace Common.Configuration.QueueCleaner;
|
||||
|
||||
public sealed record QueueCleanerConfig : IJobConfig
|
||||
public sealed record QueueCleanerConfig : IJobConfig, IIgnoredDownloadsConfig
|
||||
{
|
||||
public const string SectionName = "QueueCleaner";
|
||||
|
||||
@@ -10,22 +12,108 @@ public sealed record QueueCleanerConfig : IJobConfig
|
||||
|
||||
public required bool RunSequentially { get; init; }
|
||||
|
||||
[ConfigurationKeyName("IGNORED_DOWNLOADS_PATH")]
|
||||
public string? IgnoredDownloadsPath { get; init; }
|
||||
|
||||
[ConfigurationKeyName("IMPORT_FAILED_MAX_STRIKES")]
|
||||
public ushort ImportFailedMaxStrikes { get; init; }
|
||||
|
||||
[ConfigurationKeyName("IMPORT_FAILED_IGNORE_PRIVATE")]
|
||||
public bool ImportFailedIgnorePrivate { get; init; }
|
||||
|
||||
[ConfigurationKeyName("IMPORT_FAILED_DELETE_PRIVATE")]
|
||||
public bool ImportFailedDeletePrivate { get; init; }
|
||||
|
||||
[ConfigurationKeyName("IMPORT_FAILED_IGNORE_PATTERNS")]
|
||||
public List<string>? ImportFailedIgnorePatterns { get; init; }
|
||||
public IReadOnlyList<string>? ImportFailedIgnorePatterns { get; init; }
|
||||
|
||||
[ConfigurationKeyName("STALLED_MAX_STRIKES")]
|
||||
public ushort StalledMaxStrikes { get; init; }
|
||||
|
||||
[ConfigurationKeyName("STALLED_RESET_STRIKES_ON_PROGRESS")]
|
||||
public bool StalledResetStrikesOnProgress { get; init; }
|
||||
|
||||
[ConfigurationKeyName("STALLED_IGNORE_PRIVATE")]
|
||||
public bool StalledIgnorePrivate { get; init; }
|
||||
|
||||
[ConfigurationKeyName("STALLED_DELETE_PRIVATE")]
|
||||
public bool StalledDeletePrivate { get; init; }
|
||||
|
||||
[ConfigurationKeyName("DOWNLOADING_METADATA_MAX_STRIKES")]
|
||||
public ushort DownloadingMetadataMaxStrikes { get; init; }
|
||||
|
||||
[ConfigurationKeyName("SLOW_MAX_STRIKES")]
|
||||
public ushort SlowMaxStrikes { get; init; }
|
||||
|
||||
[ConfigurationKeyName("SLOW_RESET_STRIKES_ON_PROGRESS")]
|
||||
public bool SlowResetStrikesOnProgress { get; init; }
|
||||
|
||||
[ConfigurationKeyName("SLOW_IGNORE_PRIVATE")]
|
||||
public bool SlowIgnorePrivate { get; init; }
|
||||
|
||||
[ConfigurationKeyName("SLOW_DELETE_PRIVATE")]
|
||||
public bool SlowDeletePrivate { get; init; }
|
||||
|
||||
[ConfigurationKeyName("SLOW_MIN_SPEED")]
|
||||
public string SlowMinSpeed { get; init; } = string.Empty;
|
||||
|
||||
public ByteSize SlowMinSpeedByteSize => string.IsNullOrEmpty(SlowMinSpeed) ? new ByteSize(0) : ByteSize.Parse(SlowMinSpeed);
|
||||
|
||||
[ConfigurationKeyName("SLOW_MAX_TIME")]
|
||||
public double SlowMaxTime { get; init; }
|
||||
|
||||
[ConfigurationKeyName("SLOW_IGNORE_ABOVE_SIZE")]
|
||||
public string SlowIgnoreAboveSize { get; init; } = string.Empty;
|
||||
|
||||
public ByteSize? SlowIgnoreAboveSizeByteSize => string.IsNullOrEmpty(SlowIgnoreAboveSize) ? null : ByteSize.Parse(SlowIgnoreAboveSize);
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (ImportFailedMaxStrikes is > 0 and < 3)
|
||||
{
|
||||
throw new ValidationException($"the minimum value for {SectionName.ToUpperInvariant()}__IMPORT_FAILED_MAX_STRIKES must be 3");
|
||||
}
|
||||
|
||||
if (StalledMaxStrikes is > 0 and < 3)
|
||||
{
|
||||
throw new ValidationException($"the minimum value for {SectionName.ToUpperInvariant()}__STALLED_MAX_STRIKES must be 3");
|
||||
}
|
||||
|
||||
if (DownloadingMetadataMaxStrikes is > 0 and < 3)
|
||||
{
|
||||
throw new ValidationException($"the minimum value for {SectionName.ToUpperInvariant()}__DOWNLOADING_METADATA_MAX_STRIKES must be 3");
|
||||
}
|
||||
|
||||
if (SlowMaxStrikes is > 0 and < 3)
|
||||
{
|
||||
throw new ValidationException($"the minimum value for {SectionName.ToUpperInvariant()}__SLOW_MAX_STRIKES must be 3");
|
||||
}
|
||||
|
||||
if (SlowMaxStrikes > 0)
|
||||
{
|
||||
bool isSlowSpeedSet = !string.IsNullOrEmpty(SlowMinSpeed);
|
||||
|
||||
if (isSlowSpeedSet && ByteSize.TryParse(SlowMinSpeed, out _) is false)
|
||||
{
|
||||
throw new ValidationException($"invalid value for {SectionName.ToUpperInvariant()}__SLOW_MIN_SPEED");
|
||||
}
|
||||
|
||||
if (SlowMaxTime < 0)
|
||||
{
|
||||
throw new ValidationException($"invalid value for {SectionName.ToUpperInvariant()}__SLOW_MAX_TIME");
|
||||
}
|
||||
|
||||
if (!isSlowSpeedSet && SlowMaxTime is 0)
|
||||
{
|
||||
throw new ValidationException($"either {SectionName.ToUpperInvariant()}__SLOW_MIN_SPEED or {SectionName.ToUpperInvariant()}__SLOW_MAX_STRIKES must be set");
|
||||
}
|
||||
|
||||
bool isSlowIgnoreAboveSizeSet = !string.IsNullOrEmpty(SlowIgnoreAboveSize);
|
||||
|
||||
if (isSlowIgnoreAboveSizeSet && ByteSize.TryParse(SlowIgnoreAboveSize, out _) is false)
|
||||
{
|
||||
throw new ValidationException($"invalid value for {SectionName.ToUpperInvariant()}__SLOW_IGNORE_ABOVE_SIZE");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
115
code/Common/CustomDataTypes/ByteSize.cs
Normal file
115
code/Common/CustomDataTypes/ByteSize.cs
Normal file
@@ -0,0 +1,115 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace Common.CustomDataTypes;
|
||||
|
||||
public readonly struct ByteSize : IComparable<ByteSize>, IEquatable<ByteSize>
|
||||
{
|
||||
public long Bytes { get; }
|
||||
|
||||
private const long BytesPerKB = 1024;
|
||||
private const long BytesPerMB = 1024 * 1024;
|
||||
private const long BytesPerGB = 1024 * 1024 * 1024;
|
||||
|
||||
public ByteSize(long bytes)
|
||||
{
|
||||
if (bytes < 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(bytes), "bytes can not be negative");
|
||||
}
|
||||
|
||||
Bytes = bytes;
|
||||
}
|
||||
|
||||
public static ByteSize FromKilobytes(double kilobytes) => new((long)(kilobytes * BytesPerKB));
|
||||
public static ByteSize FromMegabytes(double megabytes) => new((long)(megabytes * BytesPerMB));
|
||||
public static ByteSize FromGigabytes(double gigabytes) => new((long)(gigabytes * BytesPerGB));
|
||||
|
||||
public static ByteSize Parse(string input)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(input))
|
||||
{
|
||||
throw new ArgumentNullException(nameof(input));
|
||||
}
|
||||
|
||||
input = input.Trim().ToUpperInvariant();
|
||||
double value;
|
||||
if (input.EndsWith("KB", StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
value = double.Parse(input[..^2], CultureInfo.InvariantCulture);
|
||||
return FromKilobytes(value);
|
||||
}
|
||||
|
||||
if (input.EndsWith("MB", StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
value = double.Parse(input[..^2], CultureInfo.InvariantCulture);
|
||||
return FromMegabytes(value);
|
||||
}
|
||||
|
||||
if (input.EndsWith("GB", StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
value = double.Parse(input[..^2], CultureInfo.InvariantCulture);
|
||||
return FromGigabytes(value);
|
||||
}
|
||||
|
||||
throw new FormatException("invalid size format | only KB, MB and GB are supported");
|
||||
}
|
||||
|
||||
public static bool TryParse(string? input, out ByteSize? result)
|
||||
{
|
||||
result = default;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(input))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
input = input.Trim().ToUpperInvariant();
|
||||
|
||||
if (input.EndsWith("KB", StringComparison.InvariantCultureIgnoreCase) &&
|
||||
double.TryParse(input[..^2], NumberStyles.Float, CultureInfo.InvariantCulture, out double kb))
|
||||
{
|
||||
result = FromKilobytes(kb);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (input.EndsWith("MB", StringComparison.InvariantCultureIgnoreCase) &&
|
||||
double.TryParse(input[..^2], NumberStyles.Float, CultureInfo.InvariantCulture, out double mb))
|
||||
{
|
||||
result = FromMegabytes(mb);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (input.EndsWith("GB", StringComparison.InvariantCultureIgnoreCase) &&
|
||||
double.TryParse(input[..^2], NumberStyles.Float, CultureInfo.InvariantCulture, out double gb))
|
||||
{
|
||||
result = FromGigabytes(gb);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public override string ToString() =>
|
||||
Bytes switch
|
||||
{
|
||||
>= BytesPerGB => $"{Bytes / (double)BytesPerGB:0.##} GB",
|
||||
>= BytesPerMB => $"{Bytes / (double)BytesPerMB:0.##} MB",
|
||||
_ => $"{Bytes / (double)BytesPerKB:0.##} KB"
|
||||
};
|
||||
|
||||
public int CompareTo(ByteSize other) => Bytes.CompareTo(other.Bytes);
|
||||
public bool Equals(ByteSize other) => Bytes == other.Bytes;
|
||||
|
||||
public override bool Equals(object? obj) => obj is ByteSize other && Equals(other);
|
||||
public override int GetHashCode() => Bytes.GetHashCode();
|
||||
|
||||
public static bool operator ==(ByteSize left, ByteSize right) => left.Equals(right);
|
||||
public static bool operator !=(ByteSize left, ByteSize right) => !(left == right);
|
||||
public static bool operator <(ByteSize left, ByteSize right) => left.Bytes < right.Bytes;
|
||||
public static bool operator >(ByteSize left, ByteSize right) => left.Bytes > right.Bytes;
|
||||
public static bool operator <=(ByteSize left, ByteSize right) => left.Bytes <= right.Bytes;
|
||||
public static bool operator >=(ByteSize left, ByteSize right) => left.Bytes >= right.Bytes;
|
||||
|
||||
public static ByteSize operator +(ByteSize left, ByteSize right) => new(left.Bytes + right.Bytes);
|
||||
public static ByteSize operator -(ByteSize left, ByteSize right) => new(Math.Max(left.Bytes - right.Bytes, 0));
|
||||
}
|
||||
66
code/Common/CustomDataTypes/SmartTimeSpan.cs
Normal file
66
code/Common/CustomDataTypes/SmartTimeSpan.cs
Normal file
@@ -0,0 +1,66 @@
|
||||
using System.Text;
|
||||
|
||||
namespace Common.CustomDataTypes;
|
||||
|
||||
public readonly struct SmartTimeSpan : IComparable<SmartTimeSpan>, IEquatable<SmartTimeSpan>
|
||||
{
|
||||
public TimeSpan Time { get; }
|
||||
|
||||
public SmartTimeSpan(TimeSpan time)
|
||||
{
|
||||
Time = time;
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
if (Time == TimeSpan.Zero)
|
||||
{
|
||||
return "0 seconds";
|
||||
}
|
||||
|
||||
StringBuilder sb = new();
|
||||
|
||||
if (Time.Days > 0)
|
||||
{
|
||||
sb.Append($"{Time.Days} day{(Time.Days > 1 ? "s" : "")} ");
|
||||
}
|
||||
|
||||
if (Time.Hours > 0)
|
||||
{
|
||||
sb.Append($"{Time.Hours} hour{(Time.Hours > 1 ? "s" : "")} ");
|
||||
}
|
||||
|
||||
if (Time.Minutes > 0)
|
||||
{
|
||||
sb.Append($"{Time.Minutes} minute{(Time.Minutes > 1 ? "s" : "")} ");
|
||||
}
|
||||
|
||||
if (Time.Seconds > 0)
|
||||
{
|
||||
sb.Append($"{Time.Seconds} second{(Time.Seconds > 1 ? "s" : "")}");
|
||||
}
|
||||
|
||||
return sb.ToString().TrimEnd();
|
||||
}
|
||||
|
||||
public static SmartTimeSpan FromMinutes(double minutes) => new(TimeSpan.FromMinutes(minutes));
|
||||
public static SmartTimeSpan FromSeconds(double seconds) => new(TimeSpan.FromSeconds(seconds));
|
||||
public static SmartTimeSpan FromHours(double hours) => new(TimeSpan.FromHours(hours));
|
||||
public static SmartTimeSpan FromDays(double days) => new(TimeSpan.FromDays(days));
|
||||
|
||||
public int CompareTo(SmartTimeSpan other) => Time.CompareTo(other.Time);
|
||||
public bool Equals(SmartTimeSpan other) => Time.Equals(other.Time);
|
||||
|
||||
public override bool Equals(object? obj) => obj is SmartTimeSpan other && Equals(other);
|
||||
public override int GetHashCode() => Time.GetHashCode();
|
||||
|
||||
public static bool operator ==(SmartTimeSpan left, SmartTimeSpan right) => left.Equals(right);
|
||||
public static bool operator !=(SmartTimeSpan left, SmartTimeSpan right) => !left.Equals(right);
|
||||
public static bool operator <(SmartTimeSpan left, SmartTimeSpan right) => left.Time < right.Time;
|
||||
public static bool operator >(SmartTimeSpan left, SmartTimeSpan right) => left.Time > right.Time;
|
||||
public static bool operator <=(SmartTimeSpan left, SmartTimeSpan right) => left.Time <= right.Time;
|
||||
public static bool operator >=(SmartTimeSpan left, SmartTimeSpan right) => left.Time >= right.Time;
|
||||
|
||||
public static SmartTimeSpan operator +(SmartTimeSpan left, SmartTimeSpan right) => new(left.Time + right.Time);
|
||||
public static SmartTimeSpan operator -(SmartTimeSpan left, SmartTimeSpan right) => new(left.Time - right.Time);
|
||||
}
|
||||
8
code/Common/Enums/CertificateValidationType.cs
Normal file
8
code/Common/Enums/CertificateValidationType.cs
Normal file
@@ -0,0 +1,8 @@
|
||||
namespace Common.Enums;
|
||||
|
||||
public enum CertificateValidationType
|
||||
{
|
||||
Enabled = 0,
|
||||
DisabledForLocalAddresses = 1,
|
||||
Disabled = 2
|
||||
}
|
||||
@@ -5,5 +5,6 @@ public enum DownloadClient
|
||||
QBittorrent,
|
||||
Deluge,
|
||||
Transmission,
|
||||
None
|
||||
None,
|
||||
Disabled
|
||||
}
|
||||
12
code/Common/Exceptions/FatalException.cs
Normal file
12
code/Common/Exceptions/FatalException.cs
Normal file
@@ -0,0 +1,12 @@
|
||||
namespace Common.Exceptions;
|
||||
|
||||
public class FatalException : Exception
|
||||
{
|
||||
public FatalException()
|
||||
{
|
||||
}
|
||||
|
||||
public FatalException(string message) : base(message)
|
||||
{
|
||||
}
|
||||
}
|
||||
12
code/Common/Exceptions/ValidationException.cs
Normal file
12
code/Common/Exceptions/ValidationException.cs
Normal file
@@ -0,0 +1,12 @@
|
||||
namespace Common.Exceptions;
|
||||
|
||||
public sealed class ValidationException : Exception
|
||||
{
|
||||
public ValidationException()
|
||||
{
|
||||
}
|
||||
|
||||
public ValidationException(string message) : base(message)
|
||||
{
|
||||
}
|
||||
}
|
||||
8
code/Domain/Enums/CleanReason.cs
Normal file
8
code/Domain/Enums/CleanReason.cs
Normal file
@@ -0,0 +1,8 @@
|
||||
namespace Domain.Enums;
|
||||
|
||||
public enum CleanReason
|
||||
{
|
||||
None,
|
||||
MaxRatioReached,
|
||||
MaxSeedTimeReached,
|
||||
}
|
||||
14
code/Domain/Enums/DeleteReason.cs
Normal file
14
code/Domain/Enums/DeleteReason.cs
Normal file
@@ -0,0 +1,14 @@
|
||||
namespace Domain.Enums;
|
||||
|
||||
public enum DeleteReason
|
||||
{
|
||||
None,
|
||||
Stalled,
|
||||
ImportFailed,
|
||||
DownloadingMetadata,
|
||||
SlowSpeed,
|
||||
SlowTime,
|
||||
AllFilesSkipped,
|
||||
AllFilesSkippedByQBit,
|
||||
AllFilesBlocked,
|
||||
}
|
||||
@@ -3,5 +3,8 @@
|
||||
public enum StrikeType
|
||||
{
|
||||
Stalled,
|
||||
ImportFailed
|
||||
DownloadingMetadata,
|
||||
ImportFailed,
|
||||
SlowSpeed,
|
||||
SlowTime,
|
||||
}
|
||||
8
code/Domain/Models/Arr/Queue/Image.cs
Normal file
8
code/Domain/Models/Arr/Queue/Image.cs
Normal file
@@ -0,0 +1,8 @@
|
||||
namespace Domain.Models.Arr.Queue;
|
||||
|
||||
public record Image
|
||||
{
|
||||
public required string CoverType { get; init; }
|
||||
|
||||
public required Uri RemoteUrl { get; init; }
|
||||
}
|
||||
8
code/Domain/Models/Arr/Queue/LidarrImage.cs
Normal file
8
code/Domain/Models/Arr/Queue/LidarrImage.cs
Normal file
@@ -0,0 +1,8 @@
|
||||
namespace Domain.Models.Arr.Queue;
|
||||
|
||||
public record LidarrImage
|
||||
{
|
||||
public required string CoverType { get; init; }
|
||||
|
||||
public required Uri Url { get; init; }
|
||||
}
|
||||
6
code/Domain/Models/Arr/Queue/QueueAlbum.cs
Normal file
6
code/Domain/Models/Arr/Queue/QueueAlbum.cs
Normal file
@@ -0,0 +1,6 @@
|
||||
namespace Domain.Models.Arr.Queue;
|
||||
|
||||
public sealed record QueueAlbum
|
||||
{
|
||||
public List<LidarrImage> Images { get; init; } = [];
|
||||
}
|
||||
6
code/Domain/Models/Arr/Queue/QueueMovie.cs
Normal file
6
code/Domain/Models/Arr/Queue/QueueMovie.cs
Normal file
@@ -0,0 +1,6 @@
|
||||
namespace Domain.Models.Arr.Queue;
|
||||
|
||||
public sealed record QueueMovie
|
||||
{
|
||||
public List<Image> Images { get; init; } = [];
|
||||
}
|
||||
@@ -7,14 +7,20 @@ public sealed record QueueRecord
|
||||
public long EpisodeId { get; init; }
|
||||
public long SeasonNumber { get; init; }
|
||||
|
||||
public QueueSeries? Series { get; init; }
|
||||
|
||||
// Radarr
|
||||
public long MovieId { get; init; }
|
||||
|
||||
public QueueSeries? Movie { get; init; }
|
||||
|
||||
// Lidarr
|
||||
public long ArtistId { get; init; }
|
||||
|
||||
public long AlbumId { get; init; }
|
||||
|
||||
public QueueAlbum? Album { get; init; }
|
||||
|
||||
// common
|
||||
public required string Title { get; init; }
|
||||
public string Status { get; init; }
|
||||
|
||||
6
code/Domain/Models/Arr/Queue/QueueSeries.cs
Normal file
6
code/Domain/Models/Arr/Queue/QueueSeries.cs
Normal file
@@ -0,0 +1,6 @@
|
||||
namespace Domain.Models.Arr.Queue;
|
||||
|
||||
public sealed record QueueSeries
|
||||
{
|
||||
public List<Image> Images { get; init; } = [];
|
||||
}
|
||||
9
code/Domain/Models/Cache/StalledCacheItem.cs
Normal file
9
code/Domain/Models/Cache/StalledCacheItem.cs
Normal file
@@ -0,0 +1,9 @@
|
||||
namespace Domain.Models.Cache;
|
||||
|
||||
public sealed record StalledCacheItem
|
||||
{
|
||||
/// <summary>
|
||||
/// The amount of bytes that have been downloaded.
|
||||
/// </summary>
|
||||
public long Downloaded { get; set; }
|
||||
}
|
||||
42
code/Domain/Models/Deluge/Response/DownloadStatus.cs
Normal file
42
code/Domain/Models/Deluge/Response/DownloadStatus.cs
Normal file
@@ -0,0 +1,42 @@
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace Domain.Models.Deluge.Response;
|
||||
|
||||
public sealed record DownloadStatus
|
||||
{
|
||||
public string? Hash { get; init; }
|
||||
|
||||
public string? State { get; init; }
|
||||
|
||||
public string? Name { get; init; }
|
||||
|
||||
public ulong Eta { get; init; }
|
||||
|
||||
[JsonProperty("download_payload_rate")]
|
||||
public long DownloadSpeed { get; init; }
|
||||
|
||||
public bool Private { get; init; }
|
||||
|
||||
[JsonProperty("total_size")]
|
||||
public long Size { get; init; }
|
||||
|
||||
[JsonProperty("total_done")]
|
||||
public long TotalDone { get; init; }
|
||||
|
||||
public string? Label { get; set; }
|
||||
|
||||
[JsonProperty("seeding_time")]
|
||||
public long SeedingTime { get; init; }
|
||||
|
||||
public float Ratio { get; init; }
|
||||
|
||||
public required IReadOnlyList<Tracker> Trackers { get; init; }
|
||||
|
||||
[JsonProperty("download_location")]
|
||||
public required string DownloadLocation { get; init; }
|
||||
}
|
||||
|
||||
public sealed record Tracker
|
||||
{
|
||||
public required string Url { get; init; }
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
namespace Domain.Models.Deluge.Response;
|
||||
|
||||
public sealed record TorrentStatus
|
||||
{
|
||||
public string? Hash { get; init; }
|
||||
|
||||
public string? State { get; init; }
|
||||
|
||||
public string? Name { get; init; }
|
||||
|
||||
public ulong Eta { get; init; }
|
||||
|
||||
public bool Private { get; init; }
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
using Common.Configuration.Arr;
|
||||
using Common.Configuration.ContentBlocker;
|
||||
using Common.Configuration.DownloadCleaner;
|
||||
using Common.Configuration.DownloadClient;
|
||||
using Common.Configuration.General;
|
||||
using Common.Configuration.Logging;
|
||||
using Common.Configuration.QueueCleaner;
|
||||
|
||||
@@ -10,8 +12,10 @@ public static class ConfigurationDI
|
||||
{
|
||||
public static IServiceCollection AddConfiguration(this IServiceCollection services, IConfiguration configuration) =>
|
||||
services
|
||||
.Configure<DryRunConfig>(configuration)
|
||||
.Configure<QueueCleanerConfig>(configuration.GetSection(QueueCleanerConfig.SectionName))
|
||||
.Configure<ContentBlockerConfig>(configuration.GetSection(ContentBlockerConfig.SectionName))
|
||||
.Configure<DownloadCleanerConfig>(configuration.GetSection(DownloadCleanerConfig.SectionName))
|
||||
.Configure<DownloadClientConfig>(configuration)
|
||||
.Configure<QBitConfig>(configuration.GetSection(QBitConfig.SectionName))
|
||||
.Configure<DelugeConfig>(configuration.GetSection(DelugeConfig.SectionName))
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using Common.Configuration.Logging;
|
||||
using Domain.Enums;
|
||||
using Infrastructure.Verticals.ContentBlocker;
|
||||
using Infrastructure.Verticals.DownloadCleaner;
|
||||
using Infrastructure.Verticals.QueueCleaner;
|
||||
using Serilog;
|
||||
using Serilog.Events;
|
||||
@@ -33,7 +34,7 @@ public static class LoggingDI
|
||||
const string consoleOutputTemplate = $"[{{@t:yyyy-MM-dd HH:mm:ss.fff}} {{@l:u3}}]{jobNameTemplate}{instanceNameTemplate} {{@m}}\n{{@x}}";
|
||||
const string fileOutputTemplate = $"{{@t:yyyy-MM-dd HH:mm:ss.fff zzz}} [{{@l:u3}}]{jobNameTemplate}{instanceNameTemplate} {{@m:lj}}\n{{@x}}";
|
||||
LogEventLevel level = LogEventLevel.Information;
|
||||
List<string> names = [nameof(ContentBlocker), nameof(QueueCleaner)];
|
||||
List<string> names = [nameof(ContentBlocker), nameof(QueueCleaner), nameof(DownloadCleaner)];
|
||||
int jobPadding = names.Max(x => x.Length) + 2;
|
||||
names = [InstanceType.Sonarr.ToString(), InstanceType.Radarr.ToString(), InstanceType.Lidarr.ToString()];
|
||||
int arrPadding = names.Max(x => x.Length) + 2;
|
||||
@@ -63,6 +64,7 @@ public static class LoggingDI
|
||||
|
||||
Log.Logger = logConfig
|
||||
.MinimumLevel.Is(level)
|
||||
.MinimumLevel.Override("MassTransit", LogEventLevel.Warning)
|
||||
.MinimumLevel.Override("Microsoft.Hosting.Lifetime", LogEventLevel.Information)
|
||||
.MinimumLevel.Override("Microsoft.Extensions.Http", LogEventLevel.Warning)
|
||||
.MinimumLevel.Override("Quartz", LogEventLevel.Warning)
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
using System.Net;
|
||||
using Common.Configuration.General;
|
||||
using Common.Helpers;
|
||||
using Infrastructure.Services;
|
||||
using Infrastructure.Verticals.DownloadClient.Deluge;
|
||||
using Infrastructure.Verticals.Notifications.Consumers;
|
||||
using Infrastructure.Verticals.Notifications.Models;
|
||||
using MassTransit;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Polly;
|
||||
using Polly.Extensions.Http;
|
||||
|
||||
@@ -14,9 +19,36 @@ public static class MainDI
|
||||
.AddLogging(builder => builder.ClearProviders().AddConsole())
|
||||
.AddHttpClients(configuration)
|
||||
.AddConfiguration(configuration)
|
||||
.AddMemoryCache()
|
||||
.AddMemoryCache(options => {
|
||||
options.ExpirationScanFrequency = TimeSpan.FromMinutes(1);
|
||||
})
|
||||
.AddServices()
|
||||
.AddQuartzServices(configuration);
|
||||
.AddQuartzServices(configuration)
|
||||
.AddNotifications(configuration)
|
||||
.AddMassTransit(config =>
|
||||
{
|
||||
config.AddConsumer<NotificationConsumer<FailedImportStrikeNotification>>();
|
||||
config.AddConsumer<NotificationConsumer<StalledStrikeNotification>>();
|
||||
config.AddConsumer<NotificationConsumer<SlowStrikeNotification>>();
|
||||
config.AddConsumer<NotificationConsumer<QueueItemDeletedNotification>>();
|
||||
config.AddConsumer<NotificationConsumer<DownloadCleanedNotification>>();
|
||||
config.AddConsumer<NotificationConsumer<CategoryChangedNotification>>();
|
||||
|
||||
config.UsingInMemory((context, cfg) =>
|
||||
{
|
||||
cfg.ReceiveEndpoint("notification-queue", e =>
|
||||
{
|
||||
e.ConfigureConsumer<NotificationConsumer<FailedImportStrikeNotification>>(context);
|
||||
e.ConfigureConsumer<NotificationConsumer<StalledStrikeNotification>>(context);
|
||||
e.ConfigureConsumer<NotificationConsumer<SlowStrikeNotification>>(context);
|
||||
e.ConfigureConsumer<NotificationConsumer<QueueItemDeletedNotification>>(context);
|
||||
e.ConfigureConsumer<NotificationConsumer<DownloadCleanedNotification>>(context);
|
||||
e.ConfigureConsumer<NotificationConsumer<CategoryChangedNotification>>(context);
|
||||
e.ConcurrentMessageLimit = 1;
|
||||
e.PrefetchCount = 1;
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
private static IServiceCollection AddHttpClients(this IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
@@ -32,13 +64,22 @@ public static class MainDI
|
||||
{
|
||||
x.Timeout = TimeSpan.FromSeconds(config.Timeout);
|
||||
})
|
||||
.ConfigurePrimaryHttpMessageHandler(provider =>
|
||||
{
|
||||
CertificateValidationService service = provider.GetRequiredService<CertificateValidationService>();
|
||||
|
||||
return new HttpClientHandler
|
||||
{
|
||||
ServerCertificateCustomValidationCallback = service.ShouldByPassValidationError
|
||||
};
|
||||
})
|
||||
.AddRetryPolicyHandler(config);
|
||||
|
||||
// add Deluge HttpClient
|
||||
services
|
||||
.AddHttpClient(nameof(DelugeService), x =>
|
||||
{
|
||||
x.Timeout = TimeSpan.FromSeconds(5);
|
||||
x.Timeout = TimeSpan.FromSeconds(config.Timeout);
|
||||
})
|
||||
.ConfigurePrimaryHttpMessageHandler(_ =>
|
||||
{
|
||||
|
||||
20
code/Executable/DependencyInjection/NotificationsDI.cs
Normal file
20
code/Executable/DependencyInjection/NotificationsDI.cs
Normal file
@@ -0,0 +1,20 @@
|
||||
using Infrastructure.Verticals.Notifications;
|
||||
using Infrastructure.Verticals.Notifications.Apprise;
|
||||
using Infrastructure.Verticals.Notifications.Notifiarr;
|
||||
|
||||
namespace Executable.DependencyInjection;
|
||||
|
||||
public static class NotificationsDI
|
||||
{
|
||||
public static IServiceCollection AddNotifications(this IServiceCollection services, IConfiguration configuration) =>
|
||||
services
|
||||
.Configure<NotifiarrConfig>(configuration.GetSection(NotifiarrConfig.SectionName))
|
||||
.Configure<AppriseConfig>(configuration.GetSection(AppriseConfig.SectionName))
|
||||
.AddTransient<INotifiarrProxy, NotifiarrProxy>()
|
||||
.AddTransient<INotificationProvider, NotifiarrProvider>()
|
||||
.AddTransient<IAppriseProxy, AppriseProxy>()
|
||||
.AddTransient<INotificationProvider, AppriseProvider>()
|
||||
.AddTransient<INotificationPublisher, NotificationPublisher>()
|
||||
.AddTransient<INotificationFactory, NotificationFactory>()
|
||||
.AddTransient<NotificationService>();
|
||||
}
|
||||
@@ -1,10 +1,12 @@
|
||||
using Common.Configuration;
|
||||
using Common.Configuration.ContentBlocker;
|
||||
using Common.Configuration.DownloadCleaner;
|
||||
using Common.Configuration.General;
|
||||
using Common.Configuration.QueueCleaner;
|
||||
using Common.Helpers;
|
||||
using Executable.Jobs;
|
||||
using Infrastructure.Verticals.ContentBlocker;
|
||||
using Infrastructure.Verticals.DownloadCleaner;
|
||||
using Infrastructure.Verticals.Jobs;
|
||||
using Infrastructure.Verticals.QueueCleaner;
|
||||
using Quartz;
|
||||
@@ -53,12 +55,18 @@ public static class QuartzDI
|
||||
if (contentBlockerConfig?.Enabled is true && queueCleanerConfig is { Enabled: true, RunSequentially: true })
|
||||
{
|
||||
q.AddJob<QueueCleaner>(queueCleanerConfig, string.Empty);
|
||||
q.AddJobListener(new JobChainingListener(nameof(QueueCleaner)));
|
||||
q.AddJobListener(new JobChainingListener(nameof(ContentBlocker), nameof(QueueCleaner)));
|
||||
}
|
||||
else
|
||||
{
|
||||
q.AddJob<QueueCleaner>(queueCleanerConfig, triggersConfig.QueueCleaner);
|
||||
}
|
||||
|
||||
DownloadCleanerConfig? downloadCleanerConfig = configuration
|
||||
.GetRequiredSection(DownloadCleanerConfig.SectionName)
|
||||
.Get<DownloadCleanerConfig>();
|
||||
|
||||
q.AddJob<DownloadCleaner>(downloadCleanerConfig, triggersConfig.DownloadCleaner);
|
||||
}
|
||||
|
||||
private static void AddJob<T>(
|
||||
@@ -109,7 +117,7 @@ public static class QuartzDI
|
||||
|
||||
if (triggerValue > Constants.TriggerMaxLimit)
|
||||
{
|
||||
throw new Exception($"{trigger} should have a fire time of maximum 1 hour");
|
||||
throw new Exception($"{trigger} should have a fire time of maximum {Constants.TriggerMaxLimit.TotalHours} hours");
|
||||
}
|
||||
|
||||
if (triggerValue > StaticConfiguration.TriggerValue)
|
||||
|
||||
@@ -1,9 +1,17 @@
|
||||
using Infrastructure.Verticals.Arr;
|
||||
using Common.Configuration.ContentBlocker;
|
||||
using Common.Configuration.DownloadCleaner;
|
||||
using Common.Configuration.QueueCleaner;
|
||||
using Infrastructure.Interceptors;
|
||||
using Infrastructure.Providers;
|
||||
using Infrastructure.Services;
|
||||
using Infrastructure.Verticals.Arr;
|
||||
using Infrastructure.Verticals.ContentBlocker;
|
||||
using Infrastructure.Verticals.DownloadCleaner;
|
||||
using Infrastructure.Verticals.DownloadClient;
|
||||
using Infrastructure.Verticals.DownloadClient.Deluge;
|
||||
using Infrastructure.Verticals.DownloadClient.QBittorrent;
|
||||
using Infrastructure.Verticals.DownloadClient.Transmission;
|
||||
using Infrastructure.Verticals.Files;
|
||||
using Infrastructure.Verticals.ItemStriker;
|
||||
using Infrastructure.Verticals.QueueCleaner;
|
||||
|
||||
@@ -13,18 +21,27 @@ public static class ServicesDI
|
||||
{
|
||||
public static IServiceCollection AddServices(this IServiceCollection services) =>
|
||||
services
|
||||
.AddTransient<IDryRunInterceptor, DryRunInterceptor>()
|
||||
.AddTransient<CertificateValidationService>()
|
||||
.AddTransient<SonarrClient>()
|
||||
.AddTransient<RadarrClient>()
|
||||
.AddTransient<LidarrClient>()
|
||||
.AddTransient<QueueCleaner>()
|
||||
.AddTransient<ContentBlocker>()
|
||||
.AddTransient<FilenameEvaluator>()
|
||||
.AddTransient<DownloadCleaner>()
|
||||
.AddTransient<IFilenameEvaluator, FilenameEvaluator>()
|
||||
.AddTransient<IHardLinkFileService, HardLinkFileService>()
|
||||
.AddTransient<UnixHardLinkFileService>()
|
||||
.AddTransient<WindowsHardLinkFileService>()
|
||||
.AddTransient<DummyDownloadService>()
|
||||
.AddTransient<QBitService>()
|
||||
.AddTransient<DelugeService>()
|
||||
.AddTransient<TransmissionService>()
|
||||
.AddTransient<ArrQueueIterator>()
|
||||
.AddTransient<DownloadServiceFactory>()
|
||||
.AddTransient<IStriker, Striker>()
|
||||
.AddSingleton<BlocklistProvider>()
|
||||
.AddSingleton<Striker>();
|
||||
.AddSingleton<IgnoredDownloadsProvider<QueueCleanerConfig>>()
|
||||
.AddSingleton<IgnoredDownloadsProvider<ContentBlockerConfig>>()
|
||||
.AddSingleton<IgnoredDownloadsProvider<DownloadCleanerConfig>>();
|
||||
}
|
||||
@@ -10,9 +10,9 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="9.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting" Version="9.0.2" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.2" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="9.0.2" />
|
||||
<PackageReference Include="Quartz" Version="3.13.1" />
|
||||
<PackageReference Include="Quartz.Extensions.DependencyInjection" Version="3.13.1" />
|
||||
<PackageReference Include="Quartz.Extensions.Hosting" Version="3.13.1" />
|
||||
|
||||
23
code/Executable/HostExtensions.cs
Normal file
23
code/Executable/HostExtensions.cs
Normal file
@@ -0,0 +1,23 @@
|
||||
using System.Reflection;
|
||||
|
||||
namespace Executable;
|
||||
|
||||
public static class HostExtensions
|
||||
{
|
||||
public static IHost Init(this IHost host)
|
||||
{
|
||||
ILogger<Program> logger = host.Services.GetRequiredService<ILogger<Program>>();
|
||||
|
||||
Version? version = Assembly.GetExecutingAssembly().GetName().Version;
|
||||
|
||||
logger.LogInformation(
|
||||
version is null
|
||||
? "cleanuperr version not detected"
|
||||
: $"cleanuperr v{version.Major}.{version.Minor}.{version.Build}"
|
||||
);
|
||||
|
||||
logger.LogInformation("timezone: {tz}", TimeZoneInfo.Local.DisplayName);
|
||||
|
||||
return host;
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,7 @@ namespace Executable.Jobs;
|
||||
|
||||
[DisallowConcurrentExecution]
|
||||
public sealed class GenericJob<T> : IJob
|
||||
where T : GenericHandler
|
||||
where T : IHandler
|
||||
{
|
||||
private readonly ILogger<GenericJob<T>> _logger;
|
||||
private readonly T _handler;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
using System.Reflection;
|
||||
using Executable;
|
||||
using Executable.DependencyInjection;
|
||||
|
||||
var builder = Host.CreateApplicationBuilder(args);
|
||||
@@ -7,15 +7,6 @@ builder.Services.AddInfrastructure(builder.Configuration);
|
||||
builder.Logging.AddLogging(builder.Configuration);
|
||||
|
||||
var host = builder.Build();
|
||||
|
||||
var logger = host.Services.GetRequiredService<ILogger<Program>>();
|
||||
|
||||
var version = Assembly.GetExecutingAssembly().GetName().Version;
|
||||
|
||||
logger.LogInformation(
|
||||
version is null
|
||||
? "cleanuperr version not detected"
|
||||
: $"cleanuperr v{version.Major}.{version.Minor}.{version.Build}"
|
||||
);
|
||||
host.Init();
|
||||
|
||||
host.Run();
|
||||
@@ -1,8 +1,10 @@
|
||||
{
|
||||
"DRY_RUN": true,
|
||||
"HTTP_MAX_RETRIES": 0,
|
||||
"HTTP_TIMEOUT": 10,
|
||||
"HTTP_TIMEOUT": 100,
|
||||
"HTTP_VALIDATE_CERT": "enabled",
|
||||
"Logging": {
|
||||
"LogLevel": "Debug",
|
||||
"LogLevel": "Verbose",
|
||||
"Enhanced": true,
|
||||
"File": {
|
||||
"Enabled": false,
|
||||
@@ -11,35 +13,72 @@
|
||||
},
|
||||
"Triggers": {
|
||||
"QueueCleaner": "0/10 * * * * ?",
|
||||
"ContentBlocker": "0/10 * * * * ?"
|
||||
"ContentBlocker": "0/10 * * * * ?",
|
||||
"DownloadCleaner": "0/10 * * * * ?"
|
||||
},
|
||||
"ContentBlocker": {
|
||||
"Enabled": true,
|
||||
"IGNORE_PRIVATE": true
|
||||
"IGNORE_PRIVATE": true,
|
||||
"DELETE_PRIVATE": false,
|
||||
"IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads"
|
||||
},
|
||||
"QueueCleaner": {
|
||||
"Enabled": true,
|
||||
"RunSequentially": true,
|
||||
"IMPORT_FAILED_MAX_STRIKES": 5,
|
||||
"IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads",
|
||||
"IMPORT_FAILED_MAX_STRIKES": 3,
|
||||
"IMPORT_FAILED_IGNORE_PRIVATE": true,
|
||||
"IMPORT_FAILED_DELETE_PRIVATE": false,
|
||||
"IMPORT_FAILED_IGNORE_PATTERNS": [
|
||||
"file is a sample"
|
||||
],
|
||||
"STALLED_MAX_STRIKES": 5,
|
||||
"STALLED_IGNORE_PRIVATE": true
|
||||
"STALLED_MAX_STRIKES": 3,
|
||||
"STALLED_RESET_STRIKES_ON_PROGRESS": true,
|
||||
"STALLED_IGNORE_PRIVATE": true,
|
||||
"STALLED_DELETE_PRIVATE": false,
|
||||
"DOWNLOADING_METADATA_MAX_STRIKES": 3,
|
||||
"SLOW_MAX_STRIKES": 5,
|
||||
"SLOW_RESET_STRIKES_ON_PROGRESS": true,
|
||||
"SLOW_IGNORE_PRIVATE": false,
|
||||
"SLOW_DELETE_PRIVATE": false,
|
||||
"SLOW_MIN_SPEED": "1MB",
|
||||
"SLOW_MAX_TIME": 20,
|
||||
"SLOW_IGNORE_ABOVE_SIZE": "4GB"
|
||||
},
|
||||
"DownloadCleaner": {
|
||||
"Enabled": false,
|
||||
"DELETE_PRIVATE": false,
|
||||
"CATEGORIES": [
|
||||
{
|
||||
"Name": "tv-sonarr",
|
||||
"MAX_RATIO": -1,
|
||||
"MIN_SEED_TIME": 0,
|
||||
"MAX_SEED_TIME": 240
|
||||
}
|
||||
],
|
||||
"UNLINKED_TARGET_CATEGORY": "cleanuperr-unlinked",
|
||||
"UNLINKED_IGNORED_ROOT_DIR": "",
|
||||
"UNLINKED_CATEGORIES": [
|
||||
"tv-sonarr",
|
||||
"radarr"
|
||||
],
|
||||
"IGNORED_DOWNLOADS_PATH": "../test/data/cleanuperr/ignored_downloads"
|
||||
},
|
||||
"DOWNLOAD_CLIENT": "qbittorrent",
|
||||
"qBittorrent": {
|
||||
"Url": "http://localhost:8080",
|
||||
"URL_BASE": "",
|
||||
"Username": "test",
|
||||
"Password": "testing"
|
||||
},
|
||||
"Deluge": {
|
||||
"Url": "http://localhost:8112",
|
||||
"URL_BASE": "",
|
||||
"Password": "testing"
|
||||
},
|
||||
"Transmission": {
|
||||
"Url": "http://localhost:9091",
|
||||
"URL_BASE": "transmission",
|
||||
"Username": "test",
|
||||
"Password": "testing"
|
||||
},
|
||||
@@ -82,5 +121,25 @@
|
||||
"ApiKey": "7f677cfdc074414397af53dd633860c5"
|
||||
}
|
||||
]
|
||||
},
|
||||
"Notifiarr": {
|
||||
"ON_IMPORT_FAILED_STRIKE": true,
|
||||
"ON_STALLED_STRIKE": true,
|
||||
"ON_SLOW_STRIKE": true,
|
||||
"ON_QUEUE_ITEM_DELETED": true,
|
||||
"ON_DOWNLOAD_CLEANED": true,
|
||||
"ON_CATEGORY_CHANGED": true,
|
||||
"API_KEY": "",
|
||||
"CHANNEL_ID": ""
|
||||
},
|
||||
"Apprise": {
|
||||
"ON_IMPORT_FAILED_STRIKE": true,
|
||||
"ON_STALLED_STRIKE": true,
|
||||
"ON_SLOW_STRIKE": true,
|
||||
"ON_QUEUE_ITEM_DELETED": true,
|
||||
"ON_DOWNLOAD_CLEANED": true,
|
||||
"ON_CATEGORY_CHANGED": true,
|
||||
"URL": "http://localhost:8000",
|
||||
"KEY": ""
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
{
|
||||
"DRY_RUN": false,
|
||||
"HTTP_MAX_RETRIES": 0,
|
||||
"HTTP_TIMEOUT": 100,
|
||||
"HTTP_VALIDATE_CERT": "enabled",
|
||||
"Logging": {
|
||||
"LogLevel": "Information",
|
||||
"Enhanced": true,
|
||||
@@ -11,33 +13,59 @@
|
||||
},
|
||||
"Triggers": {
|
||||
"QueueCleaner": "0 0/5 * * * ?",
|
||||
"ContentBlocker": "0 0/5 * * * ?"
|
||||
"ContentBlocker": "0 0/5 * * * ?",
|
||||
"DownloadCleaner": "0 0 * * * ?"
|
||||
},
|
||||
"ContentBlocker": {
|
||||
"Enabled": false,
|
||||
"IGNORE_PRIVATE": false
|
||||
"IGNORE_PRIVATE": false,
|
||||
"IGNORED_DOWNLOADS_PATH": ""
|
||||
},
|
||||
"QueueCleaner": {
|
||||
"Enabled": true,
|
||||
"Enabled": false,
|
||||
"RunSequentially": true,
|
||||
"IMPORT_FAILED_MAX_STRIKES": 5,
|
||||
"IGNORED_DOWNLOADS_PATH": "",
|
||||
"IMPORT_FAILED_MAX_STRIKES": 0,
|
||||
"IMPORT_FAILED_IGNORE_PRIVATE": false,
|
||||
"IMPORT_FAILED_DELETE_PRIVATE": false,
|
||||
"IMPORT_FAILED_IGNORE_PATTERNS": [],
|
||||
"STALLED_MAX_STRIKES": 5,
|
||||
"STALLED_IGNORE_PRIVATE": false
|
||||
"STALLED_MAX_STRIKES": 0,
|
||||
"STALLED_RESET_STRIKES_ON_PROGRESS": false,
|
||||
"STALLED_IGNORE_PRIVATE": false,
|
||||
"STALLED_DELETE_PRIVATE": false,
|
||||
"DOWNLOADING_METADATA_MAX_STRIKES": 0,
|
||||
"SLOW_MAX_STRIKES": 0,
|
||||
"SLOW_RESET_STRIKES_ON_PROGRESS": true,
|
||||
"SLOW_IGNORE_PRIVATE": false,
|
||||
"SLOW_DELETE_PRIVATE": false,
|
||||
"SLOW_MIN_SPEED": "",
|
||||
"SLOW_MAX_TIME": 0,
|
||||
"SLOW_IGNORE_ABOVE_SIZE": ""
|
||||
},
|
||||
"DownloadCleaner": {
|
||||
"Enabled": false,
|
||||
"DELETE_PRIVATE": false,
|
||||
"CATEGORIES": [],
|
||||
"UNLINKED_TARGET_CATEGORY": "cleanuperr-unlinked",
|
||||
"UNLINKED_IGNORED_ROOT_DIR": "",
|
||||
"UNLINKED_CATEGORIES": [],
|
||||
"IGNORED_DOWNLOADS_PATH": ""
|
||||
},
|
||||
"DOWNLOAD_CLIENT": "none",
|
||||
"qBittorrent": {
|
||||
"Url": "http://localhost:8080",
|
||||
"URL_BASE": "",
|
||||
"Username": "",
|
||||
"Password": ""
|
||||
},
|
||||
"Deluge": {
|
||||
"Url": "http://localhost:8112",
|
||||
"URL_BASE": "",
|
||||
"Password": "testing"
|
||||
},
|
||||
"Transmission": {
|
||||
"Url": "http://localhost:9091",
|
||||
"URL_BASE": "transmission",
|
||||
"Username": "test",
|
||||
"Password": "testing"
|
||||
},
|
||||
@@ -80,5 +108,25 @@
|
||||
"ApiKey": ""
|
||||
}
|
||||
]
|
||||
},
|
||||
"Notifiarr": {
|
||||
"ON_IMPORT_FAILED_STRIKE": false,
|
||||
"ON_STALLED_STRIKE": false,
|
||||
"ON_SLOW_STRIKE": false,
|
||||
"ON_QUEUE_ITEM_DELETED": false,
|
||||
"ON_DOWNLOAD_CLEANED": false,
|
||||
"ON_CATEGORY_CHANGED": false,
|
||||
"API_KEY": "",
|
||||
"CHANNEL_ID": ""
|
||||
},
|
||||
"Apprise": {
|
||||
"ON_IMPORT_FAILED_STRIKE": false,
|
||||
"ON_STALLED_STRIKE": false,
|
||||
"ON_SLOW_STRIKE": false,
|
||||
"ON_QUEUE_ITEM_DELETED": false,
|
||||
"ON_DOWNLOAD_CLEANED": false,
|
||||
"ON_CATEGORY_CHANGED": false,
|
||||
"URL": "",
|
||||
"KEY": ""
|
||||
}
|
||||
}
|
||||
|
||||
33
code/Infrastructure.Tests/Infrastructure.Tests.csproj
Normal file
33
code/Infrastructure.Tests/Infrastructure.Tests.csproj
Normal file
@@ -0,0 +1,33 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net9.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="coverlet.collector" Version="6.0.4">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0" />
|
||||
<PackageReference Include="NSubstitute" Version="5.3.0" />
|
||||
<PackageReference Include="Shouldly" Version="4.3.0" />
|
||||
<PackageReference Include="xunit" Version="2.9.3" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.0.2">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Using Include="Xunit"/>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Infrastructure\Infrastructure.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,20 @@
|
||||
using Infrastructure.Verticals.ContentBlocker;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using NSubstitute;
|
||||
|
||||
namespace Infrastructure.Tests.Verticals.ContentBlocker;
|
||||
|
||||
public class FilenameEvaluatorFixture
|
||||
{
|
||||
public ILogger<FilenameEvaluator> Logger { get; }
|
||||
|
||||
public FilenameEvaluatorFixture()
|
||||
{
|
||||
Logger = Substitute.For<ILogger<FilenameEvaluator>>();
|
||||
}
|
||||
|
||||
public FilenameEvaluator CreateSut()
|
||||
{
|
||||
return new FilenameEvaluator(Logger);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.RegularExpressions;
|
||||
using Common.Configuration.ContentBlocker;
|
||||
using Shouldly;
|
||||
|
||||
namespace Infrastructure.Tests.Verticals.ContentBlocker;
|
||||
|
||||
public class FilenameEvaluatorTests : IClassFixture<FilenameEvaluatorFixture>
|
||||
{
|
||||
private readonly FilenameEvaluatorFixture _fixture;
|
||||
|
||||
public FilenameEvaluatorTests(FilenameEvaluatorFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
}
|
||||
|
||||
public class PatternTests : FilenameEvaluatorTests
|
||||
{
|
||||
public PatternTests(FilenameEvaluatorFixture fixture) : base(fixture) { }
|
||||
|
||||
[Fact]
|
||||
public void WhenNoPatterns_ShouldReturnTrue()
|
||||
{
|
||||
// Arrange
|
||||
var sut = _fixture.CreateSut();
|
||||
var patterns = new ConcurrentBag<string>();
|
||||
var regexes = new ConcurrentBag<Regex>();
|
||||
|
||||
// Act
|
||||
var result = sut.IsValid("test.txt", BlocklistType.Blacklist, patterns, regexes);
|
||||
|
||||
// Assert
|
||||
result.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("test.txt", "test.txt", true)] // Exact match
|
||||
[InlineData("test.txt", "*.txt", true)] // End wildcard
|
||||
[InlineData("test.txt", "test.*", true)] // Start wildcard
|
||||
[InlineData("test.txt", "*test*", true)] // Both wildcards
|
||||
[InlineData("test.txt", "other.txt", false)] // No match
|
||||
public void Blacklist_ShouldMatchPatterns(string filename, string pattern, bool shouldBeBlocked)
|
||||
{
|
||||
// Arrange
|
||||
var sut = _fixture.CreateSut();
|
||||
var patterns = new ConcurrentBag<string> { pattern };
|
||||
var regexes = new ConcurrentBag<Regex>();
|
||||
|
||||
// Act
|
||||
var result = sut.IsValid(filename, BlocklistType.Blacklist, patterns, regexes);
|
||||
|
||||
// Assert
|
||||
result.ShouldBe(!shouldBeBlocked);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("test.txt", "test.txt", true)] // Exact match
|
||||
[InlineData("test.txt", "*.txt", true)] // End wildcard
|
||||
[InlineData("test.txt", "test.*", true)] // Start wildcard
|
||||
[InlineData("test.txt", "*test*", true)] // Both wildcards
|
||||
[InlineData("test.txt", "other.txt", false)] // No match
|
||||
public void Whitelist_ShouldMatchPatterns(string filename, string pattern, bool shouldBeAllowed)
|
||||
{
|
||||
// Arrange
|
||||
var sut = _fixture.CreateSut();
|
||||
var patterns = new ConcurrentBag<string> { pattern };
|
||||
var regexes = new ConcurrentBag<Regex>();
|
||||
|
||||
// Act
|
||||
var result = sut.IsValid(filename, BlocklistType.Whitelist, patterns, regexes);
|
||||
|
||||
// Assert
|
||||
result.ShouldBe(shouldBeAllowed);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("TEST.TXT", "test.txt")]
|
||||
[InlineData("test.txt", "TEST.TXT")]
|
||||
public void ShouldBeCaseInsensitive(string filename, string pattern)
|
||||
{
|
||||
// Arrange
|
||||
var sut = _fixture.CreateSut();
|
||||
var patterns = new ConcurrentBag<string> { pattern };
|
||||
var regexes = new ConcurrentBag<Regex>();
|
||||
|
||||
// Act
|
||||
var result = sut.IsValid(filename, BlocklistType.Blacklist, patterns, regexes);
|
||||
|
||||
// Assert
|
||||
result.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MultiplePatterns_ShouldMatchAny()
|
||||
{
|
||||
// Arrange
|
||||
var sut = _fixture.CreateSut();
|
||||
var patterns = new ConcurrentBag<string>
|
||||
{
|
||||
"other.txt",
|
||||
"*.pdf",
|
||||
"test.*"
|
||||
};
|
||||
var regexes = new ConcurrentBag<Regex>();
|
||||
|
||||
// Act
|
||||
var result = sut.IsValid("test.txt", BlocklistType.Blacklist, patterns, regexes);
|
||||
|
||||
// Assert
|
||||
result.ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
|
||||
public class RegexTests : FilenameEvaluatorTests
|
||||
{
|
||||
public RegexTests(FilenameEvaluatorFixture fixture) : base(fixture) { }
|
||||
|
||||
[Fact]
|
||||
public void WhenNoRegexes_ShouldReturnTrue()
|
||||
{
|
||||
// Arrange
|
||||
var sut = _fixture.CreateSut();
|
||||
var patterns = new ConcurrentBag<string>();
|
||||
var regexes = new ConcurrentBag<Regex>();
|
||||
|
||||
// Act
|
||||
var result = sut.IsValid("test.txt", BlocklistType.Blacklist, patterns, regexes);
|
||||
|
||||
// Assert
|
||||
result.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(@"test\d+\.txt", "test123.txt", true)]
|
||||
[InlineData(@"test\d+\.txt", "test.txt", false)]
|
||||
public void Blacklist_ShouldMatchRegexes(string pattern, string filename, bool shouldBeBlocked)
|
||||
{
|
||||
// Arrange
|
||||
var sut = _fixture.CreateSut();
|
||||
var patterns = new ConcurrentBag<string>();
|
||||
var regexes = new ConcurrentBag<Regex> { new Regex(pattern, RegexOptions.IgnoreCase) };
|
||||
|
||||
// Act
|
||||
var result = sut.IsValid(filename, BlocklistType.Blacklist, patterns, regexes);
|
||||
|
||||
// Assert
|
||||
result.ShouldBe(!shouldBeBlocked);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(@"test\d+\.txt", "test123.txt", true)]
|
||||
[InlineData(@"test\d+\.txt", "test.txt", false)]
|
||||
public void Whitelist_ShouldMatchRegexes(string pattern, string filename, bool shouldBeAllowed)
|
||||
{
|
||||
// Arrange
|
||||
var sut = _fixture.CreateSut();
|
||||
var patterns = new ConcurrentBag<string>();
|
||||
var regexes = new ConcurrentBag<Regex> { new Regex(pattern, RegexOptions.IgnoreCase) };
|
||||
|
||||
// Act
|
||||
var result = sut.IsValid(filename, BlocklistType.Whitelist, patterns, regexes);
|
||||
|
||||
// Assert
|
||||
result.ShouldBe(shouldBeAllowed);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(@"TEST\d+\.TXT", "test123.txt")]
|
||||
[InlineData(@"test\d+\.txt", "TEST123.TXT")]
|
||||
public void ShouldBeCaseInsensitive(string pattern, string filename)
|
||||
{
|
||||
// Arrange
|
||||
var sut = _fixture.CreateSut();
|
||||
var patterns = new ConcurrentBag<string>();
|
||||
var regexes = new ConcurrentBag<Regex> { new Regex(pattern, RegexOptions.IgnoreCase) };
|
||||
|
||||
// Act
|
||||
var result = sut.IsValid(filename, BlocklistType.Blacklist, patterns, regexes);
|
||||
|
||||
// Assert
|
||||
result.ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
|
||||
public class CombinedTests : FilenameEvaluatorTests
|
||||
{
|
||||
public CombinedTests(FilenameEvaluatorFixture fixture) : base(fixture) { }
|
||||
|
||||
[Fact]
|
||||
public void WhenBothPatternsAndRegexes_ShouldMatchBoth()
|
||||
{
|
||||
// Arrange
|
||||
var sut = _fixture.CreateSut();
|
||||
var patterns = new ConcurrentBag<string> { "*.txt" };
|
||||
var regexes = new ConcurrentBag<Regex> { new Regex(@"test\d+", RegexOptions.IgnoreCase) };
|
||||
|
||||
// Act
|
||||
var result = sut.IsValid("test123.txt", BlocklistType.Blacklist, patterns, regexes);
|
||||
|
||||
// Assert
|
||||
result.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhenPatternMatchesButRegexDoesNot_ShouldReturnFalse()
|
||||
{
|
||||
// Arrange
|
||||
var sut = _fixture.CreateSut();
|
||||
var patterns = new ConcurrentBag<string> { "*.txt" };
|
||||
var regexes = new ConcurrentBag<Regex> { new Regex(@"test\d+", RegexOptions.IgnoreCase) };
|
||||
|
||||
// Act
|
||||
var result = sut.IsValid("other.txt", BlocklistType.Whitelist, patterns, regexes);
|
||||
|
||||
// Assert
|
||||
result.ShouldBeFalse();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
using Common.Configuration.ContentBlocker;
|
||||
using Common.Configuration.DownloadCleaner;
|
||||
using Common.Configuration.QueueCleaner;
|
||||
using Infrastructure.Interceptors;
|
||||
using Infrastructure.Verticals.ContentBlocker;
|
||||
using Infrastructure.Verticals.DownloadClient;
|
||||
using Infrastructure.Verticals.Files;
|
||||
using Infrastructure.Verticals.ItemStriker;
|
||||
using Infrastructure.Verticals.Notifications;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using NSubstitute;
|
||||
|
||||
namespace Infrastructure.Tests.Verticals.DownloadClient;
|
||||
|
||||
public class DownloadServiceFixture : IDisposable
|
||||
{
|
||||
public ILogger<DownloadService> Logger { get; set; }
|
||||
public IMemoryCache Cache { get; set; }
|
||||
public IStriker Striker { get; set; }
|
||||
|
||||
public DownloadServiceFixture()
|
||||
{
|
||||
Logger = Substitute.For<ILogger<DownloadService>>();
|
||||
Cache = Substitute.For<IMemoryCache>();
|
||||
Striker = Substitute.For<IStriker>();
|
||||
}
|
||||
|
||||
public TestDownloadService CreateSut(
|
||||
QueueCleanerConfig? queueCleanerConfig = null,
|
||||
ContentBlockerConfig? contentBlockerConfig = null
|
||||
)
|
||||
{
|
||||
queueCleanerConfig ??= new QueueCleanerConfig
|
||||
{
|
||||
Enabled = true,
|
||||
RunSequentially = true,
|
||||
StalledResetStrikesOnProgress = true,
|
||||
StalledMaxStrikes = 3
|
||||
};
|
||||
|
||||
var queueCleanerOptions = Substitute.For<IOptions<QueueCleanerConfig>>();
|
||||
queueCleanerOptions.Value.Returns(queueCleanerConfig);
|
||||
|
||||
contentBlockerConfig ??= new ContentBlockerConfig
|
||||
{
|
||||
Enabled = true
|
||||
};
|
||||
|
||||
var contentBlockerOptions = Substitute.For<IOptions<ContentBlockerConfig>>();
|
||||
contentBlockerOptions.Value.Returns(contentBlockerConfig);
|
||||
|
||||
var downloadCleanerOptions = Substitute.For<IOptions<DownloadCleanerConfig>>();
|
||||
downloadCleanerOptions.Value.Returns(new DownloadCleanerConfig());
|
||||
|
||||
var filenameEvaluator = Substitute.For<IFilenameEvaluator>();
|
||||
var notifier = Substitute.For<INotificationPublisher>();
|
||||
var dryRunInterceptor = Substitute.For<IDryRunInterceptor>();
|
||||
var hardlinkFileService = Substitute.For<IHardLinkFileService>();
|
||||
|
||||
return new TestDownloadService(
|
||||
Logger,
|
||||
queueCleanerOptions,
|
||||
contentBlockerOptions,
|
||||
downloadCleanerOptions,
|
||||
Cache,
|
||||
filenameEvaluator,
|
||||
Striker,
|
||||
notifier,
|
||||
dryRunInterceptor,
|
||||
hardlinkFileService
|
||||
);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
// Cleanup if needed
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
using Common.Configuration.DownloadCleaner;
|
||||
using Domain.Enums;
|
||||
using Domain.Models.Cache;
|
||||
using Infrastructure.Helpers;
|
||||
using Infrastructure.Verticals.Context;
|
||||
using Infrastructure.Verticals.DownloadClient;
|
||||
using NSubstitute;
|
||||
using NSubstitute.ClearExtensions;
|
||||
using Shouldly;
|
||||
|
||||
namespace Infrastructure.Tests.Verticals.DownloadClient;
|
||||
|
||||
public class DownloadServiceTests : IClassFixture<DownloadServiceFixture>
|
||||
{
|
||||
private readonly DownloadServiceFixture _fixture;
|
||||
|
||||
public DownloadServiceTests(DownloadServiceFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
_fixture.Cache.ClearSubstitute();
|
||||
_fixture.Striker.ClearSubstitute();
|
||||
}
|
||||
|
||||
public class ResetStrikesOnProgressTests : DownloadServiceTests
|
||||
{
|
||||
public ResetStrikesOnProgressTests(DownloadServiceFixture fixture) : base(fixture)
|
||||
{
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhenStalledStrikeDisabled_ShouldNotResetStrikes()
|
||||
{
|
||||
// Arrange
|
||||
TestDownloadService sut = _fixture.CreateSut(queueCleanerConfig: new()
|
||||
{
|
||||
Enabled = true,
|
||||
RunSequentially = true,
|
||||
StalledResetStrikesOnProgress = false,
|
||||
});
|
||||
|
||||
// Act
|
||||
sut.ResetStalledStrikesOnProgress("test-hash", 100);
|
||||
|
||||
// Assert
|
||||
_fixture.Cache.ReceivedCalls().ShouldBeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhenProgressMade_ShouldResetStrikes()
|
||||
{
|
||||
// Arrange
|
||||
const string hash = "test-hash";
|
||||
StalledCacheItem stalledCacheItem = new StalledCacheItem { Downloaded = 100 };
|
||||
|
||||
_fixture.Cache.TryGetValue(Arg.Any<object>(), out Arg.Any<object?>())
|
||||
.Returns(x =>
|
||||
{
|
||||
x[1] = stalledCacheItem;
|
||||
return true;
|
||||
});
|
||||
|
||||
TestDownloadService sut = _fixture.CreateSut();
|
||||
|
||||
// Act
|
||||
sut.ResetStalledStrikesOnProgress(hash, 200);
|
||||
|
||||
// Assert
|
||||
_fixture.Cache.Received(1).Remove(CacheKeys.Strike(StrikeType.Stalled, hash));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhenNoProgress_ShouldNotResetStrikes()
|
||||
{
|
||||
// Arrange
|
||||
const string hash = "test-hash";
|
||||
StalledCacheItem stalledCacheItem = new StalledCacheItem { Downloaded = 200 };
|
||||
|
||||
_fixture.Cache
|
||||
.TryGetValue(Arg.Any<object>(), out Arg.Any<object?>())
|
||||
.Returns(x =>
|
||||
{
|
||||
x[1] = stalledCacheItem;
|
||||
return true;
|
||||
});
|
||||
|
||||
TestDownloadService sut = _fixture.CreateSut();
|
||||
|
||||
// Act
|
||||
sut.ResetStalledStrikesOnProgress(hash, 100);
|
||||
|
||||
// Assert
|
||||
_fixture.Cache.DidNotReceive().Remove(Arg.Any<object>());
|
||||
}
|
||||
}
|
||||
|
||||
public class StrikeAndCheckLimitTests : DownloadServiceTests
|
||||
{
|
||||
public StrikeAndCheckLimitTests(DownloadServiceFixture fixture) : base(fixture)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
public class ShouldCleanDownloadTests : DownloadServiceTests
|
||||
{
|
||||
public ShouldCleanDownloadTests(DownloadServiceFixture fixture) : base(fixture)
|
||||
{
|
||||
ContextProvider.Set("downloadName", "test-download");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhenRatioAndMinSeedTimeReached_ShouldReturnTrue()
|
||||
{
|
||||
// Arrange
|
||||
CleanCategory category = new()
|
||||
{
|
||||
Name = "test",
|
||||
MaxRatio = 1.0,
|
||||
MinSeedTime = 1,
|
||||
MaxSeedTime = -1
|
||||
};
|
||||
const double ratio = 1.5;
|
||||
TimeSpan seedingTime = TimeSpan.FromHours(2);
|
||||
|
||||
TestDownloadService sut = _fixture.CreateSut();
|
||||
|
||||
// Act
|
||||
var result = sut.ShouldCleanDownload(ratio, seedingTime, category);
|
||||
|
||||
// Assert
|
||||
result.ShouldSatisfyAllConditions(
|
||||
() => result.ShouldClean.ShouldBeTrue(),
|
||||
() => result.Reason.ShouldBe(CleanReason.MaxRatioReached)
|
||||
);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhenRatioReachedAndMinSeedTimeNotReached_ShouldReturnFalse()
|
||||
{
|
||||
// Arrange
|
||||
CleanCategory category = new()
|
||||
{
|
||||
Name = "test",
|
||||
MaxRatio = 1.0,
|
||||
MinSeedTime = 3,
|
||||
MaxSeedTime = -1
|
||||
};
|
||||
const double ratio = 1.5;
|
||||
TimeSpan seedingTime = TimeSpan.FromHours(2);
|
||||
|
||||
TestDownloadService sut = _fixture.CreateSut();
|
||||
|
||||
// Act
|
||||
var result = sut.ShouldCleanDownload(ratio, seedingTime, category);
|
||||
|
||||
// Assert
|
||||
result.ShouldSatisfyAllConditions(
|
||||
() => result.ShouldClean.ShouldBeFalse(),
|
||||
() => result.Reason.ShouldBe(CleanReason.None)
|
||||
);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhenMaxSeedTimeReached_ShouldReturnTrue()
|
||||
{
|
||||
// Arrange
|
||||
CleanCategory category = new()
|
||||
{
|
||||
Name = "test",
|
||||
MaxRatio = -1,
|
||||
MinSeedTime = 0,
|
||||
MaxSeedTime = 1
|
||||
};
|
||||
const double ratio = 0.5;
|
||||
TimeSpan seedingTime = TimeSpan.FromHours(2);
|
||||
|
||||
TestDownloadService sut = _fixture.CreateSut();
|
||||
|
||||
// Act
|
||||
SeedingCheckResult result = sut.ShouldCleanDownload(ratio, seedingTime, category);
|
||||
|
||||
// Assert
|
||||
result.ShouldSatisfyAllConditions(
|
||||
() => result.ShouldClean.ShouldBeTrue(),
|
||||
() => result.Reason.ShouldBe(CleanReason.MaxSeedTimeReached)
|
||||
);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void WhenNeitherConditionMet_ShouldReturnFalse()
|
||||
{
|
||||
// Arrange
|
||||
CleanCategory category = new()
|
||||
{
|
||||
Name = "test",
|
||||
MaxRatio = 2.0,
|
||||
MinSeedTime = 0,
|
||||
MaxSeedTime = 3
|
||||
};
|
||||
const double ratio = 1.0;
|
||||
TimeSpan seedingTime = TimeSpan.FromHours(1);
|
||||
|
||||
TestDownloadService sut = _fixture.CreateSut();
|
||||
|
||||
// Act
|
||||
var result = sut.ShouldCleanDownload(ratio, seedingTime, category);
|
||||
|
||||
// Assert
|
||||
result.ShouldSatisfyAllConditions(
|
||||
() => result.ShouldClean.ShouldBeFalse(),
|
||||
() => result.Reason.ShouldBe(CleanReason.None)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.RegularExpressions;
|
||||
using Common.Configuration.ContentBlocker;
|
||||
using Common.Configuration.DownloadCleaner;
|
||||
using Common.Configuration.QueueCleaner;
|
||||
using Domain.Enums;
|
||||
using Infrastructure.Interceptors;
|
||||
using Infrastructure.Verticals.ContentBlocker;
|
||||
using Infrastructure.Verticals.DownloadClient;
|
||||
using Infrastructure.Verticals.Files;
|
||||
using Infrastructure.Verticals.ItemStriker;
|
||||
using Infrastructure.Verticals.Notifications;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Infrastructure.Tests.Verticals.DownloadClient;
|
||||
|
||||
public class TestDownloadService : DownloadService
|
||||
{
|
||||
public TestDownloadService(
|
||||
ILogger<DownloadService> logger,
|
||||
IOptions<QueueCleanerConfig> queueCleanerConfig,
|
||||
IOptions<ContentBlockerConfig> contentBlockerConfig,
|
||||
IOptions<DownloadCleanerConfig> downloadCleanerConfig,
|
||||
IMemoryCache cache,
|
||||
IFilenameEvaluator filenameEvaluator,
|
||||
IStriker striker,
|
||||
INotificationPublisher notifier,
|
||||
IDryRunInterceptor dryRunInterceptor,
|
||||
IHardLinkFileService hardLinkFileService
|
||||
) : base(
|
||||
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache,
|
||||
filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService
|
||||
)
|
||||
{
|
||||
}
|
||||
|
||||
public override void Dispose() { }
|
||||
public override Task LoginAsync() => Task.CompletedTask;
|
||||
public override Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads) => Task.FromResult(new DownloadCheckResult());
|
||||
public override Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType,
|
||||
ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads) => Task.FromResult(new BlockFilesResult());
|
||||
public override Task DeleteDownload(string hash) => Task.CompletedTask;
|
||||
public override Task CreateCategoryAsync(string name) => Task.CompletedTask;
|
||||
public override Task<List<object>?> GetSeedingDownloads() => Task.FromResult<List<object>?>(null);
|
||||
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories) => null;
|
||||
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories) => null;
|
||||
public override Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads) => Task.CompletedTask;
|
||||
public override Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads) => Task.CompletedTask;
|
||||
// Expose protected methods for testing
|
||||
public new void ResetStalledStrikesOnProgress(string hash, long downloaded) => base.ResetStalledStrikesOnProgress(hash, downloaded);
|
||||
public new SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, CleanCategory category) => base.ShouldCleanDownload(ratio, seedingTime, category);
|
||||
}
|
||||
30
code/Infrastructure/Extensions/DelugeExtensions.cs
Normal file
30
code/Infrastructure/Extensions/DelugeExtensions.cs
Normal file
@@ -0,0 +1,30 @@
|
||||
using Domain.Models.Deluge.Response;
|
||||
using Infrastructure.Helpers;
|
||||
|
||||
namespace Infrastructure.Extensions;
|
||||
|
||||
public static class DelugeExtensions
|
||||
{
|
||||
public static bool ShouldIgnore(this DownloadStatus download, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
foreach (string value in ignoredDownloads)
|
||||
{
|
||||
if (download.Hash?.Equals(value, StringComparison.InvariantCultureIgnoreCase) ?? false)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (download.Label?.Equals(value, StringComparison.InvariantCultureIgnoreCase) ?? false)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (download.Trackers.Any(x => UriService.GetDomain(x.Url)?.EndsWith(value, StringComparison.InvariantCultureIgnoreCase) ?? false))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
55
code/Infrastructure/Extensions/IpAddressExtensions.cs
Normal file
55
code/Infrastructure/Extensions/IpAddressExtensions.cs
Normal file
@@ -0,0 +1,55 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
|
||||
namespace Infrastructure.Extensions;
|
||||
|
||||
public static class IpAddressExtensions
|
||||
{
|
||||
public static bool IsLocalAddress(this IPAddress ipAddress)
|
||||
{
|
||||
// Map back to IPv4 if mapped to IPv6, for example "::ffff:1.2.3.4" to "1.2.3.4".
|
||||
if (ipAddress.IsIPv4MappedToIPv6)
|
||||
{
|
||||
ipAddress = ipAddress.MapToIPv4();
|
||||
}
|
||||
|
||||
// Checks loopback ranges for both IPv4 and IPv6.
|
||||
if (IPAddress.IsLoopback(ipAddress))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// IPv4
|
||||
if (ipAddress.AddressFamily == AddressFamily.InterNetwork)
|
||||
{
|
||||
return IsLocalIPv4(ipAddress.GetAddressBytes());
|
||||
}
|
||||
|
||||
// IPv6
|
||||
if (ipAddress.AddressFamily == AddressFamily.InterNetworkV6)
|
||||
{
|
||||
return ipAddress.IsIPv6LinkLocal ||
|
||||
ipAddress.IsIPv6UniqueLocal ||
|
||||
ipAddress.IsIPv6SiteLocal;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool IsLocalIPv4(byte[] ipv4Bytes)
|
||||
{
|
||||
// Link local (no IP assigned by DHCP): 169.254.0.0 to 169.254.255.255 (169.254.0.0/16)
|
||||
bool IsLinkLocal() => ipv4Bytes[0] == 169 && ipv4Bytes[1] == 254;
|
||||
|
||||
// Class A private range: 10.0.0.0 – 10.255.255.255 (10.0.0.0/8)
|
||||
bool IsClassA() => ipv4Bytes[0] == 10;
|
||||
|
||||
// Class B private range: 172.16.0.0 – 172.31.255.255 (172.16.0.0/12)
|
||||
bool IsClassB() => ipv4Bytes[0] == 172 && ipv4Bytes[1] >= 16 && ipv4Bytes[1] <= 31;
|
||||
|
||||
// Class C private range: 192.168.0.0 – 192.168.255.255 (192.168.0.0/16)
|
||||
bool IsClassC() => ipv4Bytes[0] == 192 && ipv4Bytes[1] == 168;
|
||||
|
||||
return IsLinkLocal() || IsClassA() || IsClassC() || IsClassB();
|
||||
}
|
||||
}
|
||||
50
code/Infrastructure/Extensions/QBitExtensions.cs
Normal file
50
code/Infrastructure/Extensions/QBitExtensions.cs
Normal file
@@ -0,0 +1,50 @@
|
||||
using Infrastructure.Helpers;
|
||||
using QBittorrent.Client;
|
||||
|
||||
namespace Infrastructure.Extensions;
|
||||
|
||||
public static class QBitExtensions
|
||||
{
|
||||
public static bool ShouldIgnore(this TorrentInfo download, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
foreach (string value in ignoredDownloads)
|
||||
{
|
||||
if (download.Hash.Equals(value, StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (download.Category.Equals(value, StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (download.Tags.Contains(value, StringComparer.InvariantCultureIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public static bool ShouldIgnore(this TorrentTracker tracker, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
string? trackerUrl = UriService.GetDomain(tracker.Url);
|
||||
|
||||
if (trackerUrl is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach (string value in ignoredDownloads)
|
||||
{
|
||||
if (trackerUrl.EndsWith(value, StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
43
code/Infrastructure/Extensions/TransmissionExtensions.cs
Normal file
43
code/Infrastructure/Extensions/TransmissionExtensions.cs
Normal file
@@ -0,0 +1,43 @@
|
||||
using Infrastructure.Helpers;
|
||||
using Transmission.API.RPC.Entity;
|
||||
|
||||
namespace Infrastructure.Extensions;
|
||||
|
||||
public static class TransmissionExtensions
|
||||
{
|
||||
public static bool ShouldIgnore(this TorrentInfo download, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
foreach (string value in ignoredDownloads)
|
||||
{
|
||||
if (download.HashString?.Equals(value, StringComparison.InvariantCultureIgnoreCase) ?? false)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (download.GetCategory().Equals(value, StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
bool? hasIgnoredTracker = download.Trackers?
|
||||
.Any(x => UriService.GetDomain(x.Announce)?.EndsWith(value, StringComparison.InvariantCultureIgnoreCase) ?? false);
|
||||
|
||||
if (hasIgnoredTracker is true)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public static string GetCategory(this TorrentInfo download)
|
||||
{
|
||||
if (string.IsNullOrEmpty(download.DownloadDir))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
return Path.GetFileName(Path.TrimEndingDirectorySeparator(download.DownloadDir));
|
||||
}
|
||||
}
|
||||
16
code/Infrastructure/Helpers/CacheKeys.cs
Normal file
16
code/Infrastructure/Helpers/CacheKeys.cs
Normal file
@@ -0,0 +1,16 @@
|
||||
using Domain.Enums;
|
||||
|
||||
namespace Infrastructure.Helpers;
|
||||
|
||||
public static class CacheKeys
|
||||
{
|
||||
public static string Strike(StrikeType strikeType, string hash) => $"{strikeType.ToString()}_{hash}";
|
||||
|
||||
public static string BlocklistType(InstanceType instanceType) => $"{instanceType.ToString()}_type";
|
||||
public static string BlocklistPatterns(InstanceType instanceType) => $"{instanceType.ToString()}_patterns";
|
||||
public static string BlocklistRegexes(InstanceType instanceType) => $"{instanceType.ToString()}_regexes";
|
||||
|
||||
public static string StrikeItem(string hash, StrikeType strikeType) => $"item_{hash}_{strikeType.ToString()}";
|
||||
|
||||
public static string IgnoredDownloads(string name) => $"{name}_ignored";
|
||||
}
|
||||
37
code/Infrastructure/Helpers/UriService.cs
Normal file
37
code/Infrastructure/Helpers/UriService.cs
Normal file
@@ -0,0 +1,37 @@
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace Infrastructure.Helpers;
|
||||
|
||||
public static class UriService
|
||||
{
|
||||
public static string? GetDomain(string? input)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(input))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
// add "http://" if scheme is missing to help Uri.TryCreate
|
||||
if (!input.StartsWith("http", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
input = "http://" + input;
|
||||
}
|
||||
|
||||
if (Uri.TryCreate(input, UriKind.Absolute, out var uri))
|
||||
{
|
||||
return uri.Host;
|
||||
}
|
||||
|
||||
// url might be malformed
|
||||
var regex = new Regex(@"^(?:https?:\/\/)?([^\/\?:]+)", RegexOptions.IgnoreCase);
|
||||
var match = regex.Match(input);
|
||||
|
||||
if (match.Success)
|
||||
{
|
||||
return match.Groups[1].Value;
|
||||
}
|
||||
|
||||
// could not extract
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -12,11 +12,15 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FLM.QBittorrent" Version="1.0.0" />
|
||||
<PackageReference Include="FLM.Transmission" Version="1.0.2" />
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.0" />
|
||||
<PackageReference Include="FLM.QBittorrent" Version="1.0.1" />
|
||||
<PackageReference Include="FLM.Transmission" Version="1.0.3" />
|
||||
<PackageReference Include="Mapster" Version="7.4.0" />
|
||||
<PackageReference Include="MassTransit" Version="8.3.6" />
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="9.0.2" />
|
||||
<PackageReference Include="Microsoft.Extensions.Http" Version="9.0.2" />
|
||||
<PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
|
||||
<PackageReference Include="Quartz" Version="3.13.1" />
|
||||
<PackageReference Include="Scrutor" Version="6.0.1" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
||||
77
code/Infrastructure/Interceptors/DryRunInterceptor.cs
Normal file
77
code/Infrastructure/Interceptors/DryRunInterceptor.cs
Normal file
@@ -0,0 +1,77 @@
|
||||
using System.Reflection;
|
||||
using Common.Attributes;
|
||||
using Common.Configuration.General;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Infrastructure.Interceptors;
|
||||
|
||||
public class DryRunInterceptor : IDryRunInterceptor
|
||||
{
|
||||
private readonly ILogger<DryRunInterceptor> _logger;
|
||||
private readonly DryRunConfig _config;
|
||||
|
||||
public DryRunInterceptor(ILogger<DryRunInterceptor> logger, IOptions<DryRunConfig> config)
|
||||
{
|
||||
_logger = logger;
|
||||
_config = config.Value;
|
||||
}
|
||||
|
||||
public void Intercept(Action action)
|
||||
{
|
||||
MethodInfo methodInfo = action.Method;
|
||||
|
||||
if (IsDryRun(methodInfo))
|
||||
{
|
||||
_logger.LogInformation("[DRY RUN] skipping method: {name}", methodInfo.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
action();
|
||||
}
|
||||
|
||||
public Task InterceptAsync(Delegate action, params object[] parameters)
|
||||
{
|
||||
MethodInfo methodInfo = action.Method;
|
||||
|
||||
if (IsDryRun(methodInfo))
|
||||
{
|
||||
_logger.LogInformation("[DRY RUN] skipping method: {name}", methodInfo.Name);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
object? result = action.DynamicInvoke(parameters);
|
||||
|
||||
if (result is Task task)
|
||||
{
|
||||
return task;
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task<T?> InterceptAsync<T>(Delegate action, params object[] parameters)
|
||||
{
|
||||
MethodInfo methodInfo = action.Method;
|
||||
|
||||
if (IsDryRun(methodInfo))
|
||||
{
|
||||
_logger.LogInformation("[DRY RUN] skipping method: {name}", methodInfo.Name);
|
||||
return Task.FromResult(default(T));
|
||||
}
|
||||
|
||||
object? result = action.DynamicInvoke(parameters);
|
||||
|
||||
if (result is Task<T?> task)
|
||||
{
|
||||
return task;
|
||||
}
|
||||
|
||||
return Task.FromResult(default(T));
|
||||
}
|
||||
|
||||
private bool IsDryRun(MethodInfo method)
|
||||
{
|
||||
return method.GetCustomAttributes(typeof(DryRunSafeguardAttribute), true).Any() && _config.IsDryRun;
|
||||
}
|
||||
}
|
||||
10
code/Infrastructure/Interceptors/IDryRunInterceptor.cs
Normal file
10
code/Infrastructure/Interceptors/IDryRunInterceptor.cs
Normal file
@@ -0,0 +1,10 @@
|
||||
namespace Infrastructure.Interceptors;
|
||||
|
||||
public interface IDryRunInterceptor
|
||||
{
|
||||
void Intercept(Action action);
|
||||
|
||||
Task InterceptAsync(Delegate action, params object[] parameters);
|
||||
|
||||
Task<T?> InterceptAsync<T>(Delegate action, params object[] parameters);
|
||||
}
|
||||
82
code/Infrastructure/Providers/IgnoredDownloadsProvider.cs
Normal file
82
code/Infrastructure/Providers/IgnoredDownloadsProvider.cs
Normal file
@@ -0,0 +1,82 @@
|
||||
using Common.Configuration;
|
||||
using Infrastructure.Helpers;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Infrastructure.Providers;
|
||||
|
||||
public sealed class IgnoredDownloadsProvider<T>
|
||||
where T : IIgnoredDownloadsConfig
|
||||
{
|
||||
private readonly ILogger<IgnoredDownloadsProvider<T>> _logger;
|
||||
private IIgnoredDownloadsConfig _config;
|
||||
private readonly IMemoryCache _cache;
|
||||
private DateTime _lastModified = DateTime.MinValue;
|
||||
|
||||
public IgnoredDownloadsProvider(ILogger<IgnoredDownloadsProvider<T>> logger, IOptionsMonitor<T> config, IMemoryCache cache)
|
||||
{
|
||||
_config = config.CurrentValue;
|
||||
config.OnChange((newValue) => _config = newValue);
|
||||
_logger = logger;
|
||||
_cache = cache;
|
||||
|
||||
if (string.IsNullOrEmpty(_config.IgnoredDownloadsPath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!File.Exists(_config.IgnoredDownloadsPath))
|
||||
{
|
||||
throw new FileNotFoundException("file not found", _config.IgnoredDownloadsPath);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<string>> GetIgnoredDownloads()
|
||||
{
|
||||
if (string.IsNullOrEmpty(_config.IgnoredDownloadsPath))
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
FileInfo fileInfo = new(_config.IgnoredDownloadsPath);
|
||||
|
||||
if (fileInfo.LastWriteTime > _lastModified ||
|
||||
!_cache.TryGetValue(CacheKeys.IgnoredDownloads(typeof(T).Name), out IReadOnlyList<string>? ignoredDownloads) ||
|
||||
ignoredDownloads is null)
|
||||
{
|
||||
_lastModified = fileInfo.LastWriteTime;
|
||||
|
||||
return await LoadFile();
|
||||
}
|
||||
|
||||
return ignoredDownloads;
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<string>> LoadFile()
|
||||
{
|
||||
try
|
||||
{
|
||||
if (string.IsNullOrEmpty(_config.IgnoredDownloadsPath))
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
string[] ignoredDownloads = (await File.ReadAllLinesAsync(_config.IgnoredDownloadsPath))
|
||||
.Where(x => !string.IsNullOrWhiteSpace(x))
|
||||
.ToArray();
|
||||
|
||||
_cache.Set(CacheKeys.IgnoredDownloads(typeof(T).Name), ignoredDownloads);
|
||||
|
||||
_logger.LogInformation("ignored downloads reloaded");
|
||||
|
||||
return ignoredDownloads;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
_logger.LogError(exception, "error while reading ignored downloads file | {file}", _config.IgnoredDownloadsPath);
|
||||
}
|
||||
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
}
|
||||
86
code/Infrastructure/Services/CertificateValidationService.cs
Normal file
86
code/Infrastructure/Services/CertificateValidationService.cs
Normal file
@@ -0,0 +1,86 @@
|
||||
using System.Net;
|
||||
using System.Net.Security;
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using Common.Configuration.General;
|
||||
using Common.Enums;
|
||||
using Infrastructure.Extensions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Infrastructure.Services;
|
||||
|
||||
public class CertificateValidationService
|
||||
{
|
||||
private readonly ILogger<CertificateValidationService> _logger;
|
||||
private readonly HttpConfig _config;
|
||||
|
||||
public CertificateValidationService(ILogger<CertificateValidationService> logger, IOptions<HttpConfig> config)
|
||||
{
|
||||
_logger = logger;
|
||||
_config = config.Value;
|
||||
}
|
||||
|
||||
public bool ShouldByPassValidationError(object sender, X509Certificate? certificate, X509Chain? chain, SslPolicyErrors sslPolicyErrors)
|
||||
{
|
||||
var targetHostName = string.Empty;
|
||||
|
||||
if (sender is not SslStream && sender is not string)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (sender is SslStream request)
|
||||
{
|
||||
targetHostName = request.TargetHostName;
|
||||
}
|
||||
|
||||
// Mailkit passes host in sender as string
|
||||
if (sender is string stringHost)
|
||||
{
|
||||
targetHostName = stringHost;
|
||||
}
|
||||
|
||||
if (certificate is X509Certificate2 cert2 && cert2.SignatureAlgorithm.FriendlyName == "md5RSA")
|
||||
{
|
||||
_logger.LogError(
|
||||
$"https://{targetHostName} uses the obsolete md5 hash in its https certificate, if that is your certificate, please (re)create certificate with better algorithm as soon as possible.");
|
||||
}
|
||||
|
||||
if (sslPolicyErrors == SslPolicyErrors.None)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (targetHostName is "localhost" or "127.0.0.1")
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var ipAddresses = GetIpAddresses(targetHostName);
|
||||
|
||||
if (_config.CertificateValidation == CertificateValidationType.Disabled)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (_config.CertificateValidation == CertificateValidationType.DisabledForLocalAddresses &&
|
||||
ipAddresses.All(i => i.IsLocalAddress()))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
_logger.LogError($"certificate validation for {targetHostName} failed. {sslPolicyErrors}");
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static IPAddress[] GetIpAddresses(string host)
|
||||
{
|
||||
if (IPAddress.TryParse(host, out var ipAddress))
|
||||
{
|
||||
return [ipAddress];
|
||||
}
|
||||
|
||||
return Dns.GetHostEntry(host).AddressList;
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,13 @@
|
||||
using Common.Configuration.Arr;
|
||||
using Common.Attributes;
|
||||
using Common.Configuration.Arr;
|
||||
using Common.Configuration.Logging;
|
||||
using Common.Configuration.QueueCleaner;
|
||||
using Common.Helpers;
|
||||
using Domain.Enums;
|
||||
using Domain.Models.Arr;
|
||||
using Domain.Models.Arr.Queue;
|
||||
using Infrastructure.Interceptors;
|
||||
using Infrastructure.Verticals.Arr.Interfaces;
|
||||
using Infrastructure.Verticals.ItemStriker;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
@@ -12,35 +15,39 @@ using Newtonsoft.Json;
|
||||
|
||||
namespace Infrastructure.Verticals.Arr;
|
||||
|
||||
public abstract class ArrClient
|
||||
public abstract class ArrClient : IArrClient
|
||||
{
|
||||
protected readonly ILogger<ArrClient> _logger;
|
||||
protected readonly HttpClient _httpClient;
|
||||
protected readonly LoggingConfig _loggingConfig;
|
||||
protected readonly QueueCleanerConfig _queueCleanerConfig;
|
||||
protected readonly Striker _striker;
|
||||
protected readonly IStriker _striker;
|
||||
protected readonly IDryRunInterceptor _dryRunInterceptor;
|
||||
|
||||
protected ArrClient(
|
||||
ILogger<ArrClient> logger,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IOptions<LoggingConfig> loggingConfig,
|
||||
IOptions<QueueCleanerConfig> queueCleanerConfig,
|
||||
Striker striker
|
||||
IStriker striker,
|
||||
IDryRunInterceptor dryRunInterceptor
|
||||
)
|
||||
{
|
||||
_logger = logger;
|
||||
_striker = striker;
|
||||
_httpClient = httpClientFactory.CreateClient(Constants.HttpClientWithRetryName);
|
||||
_loggingConfig = loggingConfig.Value;
|
||||
_queueCleanerConfig = queueCleanerConfig.Value;
|
||||
_striker = striker;
|
||||
_dryRunInterceptor = dryRunInterceptor;
|
||||
}
|
||||
|
||||
public virtual async Task<QueueListResponse> GetQueueItemsAsync(ArrInstance arrInstance, int page)
|
||||
{
|
||||
Uri uri = new(arrInstance.Url, GetQueueUrlPath(page));
|
||||
UriBuilder uriBuilder = new(arrInstance.Url);
|
||||
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/{GetQueueUrlPath().TrimStart('/')}";
|
||||
uriBuilder.Query = GetQueueUrlQuery(page);
|
||||
|
||||
using HttpRequestMessage request = new(HttpMethod.Get, uri);
|
||||
using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
|
||||
SetApiKey(request, arrInstance.ApiKey);
|
||||
|
||||
using HttpResponseMessage response = await _httpClient.SendAsync(request);
|
||||
@@ -51,7 +58,7 @@ public abstract class ArrClient
|
||||
}
|
||||
catch
|
||||
{
|
||||
_logger.LogError("queue list failed | {uri}", uri);
|
||||
_logger.LogError("queue list failed | {uri}", uriBuilder.Uri);
|
||||
throw;
|
||||
}
|
||||
|
||||
@@ -60,13 +67,13 @@ public abstract class ArrClient
|
||||
|
||||
if (queueResponse is null)
|
||||
{
|
||||
throw new Exception($"unrecognized queue list response | {uri} | {responseBody}");
|
||||
throw new Exception($"unrecognized queue list response | {uriBuilder.Uri} | {responseBody}");
|
||||
}
|
||||
|
||||
return queueResponse;
|
||||
}
|
||||
|
||||
public virtual bool ShouldRemoveFromQueue(QueueRecord record, bool isPrivateDownload)
|
||||
public virtual async Task<bool> ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload)
|
||||
{
|
||||
if (_queueCleanerConfig.ImportFailedIgnorePrivate && isPrivateDownload)
|
||||
{
|
||||
@@ -81,8 +88,14 @@ public abstract class ArrClient
|
||||
.Equals("importBlocked", StringComparison.InvariantCultureIgnoreCase);
|
||||
bool isImportPending() => record.TrackedDownloadState
|
||||
.Equals("importPending", StringComparison.InvariantCultureIgnoreCase);
|
||||
bool isImportFailed() => record.TrackedDownloadState
|
||||
.Equals("importFailed", StringComparison.InvariantCultureIgnoreCase);
|
||||
bool isFailedLidarr() => instanceType is InstanceType.Lidarr &&
|
||||
(record.Status.Equals("failed", StringComparison.InvariantCultureIgnoreCase) ||
|
||||
record.Status.Equals("completed", StringComparison.InvariantCultureIgnoreCase)) &&
|
||||
hasWarn();
|
||||
|
||||
if (hasWarn() && (isImportBlocked() || isImportPending()))
|
||||
if (hasWarn() && (isImportBlocked() || isImportPending() || isImportFailed()) || isFailedLidarr())
|
||||
{
|
||||
if (HasIgnoredPatterns(record))
|
||||
{
|
||||
@@ -90,7 +103,7 @@ public abstract class ArrClient
|
||||
return false;
|
||||
}
|
||||
|
||||
return _striker.StrikeAndCheckLimit(
|
||||
return await _striker.StrikeAndCheckLimit(
|
||||
record.DownloadId,
|
||||
record.Title,
|
||||
_queueCleanerConfig.ImportFailedMaxStrikes,
|
||||
@@ -101,24 +114,37 @@ public abstract class ArrClient
|
||||
return false;
|
||||
}
|
||||
|
||||
public virtual async Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record)
|
||||
public virtual async Task DeleteQueueItemAsync(
|
||||
ArrInstance arrInstance,
|
||||
QueueRecord record,
|
||||
bool removeFromClient,
|
||||
DeleteReason deleteReason
|
||||
)
|
||||
{
|
||||
Uri uri = new(arrInstance.Url, GetQueueDeleteUrlPath(record.Id));
|
||||
|
||||
using HttpRequestMessage request = new(HttpMethod.Delete, uri);
|
||||
SetApiKey(request, arrInstance.ApiKey);
|
||||
|
||||
using HttpResponseMessage response = await _httpClient.SendAsync(request);
|
||||
UriBuilder uriBuilder = new(arrInstance.Url);
|
||||
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/{GetQueueDeleteUrlPath(record.Id).TrimStart('/')}";
|
||||
uriBuilder.Query = GetQueueDeleteUrlQuery(removeFromClient);
|
||||
|
||||
try
|
||||
{
|
||||
response.EnsureSuccessStatusCode();
|
||||
using HttpRequestMessage request = new(HttpMethod.Delete, uriBuilder.Uri);
|
||||
SetApiKey(request, arrInstance.ApiKey);
|
||||
|
||||
HttpResponseMessage? response = await _dryRunInterceptor.InterceptAsync<HttpResponseMessage>(SendRequestAsync, request);
|
||||
response?.Dispose();
|
||||
|
||||
_logger.LogInformation("queue item deleted | {url} | {title}", arrInstance.Url, record.Title);
|
||||
_logger.LogInformation(
|
||||
removeFromClient
|
||||
? "queue item deleted with reason {reason} | {url} | {title}"
|
||||
: "queue item removed from arr with reason {reason} | {url} | {title}",
|
||||
deleteReason.ToString(),
|
||||
arrInstance.Url,
|
||||
record.Title
|
||||
);
|
||||
}
|
||||
catch
|
||||
{
|
||||
_logger.LogError("queue delete failed | {uri} | {title}", uri, record.Title);
|
||||
_logger.LogError("queue delete failed | {uri} | {title}", uriBuilder.Uri, record.Title);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -133,24 +159,32 @@ public abstract class ArrClient
|
||||
return false;
|
||||
}
|
||||
|
||||
if (record.DownloadId.Equals(record.Title, StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
_logger.LogDebug("skip | item is not ready yet | {title}", record.Title);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
protected abstract string GetQueueUrlPath(int page);
|
||||
protected abstract string GetQueueUrlPath();
|
||||
|
||||
protected abstract string GetQueueUrlQuery(int page);
|
||||
|
||||
protected abstract string GetQueueDeleteUrlPath(long recordId);
|
||||
|
||||
protected abstract string GetQueueDeleteUrlQuery(bool removeFromClient);
|
||||
|
||||
protected virtual void SetApiKey(HttpRequestMessage request, string apiKey)
|
||||
{
|
||||
request.Headers.Add("x-api-key", apiKey);
|
||||
}
|
||||
|
||||
[DryRunSafeguard]
|
||||
protected virtual async Task<HttpResponseMessage> SendRequestAsync(HttpRequestMessage request)
|
||||
{
|
||||
HttpResponseMessage response = await _httpClient.SendAsync(request);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
private bool HasIgnoredPatterns(QueueRecord record)
|
||||
{
|
||||
if (_queueCleanerConfig.ImportFailedIgnorePatterns?.Count is null or 0)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using Common.Configuration;
|
||||
using Common.Configuration.Arr;
|
||||
using Domain.Models.Arr.Queue;
|
||||
using Infrastructure.Verticals.Arr.Interfaces;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Infrastructure.Verticals.Arr;
|
||||
@@ -14,7 +15,7 @@ public sealed class ArrQueueIterator
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task Iterate(ArrClient arrClient, ArrInstance arrInstance, Func<IReadOnlyList<QueueRecord>, Task> action)
|
||||
public async Task Iterate(IArrClient arrClient, ArrInstance arrInstance, Func<IReadOnlyList<QueueRecord>, Task> action)
|
||||
{
|
||||
const ushort maxPage = 100;
|
||||
ushort page = 1;
|
||||
|
||||
19
code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs
Normal file
19
code/Infrastructure/Verticals/Arr/Interfaces/IArrClient.cs
Normal file
@@ -0,0 +1,19 @@
|
||||
using Common.Configuration.Arr;
|
||||
using Domain.Enums;
|
||||
using Domain.Models.Arr;
|
||||
using Domain.Models.Arr.Queue;
|
||||
|
||||
namespace Infrastructure.Verticals.Arr.Interfaces;
|
||||
|
||||
public interface IArrClient
|
||||
{
|
||||
Task<QueueListResponse> GetQueueItemsAsync(ArrInstance arrInstance, int page);
|
||||
|
||||
Task<bool> ShouldRemoveFromQueue(InstanceType instanceType, QueueRecord record, bool isPrivateDownload);
|
||||
|
||||
Task DeleteQueueItemAsync(ArrInstance arrInstance, QueueRecord record, bool removeFromClient, DeleteReason deleteReason);
|
||||
|
||||
Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items);
|
||||
|
||||
bool IsRecordValid(QueueRecord record);
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
namespace Infrastructure.Verticals.Arr.Interfaces;
|
||||
|
||||
public interface ILidarrClient : IArrClient
|
||||
{
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
namespace Infrastructure.Verticals.Arr.Interfaces;
|
||||
|
||||
public interface IRadarrClient : IArrClient
|
||||
{
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
namespace Infrastructure.Verticals.Arr.Interfaces;
|
||||
|
||||
public interface ISonarrClient : IArrClient
|
||||
{
|
||||
}
|
||||
@@ -5,6 +5,8 @@ using Common.Configuration.QueueCleaner;
|
||||
using Domain.Models.Arr;
|
||||
using Domain.Models.Arr.Queue;
|
||||
using Domain.Models.Lidarr;
|
||||
using Infrastructure.Interceptors;
|
||||
using Infrastructure.Verticals.Arr.Interfaces;
|
||||
using Infrastructure.Verticals.ItemStriker;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
@@ -12,37 +14,55 @@ using Newtonsoft.Json;
|
||||
|
||||
namespace Infrastructure.Verticals.Arr;
|
||||
|
||||
public sealed class LidarrClient : ArrClient
|
||||
public class LidarrClient : ArrClient, ILidarrClient
|
||||
{
|
||||
public LidarrClient(
|
||||
ILogger<LidarrClient> logger,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IOptions<LoggingConfig> loggingConfig,
|
||||
IOptions<QueueCleanerConfig> queueCleanerConfig,
|
||||
Striker striker
|
||||
) : base(logger, httpClientFactory, loggingConfig, queueCleanerConfig, striker)
|
||||
IStriker striker,
|
||||
IDryRunInterceptor dryRunInterceptor
|
||||
) : base(logger, httpClientFactory, loggingConfig, queueCleanerConfig, striker, dryRunInterceptor)
|
||||
{
|
||||
}
|
||||
|
||||
protected override string GetQueueUrlPath(int page)
|
||||
protected override string GetQueueUrlPath()
|
||||
{
|
||||
return $"/api/v1/queue?page={page}&pageSize=200&includeUnknownArtistItems=true&includeArtist=true&includeAlbum=true";
|
||||
return "/api/v1/queue";
|
||||
}
|
||||
|
||||
protected override string GetQueueUrlQuery(int page)
|
||||
{
|
||||
return $"page={page}&pageSize=200&includeUnknownArtistItems=true&includeArtist=true&includeAlbum=true";
|
||||
}
|
||||
|
||||
protected override string GetQueueDeleteUrlPath(long recordId)
|
||||
{
|
||||
return $"/api/v1/queue/{recordId}?removeFromClient=true&blocklist=true&skipRedownload=true&changeCategory=false";
|
||||
return $"/api/v1/queue/{recordId}";
|
||||
}
|
||||
|
||||
protected override string GetQueueDeleteUrlQuery(bool removeFromClient)
|
||||
{
|
||||
string query = "blocklist=true&skipRedownload=true&changeCategory=false";
|
||||
query += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";
|
||||
|
||||
return query;
|
||||
}
|
||||
|
||||
public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items)
|
||||
{
|
||||
if (items?.Count is null or 0) return;
|
||||
if (items?.Count is null or 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Uri uri = new(arrInstance.Url, "/api/v1/command");
|
||||
UriBuilder uriBuilder = new(arrInstance.Url);
|
||||
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v1/command";
|
||||
|
||||
foreach (var command in GetSearchCommands(items))
|
||||
{
|
||||
using HttpRequestMessage request = new(HttpMethod.Post, uri);
|
||||
using HttpRequestMessage request = new(HttpMethod.Post, uriBuilder.Uri);
|
||||
request.Content = new StringContent(
|
||||
JsonConvert.SerializeObject(command, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }),
|
||||
Encoding.UTF8,
|
||||
@@ -50,13 +70,13 @@ public sealed class LidarrClient : ArrClient
|
||||
);
|
||||
SetApiKey(request, arrInstance.ApiKey);
|
||||
|
||||
using var response = await _httpClient.SendAsync(request);
|
||||
string? logContext = await ComputeCommandLogContextAsync(arrInstance, command);
|
||||
|
||||
try
|
||||
{
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
HttpResponseMessage? response = await _dryRunInterceptor.InterceptAsync<HttpResponseMessage>(SendRequestAsync, request);
|
||||
response?.Dispose();
|
||||
|
||||
_logger.LogInformation("{log}", GetSearchLog(arrInstance.Url, command, true, logContext));
|
||||
}
|
||||
catch
|
||||
@@ -125,8 +145,11 @@ public sealed class LidarrClient : ArrClient
|
||||
|
||||
private async Task<List<Album>?> GetAlbumsAsync(ArrInstance arrInstance, List<long> albumIds)
|
||||
{
|
||||
Uri uri = new(arrInstance.Url, $"api/v1/album?{string.Join('&', albumIds.Select(x => $"albumIds={x}"))}");
|
||||
using HttpRequestMessage request = new(HttpMethod.Get, uri);
|
||||
UriBuilder uriBuilder = new(arrInstance.Url);
|
||||
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v1/album";
|
||||
uriBuilder.Query = string.Join('&', albumIds.Select(x => $"albumIds={x}"));
|
||||
|
||||
using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
|
||||
SetApiKey(request, arrInstance.ApiKey);
|
||||
|
||||
using var response = await _httpClient.SendAsync(request);
|
||||
|
||||
@@ -5,6 +5,8 @@ using Common.Configuration.QueueCleaner;
|
||||
using Domain.Models.Arr;
|
||||
using Domain.Models.Arr.Queue;
|
||||
using Domain.Models.Radarr;
|
||||
using Infrastructure.Interceptors;
|
||||
using Infrastructure.Verticals.Arr.Interfaces;
|
||||
using Infrastructure.Verticals.ItemStriker;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
@@ -12,26 +14,40 @@ using Newtonsoft.Json;
|
||||
|
||||
namespace Infrastructure.Verticals.Arr;
|
||||
|
||||
public sealed class RadarrClient : ArrClient
|
||||
public class RadarrClient : ArrClient, IRadarrClient
|
||||
{
|
||||
public RadarrClient(
|
||||
ILogger<ArrClient> logger,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IOptions<LoggingConfig> loggingConfig,
|
||||
IOptions<QueueCleanerConfig> queueCleanerConfig,
|
||||
Striker striker
|
||||
) : base(logger, httpClientFactory, loggingConfig, queueCleanerConfig, striker)
|
||||
IStriker striker,
|
||||
IDryRunInterceptor dryRunInterceptor
|
||||
) : base(logger, httpClientFactory, loggingConfig, queueCleanerConfig, striker, dryRunInterceptor)
|
||||
{
|
||||
}
|
||||
|
||||
protected override string GetQueueUrlPath(int page)
|
||||
protected override string GetQueueUrlPath()
|
||||
{
|
||||
return $"/api/v3/queue?page={page}&pageSize=200&includeUnknownMovieItems=true&includeMovie=true";
|
||||
return "/api/v3/queue";
|
||||
}
|
||||
|
||||
protected override string GetQueueUrlQuery(int page)
|
||||
{
|
||||
return $"page={page}&pageSize=200&includeUnknownMovieItems=true&includeMovie=true";
|
||||
}
|
||||
|
||||
protected override string GetQueueDeleteUrlPath(long recordId)
|
||||
{
|
||||
return $"/api/v3/queue/{recordId}?removeFromClient=true&blocklist=true&skipRedownload=true&changeCategory=false";
|
||||
return $"/api/v3/queue/{recordId}";
|
||||
}
|
||||
|
||||
protected override string GetQueueDeleteUrlQuery(bool removeFromClient)
|
||||
{
|
||||
string query = "blocklist=true&skipRedownload=true&changeCategory=false";
|
||||
query += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";
|
||||
|
||||
return query;
|
||||
}
|
||||
|
||||
public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items)
|
||||
@@ -43,14 +59,16 @@ public sealed class RadarrClient : ArrClient
|
||||
|
||||
List<long> ids = items.Select(item => item.Id).ToList();
|
||||
|
||||
Uri uri = new(arrInstance.Url, "/api/v3/command");
|
||||
UriBuilder uriBuilder = new(arrInstance.Url);
|
||||
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/command";
|
||||
|
||||
RadarrCommand command = new()
|
||||
{
|
||||
Name = "MoviesSearch",
|
||||
MovieIds = ids,
|
||||
};
|
||||
|
||||
using HttpRequestMessage request = new(HttpMethod.Post, uri);
|
||||
using HttpRequestMessage request = new(HttpMethod.Post, uriBuilder.Uri);
|
||||
request.Content = new StringContent(
|
||||
JsonConvert.SerializeObject(command),
|
||||
Encoding.UTF8,
|
||||
@@ -58,12 +76,12 @@ public sealed class RadarrClient : ArrClient
|
||||
);
|
||||
SetApiKey(request, arrInstance.ApiKey);
|
||||
|
||||
using HttpResponseMessage response = await _httpClient.SendAsync(request);
|
||||
string? logContext = await ComputeCommandLogContextAsync(arrInstance, command);
|
||||
|
||||
try
|
||||
{
|
||||
response.EnsureSuccessStatusCode();
|
||||
HttpResponseMessage? response = await _dryRunInterceptor.InterceptAsync<HttpResponseMessage>(SendRequestAsync, request);
|
||||
response?.Dispose();
|
||||
|
||||
_logger.LogInformation("{log}", GetSearchLog(arrInstance.Url, command, true, logContext));
|
||||
}
|
||||
@@ -128,8 +146,10 @@ public sealed class RadarrClient : ArrClient
|
||||
|
||||
private async Task<Movie?> GetMovie(ArrInstance arrInstance, long movieId)
|
||||
{
|
||||
Uri uri = new(arrInstance.Url, $"api/v3/movie/{movieId}");
|
||||
using HttpRequestMessage request = new(HttpMethod.Get, uri);
|
||||
UriBuilder uriBuilder = new(arrInstance.Url);
|
||||
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/movie/{movieId}";
|
||||
|
||||
using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
|
||||
SetApiKey(request, arrInstance.ApiKey);
|
||||
|
||||
using HttpResponseMessage response = await _httpClient.SendAsync(request);
|
||||
|
||||
@@ -5,33 +5,50 @@ using Common.Configuration.QueueCleaner;
|
||||
using Domain.Models.Arr;
|
||||
using Domain.Models.Arr.Queue;
|
||||
using Domain.Models.Sonarr;
|
||||
using Infrastructure.Interceptors;
|
||||
using Infrastructure.Verticals.Arr.Interfaces;
|
||||
using Infrastructure.Verticals.ItemStriker;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Newtonsoft.Json;
|
||||
using Series = Domain.Models.Sonarr.Series;
|
||||
|
||||
namespace Infrastructure.Verticals.Arr;
|
||||
|
||||
public sealed class SonarrClient : ArrClient
|
||||
public class SonarrClient : ArrClient, ISonarrClient
|
||||
{
|
||||
public SonarrClient(
|
||||
ILogger<SonarrClient> logger,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IOptions<LoggingConfig> loggingConfig,
|
||||
IOptions<QueueCleanerConfig> queueCleanerConfig,
|
||||
Striker striker
|
||||
) : base(logger, httpClientFactory, loggingConfig, queueCleanerConfig, striker)
|
||||
IStriker striker,
|
||||
IDryRunInterceptor dryRunInterceptor
|
||||
) : base(logger, httpClientFactory, loggingConfig, queueCleanerConfig, striker, dryRunInterceptor)
|
||||
{
|
||||
}
|
||||
|
||||
protected override string GetQueueUrlPath(int page)
|
||||
protected override string GetQueueUrlPath()
|
||||
{
|
||||
return $"/api/v3/queue?page={page}&pageSize=200&includeUnknownSeriesItems=true&includeSeries=true";
|
||||
return "/api/v3/queue";
|
||||
}
|
||||
|
||||
|
||||
protected override string GetQueueUrlQuery(int page)
|
||||
{
|
||||
return $"page={page}&pageSize=200&includeUnknownSeriesItems=true&includeSeries=true&includeEpisode=true";
|
||||
}
|
||||
|
||||
protected override string GetQueueDeleteUrlPath(long recordId)
|
||||
{
|
||||
return $"/api/v3/queue/{recordId}?removeFromClient=true&blocklist=true&skipRedownload=true&changeCategory=false";
|
||||
return $"/api/v3/queue/{recordId}";
|
||||
}
|
||||
|
||||
protected override string GetQueueDeleteUrlQuery(bool removeFromClient)
|
||||
{
|
||||
string query = "blocklist=true&skipRedownload=true&changeCategory=false";
|
||||
query += removeFromClient ? "&removeFromClient=true" : "&removeFromClient=false";
|
||||
|
||||
return query;
|
||||
}
|
||||
|
||||
public override async Task RefreshItemsAsync(ArrInstance arrInstance, HashSet<SearchItem>? items)
|
||||
@@ -41,11 +58,12 @@ public sealed class SonarrClient : ArrClient
|
||||
return;
|
||||
}
|
||||
|
||||
Uri uri = new(arrInstance.Url, "/api/v3/command");
|
||||
UriBuilder uriBuilder = new(arrInstance.Url);
|
||||
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/command";
|
||||
|
||||
foreach (SonarrCommand command in GetSearchCommands(items.Cast<SonarrSearchItem>().ToHashSet()))
|
||||
{
|
||||
using HttpRequestMessage request = new(HttpMethod.Post, uri);
|
||||
using HttpRequestMessage request = new(HttpMethod.Post, uriBuilder.Uri);
|
||||
request.Content = new StringContent(
|
||||
JsonConvert.SerializeObject(command, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }),
|
||||
Encoding.UTF8,
|
||||
@@ -53,13 +71,13 @@ public sealed class SonarrClient : ArrClient
|
||||
);
|
||||
SetApiKey(request, arrInstance.ApiKey);
|
||||
|
||||
using HttpResponseMessage response = await _httpClient.SendAsync(request);
|
||||
string? logContext = await ComputeCommandLogContextAsync(arrInstance, command, command.SearchType);
|
||||
|
||||
try
|
||||
{
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
HttpResponseMessage? response = await _dryRunInterceptor.InterceptAsync<HttpResponseMessage>(SendRequestAsync, request);
|
||||
response?.Dispose();
|
||||
|
||||
_logger.LogInformation("{log}", GetSearchLog(command.SearchType, arrInstance.Url, command, true, logContext));
|
||||
}
|
||||
catch
|
||||
@@ -191,8 +209,11 @@ public sealed class SonarrClient : ArrClient
|
||||
|
||||
private async Task<List<Episode>?> GetEpisodesAsync(ArrInstance arrInstance, List<long> episodeIds)
|
||||
{
|
||||
Uri uri = new(arrInstance.Url, $"api/v3/episode?{string.Join('&', episodeIds.Select(x => $"episodeIds={x}"))}");
|
||||
using HttpRequestMessage request = new(HttpMethod.Get, uri);
|
||||
UriBuilder uriBuilder = new(arrInstance.Url);
|
||||
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/episode";
|
||||
uriBuilder.Query = string.Join('&', episodeIds.Select(x => $"episodeIds={x}"));
|
||||
|
||||
using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
|
||||
SetApiKey(request, arrInstance.ApiKey);
|
||||
|
||||
using HttpResponseMessage response = await _httpClient.SendAsync(request);
|
||||
@@ -204,8 +225,10 @@ public sealed class SonarrClient : ArrClient
|
||||
|
||||
private async Task<Series?> GetSeriesAsync(ArrInstance arrInstance, long seriesId)
|
||||
{
|
||||
Uri uri = new(arrInstance.Url, $"api/v3/series/{seriesId}");
|
||||
using HttpRequestMessage request = new(HttpMethod.Get, uri);
|
||||
UriBuilder uriBuilder = new(arrInstance.Url);
|
||||
uriBuilder.Path = $"{uriBuilder.Path.TrimEnd('/')}/api/v3/series/{seriesId}";
|
||||
|
||||
using HttpRequestMessage request = new(HttpMethod.Get, uriBuilder.Uri);
|
||||
SetApiKey(request, arrInstance.ApiKey);
|
||||
|
||||
using HttpResponseMessage response = await _httpClient.SendAsync(request);
|
||||
|
||||
@@ -5,6 +5,7 @@ using Common.Configuration.Arr;
|
||||
using Common.Configuration.ContentBlocker;
|
||||
using Common.Helpers;
|
||||
using Domain.Enums;
|
||||
using Infrastructure.Helpers;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
@@ -21,10 +22,6 @@ public sealed class BlocklistProvider
|
||||
private readonly IMemoryCache _cache;
|
||||
private bool _initialized;
|
||||
|
||||
private const string Type = "type";
|
||||
private const string Patterns = "patterns";
|
||||
private const string Regexes = "regexes";
|
||||
|
||||
public BlocklistProvider(
|
||||
ILogger<BlocklistProvider> logger,
|
||||
IOptions<SonarrConfig> sonarrConfig,
|
||||
@@ -46,15 +43,15 @@ public sealed class BlocklistProvider
|
||||
{
|
||||
if (_initialized)
|
||||
{
|
||||
_logger.LogDebug("blocklists already loaded");
|
||||
_logger.LogTrace("blocklists already loaded");
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await LoadPatternsAndRegexesAsync(_sonarrConfig.Block.Type, _sonarrConfig.Block.Path, InstanceType.Sonarr);
|
||||
await LoadPatternsAndRegexesAsync(_radarrConfig.Block.Type, _radarrConfig.Block.Path, InstanceType.Radarr);
|
||||
await LoadPatternsAndRegexesAsync(_lidarrConfig.Block.Type, _lidarrConfig.Block.Path, InstanceType.Lidarr);
|
||||
await LoadPatternsAndRegexesAsync(_sonarrConfig, InstanceType.Sonarr);
|
||||
await LoadPatternsAndRegexesAsync(_radarrConfig, InstanceType.Radarr);
|
||||
await LoadPatternsAndRegexesAsync(_lidarrConfig, InstanceType.Lidarr);
|
||||
|
||||
_initialized = true;
|
||||
}
|
||||
@@ -67,33 +64,38 @@ public sealed class BlocklistProvider
|
||||
|
||||
public BlocklistType GetBlocklistType(InstanceType instanceType)
|
||||
{
|
||||
_cache.TryGetValue($"{instanceType.ToString()}_{Type}", out BlocklistType? blocklistType);
|
||||
_cache.TryGetValue(CacheKeys.BlocklistType(instanceType), out BlocklistType? blocklistType);
|
||||
|
||||
return blocklistType ?? BlocklistType.Blacklist;
|
||||
}
|
||||
|
||||
public ConcurrentBag<string> GetPatterns(InstanceType instanceType)
|
||||
{
|
||||
_cache.TryGetValue($"{instanceType.ToString()}_{Patterns}", out ConcurrentBag<string>? patterns);
|
||||
_cache.TryGetValue(CacheKeys.BlocklistPatterns(instanceType), out ConcurrentBag<string>? patterns);
|
||||
|
||||
return patterns ?? [];
|
||||
}
|
||||
|
||||
public ConcurrentBag<Regex> GetRegexes(InstanceType instanceType)
|
||||
{
|
||||
_cache.TryGetValue($"{instanceType.ToString()}_{Regexes}", out ConcurrentBag<Regex>? regexes);
|
||||
_cache.TryGetValue(CacheKeys.BlocklistRegexes(instanceType), out ConcurrentBag<Regex>? regexes);
|
||||
|
||||
return regexes ?? [];
|
||||
}
|
||||
|
||||
private async Task LoadPatternsAndRegexesAsync(BlocklistType blocklistType, string? blocklistPath, InstanceType instanceType)
|
||||
private async Task LoadPatternsAndRegexesAsync(ArrConfig arrConfig, InstanceType instanceType)
|
||||
{
|
||||
if (string.IsNullOrEmpty(blocklistPath))
|
||||
if (!arrConfig.Enabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
string[] filePatterns = await ReadContentAsync(blocklistPath);
|
||||
if (string.IsNullOrEmpty(arrConfig.Block.Path))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
string[] filePatterns = await ReadContentAsync(arrConfig.Block.Path);
|
||||
|
||||
long startTime = Stopwatch.GetTimestamp();
|
||||
ParallelOptions options = new() { MaxDegreeOfParallelism = 5 };
|
||||
@@ -124,13 +126,13 @@ public sealed class BlocklistProvider
|
||||
|
||||
TimeSpan elapsed = Stopwatch.GetElapsedTime(startTime);
|
||||
|
||||
_cache.Set($"{instanceType.ToString()}_{Type}", blocklistType);
|
||||
_cache.Set($"{instanceType.ToString()}_{Patterns}", patterns);
|
||||
_cache.Set($"{instanceType.ToString()}_{Regexes}", regexes);
|
||||
_cache.Set(CacheKeys.BlocklistType(instanceType), arrConfig.Block.Type);
|
||||
_cache.Set(CacheKeys.BlocklistPatterns(instanceType), patterns);
|
||||
_cache.Set(CacheKeys.BlocklistRegexes(instanceType), regexes);
|
||||
|
||||
_logger.LogDebug("loaded {count} patterns", patterns.Count);
|
||||
_logger.LogDebug("loaded {count} regexes", regexes.Count);
|
||||
_logger.LogDebug("blocklist loaded in {elapsed} ms | {path}", elapsed.TotalMilliseconds, blocklistPath);
|
||||
_logger.LogDebug("blocklist loaded in {elapsed} ms | {path}", elapsed.TotalMilliseconds, arrConfig.Block.Path);
|
||||
}
|
||||
|
||||
private async Task<string[]> ReadContentAsync(string path)
|
||||
|
||||
@@ -6,9 +6,13 @@ using Common.Configuration.DownloadClient;
|
||||
using Domain.Enums;
|
||||
using Domain.Models.Arr;
|
||||
using Domain.Models.Arr.Queue;
|
||||
using Infrastructure.Providers;
|
||||
using Infrastructure.Verticals.Arr;
|
||||
using Infrastructure.Verticals.Arr.Interfaces;
|
||||
using Infrastructure.Verticals.Context;
|
||||
using Infrastructure.Verticals.DownloadClient;
|
||||
using Infrastructure.Verticals.Jobs;
|
||||
using Infrastructure.Verticals.Notifications;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Serilog.Context;
|
||||
@@ -17,10 +21,13 @@ namespace Infrastructure.Verticals.ContentBlocker;
|
||||
|
||||
public sealed class ContentBlocker : GenericHandler
|
||||
{
|
||||
private readonly ContentBlockerConfig _config;
|
||||
private readonly BlocklistProvider _blocklistProvider;
|
||||
|
||||
private readonly IgnoredDownloadsProvider<ContentBlockerConfig> _ignoredDownloadsProvider;
|
||||
|
||||
public ContentBlocker(
|
||||
ILogger<ContentBlocker> logger,
|
||||
IOptions<ContentBlockerConfig> config,
|
||||
IOptions<DownloadClientConfig> downloadClientConfig,
|
||||
IOptions<SonarrConfig> sonarrConfig,
|
||||
IOptions<RadarrConfig> radarrConfig,
|
||||
@@ -30,22 +37,27 @@ public sealed class ContentBlocker : GenericHandler
|
||||
LidarrClient lidarrClient,
|
||||
ArrQueueIterator arrArrQueueIterator,
|
||||
BlocklistProvider blocklistProvider,
|
||||
DownloadServiceFactory downloadServiceFactory
|
||||
DownloadServiceFactory downloadServiceFactory,
|
||||
INotificationPublisher notifier,
|
||||
IgnoredDownloadsProvider<ContentBlockerConfig> ignoredDownloadsProvider
|
||||
) : base(
|
||||
logger, downloadClientConfig,
|
||||
sonarrConfig, radarrConfig, lidarrConfig,
|
||||
sonarrClient, radarrClient, lidarrClient,
|
||||
arrArrQueueIterator, downloadServiceFactory
|
||||
arrArrQueueIterator, downloadServiceFactory,
|
||||
notifier
|
||||
)
|
||||
{
|
||||
_config = config.Value;
|
||||
_blocklistProvider = blocklistProvider;
|
||||
_ignoredDownloadsProvider = ignoredDownloadsProvider;
|
||||
}
|
||||
|
||||
public override async Task ExecuteAsync()
|
||||
{
|
||||
if (_downloadClientConfig.DownloadClient is Common.Enums.DownloadClient.None)
|
||||
if (_downloadClientConfig.DownloadClient is Common.Enums.DownloadClient.None or Common.Enums.DownloadClient.Disabled)
|
||||
{
|
||||
_logger.LogWarning("download client is set to none");
|
||||
_logger.LogWarning("download client is not set");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -65,13 +77,19 @@ public sealed class ContentBlocker : GenericHandler
|
||||
|
||||
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)
|
||||
{
|
||||
IReadOnlyList<string> ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads();
|
||||
|
||||
using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString());
|
||||
|
||||
HashSet<SearchItem> itemsToBeRefreshed = [];
|
||||
ArrClient arrClient = GetClient(instanceType);
|
||||
IArrClient arrClient = GetClient(instanceType);
|
||||
BlocklistType blocklistType = _blocklistProvider.GetBlocklistType(instanceType);
|
||||
ConcurrentBag<string> patterns = _blocklistProvider.GetPatterns(instanceType);
|
||||
ConcurrentBag<Regex> regexes = _blocklistProvider.GetRegexes(instanceType);
|
||||
|
||||
// push to context
|
||||
ContextProvider.Set(nameof(ArrInstance) + nameof(ArrInstance.Url), instance.Url);
|
||||
ContextProvider.Set(nameof(InstanceType), instanceType);
|
||||
|
||||
await _arrArrQueueIterator.Iterate(arrClient, instance, async items =>
|
||||
{
|
||||
@@ -94,9 +112,21 @@ public sealed class ContentBlocker : GenericHandler
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ignoredDownloads.Contains(record.DownloadId, StringComparer.InvariantCultureIgnoreCase))
|
||||
{
|
||||
_logger.LogInformation("skip | {title} | ignored", record.Title);
|
||||
continue;
|
||||
}
|
||||
|
||||
// push record to context
|
||||
ContextProvider.Set(nameof(QueueRecord), record);
|
||||
|
||||
_logger.LogDebug("searching unwanted files for {title}", record.Title);
|
||||
|
||||
if (!await _downloadService.BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes))
|
||||
BlockFilesResult result = await _downloadService
|
||||
.BlockUnwantedFilesAsync(record.DownloadId, blocklistType, patterns, regexes, ignoredDownloads);
|
||||
|
||||
if (!result.ShouldRemove)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
@@ -104,7 +134,16 @@ public sealed class ContentBlocker : GenericHandler
|
||||
_logger.LogDebug("all files are marked as unwanted | {hash}", record.Title);
|
||||
|
||||
itemsToBeRefreshed.Add(GetRecordSearchItem(instanceType, record, group.Count() > 1));
|
||||
await arrClient.DeleteQueueItemAsync(instance, record);
|
||||
|
||||
bool removeFromClient = true;
|
||||
|
||||
if (result.IsPrivate && !_config.DeletePrivate)
|
||||
{
|
||||
removeFromClient = false;
|
||||
}
|
||||
|
||||
await arrClient.DeleteQueueItemAsync(instance, record, removeFromClient, DeleteReason.AllFilesBlocked);
|
||||
await _notifier.NotifyQueueItemDeleted(removeFromClient, DeleteReason.AllFilesBlocked);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Infrastructure.Verticals.ContentBlocker;
|
||||
|
||||
public sealed class FilenameEvaluator
|
||||
public class FilenameEvaluator : IFilenameEvaluator
|
||||
{
|
||||
private readonly ILogger<FilenameEvaluator> _logger;
|
||||
|
||||
@@ -31,7 +31,6 @@ public sealed class FilenameEvaluator
|
||||
{
|
||||
BlocklistType.Blacklist => !patterns.Any(pattern => MatchesPattern(filename, pattern)),
|
||||
BlocklistType.Whitelist => patterns.Any(pattern => MatchesPattern(filename, pattern)),
|
||||
_ => true
|
||||
};
|
||||
}
|
||||
|
||||
@@ -46,7 +45,6 @@ public sealed class FilenameEvaluator
|
||||
{
|
||||
BlocklistType.Blacklist => !regexes.Any(regex => regex.IsMatch(filename)),
|
||||
BlocklistType.Whitelist => regexes.Any(regex => regex.IsMatch(filename)),
|
||||
_ => true
|
||||
};
|
||||
}
|
||||
|
||||
@@ -76,6 +74,6 @@ public sealed class FilenameEvaluator
|
||||
);
|
||||
}
|
||||
|
||||
return filename == pattern;
|
||||
return filename.Equals(pattern, StringComparison.InvariantCultureIgnoreCase);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.RegularExpressions;
|
||||
using Common.Configuration.ContentBlocker;
|
||||
|
||||
namespace Infrastructure.Verticals.ContentBlocker;
|
||||
|
||||
public interface IFilenameEvaluator
|
||||
{
|
||||
bool IsValid(string filename, BlocklistType type, ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes);
|
||||
}
|
||||
24
code/Infrastructure/Verticals/Context/ContextProvider.cs
Normal file
24
code/Infrastructure/Verticals/Context/ContextProvider.cs
Normal file
@@ -0,0 +1,24 @@
|
||||
using System.Collections.Immutable;
|
||||
|
||||
namespace Infrastructure.Verticals.Context;
|
||||
|
||||
public static class ContextProvider
|
||||
{
|
||||
private static readonly AsyncLocal<ImmutableDictionary<string, object>> _asyncLocalDict = new();
|
||||
|
||||
public static void Set(string key, object value)
|
||||
{
|
||||
ImmutableDictionary<string, object> currentDict = _asyncLocalDict.Value ?? ImmutableDictionary<string, object>.Empty;
|
||||
_asyncLocalDict.Value = currentDict.SetItem(key, value);
|
||||
}
|
||||
|
||||
public static object? Get(string key)
|
||||
{
|
||||
return _asyncLocalDict.Value?.TryGetValue(key, out object? value) is true ? value : null;
|
||||
}
|
||||
|
||||
public static T Get<T>(string key) where T : class
|
||||
{
|
||||
return Get(key) as T ?? throw new Exception($"failed to get \"{key}\" from context");
|
||||
}
|
||||
}
|
||||
129
code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs
Normal file
129
code/Infrastructure/Verticals/DownloadCleaner/DownloadCleaner.cs
Normal file
@@ -0,0 +1,129 @@
|
||||
using Common.Configuration.Arr;
|
||||
using Common.Configuration.DownloadCleaner;
|
||||
using Common.Configuration.DownloadClient;
|
||||
using Domain.Enums;
|
||||
using Domain.Models.Arr.Queue;
|
||||
using Infrastructure.Providers;
|
||||
using Infrastructure.Verticals.Arr;
|
||||
using Infrastructure.Verticals.Arr.Interfaces;
|
||||
using Infrastructure.Verticals.DownloadClient;
|
||||
using Infrastructure.Verticals.Jobs;
|
||||
using Infrastructure.Verticals.Notifications;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Serilog.Context;
|
||||
|
||||
namespace Infrastructure.Verticals.DownloadCleaner;
|
||||
|
||||
public sealed class DownloadCleaner : GenericHandler
|
||||
{
|
||||
private readonly DownloadCleanerConfig _config;
|
||||
private readonly IgnoredDownloadsProvider<DownloadCleanerConfig> _ignoredDownloadsProvider;
|
||||
private readonly HashSet<string> _excludedHashes = [];
|
||||
|
||||
private static bool _hardLinkCategoryCreated;
|
||||
|
||||
public DownloadCleaner(
|
||||
ILogger<DownloadCleaner> logger,
|
||||
IOptions<DownloadCleanerConfig> config,
|
||||
IOptions<DownloadClientConfig> downloadClientConfig,
|
||||
IOptions<SonarrConfig> sonarrConfig,
|
||||
IOptions<RadarrConfig> radarrConfig,
|
||||
IOptions<LidarrConfig> lidarrConfig,
|
||||
SonarrClient sonarrClient,
|
||||
RadarrClient radarrClient,
|
||||
LidarrClient lidarrClient,
|
||||
ArrQueueIterator arrArrQueueIterator,
|
||||
DownloadServiceFactory downloadServiceFactory,
|
||||
INotificationPublisher notifier,
|
||||
IgnoredDownloadsProvider<DownloadCleanerConfig> ignoredDownloadsProvider
|
||||
) : base(
|
||||
logger, downloadClientConfig,
|
||||
sonarrConfig, radarrConfig, lidarrConfig,
|
||||
sonarrClient, radarrClient, lidarrClient,
|
||||
arrArrQueueIterator, downloadServiceFactory,
|
||||
notifier
|
||||
)
|
||||
{
|
||||
_config = config.Value;
|
||||
_config.Validate();
|
||||
_ignoredDownloadsProvider = ignoredDownloadsProvider;
|
||||
}
|
||||
|
||||
public override async Task ExecuteAsync()
|
||||
{
|
||||
if (_downloadClientConfig.DownloadClient is Common.Enums.DownloadClient.None or Common.Enums.DownloadClient.Disabled)
|
||||
{
|
||||
_logger.LogWarning("download client is not set");
|
||||
return;
|
||||
}
|
||||
|
||||
if (_config.Categories?.Count is null or 0)
|
||||
{
|
||||
_logger.LogWarning("no categories configured");
|
||||
return;
|
||||
}
|
||||
|
||||
IReadOnlyList<string> ignoredDownloads = await _ignoredDownloadsProvider.GetIgnoredDownloads();
|
||||
|
||||
await _downloadService.LoginAsync();
|
||||
List<object>? downloads = await _downloadService.GetSeedingDownloads();
|
||||
List<object>? downloadsToChangeCategory = null;
|
||||
|
||||
if (!string.IsNullOrEmpty(_config.UnlinkedTargetCategory) && _config.UnlinkedCategories?.Count > 0)
|
||||
{
|
||||
if (!_hardLinkCategoryCreated)
|
||||
{
|
||||
_logger.LogDebug("creating category {cat}", _config.UnlinkedTargetCategory);
|
||||
|
||||
await _downloadService.CreateCategoryAsync(_config.UnlinkedTargetCategory);
|
||||
_hardLinkCategoryCreated = true;
|
||||
}
|
||||
|
||||
downloadsToChangeCategory = _downloadService.FilterDownloadsToChangeCategoryAsync(downloads, _config.UnlinkedCategories);
|
||||
}
|
||||
|
||||
// wait for the downloads to appear in the arr queue
|
||||
await Task.Delay(10 * 1000);
|
||||
|
||||
await ProcessArrConfigAsync(_sonarrConfig, InstanceType.Sonarr, true);
|
||||
await ProcessArrConfigAsync(_radarrConfig, InstanceType.Radarr, true);
|
||||
await ProcessArrConfigAsync(_lidarrConfig, InstanceType.Lidarr, true);
|
||||
|
||||
_logger.LogTrace("looking for downloads to change category");
|
||||
await _downloadService.ChangeCategoryForNoHardLinksAsync(downloadsToChangeCategory, _excludedHashes, ignoredDownloads);
|
||||
|
||||
List<object>? downloadsToClean = _downloadService.FilterDownloadsToBeCleanedAsync(downloads, _config.Categories);
|
||||
|
||||
// release unused objects
|
||||
downloads = null;
|
||||
|
||||
_logger.LogTrace("looking for downloads to clean");
|
||||
await _downloadService.CleanDownloadsAsync(downloadsToClean, _config.Categories, _excludedHashes, ignoredDownloads);
|
||||
}
|
||||
|
||||
protected override async Task ProcessInstanceAsync(ArrInstance instance, InstanceType instanceType)
|
||||
{
|
||||
using var _ = LogContext.PushProperty("InstanceName", instanceType.ToString());
|
||||
|
||||
IArrClient arrClient = GetClient(instanceType);
|
||||
|
||||
await _arrArrQueueIterator.Iterate(arrClient, instance, async items =>
|
||||
{
|
||||
var groups = items
|
||||
.Where(x => !string.IsNullOrEmpty(x.DownloadId))
|
||||
.GroupBy(x => x.DownloadId)
|
||||
.ToList();
|
||||
|
||||
foreach (QueueRecord record in groups.Select(group => group.First()))
|
||||
{
|
||||
_excludedHashes.Add(record.DownloadId.ToLowerInvariant());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
_downloadService.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
namespace Infrastructure.Verticals.DownloadClient;
|
||||
|
||||
public sealed record RemoveResult
|
||||
public sealed record BlockFilesResult
|
||||
{
|
||||
/// <summary>
|
||||
/// True if the download should be removed; otherwise false.
|
||||
@@ -2,6 +2,7 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using Common.Configuration;
|
||||
using Common.Configuration.DownloadClient;
|
||||
using Common.Exceptions;
|
||||
using Domain.Models.Deluge.Exceptions;
|
||||
using Domain.Models.Deluge.Request;
|
||||
using Domain.Models.Deluge.Response;
|
||||
@@ -16,9 +17,27 @@ public sealed class DelugeClient
|
||||
private readonly DelugeConfig _config;
|
||||
private readonly HttpClient _httpClient;
|
||||
|
||||
private static readonly IReadOnlyList<string> Fields =
|
||||
[
|
||||
"hash",
|
||||
"state",
|
||||
"name",
|
||||
"eta",
|
||||
"private",
|
||||
"total_done",
|
||||
"label",
|
||||
"seeding_time",
|
||||
"ratio",
|
||||
"trackers",
|
||||
"download_payload_rate",
|
||||
"total_size",
|
||||
"download_location"
|
||||
];
|
||||
|
||||
public DelugeClient(IOptions<DelugeConfig> config, IHttpClientFactory httpClientFactory)
|
||||
{
|
||||
_config = config.Value;
|
||||
_config.Validate();
|
||||
_httpClient = httpClientFactory.CreateClient(nameof(DelugeService));
|
||||
}
|
||||
|
||||
@@ -27,11 +46,42 @@ public sealed class DelugeClient
|
||||
return await SendRequest<bool>("auth.login", _config.Password);
|
||||
}
|
||||
|
||||
public async Task<bool> IsConnected()
|
||||
{
|
||||
return await SendRequest<bool>("web.connected");
|
||||
}
|
||||
|
||||
public async Task<bool> Connect()
|
||||
{
|
||||
string? firstHost = await GetHost();
|
||||
|
||||
if (string.IsNullOrEmpty(firstHost))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var result = await SendRequest<List<string>?>("web.connect", firstHost);
|
||||
|
||||
return result?.Count > 0;
|
||||
}
|
||||
|
||||
public async Task<bool> Logout()
|
||||
{
|
||||
return await SendRequest<bool>("auth.delete_session");
|
||||
}
|
||||
|
||||
public async Task<string?> GetHost()
|
||||
{
|
||||
var hosts = await SendRequest<List<List<string>?>?>("web.get_hosts");
|
||||
|
||||
if (hosts?.Count > 1)
|
||||
{
|
||||
throw new FatalException("multiple Deluge hosts found - please connect to only one host");
|
||||
}
|
||||
|
||||
return hosts?.FirstOrDefault()?.FirstOrDefault();
|
||||
}
|
||||
|
||||
public async Task<List<DelugeTorrent>> ListTorrents(Dictionary<string, string>? filters = null)
|
||||
{
|
||||
filters ??= new Dictionary<string, string>();
|
||||
@@ -62,6 +112,39 @@ public sealed class DelugeClient
|
||||
await ListTorrentsExtended(new Dictionary<string, string> { { "hash", hash } });
|
||||
return torrents.FirstOrDefault();
|
||||
}
|
||||
|
||||
public async Task<DownloadStatus?> GetTorrentStatus(string hash)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await SendRequest<DownloadStatus?>(
|
||||
"web.get_torrent_status",
|
||||
hash,
|
||||
Fields
|
||||
);
|
||||
}
|
||||
catch (DelugeClientException e)
|
||||
{
|
||||
// Deluge returns an error when the torrent is not found
|
||||
if (e.Message == "AttributeError: 'NoneType' object has no attribute 'call'")
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<List<DownloadStatus>?> GetStatusForAllTorrents()
|
||||
{
|
||||
Dictionary<string, DownloadStatus>? downloads = await SendRequest<Dictionary<string, DownloadStatus>?>(
|
||||
"core.get_torrents_status",
|
||||
"",
|
||||
Fields
|
||||
);
|
||||
|
||||
return downloads?.Values.ToList();
|
||||
}
|
||||
|
||||
public async Task<DelugeContents?> GetTorrentFiles(string hash)
|
||||
{
|
||||
@@ -78,24 +161,28 @@ public sealed class DelugeClient
|
||||
await SendRequest<DelugeResponse<object>>("core.set_torrent_options", hash, filePriorities);
|
||||
}
|
||||
|
||||
public async Task<DelugeResponse<object>> DeleteTorrent(string hash)
|
||||
public async Task DeleteTorrents(List<string> hashes)
|
||||
{
|
||||
return await SendRequest<DelugeResponse<object>>("core.remove_torrents", new List<string> { hash }, true);
|
||||
await SendRequest<DelugeResponse<object>>("core.remove_torrents", hashes, true);
|
||||
}
|
||||
|
||||
private async Task<String> PostJson(String json)
|
||||
{
|
||||
StringContent content = new StringContent(json);
|
||||
content.Headers.ContentType = new MediaTypeWithQualityHeaderValue("application/json");
|
||||
|
||||
var responseMessage = await _httpClient.PostAsync(new Uri(_config.Url, "/json"), content);
|
||||
|
||||
UriBuilder uriBuilder = new(_config.Url);
|
||||
uriBuilder.Path = string.IsNullOrEmpty(_config.UrlBase)
|
||||
? $"{uriBuilder.Path.TrimEnd('/')}/json"
|
||||
: $"{uriBuilder.Path.TrimEnd('/')}/{_config.UrlBase.TrimStart('/').TrimEnd('/')}/json";
|
||||
var responseMessage = await _httpClient.PostAsync(uriBuilder.Uri, content);
|
||||
responseMessage.EnsureSuccessStatusCode();
|
||||
|
||||
var responseJson = await responseMessage.Content.ReadAsStringAsync();
|
||||
return responseJson;
|
||||
}
|
||||
|
||||
private DelugeRequest CreateRequest(string method, params object[] parameters)
|
||||
private static DelugeRequest CreateRequest(string method, params object[] parameters)
|
||||
{
|
||||
if (String.IsNullOrWhiteSpace(method))
|
||||
{
|
||||
@@ -141,4 +228,19 @@ public sealed class DelugeClient
|
||||
|
||||
return webResponse.Result;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<string>> GetLabels()
|
||||
{
|
||||
return await SendRequest<IReadOnlyList<string>>("label.get_labels");
|
||||
}
|
||||
|
||||
public async Task CreateLabel(string label)
|
||||
{
|
||||
await SendRequest<DelugeResponse<object>>("label.add", label);
|
||||
}
|
||||
|
||||
public async Task SetTorrentLabel(string hash, string newLabel)
|
||||
{
|
||||
await SendRequest<DelugeResponse<object>>("label.set_torrent", hash, newLabel);
|
||||
}
|
||||
}
|
||||
@@ -1,29 +1,49 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
using Common.Attributes;
|
||||
using Common.Configuration.ContentBlocker;
|
||||
using Common.Configuration.DownloadCleaner;
|
||||
using Common.Configuration.DownloadClient;
|
||||
using Common.Configuration.QueueCleaner;
|
||||
using Common.CustomDataTypes;
|
||||
using Common.Exceptions;
|
||||
using Domain.Enums;
|
||||
using Domain.Models.Deluge.Response;
|
||||
using Infrastructure.Extensions;
|
||||
using Infrastructure.Interceptors;
|
||||
using Infrastructure.Verticals.ContentBlocker;
|
||||
using Infrastructure.Verticals.Context;
|
||||
using Infrastructure.Verticals.Files;
|
||||
using Infrastructure.Verticals.ItemStriker;
|
||||
using Infrastructure.Verticals.Notifications;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Infrastructure.Verticals.DownloadClient.Deluge;
|
||||
|
||||
public sealed class DelugeService : DownloadServiceBase
|
||||
public class DelugeService : DownloadService, IDelugeService
|
||||
{
|
||||
private readonly DelugeClient _client;
|
||||
|
||||
|
||||
public DelugeService(
|
||||
ILogger<DelugeService> logger,
|
||||
IOptions<DelugeConfig> config,
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IOptions<QueueCleanerConfig> queueCleanerConfig,
|
||||
IOptions<ContentBlockerConfig> contentBlockerConfig,
|
||||
FilenameEvaluator filenameEvaluator,
|
||||
Striker striker
|
||||
) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
|
||||
IOptions<DownloadCleanerConfig> downloadCleanerConfig,
|
||||
IMemoryCache cache,
|
||||
IFilenameEvaluator filenameEvaluator,
|
||||
IStriker striker,
|
||||
INotificationPublisher notifier,
|
||||
IDryRunInterceptor dryRunInterceptor,
|
||||
IHardLinkFileService hardLinkFileService
|
||||
) : base(
|
||||
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache,
|
||||
filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService
|
||||
)
|
||||
{
|
||||
config.Value.Validate();
|
||||
_client = new (config, httpClientFactory);
|
||||
@@ -32,23 +52,36 @@ public sealed class DelugeService : DownloadServiceBase
|
||||
public override async Task LoginAsync()
|
||||
{
|
||||
await _client.LoginAsync();
|
||||
|
||||
if (!await _client.IsConnected() && !await _client.Connect())
|
||||
{
|
||||
throw new FatalException("Deluge WebUI is not connected to the daemon");
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
|
||||
public override async Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
hash = hash.ToLowerInvariant();
|
||||
|
||||
DelugeContents? contents = null;
|
||||
RemoveResult result = new();
|
||||
DownloadCheckResult result = new();
|
||||
|
||||
TorrentStatus? status = await GetTorrentStatus(hash);
|
||||
DownloadStatus? download = await _client.GetTorrentStatus(hash);
|
||||
|
||||
if (status?.Hash is null)
|
||||
if (download?.Hash is null)
|
||||
{
|
||||
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
|
||||
return result;
|
||||
}
|
||||
|
||||
result.IsPrivate = download.Private;
|
||||
|
||||
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
|
||||
{
|
||||
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
|
||||
return result;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
@@ -58,6 +91,7 @@ public sealed class DelugeService : DownloadServiceBase
|
||||
{
|
||||
_logger.LogDebug(exception, "failed to find torrent {hash} in the download client", hash);
|
||||
}
|
||||
|
||||
|
||||
bool shouldRemove = contents?.Contents?.Count > 0;
|
||||
|
||||
@@ -69,35 +103,50 @@ public sealed class DelugeService : DownloadServiceBase
|
||||
}
|
||||
});
|
||||
|
||||
result.ShouldRemove = shouldRemove || IsItemStuckAndShouldRemove(status);
|
||||
result.IsPrivate = status.Private;
|
||||
if (shouldRemove)
|
||||
{
|
||||
// remove if all files are unwanted
|
||||
result.ShouldRemove = true;
|
||||
result.DeleteReason = DeleteReason.AllFilesSkipped;
|
||||
return result;
|
||||
}
|
||||
|
||||
// remove if download is stuck
|
||||
(result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override async Task<bool> BlockUnwantedFilesAsync(
|
||||
string hash,
|
||||
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
|
||||
BlocklistType blocklistType,
|
||||
ConcurrentBag<string> patterns,
|
||||
ConcurrentBag<Regex> regexes
|
||||
)
|
||||
ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
hash = hash.ToLowerInvariant();
|
||||
|
||||
TorrentStatus? status = await GetTorrentStatus(hash);
|
||||
DownloadStatus? download = await _client.GetTorrentStatus(hash);
|
||||
BlockFilesResult result = new();
|
||||
|
||||
if (status?.Hash is null)
|
||||
if (download?.Hash is null)
|
||||
{
|
||||
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
|
||||
return false;
|
||||
return result;
|
||||
}
|
||||
|
||||
if (_contentBlockerConfig.IgnorePrivate && status.Private)
|
||||
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
|
||||
{
|
||||
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
|
||||
return result;
|
||||
}
|
||||
|
||||
result.IsPrivate = download.Private;
|
||||
|
||||
if (_contentBlockerConfig.IgnorePrivate && download.Private)
|
||||
{
|
||||
// ignore private trackers
|
||||
_logger.LogDebug("skip files check | download is private | {name}", status.Name);
|
||||
return false;
|
||||
_logger.LogDebug("skip files check | download is private | {name}", download.Name);
|
||||
return result;
|
||||
}
|
||||
|
||||
DelugeContents? contents = null;
|
||||
@@ -113,7 +162,7 @@ public sealed class DelugeService : DownloadServiceBase
|
||||
|
||||
if (contents is null)
|
||||
{
|
||||
return false;
|
||||
return result;
|
||||
}
|
||||
|
||||
Dictionary<int, int> priorities = [];
|
||||
@@ -144,7 +193,7 @@ public sealed class DelugeService : DownloadServiceBase
|
||||
|
||||
if (!hasPriorityUpdates)
|
||||
{
|
||||
return false;
|
||||
return result;
|
||||
}
|
||||
|
||||
_logger.LogDebug("changing priorities | torrent {hash}", hash);
|
||||
@@ -157,56 +206,318 @@ public sealed class DelugeService : DownloadServiceBase
|
||||
if (totalUnwantedFiles == totalFiles)
|
||||
{
|
||||
// Skip marking files as unwanted. The download will be removed completely.
|
||||
return true;
|
||||
result.ShouldRemove = true;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
await _client.ChangeFilesPriority(hash, sortedPriorities);
|
||||
await _dryRunInterceptor.InterceptAsync(ChangeFilesPriority, hash, sortedPriorities);
|
||||
|
||||
return false;
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
public override async Task<List<object>?> GetSeedingDownloads()
|
||||
{
|
||||
return (await _client.GetStatusForAllTorrents())
|
||||
?.Where(x => !string.IsNullOrEmpty(x.Hash))
|
||||
.Where(x => x.State?.Equals("seeding", StringComparison.InvariantCultureIgnoreCase) is true)
|
||||
.Cast<object>()
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories) =>
|
||||
downloads
|
||||
?.Cast<DownloadStatus>()
|
||||
.Where(x => categories.Any(cat => cat.Name.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase)))
|
||||
.Cast<object>()
|
||||
.ToList();
|
||||
|
||||
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories) =>
|
||||
downloads
|
||||
?.Cast<DownloadStatus>()
|
||||
.Where(x => !string.IsNullOrEmpty(x.Hash))
|
||||
.Where(x => categories.Any(cat => cat.Equals(x.Label, StringComparison.InvariantCultureIgnoreCase)))
|
||||
.Cast<object>()
|
||||
.ToList();
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override async Task Delete(string hash)
|
||||
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes,
|
||||
IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
if (downloads?.Count is null or 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (DownloadStatus download in downloads)
|
||||
{
|
||||
if (string.IsNullOrEmpty(download.Hash))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
|
||||
{
|
||||
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
|
||||
{
|
||||
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
CleanCategory? category = categoriesToClean
|
||||
.FirstOrDefault(x => x.Name.Equals(download.Label, StringComparison.InvariantCultureIgnoreCase));
|
||||
|
||||
if (category is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!_downloadCleanerConfig.DeletePrivate && download.Private)
|
||||
{
|
||||
_logger.LogDebug("skip | download is private | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
ContextProvider.Set("downloadName", download.Name);
|
||||
ContextProvider.Set("hash", download.Hash);
|
||||
|
||||
TimeSpan seedingTime = TimeSpan.FromSeconds(download.SeedingTime);
|
||||
SeedingCheckResult result = ShouldCleanDownload(download.Ratio, seedingTime, category);
|
||||
|
||||
if (!result.ShouldClean)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
await _dryRunInterceptor.InterceptAsync(DeleteDownload, download.Hash);
|
||||
|
||||
_logger.LogInformation(
|
||||
"download cleaned | {reason} reached | {name}",
|
||||
result.Reason is CleanReason.MaxRatioReached
|
||||
? "MAX_RATIO & MIN_SEED_TIME"
|
||||
: "MAX_SEED_TIME",
|
||||
download.Name
|
||||
);
|
||||
|
||||
await _notifier.NotifyDownloadCleaned(download.Ratio, seedingTime, category.Name, result.Reason);
|
||||
}
|
||||
}
|
||||
|
||||
public override async Task CreateCategoryAsync(string name)
|
||||
{
|
||||
IReadOnlyList<string> existingLabels = await _client.GetLabels();
|
||||
|
||||
if (existingLabels.Contains(name, StringComparer.InvariantCultureIgnoreCase))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _dryRunInterceptor.InterceptAsync(CreateLabel, name);
|
||||
}
|
||||
|
||||
public override async Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
if (downloads?.Count is null or 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir))
|
||||
{
|
||||
_hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir);
|
||||
}
|
||||
|
||||
foreach (DownloadStatus download in downloads.Cast<DownloadStatus>())
|
||||
{
|
||||
if (string.IsNullOrEmpty(download.Hash) || string.IsNullOrEmpty(download.Name) || string.IsNullOrEmpty(download.Label))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
|
||||
{
|
||||
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
|
||||
{
|
||||
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
ContextProvider.Set("downloadName", download.Name);
|
||||
ContextProvider.Set("hash", download.Hash);
|
||||
|
||||
DelugeContents? contents = null;
|
||||
try
|
||||
{
|
||||
contents = await _client.GetTorrentFiles(download.Hash);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
_logger.LogDebug(exception, "failed to find torrent files for {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
bool hasHardlinks = false;
|
||||
|
||||
ProcessFiles(contents?.Contents, (_, file) =>
|
||||
{
|
||||
string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadLocation, file.Path).Split(['\\', '/']));
|
||||
|
||||
if (file.Priority <= 0)
|
||||
{
|
||||
_logger.LogDebug("skip | file is not downloaded | {file}", filePath);
|
||||
return;
|
||||
}
|
||||
|
||||
long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir));
|
||||
|
||||
if (hardlinkCount < 0)
|
||||
{
|
||||
_logger.LogDebug("skip | could not get file properties | {file}", filePath);
|
||||
hasHardlinks = true;
|
||||
return;
|
||||
}
|
||||
|
||||
if (hardlinkCount > 0)
|
||||
{
|
||||
hasHardlinks = true;
|
||||
}
|
||||
});
|
||||
|
||||
if (hasHardlinks)
|
||||
{
|
||||
_logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
await _dryRunInterceptor.InterceptAsync(ChangeLabel, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory);
|
||||
|
||||
_logger.LogInformation("category changed for {name}", download.Name);
|
||||
|
||||
await _notifier.NotifyCategoryChanged(download.Label, _downloadCleanerConfig.UnlinkedTargetCategory);
|
||||
|
||||
download.Label = _downloadCleanerConfig.UnlinkedTargetCategory;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
[DryRunSafeguard]
|
||||
public override async Task DeleteDownload(string hash)
|
||||
{
|
||||
hash = hash.ToLowerInvariant();
|
||||
|
||||
await _client.DeleteTorrent(hash);
|
||||
await _client.DeleteTorrents([hash]);
|
||||
}
|
||||
|
||||
[DryRunSafeguard]
|
||||
protected async Task CreateLabel(string name)
|
||||
{
|
||||
await _client.CreateLabel(name);
|
||||
}
|
||||
|
||||
private bool IsItemStuckAndShouldRemove(TorrentStatus status)
|
||||
[DryRunSafeguard]
|
||||
protected virtual async Task ChangeFilesPriority(string hash, List<int> sortedPriorities)
|
||||
{
|
||||
await _client.ChangeFilesPriority(hash, sortedPriorities);
|
||||
}
|
||||
|
||||
[DryRunSafeguard]
|
||||
protected virtual async Task ChangeLabel(string hash, string newLabel)
|
||||
{
|
||||
await _client.SetTorrentLabel(hash, newLabel);
|
||||
}
|
||||
|
||||
private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(DownloadStatus status)
|
||||
{
|
||||
(bool ShouldRemove, DeleteReason Reason) result = await CheckIfSlow(status);
|
||||
|
||||
if (result.ShouldRemove)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
return await CheckIfStuck(status);
|
||||
}
|
||||
|
||||
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(DownloadStatus download)
|
||||
{
|
||||
if (_queueCleanerConfig.SlowMaxStrikes is 0)
|
||||
{
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (download.State is null || !download.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (download.DownloadSpeed <= 0)
|
||||
{
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (_queueCleanerConfig.SlowIgnorePrivate && download.Private)
|
||||
{
|
||||
// ignore private trackers
|
||||
_logger.LogDebug("skip slow check | download is private | {name}", download.Name);
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (download.Size > (_queueCleanerConfig.SlowIgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue))
|
||||
{
|
||||
_logger.LogDebug("skip slow check | download is too large | {name}", download.Name);
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
ByteSize minSpeed = _queueCleanerConfig.SlowMinSpeedByteSize;
|
||||
ByteSize currentSpeed = new ByteSize(download.DownloadSpeed);
|
||||
SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.SlowMaxTime);
|
||||
SmartTimeSpan currentTime = SmartTimeSpan.FromSeconds(download.Eta);
|
||||
|
||||
return await CheckIfSlow(
|
||||
download.Hash!,
|
||||
download.Name!,
|
||||
minSpeed,
|
||||
currentSpeed,
|
||||
maxTime,
|
||||
currentTime
|
||||
);
|
||||
}
|
||||
|
||||
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(DownloadStatus status)
|
||||
{
|
||||
if (_queueCleanerConfig.StalledMaxStrikes is 0)
|
||||
{
|
||||
return false;
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (_queueCleanerConfig.StalledIgnorePrivate && status.Private)
|
||||
{
|
||||
// ignore private trackers
|
||||
_logger.LogDebug("skip stalled check | download is private | {name}", status.Name);
|
||||
return false;
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (status.State is null || !status.State.Equals("Downloading", StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (status.Eta > 0)
|
||||
{
|
||||
return false;
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
return StrikeAndCheckLimit(status.Hash!, status.Name!);
|
||||
}
|
||||
|
||||
private async Task<TorrentStatus?> GetTorrentStatus(string hash)
|
||||
{
|
||||
return await _client.SendRequest<TorrentStatus?>(
|
||||
"web.get_torrent_status",
|
||||
hash,
|
||||
new[] { "hash", "state", "name", "eta", "private" }
|
||||
);
|
||||
|
||||
ResetStalledStrikesOnProgress(status.Hash!, status.TotalDone);
|
||||
|
||||
return (await _striker.StrikeAndCheckLimit(status.Hash!, status.Name!, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled), DeleteReason.Stalled);
|
||||
}
|
||||
|
||||
private static void ProcessFiles(Dictionary<string, DelugeFileOrDirectory>? contents, Action<string, DelugeFileOrDirectory> processFile)
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
namespace Infrastructure.Verticals.DownloadClient.Deluge;
|
||||
|
||||
public interface IDelugeService : IDownloadService
|
||||
{
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
using Domain.Enums;
|
||||
|
||||
namespace Infrastructure.Verticals.DownloadClient;
|
||||
|
||||
public sealed record DownloadCheckResult
|
||||
{
|
||||
/// <summary>
|
||||
/// True if the download should be removed; otherwise false.
|
||||
/// </summary>
|
||||
public bool ShouldRemove { get; set; }
|
||||
|
||||
public DeleteReason DeleteReason { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// True if the download is private; otherwise false.
|
||||
/// </summary>
|
||||
public bool IsPrivate { get; set; }
|
||||
}
|
||||
287
code/Infrastructure/Verticals/DownloadClient/DownloadService.cs
Normal file
287
code/Infrastructure/Verticals/DownloadClient/DownloadService.cs
Normal file
@@ -0,0 +1,287 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.RegularExpressions;
|
||||
using Common.Configuration.ContentBlocker;
|
||||
using Common.Configuration.DownloadCleaner;
|
||||
using Common.Configuration.QueueCleaner;
|
||||
using Common.CustomDataTypes;
|
||||
using Common.Helpers;
|
||||
using Domain.Enums;
|
||||
using Domain.Models.Cache;
|
||||
using Infrastructure.Helpers;
|
||||
using Infrastructure.Interceptors;
|
||||
using Infrastructure.Verticals.ContentBlocker;
|
||||
using Infrastructure.Verticals.Context;
|
||||
using Infrastructure.Verticals.Files;
|
||||
using Infrastructure.Verticals.ItemStriker;
|
||||
using Infrastructure.Verticals.Notifications;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Infrastructure.Verticals.DownloadClient;
|
||||
|
||||
public abstract class DownloadService : IDownloadService
|
||||
{
|
||||
protected readonly ILogger<DownloadService> _logger;
|
||||
protected readonly QueueCleanerConfig _queueCleanerConfig;
|
||||
protected readonly ContentBlockerConfig _contentBlockerConfig;
|
||||
protected readonly DownloadCleanerConfig _downloadCleanerConfig;
|
||||
protected readonly IMemoryCache _cache;
|
||||
protected readonly IFilenameEvaluator _filenameEvaluator;
|
||||
protected readonly IStriker _striker;
|
||||
protected readonly MemoryCacheEntryOptions _cacheOptions;
|
||||
protected readonly INotificationPublisher _notifier;
|
||||
protected readonly IDryRunInterceptor _dryRunInterceptor;
|
||||
protected readonly IHardLinkFileService _hardLinkFileService;
|
||||
|
||||
protected DownloadService(
|
||||
ILogger<DownloadService> logger,
|
||||
IOptions<QueueCleanerConfig> queueCleanerConfig,
|
||||
IOptions<ContentBlockerConfig> contentBlockerConfig,
|
||||
IOptions<DownloadCleanerConfig> downloadCleanerConfig,
|
||||
IMemoryCache cache,
|
||||
IFilenameEvaluator filenameEvaluator,
|
||||
IStriker striker,
|
||||
INotificationPublisher notifier,
|
||||
IDryRunInterceptor dryRunInterceptor,
|
||||
IHardLinkFileService hardLinkFileService
|
||||
)
|
||||
{
|
||||
_logger = logger;
|
||||
_queueCleanerConfig = queueCleanerConfig.Value;
|
||||
_contentBlockerConfig = contentBlockerConfig.Value;
|
||||
_downloadCleanerConfig = downloadCleanerConfig.Value;
|
||||
_cache = cache;
|
||||
_filenameEvaluator = filenameEvaluator;
|
||||
_striker = striker;
|
||||
_notifier = notifier;
|
||||
_dryRunInterceptor = dryRunInterceptor;
|
||||
_hardLinkFileService = hardLinkFileService;
|
||||
_cacheOptions = new MemoryCacheEntryOptions()
|
||||
.SetSlidingExpiration(StaticConfiguration.TriggerValue + Constants.CacheLimitBuffer);
|
||||
}
|
||||
|
||||
public abstract void Dispose();
|
||||
|
||||
public abstract Task LoginAsync();
|
||||
|
||||
public abstract Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public abstract Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
|
||||
BlocklistType blocklistType,
|
||||
ConcurrentBag<string> patterns,
|
||||
ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public abstract Task DeleteDownload(string hash);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public abstract Task<List<object>?> GetSeedingDownloads();
|
||||
|
||||
/// <inheritdoc/>
|
||||
public abstract List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public abstract List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public abstract Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public abstract Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public abstract Task CreateCategoryAsync(string name);
|
||||
|
||||
protected void ResetStalledStrikesOnProgress(string hash, long downloaded)
|
||||
{
|
||||
if (!_queueCleanerConfig.StalledResetStrikesOnProgress)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (_cache.TryGetValue(CacheKeys.StrikeItem(hash, StrikeType.Stalled), out StalledCacheItem? cachedItem) &&
|
||||
cachedItem is not null && downloaded > cachedItem.Downloaded)
|
||||
{
|
||||
// cache item found
|
||||
_cache.Remove(CacheKeys.Strike(StrikeType.Stalled, hash));
|
||||
_logger.LogDebug("resetting stalled strikes for {hash} due to progress", hash);
|
||||
}
|
||||
|
||||
_cache.Set(CacheKeys.StrikeItem(hash, StrikeType.Stalled), new StalledCacheItem { Downloaded = downloaded }, _cacheOptions);
|
||||
}
|
||||
|
||||
protected void ResetSlowSpeedStrikesOnProgress(string downloadName, string hash)
|
||||
{
|
||||
if (!_queueCleanerConfig.SlowResetStrikesOnProgress)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
string key = CacheKeys.Strike(StrikeType.SlowSpeed, hash);
|
||||
|
||||
if (!_cache.TryGetValue(key, out object? value) || value is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_cache.Remove(key);
|
||||
_logger.LogDebug("resetting slow speed strikes due to progress | {name}", downloadName);
|
||||
}
|
||||
|
||||
protected void ResetSlowTimeStrikesOnProgress(string downloadName, string hash)
|
||||
{
|
||||
if (!_queueCleanerConfig.SlowResetStrikesOnProgress)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
string key = CacheKeys.Strike(StrikeType.SlowTime, hash);
|
||||
|
||||
if (!_cache.TryGetValue(key, out object? value) || value is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_cache.Remove(key);
|
||||
_logger.LogDebug("resetting slow time strikes due to progress | {name}", downloadName);
|
||||
}
|
||||
|
||||
protected async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(
|
||||
string downloadHash,
|
||||
string downloadName,
|
||||
ByteSize minSpeed,
|
||||
ByteSize currentSpeed,
|
||||
SmartTimeSpan maxTime,
|
||||
SmartTimeSpan currentTime
|
||||
)
|
||||
{
|
||||
if (minSpeed.Bytes > 0 && currentSpeed < minSpeed)
|
||||
{
|
||||
_logger.LogTrace("slow speed | {speed}/s | {name}", currentSpeed.ToString(), downloadName);
|
||||
|
||||
bool shouldRemove = await _striker
|
||||
.StrikeAndCheckLimit(downloadHash, downloadName, _queueCleanerConfig.SlowMaxStrikes, StrikeType.SlowSpeed);
|
||||
|
||||
if (shouldRemove)
|
||||
{
|
||||
return (true, DeleteReason.SlowSpeed);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
ResetSlowSpeedStrikesOnProgress(downloadName, downloadHash);
|
||||
}
|
||||
|
||||
if (maxTime.Time > TimeSpan.Zero && currentTime > maxTime)
|
||||
{
|
||||
_logger.LogTrace("slow estimated time | {time} | {name}", currentTime.ToString(), downloadName);
|
||||
|
||||
bool shouldRemove = await _striker
|
||||
.StrikeAndCheckLimit(downloadHash, downloadName, _queueCleanerConfig.SlowMaxStrikes, StrikeType.SlowTime);
|
||||
|
||||
if (shouldRemove)
|
||||
{
|
||||
return (true, DeleteReason.SlowTime);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
ResetSlowTimeStrikesOnProgress(downloadName, downloadHash);
|
||||
}
|
||||
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
protected SeedingCheckResult ShouldCleanDownload(double ratio, TimeSpan seedingTime, CleanCategory category)
|
||||
{
|
||||
// check ratio
|
||||
if (DownloadReachedRatio(ratio, seedingTime, category))
|
||||
{
|
||||
return new()
|
||||
{
|
||||
ShouldClean = true,
|
||||
Reason = CleanReason.MaxRatioReached
|
||||
};
|
||||
}
|
||||
|
||||
// check max seed time
|
||||
if (DownloadReachedMaxSeedTime(seedingTime, category))
|
||||
{
|
||||
return new()
|
||||
{
|
||||
ShouldClean = true,
|
||||
Reason = CleanReason.MaxSeedTimeReached
|
||||
};
|
||||
}
|
||||
|
||||
return new();
|
||||
}
|
||||
|
||||
protected string? GetRootWithFirstDirectory(string path)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
string? root = Path.GetPathRoot(path);
|
||||
|
||||
if (root is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
string relativePath = path[root.Length..].TrimStart(Path.DirectorySeparatorChar);
|
||||
string[] parts = relativePath.Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries);
|
||||
|
||||
return parts.Length > 0 ? Path.Combine(root, parts[0]) : root;
|
||||
}
|
||||
|
||||
private bool DownloadReachedRatio(double ratio, TimeSpan seedingTime, CleanCategory category)
|
||||
{
|
||||
if (category.MaxRatio < 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
string downloadName = ContextProvider.Get<string>("downloadName");
|
||||
TimeSpan minSeedingTime = TimeSpan.FromHours(category.MinSeedTime);
|
||||
|
||||
if (category.MinSeedTime > 0 && seedingTime < minSeedingTime)
|
||||
{
|
||||
_logger.LogDebug("skip | download has not reached MIN_SEED_TIME | {name}", downloadName);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (ratio < category.MaxRatio)
|
||||
{
|
||||
_logger.LogDebug("skip | download has not reached MAX_RATIO | {name}", downloadName);
|
||||
return false;
|
||||
}
|
||||
|
||||
// max ration is 0 or reached
|
||||
return true;
|
||||
}
|
||||
|
||||
private bool DownloadReachedMaxSeedTime(TimeSpan seedingTime, CleanCategory category)
|
||||
{
|
||||
if (category.MaxSeedTime < 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
string downloadName = ContextProvider.Get<string>("downloadName");
|
||||
TimeSpan maxSeedingTime = TimeSpan.FromHours(category.MaxSeedTime);
|
||||
|
||||
if (category.MaxSeedTime > 0 && seedingTime < maxSeedingTime)
|
||||
{
|
||||
_logger.LogDebug("skip | download has not reached MAX_SEED_TIME | {name}", downloadName);
|
||||
return false;
|
||||
}
|
||||
|
||||
// max seed time is 0 or reached
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.RegularExpressions;
|
||||
using Common.Configuration.ContentBlocker;
|
||||
using Common.Configuration.QueueCleaner;
|
||||
using Domain.Enums;
|
||||
using Infrastructure.Verticals.ContentBlocker;
|
||||
using Infrastructure.Verticals.ItemStriker;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Infrastructure.Verticals.DownloadClient;
|
||||
|
||||
public abstract class DownloadServiceBase : IDownloadService
|
||||
{
|
||||
protected readonly ILogger<DownloadServiceBase> _logger;
|
||||
protected readonly QueueCleanerConfig _queueCleanerConfig;
|
||||
protected readonly ContentBlockerConfig _contentBlockerConfig;
|
||||
protected readonly FilenameEvaluator _filenameEvaluator;
|
||||
protected readonly Striker _striker;
|
||||
|
||||
protected DownloadServiceBase(
|
||||
ILogger<DownloadServiceBase> logger,
|
||||
IOptions<QueueCleanerConfig> queueCleanerConfig,
|
||||
IOptions<ContentBlockerConfig> contentBlockerConfig,
|
||||
FilenameEvaluator filenameEvaluator,
|
||||
Striker striker
|
||||
)
|
||||
{
|
||||
_logger = logger;
|
||||
_queueCleanerConfig = queueCleanerConfig.Value;
|
||||
_contentBlockerConfig = contentBlockerConfig.Value;
|
||||
_filenameEvaluator = filenameEvaluator;
|
||||
_striker = striker;
|
||||
}
|
||||
|
||||
public abstract void Dispose();
|
||||
|
||||
public abstract Task LoginAsync();
|
||||
|
||||
public abstract Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public abstract Task<bool> BlockUnwantedFilesAsync(
|
||||
string hash,
|
||||
BlocklistType blocklistType,
|
||||
ConcurrentBag<string> patterns,
|
||||
ConcurrentBag<Regex> regexes
|
||||
);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public abstract Task Delete(string hash);
|
||||
|
||||
/// <summary>
|
||||
/// Strikes an item and checks if the limit has been reached.
|
||||
/// </summary>
|
||||
/// <param name="hash">The torrent hash.</param>
|
||||
/// <param name="itemName">The name or title of the item.</param>
|
||||
/// <returns>True if the limit has been reached; otherwise, false.</returns>
|
||||
protected bool StrikeAndCheckLimit(string hash, string itemName)
|
||||
{
|
||||
return _striker.StrikeAndCheckLimit(hash, itemName, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled);
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@ public sealed class DownloadServiceFactory
|
||||
Common.Enums.DownloadClient.Deluge => _serviceProvider.GetRequiredService<DelugeService>(),
|
||||
Common.Enums.DownloadClient.Transmission => _serviceProvider.GetRequiredService<TransmissionService>(),
|
||||
Common.Enums.DownloadClient.None => _serviceProvider.GetRequiredService<DummyDownloadService>(),
|
||||
Common.Enums.DownloadClient.Disabled => _serviceProvider.GetRequiredService<DummyDownloadService>(),
|
||||
_ => throw new ArgumentOutOfRangeException()
|
||||
};
|
||||
}
|
||||
@@ -1,17 +1,36 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.RegularExpressions;
|
||||
using Common.Configuration.ContentBlocker;
|
||||
using Common.Configuration.DownloadCleaner;
|
||||
using Common.Configuration.QueueCleaner;
|
||||
using Infrastructure.Interceptors;
|
||||
using Infrastructure.Verticals.ContentBlocker;
|
||||
using Infrastructure.Verticals.Files;
|
||||
using Infrastructure.Verticals.ItemStriker;
|
||||
using Infrastructure.Verticals.Notifications;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace Infrastructure.Verticals.DownloadClient;
|
||||
|
||||
public sealed class DummyDownloadService : DownloadServiceBase
|
||||
public class DummyDownloadService : DownloadService
|
||||
{
|
||||
public DummyDownloadService(ILogger<DownloadServiceBase> logger, IOptions<QueueCleanerConfig> queueCleanerConfig, IOptions<ContentBlockerConfig> contentBlockerConfig, FilenameEvaluator filenameEvaluator, Striker striker) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
|
||||
public DummyDownloadService(
|
||||
ILogger<DownloadService> logger,
|
||||
IOptions<QueueCleanerConfig> queueCleanerConfig,
|
||||
IOptions<ContentBlockerConfig> contentBlockerConfig,
|
||||
IOptions<DownloadCleanerConfig> downloadCleanerConfig,
|
||||
IMemoryCache cache,
|
||||
IFilenameEvaluator filenameEvaluator,
|
||||
IStriker striker,
|
||||
INotificationPublisher notifier,
|
||||
IDryRunInterceptor dryRunInterceptor,
|
||||
IHardLinkFileService hardLinkFileService
|
||||
) : base(
|
||||
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig,
|
||||
cache, filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService
|
||||
)
|
||||
{
|
||||
}
|
||||
|
||||
@@ -24,17 +43,48 @@ public sealed class DummyDownloadService : DownloadServiceBase
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public override Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
|
||||
public override Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public override Task<bool> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag<string> patterns, ConcurrentBag<Regex> regexes)
|
||||
public override Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash, BlocklistType blocklistType, ConcurrentBag<string> patterns,
|
||||
ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public override Task Delete(string hash)
|
||||
public override Task<List<object>?> GetSeedingDownloads()
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public override Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public override Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public override Task CreateCategoryAsync(string name)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public override Task DeleteDownload(string hash)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.RegularExpressions;
|
||||
using Common.Configuration.ContentBlocker;
|
||||
using Common.Configuration.DownloadCleaner;
|
||||
using Infrastructure.Interceptors;
|
||||
|
||||
namespace Infrastructure.Verticals.DownloadClient;
|
||||
|
||||
@@ -12,7 +14,8 @@ public interface IDownloadService : IDisposable
|
||||
/// Checks whether the download should be removed from the *arr queue.
|
||||
/// </summary>
|
||||
/// <param name="hash">The download hash.</param>
|
||||
public Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash);
|
||||
/// <param name="ignoredDownloads">Downloads to ignore from processing.</param>
|
||||
public Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads);
|
||||
|
||||
/// <summary>
|
||||
/// Blocks unwanted files from being fully downloaded.
|
||||
@@ -21,16 +24,62 @@ public interface IDownloadService : IDisposable
|
||||
/// <param name="blocklistType">The <see cref="BlocklistType"/>.</param>
|
||||
/// <param name="patterns">The patterns to test the files against.</param>
|
||||
/// <param name="regexes">The regexes to test the files against.</param>
|
||||
/// <param name="ignoredDownloads">Downloads to ignore from processing.</param>
|
||||
/// <returns>True if all files have been blocked; otherwise false.</returns>
|
||||
public Task<bool> BlockUnwantedFilesAsync(
|
||||
string hash,
|
||||
public Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
|
||||
BlocklistType blocklistType,
|
||||
ConcurrentBag<string> patterns,
|
||||
ConcurrentBag<Regex> regexes
|
||||
ConcurrentBag<Regex> regexes,
|
||||
IReadOnlyList<string> ignoredDownloads
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Fetches all seeding downloads.
|
||||
/// </summary>
|
||||
/// <returns>A list of downloads that are seeding.</returns>
|
||||
Task<List<object>?> GetSeedingDownloads();
|
||||
|
||||
/// <summary>
|
||||
/// Filters downloads that should be cleaned.
|
||||
/// </summary>
|
||||
/// <param name="downloads">The downloads to filter.</param>
|
||||
/// <param name="categories">The categories by which to filter the downloads.</param>
|
||||
/// <returns>A list of downloads for the provided categories.</returns>
|
||||
List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories);
|
||||
|
||||
/// <summary>
|
||||
/// Filters downloads that should have their category changed.
|
||||
/// </summary>
|
||||
/// <param name="downloads">The downloads to filter.</param>
|
||||
/// <param name="categories">The categories by which to filter the downloads.</param>
|
||||
/// <returns>A list of downloads for the provided categories.</returns>
|
||||
List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories);
|
||||
|
||||
/// <summary>
|
||||
/// Cleans the downloads.
|
||||
/// </summary>
|
||||
/// <param name="downloads">The downloads to clean.</param>
|
||||
/// <param name="categoriesToClean">The categories that should be cleaned.</param>
|
||||
/// <param name="excludedHashes">The hashes that should not be cleaned.</param>
|
||||
/// <param name="ignoredDownloads">The downloads to ignore from processing.</param>
|
||||
Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads);
|
||||
|
||||
/// <summary>
|
||||
/// Changes the category for downloads that have no hardlinks.
|
||||
/// </summary>
|
||||
/// <param name="downloads">The downloads to change.</param>
|
||||
/// <param name="excludedHashes">The hashes that should not be cleaned.</param>
|
||||
/// <param name="ignoredDownloads">The downloads to ignore from processing.</param>
|
||||
Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a download item.
|
||||
/// </summary>
|
||||
public Task Delete(string hash);
|
||||
public Task DeleteDownload(string hash);
|
||||
|
||||
/// <summary>
|
||||
/// Creates a category.
|
||||
/// </summary>
|
||||
/// <param name="name">The category name.</param>
|
||||
public Task CreateCategoryAsync(string name);
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
namespace Infrastructure.Verticals.DownloadClient.QBittorrent;
|
||||
|
||||
public interface IQBitService : IDownloadService, IDisposable
|
||||
{
|
||||
}
|
||||
@@ -1,18 +1,28 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.RegularExpressions;
|
||||
using Common.Attributes;
|
||||
using Common.Configuration.ContentBlocker;
|
||||
using Common.Configuration.DownloadCleaner;
|
||||
using Common.Configuration.DownloadClient;
|
||||
using Common.Configuration.QueueCleaner;
|
||||
using Common.CustomDataTypes;
|
||||
using Common.Helpers;
|
||||
using Domain.Enums;
|
||||
using Infrastructure.Extensions;
|
||||
using Infrastructure.Interceptors;
|
||||
using Infrastructure.Verticals.ContentBlocker;
|
||||
using Infrastructure.Verticals.Context;
|
||||
using Infrastructure.Verticals.Files;
|
||||
using Infrastructure.Verticals.ItemStriker;
|
||||
using Infrastructure.Verticals.Notifications;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using QBittorrent.Client;
|
||||
|
||||
namespace Infrastructure.Verticals.DownloadClient.QBittorrent;
|
||||
|
||||
public sealed class QBitService : DownloadServiceBase
|
||||
public class QBitService : DownloadService, IQBitService
|
||||
{
|
||||
private readonly QBitConfig _config;
|
||||
private readonly QBittorrentClient _client;
|
||||
@@ -23,13 +33,25 @@ public sealed class QBitService : DownloadServiceBase
|
||||
IOptions<QBitConfig> config,
|
||||
IOptions<QueueCleanerConfig> queueCleanerConfig,
|
||||
IOptions<ContentBlockerConfig> contentBlockerConfig,
|
||||
FilenameEvaluator filenameEvaluator,
|
||||
Striker striker
|
||||
) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
|
||||
IOptions<DownloadCleanerConfig> downloadCleanerConfig,
|
||||
IMemoryCache cache,
|
||||
IFilenameEvaluator filenameEvaluator,
|
||||
IStriker striker,
|
||||
INotificationPublisher notifier,
|
||||
IDryRunInterceptor dryRunInterceptor,
|
||||
IHardLinkFileService hardLinkFileService
|
||||
) : base(
|
||||
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache,
|
||||
filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService
|
||||
)
|
||||
{
|
||||
_config = config.Value;
|
||||
_config.Validate();
|
||||
_client = new(httpClientFactory.CreateClient(Constants.HttpClientWithRetryName), _config.Url);
|
||||
UriBuilder uriBuilder = new(_config.Url);
|
||||
uriBuilder.Path = string.IsNullOrEmpty(_config.UrlBase)
|
||||
? uriBuilder.Path
|
||||
: $"{uriBuilder.Path.TrimEnd('/')}/{_config.UrlBase.TrimStart('/')}";
|
||||
_client = new(httpClientFactory.CreateClient(Constants.HttpClientWithRetryName), uriBuilder.Uri);
|
||||
}
|
||||
|
||||
public override async Task LoginAsync()
|
||||
@@ -43,18 +65,27 @@ public sealed class QBitService : DownloadServiceBase
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
|
||||
public override async Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
RemoveResult result = new();
|
||||
TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
|
||||
DownloadCheckResult result = new();
|
||||
TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
|
||||
.FirstOrDefault();
|
||||
|
||||
if (torrent is null)
|
||||
if (download is null)
|
||||
{
|
||||
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
|
||||
return result;
|
||||
}
|
||||
|
||||
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(hash);
|
||||
|
||||
if (ignoredDownloads.Count > 0 &&
|
||||
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true))
|
||||
{
|
||||
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
|
||||
return result;
|
||||
}
|
||||
|
||||
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash);
|
||||
|
||||
if (torrentProperties is null)
|
||||
@@ -67,42 +98,54 @@ public sealed class QBitService : DownloadServiceBase
|
||||
bool.TryParse(dictValue?.ToString(), out bool boolValue)
|
||||
&& boolValue;
|
||||
|
||||
// if all files were blocked by qBittorrent
|
||||
if (torrent is { CompletionOn: not null, Downloaded: null or 0 })
|
||||
{
|
||||
result.ShouldRemove = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(hash);
|
||||
|
||||
// if all files are marked as skip
|
||||
if (files?.Count is > 0 && files.All(x => x.Priority is TorrentContentPriority.Skip))
|
||||
{
|
||||
result.ShouldRemove = true;
|
||||
|
||||
// if all files were blocked by qBittorrent
|
||||
if (download is { CompletionOn: not null, Downloaded: null or 0 })
|
||||
{
|
||||
result.DeleteReason = DeleteReason.AllFilesSkippedByQBit;
|
||||
return result;
|
||||
}
|
||||
|
||||
// remove if all files are unwanted
|
||||
result.DeleteReason = DeleteReason.AllFilesSkipped;
|
||||
return result;
|
||||
}
|
||||
|
||||
result.ShouldRemove = IsItemStuckAndShouldRemove(torrent, result.IsPrivate);
|
||||
(result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download, result.IsPrivate);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override async Task<bool> BlockUnwantedFilesAsync(
|
||||
string hash,
|
||||
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
|
||||
BlocklistType blocklistType,
|
||||
ConcurrentBag<string> patterns,
|
||||
ConcurrentBag<Regex> regexes
|
||||
ConcurrentBag<Regex> regexes,
|
||||
IReadOnlyList<string> ignoredDownloads
|
||||
)
|
||||
{
|
||||
TorrentInfo? torrent = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
|
||||
TorrentInfo? download = (await _client.GetTorrentListAsync(new TorrentListQuery { Hashes = [hash] }))
|
||||
.FirstOrDefault();
|
||||
BlockFilesResult result = new();
|
||||
|
||||
if (torrent is null)
|
||||
if (download is null)
|
||||
{
|
||||
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
|
||||
return false;
|
||||
return result;
|
||||
}
|
||||
|
||||
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(hash);
|
||||
|
||||
if (ignoredDownloads.Count > 0 &&
|
||||
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads)) is true))
|
||||
{
|
||||
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
|
||||
return result;
|
||||
}
|
||||
|
||||
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(hash);
|
||||
@@ -110,25 +153,27 @@ public sealed class QBitService : DownloadServiceBase
|
||||
if (torrentProperties is null)
|
||||
{
|
||||
_logger.LogDebug("failed to find torrent properties {hash} in the download client", hash);
|
||||
return false;
|
||||
return result;
|
||||
}
|
||||
|
||||
bool isPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) &&
|
||||
bool.TryParse(dictValue?.ToString(), out bool boolValue)
|
||||
&& boolValue;
|
||||
|
||||
result.IsPrivate = isPrivate;
|
||||
|
||||
if (_contentBlockerConfig.IgnorePrivate && isPrivate)
|
||||
{
|
||||
// ignore private trackers
|
||||
_logger.LogDebug("skip files check | download is private | {name}", torrent.Name);
|
||||
return false;
|
||||
_logger.LogDebug("skip files check | download is private | {name}", download.Name);
|
||||
return result;
|
||||
}
|
||||
|
||||
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(hash);
|
||||
|
||||
if (files is null)
|
||||
{
|
||||
return false;
|
||||
return result;
|
||||
}
|
||||
|
||||
List<int> unwantedFiles = [];
|
||||
@@ -162,55 +207,372 @@ public sealed class QBitService : DownloadServiceBase
|
||||
|
||||
if (unwantedFiles.Count is 0)
|
||||
{
|
||||
return false;
|
||||
return result;
|
||||
}
|
||||
|
||||
if (totalUnwantedFiles == totalFiles)
|
||||
{
|
||||
// Skip marking files as unwanted. The download will be removed completely.
|
||||
return true;
|
||||
result.ShouldRemove = true;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
foreach (int fileIndex in unwantedFiles)
|
||||
{
|
||||
await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip);
|
||||
await _dryRunInterceptor.InterceptAsync(SkipFile, hash, fileIndex);
|
||||
}
|
||||
|
||||
return false;
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override async Task<List<object>?> GetSeedingDownloads() =>
|
||||
(await _client.GetTorrentListAsync(new()
|
||||
{
|
||||
Filter = TorrentListFilter.Seeding
|
||||
}))
|
||||
?.Where(x => !string.IsNullOrEmpty(x.Hash))
|
||||
.Cast<object>()
|
||||
.ToList();
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override async Task Delete(string hash)
|
||||
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories) =>
|
||||
downloads
|
||||
?.Cast<TorrentInfo>()
|
||||
.Where(x => !string.IsNullOrEmpty(x.Hash))
|
||||
.Where(x => categories.Any(cat => cat.Name.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase)))
|
||||
.Cast<object>()
|
||||
.ToList();
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories) =>
|
||||
downloads
|
||||
?.Cast<TorrentInfo>()
|
||||
.Where(x => !string.IsNullOrEmpty(x.Hash))
|
||||
.Where(x => categories.Any(cat => cat.Equals(x.Category, StringComparison.InvariantCultureIgnoreCase)))
|
||||
.Cast<object>()
|
||||
.ToList();
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean,
|
||||
HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
if (downloads?.Count is null or 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (TorrentInfo download in downloads)
|
||||
{
|
||||
if (string.IsNullOrEmpty(download.Hash))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
|
||||
{
|
||||
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(download.Hash);
|
||||
|
||||
if (ignoredDownloads.Count > 0 &&
|
||||
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads))))
|
||||
{
|
||||
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
CleanCategory? category = categoriesToClean
|
||||
.FirstOrDefault(x => download.Category.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase));
|
||||
|
||||
if (category is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!_downloadCleanerConfig.DeletePrivate)
|
||||
{
|
||||
TorrentProperties? torrentProperties = await _client.GetTorrentPropertiesAsync(download.Hash);
|
||||
|
||||
if (torrentProperties is null)
|
||||
{
|
||||
_logger.LogDebug("failed to find torrent properties in the download client | {name}", download.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
bool isPrivate = torrentProperties.AdditionalData.TryGetValue("is_private", out var dictValue) &&
|
||||
bool.TryParse(dictValue?.ToString(), out bool boolValue)
|
||||
&& boolValue;
|
||||
|
||||
if (isPrivate)
|
||||
{
|
||||
_logger.LogDebug("skip | download is private | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
ContextProvider.Set("downloadName", download.Name);
|
||||
ContextProvider.Set("hash", download.Hash);
|
||||
|
||||
SeedingCheckResult result = ShouldCleanDownload(download.Ratio, download.SeedingTime ?? TimeSpan.Zero, category);
|
||||
|
||||
if (!result.ShouldClean)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
await _dryRunInterceptor.InterceptAsync(DeleteDownload, download.Hash);
|
||||
|
||||
_logger.LogInformation(
|
||||
"download cleaned | {reason} reached | {name}",
|
||||
result.Reason is CleanReason.MaxRatioReached
|
||||
? "MAX_RATIO & MIN_SEED_TIME"
|
||||
: "MAX_SEED_TIME",
|
||||
download.Name
|
||||
);
|
||||
|
||||
await _notifier.NotifyDownloadCleaned(download.Ratio, download.SeedingTime ?? TimeSpan.Zero, category.Name, result.Reason);
|
||||
}
|
||||
}
|
||||
|
||||
public override async Task CreateCategoryAsync(string name)
|
||||
{
|
||||
IReadOnlyDictionary<string, Category>? existingCategories = await _client.GetCategoriesAsync();
|
||||
|
||||
if (existingCategories.Any(x => x.Value.Name.Equals(name, StringComparison.InvariantCultureIgnoreCase)))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _dryRunInterceptor.InterceptAsync(CreateCategory, name);
|
||||
}
|
||||
|
||||
public override async Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
if (downloads?.Count is null or 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir))
|
||||
{
|
||||
_hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir);
|
||||
}
|
||||
|
||||
foreach (TorrentInfo download in downloads)
|
||||
{
|
||||
if (string.IsNullOrEmpty(download.Hash))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (excludedHashes.Any(x => x.Equals(download.Hash, StringComparison.InvariantCultureIgnoreCase)))
|
||||
{
|
||||
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
IReadOnlyList<TorrentTracker> trackers = await GetTrackersAsync(download.Hash);
|
||||
|
||||
if (ignoredDownloads.Count > 0 &&
|
||||
(download.ShouldIgnore(ignoredDownloads) || trackers.Any(x => x.ShouldIgnore(ignoredDownloads))))
|
||||
{
|
||||
_logger.LogInformation("skip | download is ignored | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
IReadOnlyList<TorrentContent>? files = await _client.GetTorrentContentsAsync(download.Hash);
|
||||
|
||||
if (files is null)
|
||||
{
|
||||
_logger.LogDebug("failed to find files for {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
ContextProvider.Set("downloadName", download.Name);
|
||||
ContextProvider.Set("hash", download.Hash);
|
||||
bool hasHardlinks = false;
|
||||
|
||||
foreach (TorrentContent file in files)
|
||||
{
|
||||
if (!file.Index.HasValue)
|
||||
{
|
||||
_logger.LogDebug("skip | file index is null for {name}", download.Name);
|
||||
hasHardlinks = true;
|
||||
break;
|
||||
}
|
||||
|
||||
string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.SavePath, file.Name).Split(['\\', '/']));
|
||||
|
||||
if (file.Priority is TorrentContentPriority.Skip)
|
||||
{
|
||||
_logger.LogDebug("skip | file is not downloaded | {file}", filePath);
|
||||
continue;
|
||||
}
|
||||
|
||||
long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir));
|
||||
|
||||
if (hardlinkCount < 0)
|
||||
{
|
||||
_logger.LogDebug("skip | could not get file properties | {file}", filePath);
|
||||
hasHardlinks = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (hardlinkCount > 0)
|
||||
{
|
||||
hasHardlinks = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (hasHardlinks)
|
||||
{
|
||||
_logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
await _dryRunInterceptor.InterceptAsync(ChangeCategory, download.Hash, _downloadCleanerConfig.UnlinkedTargetCategory);
|
||||
|
||||
_logger.LogInformation("category changed for {name}", download.Name);
|
||||
|
||||
await _notifier.NotifyCategoryChanged(download.Category, _downloadCleanerConfig.UnlinkedTargetCategory);
|
||||
|
||||
download.Category = _downloadCleanerConfig.UnlinkedTargetCategory;
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
[DryRunSafeguard]
|
||||
public override async Task DeleteDownload(string hash)
|
||||
{
|
||||
await _client.DeleteAsync(hash, deleteDownloadedData: true);
|
||||
}
|
||||
|
||||
[DryRunSafeguard]
|
||||
protected async Task CreateCategory(string name)
|
||||
{
|
||||
await _client.AddCategoryAsync(name);
|
||||
}
|
||||
|
||||
[DryRunSafeguard]
|
||||
protected virtual async Task SkipFile(string hash, int fileIndex)
|
||||
{
|
||||
await _client.SetFilePriorityAsync(hash, fileIndex, TorrentContentPriority.Skip);
|
||||
}
|
||||
|
||||
[DryRunSafeguard]
|
||||
protected virtual async Task ChangeCategory(string hash, string newCategory)
|
||||
{
|
||||
await _client.SetTorrentCategoryAsync([hash], newCategory);
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
_client.Dispose();
|
||||
}
|
||||
|
||||
private bool IsItemStuckAndShouldRemove(TorrentInfo torrent, bool isPrivate)
|
||||
|
||||
private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(TorrentInfo torrent, bool isPrivate)
|
||||
{
|
||||
if (_queueCleanerConfig.StalledMaxStrikes is 0)
|
||||
(bool ShouldRemove, DeleteReason Reason) result = await CheckIfSlow(torrent, isPrivate);
|
||||
|
||||
if (result.ShouldRemove)
|
||||
{
|
||||
return false;
|
||||
return result;
|
||||
}
|
||||
|
||||
return await CheckIfStuck(torrent, isPrivate);
|
||||
}
|
||||
|
||||
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(TorrentInfo download, bool isPrivate)
|
||||
{
|
||||
if (_queueCleanerConfig.SlowMaxStrikes is 0)
|
||||
{
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (_queueCleanerConfig.StalledIgnorePrivate && isPrivate)
|
||||
if (download.State is not (TorrentState.Downloading or TorrentState.ForcedDownload))
|
||||
{
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (download.DownloadSpeed <= 0)
|
||||
{
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (_queueCleanerConfig.SlowIgnorePrivate && isPrivate)
|
||||
{
|
||||
// ignore private trackers
|
||||
_logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name);
|
||||
return false;
|
||||
_logger.LogDebug("skip slow check | download is private | {name}", download.Name);
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (download.Size > (_queueCleanerConfig.SlowIgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue))
|
||||
{
|
||||
_logger.LogDebug("skip slow check | download is too large | {name}", download.Name);
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
ByteSize minSpeed = _queueCleanerConfig.SlowMinSpeedByteSize;
|
||||
ByteSize currentSpeed = new ByteSize(download.DownloadSpeed);
|
||||
SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.SlowMaxTime);
|
||||
SmartTimeSpan currentTime = new SmartTimeSpan(download.EstimatedTime ?? TimeSpan.Zero);
|
||||
|
||||
return await CheckIfSlow(
|
||||
download.Hash,
|
||||
download.Name,
|
||||
minSpeed,
|
||||
currentSpeed,
|
||||
maxTime,
|
||||
currentTime
|
||||
);
|
||||
}
|
||||
|
||||
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(TorrentInfo torrent, bool isPrivate)
|
||||
{
|
||||
if (_queueCleanerConfig.StalledMaxStrikes is 0 && _queueCleanerConfig.DownloadingMetadataMaxStrikes is 0)
|
||||
{
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (torrent.State is not TorrentState.StalledDownload and not TorrentState.FetchingMetadata
|
||||
and not TorrentState.ForcedFetchingMetadata)
|
||||
{
|
||||
// ignore other states
|
||||
return false;
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
return StrikeAndCheckLimit(torrent.Hash, torrent.Name);
|
||||
if (_queueCleanerConfig.StalledMaxStrikes > 0 && torrent.State is TorrentState.StalledDownload)
|
||||
{
|
||||
if (_queueCleanerConfig.StalledIgnorePrivate && isPrivate)
|
||||
{
|
||||
// ignore private trackers
|
||||
_logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name);
|
||||
}
|
||||
else
|
||||
{
|
||||
ResetStalledStrikesOnProgress(torrent.Hash, torrent.Downloaded ?? 0);
|
||||
|
||||
return (await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled), DeleteReason.Stalled);
|
||||
}
|
||||
}
|
||||
|
||||
if (_queueCleanerConfig.DownloadingMetadataMaxStrikes > 0 && torrent.State is not TorrentState.StalledDownload)
|
||||
{
|
||||
return (await _striker.StrikeAndCheckLimit(torrent.Hash, torrent.Name, _queueCleanerConfig.DownloadingMetadataMaxStrikes, StrikeType.DownloadingMetadata), DeleteReason.DownloadingMetadata);
|
||||
}
|
||||
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<TorrentTracker>> GetTrackersAsync(string hash)
|
||||
{
|
||||
return (await _client.GetTorrentTrackersAsync(hash))
|
||||
.Where(x => x.Url.Contains("**"))
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
using Domain.Enums;
|
||||
|
||||
namespace Infrastructure.Verticals.DownloadClient;
|
||||
|
||||
public sealed record SeedingCheckResult
|
||||
{
|
||||
public bool ShouldClean { get; set; }
|
||||
public CleanReason Reason { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
namespace Infrastructure.Verticals.DownloadClient.Transmission;
|
||||
|
||||
public interface ITransmissionService : IDownloadService
|
||||
{
|
||||
}
|
||||
@@ -1,11 +1,21 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Text.RegularExpressions;
|
||||
using Common.Attributes;
|
||||
using Common.Configuration.ContentBlocker;
|
||||
using Common.Configuration.DownloadCleaner;
|
||||
using Common.Configuration.DownloadClient;
|
||||
using Common.Configuration.QueueCleaner;
|
||||
using Common.CustomDataTypes;
|
||||
using Common.Helpers;
|
||||
using Domain.Enums;
|
||||
using Infrastructure.Extensions;
|
||||
using Infrastructure.Interceptors;
|
||||
using Infrastructure.Verticals.ContentBlocker;
|
||||
using Infrastructure.Verticals.Context;
|
||||
using Infrastructure.Verticals.Files;
|
||||
using Infrastructure.Verticals.ItemStriker;
|
||||
using Infrastructure.Verticals.Notifications;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Transmission.API.RPC;
|
||||
@@ -14,11 +24,29 @@ using Transmission.API.RPC.Entity;
|
||||
|
||||
namespace Infrastructure.Verticals.DownloadClient.Transmission;
|
||||
|
||||
public sealed class TransmissionService : DownloadServiceBase
|
||||
public class TransmissionService : DownloadService, ITransmissionService
|
||||
{
|
||||
private readonly TransmissionConfig _config;
|
||||
private readonly Client _client;
|
||||
private TorrentInfo[]? _torrentsCache;
|
||||
|
||||
private static readonly string[] Fields =
|
||||
[
|
||||
TorrentFields.FILES,
|
||||
TorrentFields.FILE_STATS,
|
||||
TorrentFields.HASH_STRING,
|
||||
TorrentFields.ID,
|
||||
TorrentFields.ETA,
|
||||
TorrentFields.NAME,
|
||||
TorrentFields.STATUS,
|
||||
TorrentFields.IS_PRIVATE,
|
||||
TorrentFields.DOWNLOADED_EVER,
|
||||
TorrentFields.DOWNLOAD_DIR,
|
||||
TorrentFields.SECONDS_SEEDING,
|
||||
TorrentFields.UPLOAD_RATIO,
|
||||
TorrentFields.TRACKERS,
|
||||
TorrentFields.RATE_DOWNLOAD,
|
||||
TorrentFields.TOTAL_SIZE,
|
||||
];
|
||||
|
||||
public TransmissionService(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
@@ -26,15 +54,27 @@ public sealed class TransmissionService : DownloadServiceBase
|
||||
IOptions<TransmissionConfig> config,
|
||||
IOptions<QueueCleanerConfig> queueCleanerConfig,
|
||||
IOptions<ContentBlockerConfig> contentBlockerConfig,
|
||||
FilenameEvaluator filenameEvaluator,
|
||||
Striker striker
|
||||
) : base(logger, queueCleanerConfig, contentBlockerConfig, filenameEvaluator, striker)
|
||||
IOptions<DownloadCleanerConfig> downloadCleanerConfig,
|
||||
IMemoryCache cache,
|
||||
IFilenameEvaluator filenameEvaluator,
|
||||
IStriker striker,
|
||||
INotificationPublisher notifier,
|
||||
IDryRunInterceptor dryRunInterceptor,
|
||||
IHardLinkFileService hardLinkFileService
|
||||
) : base(
|
||||
logger, queueCleanerConfig, contentBlockerConfig, downloadCleanerConfig, cache,
|
||||
filenameEvaluator, striker, notifier, dryRunInterceptor, hardLinkFileService
|
||||
)
|
||||
{
|
||||
_config = config.Value;
|
||||
_config.Validate();
|
||||
UriBuilder uriBuilder = new(_config.Url);
|
||||
uriBuilder.Path = string.IsNullOrEmpty(_config.UrlBase)
|
||||
? $"{uriBuilder.Path.TrimEnd('/')}/rpc"
|
||||
: $"{uriBuilder.Path.TrimEnd('/')}/{_config.UrlBase.TrimStart('/').TrimEnd('/')}/rpc";
|
||||
_client = new(
|
||||
httpClientFactory.CreateClient(Constants.HttpClientWithRetryName),
|
||||
new Uri(_config.Url, "/transmission/rpc").ToString(),
|
||||
uriBuilder.Uri.ToString(),
|
||||
login: _config.Username,
|
||||
password: _config.Password
|
||||
);
|
||||
@@ -46,21 +86,27 @@ public sealed class TransmissionService : DownloadServiceBase
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override async Task<RemoveResult> ShouldRemoveFromArrQueueAsync(string hash)
|
||||
public override async Task<DownloadCheckResult> ShouldRemoveFromArrQueueAsync(string hash, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
RemoveResult result = new();
|
||||
TorrentInfo? torrent = await GetTorrentAsync(hash);
|
||||
DownloadCheckResult result = new();
|
||||
TorrentInfo? download = await GetTorrentAsync(hash);
|
||||
|
||||
if (torrent is null)
|
||||
if (download is null)
|
||||
{
|
||||
_logger.LogDebug("failed to find torrent {hash} in the download client", hash);
|
||||
return result;
|
||||
}
|
||||
|
||||
bool shouldRemove = torrent.FileStats?.Length > 0;
|
||||
result.IsPrivate = torrent.IsPrivate ?? false;
|
||||
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
|
||||
{
|
||||
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
|
||||
return result;
|
||||
}
|
||||
|
||||
bool shouldRemove = download.FileStats?.Length > 0;
|
||||
result.IsPrivate = download.IsPrivate ?? false;
|
||||
|
||||
foreach (TransmissionTorrentFileStats? stats in torrent.FileStats ?? [])
|
||||
foreach (TransmissionTorrentFileStats? stats in download.FileStats ?? [])
|
||||
{
|
||||
if (!stats.Wanted.HasValue)
|
||||
{
|
||||
@@ -74,87 +120,308 @@ public sealed class TransmissionService : DownloadServiceBase
|
||||
shouldRemove = false;
|
||||
}
|
||||
}
|
||||
|
||||
// remove if all files are unwanted or download is stuck
|
||||
result.ShouldRemove = shouldRemove || IsItemStuckAndShouldRemove(torrent);
|
||||
|
||||
if (shouldRemove)
|
||||
{
|
||||
// remove if all files are unwanted
|
||||
result.ShouldRemove = true;
|
||||
result.DeleteReason = DeleteReason.AllFilesBlocked;
|
||||
return result;
|
||||
}
|
||||
|
||||
// remove if download is stuck
|
||||
(result.ShouldRemove, result.DeleteReason) = await EvaluateDownloadRemoval(download);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override async Task<bool> BlockUnwantedFilesAsync(
|
||||
string hash,
|
||||
public override async Task<BlockFilesResult> BlockUnwantedFilesAsync(string hash,
|
||||
BlocklistType blocklistType,
|
||||
ConcurrentBag<string> patterns,
|
||||
ConcurrentBag<Regex> regexes
|
||||
)
|
||||
ConcurrentBag<Regex> regexes, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
TorrentInfo? torrent = await GetTorrentAsync(hash);
|
||||
TorrentInfo? download = await GetTorrentAsync(hash);
|
||||
BlockFilesResult result = new();
|
||||
|
||||
if (torrent?.FileStats is null || torrent.Files is null)
|
||||
if (download?.FileStats is null || download.Files is null)
|
||||
{
|
||||
return false;
|
||||
return result;
|
||||
}
|
||||
|
||||
if (_contentBlockerConfig.IgnorePrivate && (torrent.IsPrivate ?? false))
|
||||
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
|
||||
{
|
||||
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
|
||||
return result;
|
||||
}
|
||||
|
||||
bool isPrivate = download.IsPrivate ?? false;
|
||||
result.IsPrivate = isPrivate;
|
||||
|
||||
if (_contentBlockerConfig.IgnorePrivate && isPrivate)
|
||||
{
|
||||
// ignore private trackers
|
||||
_logger.LogDebug("skip files check | download is private | {name}", torrent.Name);
|
||||
return false;
|
||||
_logger.LogDebug("skip files check | download is private | {name}", download.Name);
|
||||
return result;
|
||||
}
|
||||
|
||||
List<long> unwantedFiles = [];
|
||||
long totalFiles = 0;
|
||||
long totalUnwantedFiles = 0;
|
||||
|
||||
for (int i = 0; i < torrent.Files.Length; i++)
|
||||
for (int i = 0; i < download.Files.Length; i++)
|
||||
{
|
||||
if (torrent.FileStats?[i].Wanted == null)
|
||||
if (download.FileStats?[i].Wanted == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
totalFiles++;
|
||||
|
||||
if (!torrent.FileStats[i].Wanted.Value)
|
||||
if (!download.FileStats[i].Wanted.Value)
|
||||
{
|
||||
totalUnwantedFiles++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (_filenameEvaluator.IsValid(torrent.Files[i].Name, blocklistType, patterns, regexes))
|
||||
if (_filenameEvaluator.IsValid(download.Files[i].Name, blocklistType, patterns, regexes))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
_logger.LogInformation("unwanted file found | {file}", torrent.Files[i].Name);
|
||||
_logger.LogInformation("unwanted file found | {file}", download.Files[i].Name);
|
||||
unwantedFiles.Add(i);
|
||||
totalUnwantedFiles++;
|
||||
}
|
||||
|
||||
if (unwantedFiles.Count is 0)
|
||||
{
|
||||
return false;
|
||||
return result;
|
||||
}
|
||||
|
||||
if (totalUnwantedFiles == totalFiles)
|
||||
{
|
||||
// Skip marking files as unwanted. The download will be removed completely.
|
||||
return true;
|
||||
result.ShouldRemove = true;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
_logger.LogDebug("changing priorities | torrent {hash}", hash);
|
||||
|
||||
await _client.TorrentSetAsync(new TorrentSettings
|
||||
{
|
||||
Ids = [ torrent.Id ],
|
||||
FilesUnwanted = unwantedFiles.ToArray(),
|
||||
});
|
||||
|
||||
return false;
|
||||
await _dryRunInterceptor.InterceptAsync(SetUnwantedFiles, download.Id, unwantedFiles.ToArray());
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
public override async Task<List<object>?> GetSeedingDownloads() =>
|
||||
(await _client.TorrentGetAsync(Fields))
|
||||
?.Torrents
|
||||
?.Where(x => !string.IsNullOrEmpty(x.HashString))
|
||||
.Where(x => x.Status is 5 or 6)
|
||||
.Cast<object>()
|
||||
.ToList();
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override List<object>? FilterDownloadsToBeCleanedAsync(List<object>? downloads, List<CleanCategory> categories)
|
||||
{
|
||||
return downloads
|
||||
?
|
||||
.Cast<TorrentInfo>()
|
||||
.Where(x => categories
|
||||
.Any(cat => cat.Name.Equals(x.GetCategory(), StringComparison.InvariantCultureIgnoreCase))
|
||||
)
|
||||
.Cast<object>()
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public override async Task Delete(string hash)
|
||||
public override List<object>? FilterDownloadsToChangeCategoryAsync(List<object>? downloads, List<string> categories)
|
||||
{
|
||||
return downloads
|
||||
?.Cast<TorrentInfo>()
|
||||
.Where(x => !string.IsNullOrEmpty(x.HashString))
|
||||
.Where(x => categories.Any(cat => cat.Equals(x.GetCategory(), StringComparison.InvariantCultureIgnoreCase)))
|
||||
.Cast<object>()
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public override async Task CleanDownloadsAsync(List<object>? downloads, List<CleanCategory> categoriesToClean,
|
||||
HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
if (downloads?.Count is null or 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (TorrentInfo download in downloads)
|
||||
{
|
||||
if (string.IsNullOrEmpty(download.HashString))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase)))
|
||||
{
|
||||
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
|
||||
{
|
||||
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
CleanCategory? category = categoriesToClean
|
||||
.FirstOrDefault(x =>
|
||||
{
|
||||
if (download.DownloadDir is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return Path.GetFileName(Path.TrimEndingDirectorySeparator(download.DownloadDir))
|
||||
.Equals(x.Name, StringComparison.InvariantCultureIgnoreCase);
|
||||
});
|
||||
|
||||
if (category is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!_downloadCleanerConfig.DeletePrivate && download.IsPrivate is true)
|
||||
{
|
||||
_logger.LogDebug("skip | download is private | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
ContextProvider.Set("downloadName", download.Name);
|
||||
ContextProvider.Set("hash", download.HashString);
|
||||
|
||||
TimeSpan seedingTime = TimeSpan.FromSeconds(download.SecondsSeeding ?? 0);
|
||||
SeedingCheckResult result = ShouldCleanDownload(download.uploadRatio ?? 0, seedingTime, category);
|
||||
|
||||
if (!result.ShouldClean)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
await _dryRunInterceptor.InterceptAsync(RemoveDownloadAsync, download.Id);
|
||||
|
||||
_logger.LogInformation(
|
||||
"download cleaned | {reason} reached | {name}",
|
||||
result.Reason is CleanReason.MaxRatioReached
|
||||
? "MAX_RATIO & MIN_SEED_TIME"
|
||||
: "MAX_SEED_TIME",
|
||||
download.Name
|
||||
);
|
||||
|
||||
await _notifier.NotifyDownloadCleaned(download.uploadRatio ?? 0, seedingTime, category.Name, result.Reason);
|
||||
}
|
||||
}
|
||||
|
||||
public override async Task CreateCategoryAsync(string name)
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
public override async Task ChangeCategoryForNoHardLinksAsync(List<object>? downloads, HashSet<string> excludedHashes, IReadOnlyList<string> ignoredDownloads)
|
||||
{
|
||||
if (downloads?.Count is null or 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir))
|
||||
{
|
||||
_hardLinkFileService.PopulateFileCounts(_downloadCleanerConfig.UnlinkedIgnoredRootDir);
|
||||
}
|
||||
|
||||
foreach (TorrentInfo download in downloads.Cast<TorrentInfo>())
|
||||
{
|
||||
if (string.IsNullOrEmpty(download.HashString) || string.IsNullOrEmpty(download.Name) || download.DownloadDir == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (excludedHashes.Any(x => x.Equals(download.HashString, StringComparison.InvariantCultureIgnoreCase)))
|
||||
{
|
||||
_logger.LogDebug("skip | download is used by an arr | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ignoredDownloads.Count > 0 && download.ShouldIgnore(ignoredDownloads))
|
||||
{
|
||||
_logger.LogDebug("skip | download is ignored | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
ContextProvider.Set("downloadName", download.Name);
|
||||
ContextProvider.Set("hash", download.HashString);
|
||||
|
||||
bool hasHardlinks = false;
|
||||
|
||||
if (download.Files is null || download.FileStats is null)
|
||||
{
|
||||
_logger.LogDebug("skip | download has no files | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
for (int i = 0; i < download.Files.Length; i++)
|
||||
{
|
||||
TransmissionTorrentFiles file = download.Files[i];
|
||||
TransmissionTorrentFileStats stats = download.FileStats[i];
|
||||
|
||||
if (stats.Wanted is null or false || string.IsNullOrEmpty(file.Name))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
string filePath = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadDir, file.Name).Split(['\\', '/']));
|
||||
|
||||
long hardlinkCount = _hardLinkFileService.GetHardLinkCount(filePath, !string.IsNullOrEmpty(_downloadCleanerConfig.UnlinkedIgnoredRootDir));
|
||||
|
||||
if (hardlinkCount < 0)
|
||||
{
|
||||
_logger.LogDebug("skip | could not get file properties | {file}", filePath);
|
||||
hasHardlinks = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (hardlinkCount > 0)
|
||||
{
|
||||
hasHardlinks = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (hasHardlinks)
|
||||
{
|
||||
_logger.LogDebug("skip | download has hardlinks | {name}", download.Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
string currentCategory = download.GetCategory();
|
||||
string newLocation = string.Join(Path.DirectorySeparatorChar, Path.Combine(download.DownloadDir, _downloadCleanerConfig.UnlinkedTargetCategory).Split(['\\', '/']));
|
||||
|
||||
await _dryRunInterceptor.InterceptAsync(ChangeDownloadLocation, download.Id, newLocation);
|
||||
|
||||
_logger.LogInformation("category changed for {name}", download.Name);
|
||||
|
||||
await _notifier.NotifyCategoryChanged(currentCategory, _downloadCleanerConfig.UnlinkedTargetCategory);
|
||||
|
||||
download.DownloadDir = newLocation;
|
||||
}
|
||||
}
|
||||
|
||||
[DryRunSafeguard]
|
||||
protected virtual async Task ChangeDownloadLocation(long downloadId, string newLocation)
|
||||
{
|
||||
await _client.TorrentSetLocationAsync([downloadId], newLocation, true);
|
||||
}
|
||||
|
||||
public override async Task DeleteDownload(string hash)
|
||||
{
|
||||
TorrentInfo? torrent = await GetTorrentAsync(hash);
|
||||
|
||||
@@ -170,69 +437,112 @@ public sealed class TransmissionService : DownloadServiceBase
|
||||
{
|
||||
}
|
||||
|
||||
private bool IsItemStuckAndShouldRemove(TorrentInfo torrent)
|
||||
[DryRunSafeguard]
|
||||
protected virtual async Task RemoveDownloadAsync(long downloadId)
|
||||
{
|
||||
await _client.TorrentRemoveAsync([downloadId], true);
|
||||
}
|
||||
|
||||
[DryRunSafeguard]
|
||||
protected virtual async Task SetUnwantedFiles(long downloadId, long[] unwantedFiles)
|
||||
{
|
||||
await _client.TorrentSetAsync(new TorrentSettings
|
||||
{
|
||||
Ids = [downloadId],
|
||||
FilesUnwanted = unwantedFiles,
|
||||
});
|
||||
}
|
||||
|
||||
private async Task<(bool, DeleteReason)> EvaluateDownloadRemoval(TorrentInfo torrent)
|
||||
{
|
||||
(bool ShouldRemove, DeleteReason Reason) result = await CheckIfSlow(torrent);
|
||||
|
||||
if (result.ShouldRemove)
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
return await CheckIfStuck(torrent);
|
||||
}
|
||||
|
||||
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfSlow(TorrentInfo download)
|
||||
{
|
||||
if (_queueCleanerConfig.SlowMaxStrikes is 0)
|
||||
{
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (download.Status is not 4)
|
||||
{
|
||||
// not in downloading state
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (download.RateDownload <= 0)
|
||||
{
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (_queueCleanerConfig.SlowIgnorePrivate && download.IsPrivate is true)
|
||||
{
|
||||
// ignore private trackers
|
||||
_logger.LogDebug("skip slow check | download is private | {name}", download.Name);
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (download.TotalSize > (_queueCleanerConfig.SlowIgnoreAboveSizeByteSize?.Bytes ?? long.MaxValue))
|
||||
{
|
||||
_logger.LogDebug("skip slow check | download is too large | {name}", download.Name);
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
ByteSize minSpeed = _queueCleanerConfig.SlowMinSpeedByteSize;
|
||||
ByteSize currentSpeed = new ByteSize(download.RateDownload ?? long.MaxValue);
|
||||
SmartTimeSpan maxTime = SmartTimeSpan.FromHours(_queueCleanerConfig.SlowMaxTime);
|
||||
SmartTimeSpan currentTime = SmartTimeSpan.FromSeconds(download.Eta ?? 0);
|
||||
|
||||
return await CheckIfSlow(
|
||||
download.HashString!,
|
||||
download.Name!,
|
||||
minSpeed,
|
||||
currentSpeed,
|
||||
maxTime,
|
||||
currentTime
|
||||
);
|
||||
}
|
||||
|
||||
private async Task<(bool ShouldRemove, DeleteReason Reason)> CheckIfStuck(TorrentInfo download)
|
||||
{
|
||||
if (_queueCleanerConfig.StalledMaxStrikes is 0)
|
||||
{
|
||||
return false;
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (_queueCleanerConfig.StalledIgnorePrivate && (torrent.IsPrivate ?? false))
|
||||
{
|
||||
// ignore private trackers
|
||||
_logger.LogDebug("skip stalled check | download is private | {name}", torrent.Name);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (torrent.Status is not 4)
|
||||
if (download.Status is not 4)
|
||||
{
|
||||
// not in downloading state
|
||||
return false;
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
if (torrent.Eta > 0)
|
||||
|
||||
if (download.RateDownload > 0 || download.Eta > 0)
|
||||
{
|
||||
return false;
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
return StrikeAndCheckLimit(torrent.HashString!, torrent.Name!);
|
||||
|
||||
if (_queueCleanerConfig.StalledIgnorePrivate && (download.IsPrivate ?? false))
|
||||
{
|
||||
// ignore private trackers
|
||||
_logger.LogDebug("skip stalled check | download is private | {name}", download.Name);
|
||||
return (false, DeleteReason.None);
|
||||
}
|
||||
|
||||
ResetStalledStrikesOnProgress(download.HashString!, download.DownloadedEver ?? 0);
|
||||
|
||||
return (await _striker.StrikeAndCheckLimit(download.HashString!, download.Name!, _queueCleanerConfig.StalledMaxStrikes, StrikeType.Stalled), DeleteReason.Stalled);
|
||||
}
|
||||
|
||||
private async Task<TorrentInfo?> GetTorrentAsync(string hash)
|
||||
{
|
||||
TorrentInfo? torrent = _torrentsCache?
|
||||
.FirstOrDefault(x => x.HashString.Equals(hash, StringComparison.InvariantCultureIgnoreCase));
|
||||
|
||||
if (_torrentsCache is null || torrent is null)
|
||||
{
|
||||
string[] fields = [
|
||||
TorrentFields.FILES,
|
||||
TorrentFields.FILE_STATS,
|
||||
TorrentFields.HASH_STRING,
|
||||
TorrentFields.ID,
|
||||
TorrentFields.ETA,
|
||||
TorrentFields.NAME,
|
||||
TorrentFields.STATUS,
|
||||
TorrentFields.IS_PRIVATE
|
||||
];
|
||||
|
||||
// refresh cache
|
||||
_torrentsCache = (await _client.TorrentGetAsync(fields))
|
||||
?.Torrents;
|
||||
}
|
||||
|
||||
if (_torrentsCache?.Length is null or 0)
|
||||
{
|
||||
_logger.LogDebug("could not list torrents | {url}", _config.Url);
|
||||
}
|
||||
|
||||
torrent = _torrentsCache?.FirstOrDefault(x => x.HashString.Equals(hash, StringComparison.InvariantCultureIgnoreCase));
|
||||
|
||||
if (torrent is null)
|
||||
{
|
||||
_logger.LogDebug("could not find torrent | {hash} | {url}", hash, _config.Url);
|
||||
}
|
||||
|
||||
return torrent;
|
||||
}
|
||||
private async Task<TorrentInfo?> GetTorrentAsync(string hash) =>
|
||||
(await _client.TorrentGetAsync(Fields, hash))
|
||||
?.Torrents
|
||||
?.FirstOrDefault();
|
||||
}
|
||||
51
code/Infrastructure/Verticals/Files/HardLinkFileService.cs
Normal file
51
code/Infrastructure/Verticals/Files/HardLinkFileService.cs
Normal file
@@ -0,0 +1,51 @@
|
||||
using System.Runtime.InteropServices;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace Infrastructure.Verticals.Files;
|
||||
|
||||
public class HardLinkFileService : IHardLinkFileService
|
||||
{
|
||||
private readonly ILogger<HardLinkFileService> _logger;
|
||||
private readonly UnixHardLinkFileService _unixHardLinkFileService;
|
||||
private readonly WindowsHardLinkFileService _windowsHardLinkFileService;
|
||||
|
||||
public HardLinkFileService(
|
||||
ILogger<HardLinkFileService> logger,
|
||||
UnixHardLinkFileService unixHardLinkFileService,
|
||||
WindowsHardLinkFileService windowsHardLinkFileService
|
||||
)
|
||||
{
|
||||
_logger = logger;
|
||||
_unixHardLinkFileService = unixHardLinkFileService;
|
||||
_windowsHardLinkFileService = windowsHardLinkFileService;
|
||||
}
|
||||
|
||||
public void PopulateFileCounts(string directoryPath)
|
||||
{
|
||||
_logger.LogTrace("populating file counts from {dir}", directoryPath);
|
||||
|
||||
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
|
||||
{
|
||||
_windowsHardLinkFileService.PopulateFileCounts(directoryPath);
|
||||
return;
|
||||
}
|
||||
|
||||
_unixHardLinkFileService.PopulateFileCounts(directoryPath);
|
||||
}
|
||||
|
||||
public long GetHardLinkCount(string filePath, bool ignoreRootDir)
|
||||
{
|
||||
if (!File.Exists(filePath))
|
||||
{
|
||||
_logger.LogDebug("file {file} does not exist", filePath);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
|
||||
{
|
||||
return _windowsHardLinkFileService.GetHardLinkCount(filePath, ignoreRootDir);
|
||||
}
|
||||
|
||||
return _unixHardLinkFileService.GetHardLinkCount(filePath, ignoreRootDir);
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user