mirror of
https://github.com/calibrain/shelfmark.git
synced 2026-04-20 05:51:21 -04:00
5e04b6bfb8
## Summary
This PR completely overhauls the download system to support concurrent
downloads, cancellation, and advanced queue
management. No more waiting for stuck downloads to block the entire
queue!
## New Features
### **Concurrent Downloads**
- **3 simultaneous downloads** by default (configurable)
- ThreadPoolExecutor-based architecture replaces single-threaded
bottleneck
- Downloads no longer block each other
### **Download Cancellation**
- **Cancel button** for active/queued downloads
- Clean cancellation with partial file cleanup
- Thread-safe cancellation flags
### **Queue Priority & Reordering**
- **Priority-based queue** (lower number = higher priority)
- **Editable priority inputs** in UI for queued items
- **Bulk reordering** API support
## UI Enhancements
- **Enhanced status table** with Priority, Progress, and Actions columns
- **Progress bars** for active downloads
- **Cancel buttons** for downloads
- **Queue management controls**: Refresh, Clear Completed, Active
Counter
- **Real-time updates** every 60 seconds
## Implementation
### New Architecture
- `queue.PriorityQueue` replaces simple `set()` for proper ordering
- `concurrent.futures.ThreadPoolExecutor` handles worker threads
- `threading.Event` flags enable clean cancellation
- Thread-safe operations with proper locking
### New API Endpoints
DELETE /api/download/{id}/cancel # Cancel downloads
PUT /api/queue/{id}/priority # Set priorityPOST /api/queue/reorder #
Bulk reorder
GET /api/downloads/active # Active tracking
DELETE /api/queue/clear # Clear completed
### New Environment Variables
```bash
MAX_CONCURRENT_DOWNLOADS=3 # Worker thread count
DOWNLOAD_PROGRESS_UPDATE_INTERVAL=5 # Progress frequency
```
### Fully backward compatible - existing functionality unchanged
### Fixes
- Resolves: Downloads blocking when one gets stuck
- Resolves: No way to cancel problematic downloads
- Resolves: No queue management or prioritization
#183
---------
Co-authored-by: CaliBrain <calibrain@l4n.xyz>
55 lines
2.4 KiB
Python
55 lines
2.4 KiB
Python
import os
|
|
from pathlib import Path
|
|
|
|
def string_to_bool(s: str) -> bool:
|
|
return s.lower() in ["true", "yes", "1", "y"]
|
|
|
|
CWA_DB = os.getenv("CWA_DB_PATH")
|
|
CWA_DB_PATH = Path(CWA_DB) if CWA_DB else None
|
|
LOG_ROOT = Path(os.getenv("LOG_ROOT", "/var/log/"))
|
|
LOG_DIR = LOG_ROOT / "cwa-book-downloader"
|
|
TMP_DIR = Path(os.getenv("TMP_DIR", "/tmp/cwa-book-downloader"))
|
|
INGEST_DIR = Path(os.getenv("INGEST_DIR", "/cwa-book-ingest"))
|
|
STATUS_TIMEOUT = int(os.getenv("STATUS_TIMEOUT", "3600"))
|
|
USE_BOOK_TITLE = string_to_bool(os.getenv("USE_BOOK_TITLE", "false"))
|
|
MAX_RETRY = int(os.getenv("MAX_RETRY", "10"))
|
|
DEFAULT_SLEEP = int(os.getenv("DEFAULT_SLEEP", "5"))
|
|
USE_CF_BYPASS = string_to_bool(os.getenv("USE_CF_BYPASS", "true"))
|
|
HTTP_PROXY = os.getenv("HTTP_PROXY", "").strip()
|
|
HTTPS_PROXY = os.getenv("HTTPS_PROXY", "").strip()
|
|
AA_DONATOR_KEY = os.getenv("AA_DONATOR_KEY", "").strip()
|
|
_AA_BASE_URL = os.getenv("AA_BASE_URL", "auto").strip()
|
|
_AA_ADDITIONAL_URLS = os.getenv("AA_ADDITIONAL_URLS", "").strip()
|
|
_SUPPORTED_FORMATS = os.getenv("SUPPORTED_FORMATS", "epub,mobi,azw3,fb2,djvu,cbz,cbr").lower()
|
|
_BOOK_LANGUAGE = os.getenv("BOOK_LANGUAGE", "en").lower()
|
|
_CUSTOM_SCRIPT = os.getenv("CUSTOM_SCRIPT", "").strip()
|
|
FLASK_HOST = os.getenv("FLASK_HOST", "0.0.0.0")
|
|
FLASK_PORT = int(os.getenv("FLASK_PORT", "8084"))
|
|
DEBUG = string_to_bool(os.getenv("DEBUG", "false"))
|
|
PRIORITIZE_WELIB = string_to_bool(os.getenv("PRIORITIZE_WELIB", "false"))
|
|
|
|
# If debug is true, we want to log everything
|
|
if DEBUG:
|
|
LOG_LEVEL = "DEBUG"
|
|
else:
|
|
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
|
|
ENABLE_LOGGING = string_to_bool(os.getenv("ENABLE_LOGGING", "true"))
|
|
MAIN_LOOP_SLEEP_TIME = int(os.getenv("MAIN_LOOP_SLEEP_TIME", "5"))
|
|
MAX_CONCURRENT_DOWNLOADS = int(os.getenv("MAX_CONCURRENT_DOWNLOADS", "3"))
|
|
DOWNLOAD_PROGRESS_UPDATE_INTERVAL = int(os.getenv("DOWNLOAD_PROGRESS_UPDATE_INTERVAL", "5"))
|
|
DOCKERMODE = string_to_bool(os.getenv("DOCKERMODE", "false"))
|
|
_CUSTOM_DNS = os.getenv("CUSTOM_DNS", "").strip()
|
|
USE_DOH = string_to_bool(os.getenv("USE_DOH", "false"))
|
|
BYPASS_RELEASE_INACTIVE_MIN = int(os.getenv("BYPASS_RELEASE_INACTIVE_MIN", "5"))
|
|
APP_ENV = os.getenv("APP_ENV", "prod").lower()
|
|
# Logging settings
|
|
LOG_FILE = LOG_DIR / "cwa-book-downloader.log"
|
|
|
|
USING_TOR = string_to_bool(os.getenv("USING_TOR", "false"))
|
|
# If using Tor, we don't need to set custom DNS, use DOH, or proxy
|
|
if USING_TOR:
|
|
_CUSTOM_DNS = ""
|
|
USE_DOH = False
|
|
HTTP_PROXY = ""
|
|
HTTPS_PROXY = ""
|
|
|