From f08d3918dccfa2a4c2b7df2852c6a501cff92689 Mon Sep 17 00:00:00 2001 From: Andrey Prygunkov Date: Mon, 9 May 2016 19:17:17 +0200 Subject: [PATCH] #205: retry for deleted downloads too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commands “Retry failed articles” and “Download remaining files“ now work for deleted downloads too (including deleted by health check). --- daemon/nntp/ArticleWriter.cpp | 2 +- daemon/postprocess/PrePostProcessor.cpp | 72 +++--- daemon/postprocess/PrePostProcessor.h | 2 +- daemon/queue/DiskState.cpp | 2 +- daemon/queue/DownloadInfo.h | 12 +- daemon/queue/HistoryCoordinator.cpp | 311 ++++++++++++++---------- daemon/queue/HistoryCoordinator.h | 7 +- daemon/queue/QueueCoordinator.cpp | 39 +-- daemon/queue/QueueEditor.cpp | 10 + daemon/queue/QueueEditor.h | 1 + nzbget.conf | 4 + webui/edit.js | 3 +- 12 files changed, 279 insertions(+), 186 deletions(-) diff --git a/daemon/nntp/ArticleWriter.cpp b/daemon/nntp/ArticleWriter.cpp index 9561d4e6..efb0bb7f 100644 --- a/daemon/nntp/ArticleWriter.cpp +++ b/daemon/nntp/ArticleWriter.cpp @@ -588,7 +588,7 @@ void ArticleWriter::FlushCache() for (ArticleInfo* pa : cachedArticles) { - if (m_fileInfo->GetDeleted()) + if (m_fileInfo->GetDeleted() && !m_fileInfo->GetNzbInfo()->GetParking()) { // the file was deleted during flushing: stop flushing immediately break; diff --git a/daemon/postprocess/PrePostProcessor.cpp b/daemon/postprocess/PrePostProcessor.cpp index 6d3eb007..8518b960 100644 --- a/daemon/postprocess/PrePostProcessor.cpp +++ b/daemon/postprocess/PrePostProcessor.cpp @@ -128,34 +128,34 @@ void PrePostProcessor::DownloadQueueUpdate(Subject* Caller, void* Aspect) g_QueueScriptCoordinator->EnqueueScript(queueAspect->nzbInfo, QueueScriptCoordinator::qeFileDownloaded); } - if ( #ifndef DISABLE_PARCHECK - !m_parCoordinator.AddPar(queueAspect->fileInfo, queueAspect->action == DownloadQueue::eaFileDeleted) && -#endif - IsNzbFileCompleted(queueAspect->nzbInfo, true, false) && - !queueAspect->nzbInfo->GetPostInfo() && - (!queueAspect->fileInfo->GetPaused() || IsNzbFileCompleted(queueAspect->nzbInfo, false, false))) + if (m_parCoordinator.AddPar(queueAspect->fileInfo, queueAspect->action == DownloadQueue::eaFileDeleted)) { - if ((queueAspect->action == DownloadQueue::eaFileCompleted || - (queueAspect->fileInfo->GetAutoDeleted() && - IsNzbFileCompleted(queueAspect->nzbInfo, false, true))) && - queueAspect->fileInfo->GetNzbInfo()->GetDeleteStatus() != NzbInfo::dsHealth) - { - queueAspect->nzbInfo->PrintMessage(Message::mkInfo, - "Collection %s completely downloaded", queueAspect->nzbInfo->GetName()); - g_QueueScriptCoordinator->EnqueueScript(queueAspect->nzbInfo, QueueScriptCoordinator::qeNzbDownloaded); - NzbDownloaded(queueAspect->downloadQueue, queueAspect->nzbInfo); - } - else if ((queueAspect->action == DownloadQueue::eaFileDeleted || - (queueAspect->action == DownloadQueue::eaFileCompleted && - queueAspect->fileInfo->GetNzbInfo()->GetDeleteStatus() > NzbInfo::dsNone)) && - !queueAspect->nzbInfo->GetParCleanup() && - IsNzbFileCompleted(queueAspect->nzbInfo, false, true)) - { - queueAspect->nzbInfo->PrintMessage(Message::mkInfo, - "Collection %s deleted from queue", queueAspect->nzbInfo->GetName()); - NzbDeleted(queueAspect->downloadQueue, queueAspect->nzbInfo); - } + return; + } +#endif + + if ((queueAspect->action == DownloadQueue::eaFileCompleted || + queueAspect->fileInfo->GetDupeDeleted()) && + queueAspect->fileInfo->GetNzbInfo()->GetDeleteStatus() != NzbInfo::dsHealth && + !queueAspect->nzbInfo->GetPostInfo() && + IsNzbFileCompleted(queueAspect->nzbInfo, true)) + { + queueAspect->nzbInfo->PrintMessage(Message::mkInfo, + "Collection %s completely downloaded", queueAspect->nzbInfo->GetName()); + g_QueueScriptCoordinator->EnqueueScript(queueAspect->nzbInfo, QueueScriptCoordinator::qeNzbDownloaded); + NzbDownloaded(queueAspect->downloadQueue, queueAspect->nzbInfo); + } + else if ((queueAspect->action == DownloadQueue::eaFileDeleted || + (queueAspect->action == DownloadQueue::eaFileCompleted && + queueAspect->fileInfo->GetNzbInfo()->GetDeleteStatus() > NzbInfo::dsNone)) && + !queueAspect->nzbInfo->GetPostInfo() && + !queueAspect->nzbInfo->GetParCleanup() && + IsNzbFileCompleted(queueAspect->nzbInfo, false)) + { + queueAspect->nzbInfo->PrintMessage(Message::mkInfo, + "Collection %s deleted from queue", queueAspect->nzbInfo->GetName()); + NzbDeleted(queueAspect->downloadQueue, queueAspect->nzbInfo); } } } @@ -613,7 +613,7 @@ void PrePostProcessor::JobCompleted(DownloadQueue* downloadQueue, PostInfo* post DeletePostThread(postInfo); nzbInfo->LeavePostProcess(); - if (IsNzbFileCompleted(nzbInfo, true, false)) + if (IsNzbFileCompleted(nzbInfo, true)) { // Cleaning up queue if par-check was successful or unpack was successful or // health is 100% (if unpack and par-check were not performed) @@ -655,19 +655,21 @@ void PrePostProcessor::JobCompleted(DownloadQueue* downloadQueue, PostInfo* post downloadQueue->Save(); } -bool PrePostProcessor::IsNzbFileCompleted(NzbInfo* nzbInfo, bool ignorePausedPars, bool allowOnlyOneDeleted) +bool PrePostProcessor::IsNzbFileCompleted(NzbInfo* nzbInfo, bool ignorePausedPars) { - int deleted = 0; + if (nzbInfo->GetActiveDownloads()) + { + return false; + } for (FileInfo* fileInfo : nzbInfo->GetFileList()) { - if (fileInfo->GetDeleted()) + if (fileInfo->GetParking()) { - deleted++; + continue; } - if (((!fileInfo->GetPaused() || !ignorePausedPars || !fileInfo->GetParFile()) && - !fileInfo->GetDeleted()) || - (allowOnlyOneDeleted && deleted > 1)) + if ((!fileInfo->GetPaused() || !ignorePausedPars || !fileInfo->GetParFile()) && + !fileInfo->GetDeleted()) { return false; } @@ -685,7 +687,7 @@ bool PrePostProcessor::IsNzbFileDownloading(NzbInfo* nzbInfo) for (FileInfo* fileInfo : nzbInfo->GetFileList()) { - if (!fileInfo->GetPaused()) + if (!fileInfo->GetPaused() && !fileInfo->GetParking()) { return true; } diff --git a/daemon/postprocess/PrePostProcessor.h b/daemon/postprocess/PrePostProcessor.h index b2f6447b..bfdd6bae 100644 --- a/daemon/postprocess/PrePostProcessor.h +++ b/daemon/postprocess/PrePostProcessor.h @@ -53,7 +53,7 @@ private: NzbInfo* m_curJob = nullptr; const char* m_pauseReason = nullptr; - bool IsNzbFileCompleted(NzbInfo* nzbInfo, bool ignorePausedPars, bool allowOnlyOneDeleted); + bool IsNzbFileCompleted(NzbInfo* nzbInfo, bool ignorePausedPars); bool IsNzbFileDownloading(NzbInfo* nzbInfo); void CheckPostQueue(); void JobCompleted(DownloadQueue* downloadQueue, PostInfo* postInfo); diff --git a/daemon/queue/DiskState.cpp b/daemon/queue/DiskState.cpp index 11c83a43..20a08bc5 100644 --- a/daemon/queue/DiskState.cpp +++ b/daemon/queue/DiskState.cpp @@ -1023,7 +1023,7 @@ bool DiskState::LoadFileState(FileInfo* fileInfo, Servers* servers, StateDiskFil uint32 High1, Low1, High2, Low2, High3, Low3; if (infile.ScanLine("%u,%u,%u,%u,%u,%u", &High1, &Low1, &High2, &Low2, &High3, &Low3) != 6) goto error; fileInfo->SetRemainingSize(Util::JoinInt64(High1, Low1)); - fileInfo->SetSuccessSize(Util::JoinInt64(High2, Low3)); + fileInfo->SetSuccessSize(Util::JoinInt64(High2, Low2)); fileInfo->SetFailedSize(Util::JoinInt64(High3, Low3)); if (!LoadServerStats(fileInfo->GetServerStats(), servers, infile)) goto error; diff --git a/daemon/queue/DownloadInfo.h b/daemon/queue/DownloadInfo.h index 4d24cec8..028fd82a 100644 --- a/daemon/queue/DownloadInfo.h +++ b/daemon/queue/DownloadInfo.h @@ -173,14 +173,16 @@ public: void SetExtraPriority(bool extraPriority) { m_extraPriority = extraPriority; } int GetActiveDownloads() { return m_activeDownloads; } void SetActiveDownloads(int activeDownloads); - bool GetAutoDeleted() { return m_autoDeleted; } - void SetAutoDeleted(bool autoDeleted) { m_autoDeleted = autoDeleted; } + bool GetDupeDeleted() { return m_dupeDeleted; } + void SetDupeDeleted(bool dupeDeleted) { m_dupeDeleted = dupeDeleted; } int GetCachedArticles() { return m_cachedArticles; } void SetCachedArticles(int cachedArticles) { m_cachedArticles = cachedArticles; } bool GetPartialChanged() { return m_partialChanged; } void SetPartialChanged(bool partialChanged) { m_partialChanged = partialChanged; } bool GetForceDirectWrite() { return m_forceDirectWrite; } void SetForceDirectWrite(bool forceDirectWrite) { m_forceDirectWrite = forceDirectWrite; } + bool GetParking() { return m_parking; } + void SetParking(bool parking) { m_parking = parking; } ServerStatList* GetServerStats() { return &m_serverStats; } private: @@ -211,10 +213,11 @@ private: std::unique_ptr m_outputFileMutex; bool m_extraPriority = false; int m_activeDownloads = 0; - bool m_autoDeleted = false; + bool m_dupeDeleted = false; int m_cachedArticles = 0; bool m_partialChanged = false; bool m_forceDirectWrite = false; + bool m_parking = false; static int m_idGen; static int m_idMax; @@ -494,6 +497,8 @@ public: void SetQueuedFilename(const char* queuedFilename) { m_queuedFilename = queuedFilename; } bool GetDeleting() { return m_deleting; } void SetDeleting(bool deleting) { m_deleting = deleting; } + bool GetParking() { return m_parking; } + void SetParking(bool parking) { m_parking = parking; } bool GetDeletePaused() { return m_deletePaused; } void SetDeletePaused(bool deletePaused) { m_deletePaused = deletePaused; } bool GetManyDupeFiles() { return m_manyDupeFiles; } @@ -614,6 +619,7 @@ private: bool m_manyDupeFiles = false; CString m_queuedFilename = ""; bool m_deleting = false; + bool m_parking = false; bool m_avoidHistory = false; bool m_healthPaused = false; bool m_parCleanup = false; diff --git a/daemon/queue/HistoryCoordinator.cpp b/daemon/queue/HistoryCoordinator.cpp index aef01061..db37bfe8 100644 --- a/daemon/queue/HistoryCoordinator.cpp +++ b/daemon/queue/HistoryCoordinator.cpp @@ -126,34 +126,38 @@ void HistoryCoordinator::AddToHistory(DownloadQueue* downloadQueue, NzbInfo* nzb downloadQueue->GetHistory()->Add(std::move(historyInfo), true); downloadQueue->HistoryChanged(); - if (nzbInfo->GetDeleteStatus() == NzbInfo::dsNone) + if (nzbInfo->GetDeleteStatus() == NzbInfo::dsNone || nzbInfo->GetParking()) { - // park files and delete files marked for deletion - int parkedFiles = 0; - for (FileList::iterator it = nzbInfo->GetFileList()->begin(); it != nzbInfo->GetFileList()->end(); ) + // park remaining files + nzbInfo->SetParkedFileCount(0); + for (FileInfo* fileInfo : nzbInfo->GetFileList()) { - FileInfo* fileInfo = (*it).get(); - if (!fileInfo->GetDeleted()) + if (!fileInfo->GetDeleted() || fileInfo->GetParking()) { - detail("Parking file %s", fileInfo->GetFilename()); - g_QueueCoordinator->DiscardDiskFile(fileInfo); - parkedFiles++; - it++; - } - else - { - // since we removed nzbInfo from queue we need to take care of removing file infos marked for deletion - nzbInfo->GetFileList()->erase(it); - it = nzbInfo->GetFileList()->begin() + parkedFiles; + nzbInfo->PrintMessage(Message::mkDetail, "Parking file %s", fileInfo->GetFilename()); + + CompletedFileList::iterator pos = std::find_if(nzbInfo->GetCompletedFiles()->begin(), nzbInfo->GetCompletedFiles()->end(), + [fileInfo](CompletedFile& completedFile) + { + return completedFile.GetId() == fileInfo->GetId(); + }); + + if (pos == nzbInfo->GetCompletedFiles()->end()) + { + nzbInfo->GetCompletedFiles()->emplace_back(fileInfo->GetId(), fileInfo->GetFilename(), + fileInfo->GetSuccessArticles() > 0 || fileInfo->GetFailedArticles() > 0 ? CompletedFile::cfPartial : CompletedFile::cfUnknown, + 0); + } + + nzbInfo->SetParkedFileCount(nzbInfo->GetParkedFileCount() + 1); } } - nzbInfo->SetParkedFileCount(parkedFiles); - } - else - { - nzbInfo->GetFileList()->clear(); } + nzbInfo->GetFileList()->clear(); + nzbInfo->SetRemainingParCount(0); + nzbInfo->SetParking(false); + nzbInfo->PrintMessage(Message::mkInfo, "Collection %s added to history", nzbInfo->GetName()); } @@ -230,8 +234,11 @@ bool HistoryCoordinator::EditList(DownloadQueue* downloadQueue, IdList* idList, break; case DownloadQueue::eaHistoryReturn: + HistoryReturn(downloadQueue, itHistory, historyInfo); + break; + case DownloadQueue::eaHistoryProcess: - HistoryReturn(downloadQueue, itHistory, historyInfo, action == DownloadQueue::eaHistoryProcess); + HistoryProcess(downloadQueue, itHistory, historyInfo); break; case DownloadQueue::eaHistoryRedownload: @@ -239,7 +246,7 @@ bool HistoryCoordinator::EditList(DownloadQueue* downloadQueue, IdList* idList, break; case DownloadQueue::eaHistoryRetryFailed: - HistoryRetryFailed(downloadQueue, itHistory, historyInfo); + HistoryRetry(downloadQueue, itHistory, historyInfo, true, false); break; case DownloadQueue::eaHistorySetParameter: @@ -333,79 +340,51 @@ void HistoryCoordinator::HistoryDelete(DownloadQueue* downloadQueue, HistoryList } } -void HistoryCoordinator::HistoryReturn(DownloadQueue* downloadQueue, HistoryList::iterator itHistory, HistoryInfo* historyInfo, bool reprocess) +void HistoryCoordinator::MoveToQueue(DownloadQueue* downloadQueue, HistoryList::iterator itHistory, HistoryInfo* historyInfo, bool reprocess) { debug("Returning %s from history back to download queue", historyInfo->GetName()); - NzbInfo* nzbInfo = nullptr; CString nicename = historyInfo->GetName(); + NzbInfo* nzbInfo = historyInfo->GetNzbInfo(); - if (reprocess && historyInfo->GetKind() != HistoryInfo::hkNzb) + // unpark files + for (FileInfo* fileInfo : nzbInfo->GetFileList()) { - error("Could not restart postprocessing for %s: history item has wrong type", historyInfo->GetName()); - return; + nzbInfo->PrintMessage(Message::mkDetail, "Unparking file %s", fileInfo->GetFilename()); } - if (historyInfo->GetKind() == HistoryInfo::hkNzb) + downloadQueue->GetQueue()->Add(std::unique_ptr(nzbInfo), true); + historyInfo->DiscardNzbInfo(); + + // reset postprocessing status variables + nzbInfo->SetParCleanup(false); + if (!nzbInfo->GetUnpackCleanedUpDisk()) { - nzbInfo = historyInfo->GetNzbInfo(); + nzbInfo->SetUnpackStatus(NzbInfo::usNone); + nzbInfo->SetCleanupStatus(NzbInfo::csNone); + nzbInfo->SetRenameStatus(NzbInfo::rsNone); + nzbInfo->SetPostTotalSec(nzbInfo->GetPostTotalSec() - nzbInfo->GetUnpackSec()); + nzbInfo->SetUnpackSec(0); - // unpark files - bool unparked = false; - for (FileInfo* fileInfo : nzbInfo->GetFileList()) + if (ParParser::FindMainPars(nzbInfo->GetDestDir(), nullptr)) { - detail("Unparking file %s", fileInfo->GetFilename()); - unparked = true; + nzbInfo->SetParStatus(NzbInfo::psNone); + nzbInfo->SetPostTotalSec(nzbInfo->GetPostTotalSec() - nzbInfo->GetParSec()); + nzbInfo->SetParSec(0); + nzbInfo->SetRepairSec(0); + nzbInfo->SetParFull(false); } - - if (!(unparked || reprocess)) - { - warn("Could not return %s back from history to download queue: history item does not have any files left for download", historyInfo->GetName()); - return; - } - - downloadQueue->GetQueue()->Add(std::unique_ptr(nzbInfo), true); - historyInfo->DiscardNzbInfo(); - - // reset postprocessing status variables - nzbInfo->SetParCleanup(false); - if (!nzbInfo->GetUnpackCleanedUpDisk()) - { - nzbInfo->SetUnpackStatus(NzbInfo::usNone); - nzbInfo->SetCleanupStatus(NzbInfo::csNone); - nzbInfo->SetRenameStatus(NzbInfo::rsNone); - nzbInfo->SetPostTotalSec(nzbInfo->GetPostTotalSec() - nzbInfo->GetUnpackSec()); - nzbInfo->SetUnpackSec(0); - - if (ParParser::FindMainPars(nzbInfo->GetDestDir(), nullptr)) - { - nzbInfo->SetParStatus(NzbInfo::psNone); - nzbInfo->SetPostTotalSec(nzbInfo->GetPostTotalSec() - nzbInfo->GetParSec()); - nzbInfo->SetParSec(0); - nzbInfo->SetRepairSec(0); - nzbInfo->SetParFull(false); - } - } - nzbInfo->SetDeleteStatus(NzbInfo::dsNone); - nzbInfo->SetDeletePaused(false); - nzbInfo->SetMarkStatus(NzbInfo::ksNone); - nzbInfo->GetScriptStatuses()->clear(); - nzbInfo->SetParkedFileCount(0); - if (nzbInfo->GetMoveStatus() == NzbInfo::msFailure) - { - nzbInfo->SetMoveStatus(NzbInfo::msNone); - } - nzbInfo->SetReprocess(reprocess); - nzbInfo->SetFinalDir(""); } - - if (historyInfo->GetKind() == HistoryInfo::hkUrl) + nzbInfo->SetDeleteStatus(NzbInfo::dsNone); + nzbInfo->SetDeletePaused(false); + nzbInfo->SetMarkStatus(NzbInfo::ksNone); + nzbInfo->GetScriptStatuses()->clear(); + nzbInfo->SetParkedFileCount(0); + if (nzbInfo->GetMoveStatus() == NzbInfo::msFailure) { - nzbInfo = historyInfo->GetNzbInfo(); - historyInfo->DiscardNzbInfo(); - nzbInfo->SetUrlStatus(NzbInfo::lsNone); - nzbInfo->SetDeleteStatus(NzbInfo::dsNone); - downloadQueue->GetQueue()->Add(std::unique_ptr(nzbInfo), true); + nzbInfo->SetMoveStatus(NzbInfo::msNone); } + nzbInfo->SetReprocess(reprocess); + nzbInfo->SetFinalDir(""); downloadQueue->GetHistory()->erase(itHistory); // the object "pHistoryInfo" is released few lines later, after the call to "NZBDownloaded" @@ -424,13 +403,17 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History { if (historyInfo->GetKind() == HistoryInfo::hkUrl) { - HistoryReturn(downloadQueue, itHistory, historyInfo, false); + NzbInfo* nzbInfo = historyInfo->GetNzbInfo(); + historyInfo->DiscardNzbInfo(); + nzbInfo->SetUrlStatus(NzbInfo::lsNone); + nzbInfo->SetDeleteStatus(NzbInfo::dsNone); + downloadQueue->GetQueue()->Add(std::unique_ptr(nzbInfo), true); return; } if (historyInfo->GetKind() != HistoryInfo::hkNzb) { - error("Could not return %s from history back to queue: history item has wrong type", historyInfo->GetName()); + error("Could not download again %s: history item has wrong type", historyInfo->GetName()); return; } @@ -439,7 +422,7 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History if (!FileSystem::FileExists(nzbInfo->GetQueuedFilename())) { - error("Could not return %s from history back to queue: could not find source nzb-file %s", + error("Could not download again %s: could not find source nzb-file %s", nzbInfo->GetName(), nzbInfo->GetQueuedFilename()); return; } @@ -447,12 +430,11 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History NzbFile nzbFile(nzbInfo->GetQueuedFilename(), ""); if (!nzbFile.Parse()) { - error("Could not return %s from history back to queue: could not parse nzb-file", - nzbInfo->GetName()); + error("Could not download again %s: could not parse nzb-file", nzbInfo->GetName()); return; } - info("Returning %s from history back to queue", nzbInfo->GetName()); + info("Downloading again %s", nzbInfo->GetName()); std::unique_ptr newNzbInfo = nzbFile.DetachNzbInfo(); @@ -484,7 +466,7 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History g_DiskState->DiscardFiles(nzbInfo); - // reset status fields (which are not reset by "HistoryReturn") + // reset status fields (which are not reset by "MoveToQueue") nzbInfo->SetMoveStatus(NzbInfo::msNone); nzbInfo->SetUnpackCleanedUpDisk(false); nzbInfo->SetParStatus(NzbInfo::psNone); @@ -504,17 +486,51 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History g_QueueCoordinator->CheckDupeFileInfos(nzbInfo); - HistoryReturn(downloadQueue, itHistory, historyInfo, false); + MoveToQueue(downloadQueue, itHistory, historyInfo, false); g_PrePostProcessor->NzbAdded(downloadQueue, nzbInfo); } -void HistoryCoordinator::HistoryRetryFailed(DownloadQueue* downloadQueue, HistoryList::iterator itHistory, +void HistoryCoordinator::HistoryReturn(DownloadQueue* downloadQueue, HistoryList::iterator itHistory, HistoryInfo* historyInfo) +{ + if (historyInfo->GetKind() == HistoryInfo::hkUrl) + { + HistoryRedownload(downloadQueue, itHistory, historyInfo, false); + } + else if (historyInfo->GetKind() == HistoryInfo::hkNzb && historyInfo->GetNzbInfo()->GetParkedFileCount() == 0) + { + warn("Could not download remaining files for %s: history item does not have any files left for download", historyInfo->GetName()); + } + else if (historyInfo->GetKind() == HistoryInfo::hkNzb) + { + HistoryRetry(downloadQueue, itHistory, historyInfo, false, false); + } + else + { + error("Could not download remaining files for %s: history item has wrong type", historyInfo->GetName()); + } +} + +void HistoryCoordinator::HistoryProcess(DownloadQueue* downloadQueue, HistoryList::iterator itHistory, HistoryInfo* historyInfo) { if (historyInfo->GetKind() != HistoryInfo::hkNzb) { - error("Could not retry failed articles for %s: history item has wrong type", historyInfo->GetName()); + error("Could not post-process again %s: history item has wrong type", historyInfo->GetName()); + return; + } + + HistoryRetry(downloadQueue, itHistory, historyInfo, false, true); +} + +void HistoryCoordinator::HistoryRetry(DownloadQueue* downloadQueue, HistoryList::iterator itHistory, + HistoryInfo* historyInfo, bool resetFailed, bool reprocess) +{ + if (historyInfo->GetKind() != HistoryInfo::hkNzb) + { + error("Could not %s for %s: history item has wrong type", + (resetFailed ? "retry failed articles" : "download remaining files"), + historyInfo->GetName()); return; } @@ -522,31 +538,42 @@ void HistoryCoordinator::HistoryRetryFailed(DownloadQueue* downloadQueue, Histor if (!FileSystem::DirectoryExists(nzbInfo->GetDestDir())) { - error("Could not retry failed articles for %s: destination directory %s doesn't exist", historyInfo->GetName(), nzbInfo->GetDestDir()); + error("Could not %s %s: destination directory %s doesn't exist", + (resetFailed ? "retry failed articles for" : reprocess ? "post-process again" : "download remaining files for"), + historyInfo->GetName(), nzbInfo->GetDestDir()); return; } - nzbInfo->PrintMessage(Message::mkInfo, "Retrying failed articles for %s", nzbInfo->GetName()); + nzbInfo->PrintMessage(Message::mkInfo, "%s %s", + (resetFailed ? "Retrying failed articles for" : reprocess ? "Post-processing again" : "Downloading remaining files for"), + nzbInfo->GetName()); // move failed completed files to (parked) file list for (CompletedFileList::iterator it = nzbInfo->GetCompletedFiles()->begin(); it != nzbInfo->GetCompletedFiles()->end(); ) { CompletedFile& completedFile = *it; - if (completedFile.GetStatus() != CompletedFile::cfSuccess && completedFile.GetId() > 0) + if (completedFile.GetStatus() != CompletedFile::cfSuccess && + (resetFailed || completedFile.GetStatus() != CompletedFile::cfFailure) && + completedFile.GetId() > 0) { - nzbInfo->PrintMessage(Message::mkDetail, "Retrying %s", completedFile.GetFileName()); std::unique_ptr fileInfo = std::make_unique(); fileInfo->SetId(completedFile.GetId()); if (g_DiskState->LoadFile(fileInfo.get(), true, true) && - g_DiskState->LoadFileState(fileInfo.get(), g_ServerPool->GetServers(), true)) + (completedFile.GetStatus() == CompletedFile::cfUnknown || + g_DiskState->LoadFileState(fileInfo.get(), g_ServerPool->GetServers(), true)) && + (resetFailed || fileInfo->GetRemainingSize() > 0)) { fileInfo->SetFilename(completedFile.GetFileName()); + fileInfo->SetPaused(fileInfo->GetParFile()); + fileInfo->SetNzbInfo(nzbInfo); BString<1024> outputFilename("%s%c%s", nzbInfo->GetDestDir(), PATH_SEPARATOR, fileInfo->GetFilename()); + if (FileSystem::FileSize(outputFilename) == 0) { FileSystem::DeleteFile(outputFilename); } + if (fileInfo->GetSuccessArticles() > 0 && FileSystem::FileExists(outputFilename)) { fileInfo->SetOutputFilename(outputFilename); @@ -554,38 +581,19 @@ void HistoryCoordinator::HistoryRetryFailed(DownloadQueue* downloadQueue, Histor fileInfo->SetForceDirectWrite(true); fileInfo->SetFilenameConfirmed(true); } + + ResetArticles(fileInfo.get(), resetFailed); - // reset articles state - for (ArticleInfo* pa : fileInfo->GetArticles()) + if (completedFile.GetStatus() != CompletedFile::cfUnknown) { - if (pa->GetStatus() == ArticleInfo::aiFailed) + if (g_Options->GetContinuePartial()) { - pa->SetStatus(ArticleInfo::aiUndefined); - fileInfo->SetCompletedArticles(fileInfo->GetCompletedArticles() - 1); - - fileInfo->SetFailedArticles(fileInfo->GetFailedArticles() - 1); - fileInfo->SetFailedSize(fileInfo->GetFailedSize() - pa->GetSize()); - fileInfo->SetRemainingSize(fileInfo->GetRemainingSize() + pa->GetSize()); - - nzbInfo->SetFailedArticles(nzbInfo->GetFailedArticles() - 1); - nzbInfo->SetCurrentFailedArticles(nzbInfo->GetCurrentFailedArticles() - 1); - nzbInfo->SetFailedSize(nzbInfo->GetFailedSize() - pa->GetSize()); - nzbInfo->SetCurrentFailedSize(nzbInfo->GetCurrentFailedSize() - pa->GetSize()); - nzbInfo->SetRemainingSize(nzbInfo->GetRemainingSize() + pa->GetSize()); - - if (fileInfo->GetParFile()) - { - nzbInfo->SetParFailedSize(nzbInfo->GetParFailedSize() - pa->GetSize()); - nzbInfo->SetParCurrentFailedSize(nzbInfo->GetParCurrentFailedSize() - pa->GetSize()); - nzbInfo->SetRemainingParCount(nzbInfo->GetRemainingParCount() + 1); - } + g_DiskState->SaveFileState(fileInfo.get(), false); } + g_DiskState->DiscardFile(fileInfo.get(), false, false, true); } - g_DiskState->SaveFileState(fileInfo.get(), false); - - fileInfo->SetNzbInfo(nzbInfo); - nzbInfo->GetFileList()->Add(std::move(fileInfo), true); + nzbInfo->GetFileList()->Add(std::move(fileInfo), false); it = nzbInfo->GetCompletedFiles()->erase(it); continue; @@ -594,7 +602,62 @@ void HistoryCoordinator::HistoryRetryFailed(DownloadQueue* downloadQueue, Histor it++; } - HistoryReturn(downloadQueue, itHistory, historyInfo, false); + MoveToQueue(downloadQueue, itHistory, historyInfo, reprocess); +} + +void HistoryCoordinator::ResetArticles(FileInfo* fileInfo, bool resetFailed) +{ + NzbInfo* nzbInfo = fileInfo->GetNzbInfo(); + + for (ArticleInfo* pa : fileInfo->GetArticles()) + { + if ((pa->GetStatus() == ArticleInfo::aiFailed && resetFailed) || + (pa->GetStatus() == ArticleInfo::aiFinished && !fileInfo->GetOutputInitialized())) + { + fileInfo->SetCompletedArticles(fileInfo->GetCompletedArticles() - 1); + fileInfo->SetRemainingSize(fileInfo->GetRemainingSize() + pa->GetSize()); + nzbInfo->SetRemainingSize(nzbInfo->GetRemainingSize() + pa->GetSize()); + + if (pa->GetStatus() == ArticleInfo::aiFailed) + { + fileInfo->SetFailedArticles(fileInfo->GetFailedArticles() - 1); + fileInfo->SetFailedSize(fileInfo->GetFailedSize() - pa->GetSize()); + nzbInfo->SetFailedArticles(nzbInfo->GetFailedArticles() - 1); + nzbInfo->SetCurrentFailedArticles(nzbInfo->GetCurrentFailedArticles() - 1); + nzbInfo->SetFailedSize(nzbInfo->GetFailedSize() - pa->GetSize()); + nzbInfo->SetCurrentFailedSize(nzbInfo->GetCurrentFailedSize() - pa->GetSize()); + if (fileInfo->GetParFile()) + { + nzbInfo->SetParFailedSize(nzbInfo->GetParFailedSize() - pa->GetSize()); + nzbInfo->SetParCurrentFailedSize(nzbInfo->GetParCurrentFailedSize() - pa->GetSize()); + } + } + else if (pa->GetStatus() == ArticleInfo::aiFinished) + { + fileInfo->SetSuccessArticles(fileInfo->GetSuccessArticles() - 1); + fileInfo->SetSuccessSize(fileInfo->GetSuccessSize() - pa->GetSize()); + nzbInfo->SetSuccessArticles(nzbInfo->GetSuccessArticles() - 1); + nzbInfo->SetCurrentSuccessArticles(nzbInfo->GetCurrentSuccessArticles() - 1); + nzbInfo->SetSuccessSize(nzbInfo->GetSuccessSize() - pa->GetSize()); + nzbInfo->SetCurrentSuccessSize(nzbInfo->GetCurrentSuccessSize() - pa->GetSize()); + if (fileInfo->GetParFile()) + { + nzbInfo->SetParSuccessSize(nzbInfo->GetParSuccessSize() - pa->GetSize()); + nzbInfo->SetParCurrentSuccessSize(nzbInfo->GetParCurrentSuccessSize() - pa->GetSize()); + } + } + + pa->SetStatus(ArticleInfo::aiUndefined); + pa->SetCrc(0); + pa->SetSegmentOffset(0); + pa->SetSegmentSize(0); + } + } + + if (fileInfo->GetParFile()) + { + nzbInfo->SetRemainingParCount(nzbInfo->GetRemainingParCount() + 1); + } } bool HistoryCoordinator::HistorySetParameter(HistoryInfo* historyInfo, const char* text) diff --git a/daemon/queue/HistoryCoordinator.h b/daemon/queue/HistoryCoordinator.h index 9f8514d5..e953bb71 100644 --- a/daemon/queue/HistoryCoordinator.h +++ b/daemon/queue/HistoryCoordinator.h @@ -39,14 +39,17 @@ protected: private: void HistoryDelete(DownloadQueue* downloadQueue, HistoryList::iterator itHistory, HistoryInfo* historyInfo, bool final); - void HistoryReturn(DownloadQueue* downloadQueue, HistoryList::iterator itHistory, HistoryInfo* historyInfo, bool reprocess); + void HistoryReturn(DownloadQueue* downloadQueue, HistoryList::iterator itHistory, HistoryInfo* historyInfo); + void HistoryProcess(DownloadQueue* downloadQueue, HistoryList::iterator itHistory, HistoryInfo* historyInfo); void HistoryRedownload(DownloadQueue* downloadQueue, HistoryList::iterator itHistory, HistoryInfo* historyInfo, bool restorePauseState); - void HistoryRetryFailed(DownloadQueue* downloadQueue, HistoryList::iterator itHistory, HistoryInfo* historyInfo); + void HistoryRetry(DownloadQueue* downloadQueue, HistoryList::iterator itHistory, HistoryInfo* historyInfo, bool resetFailed, bool reprocess); bool HistorySetParameter(HistoryInfo* historyInfo, const char* text); void HistorySetDupeParam(HistoryInfo* historyInfo, DownloadQueue::EEditAction action, const char* text); bool HistorySetCategory(HistoryInfo* historyInfo, const char* text); bool HistorySetName(HistoryInfo* historyInfo, const char* text); + void MoveToQueue(DownloadQueue* downloadQueue, HistoryList::iterator itHistory, HistoryInfo* historyInfo, bool reprocess); void PrepareEdit(DownloadQueue* downloadQueue, IdList* idList, DownloadQueue::EEditAction action); + void ResetArticles(FileInfo* fileInfo, bool resetFailed); }; extern HistoryCoordinator* g_HistoryCoordinator; diff --git a/daemon/queue/QueueCoordinator.cpp b/daemon/queue/QueueCoordinator.cpp index 50fbd370..d353c319 100644 --- a/daemon/queue/QueueCoordinator.cpp +++ b/daemon/queue/QueueCoordinator.cpp @@ -146,10 +146,12 @@ void QueueCoordinator::Load() for (CompletedFile& completedFile : nzbInfo->GetCompletedFiles()) { - if (completedFile.GetStatus() != CompletedFile::cfSuccess && completedFile.GetId() > 0) + if ((completedFile.GetStatus() == CompletedFile::cfPartial || + completedFile.GetStatus() == CompletedFile::cfFailure) && + completedFile.GetId() > 0) { FileInfo fileInfo(completedFile.GetId()); - if (g_DiskState->LoadFileState(&fileInfo, g_ServerPool->GetServers(), false)) + if (g_DiskState->LoadFileState(&fileInfo, g_ServerPool->GetServers(), true)) { g_DiskState->SaveFileState(&fileInfo, true); } @@ -615,37 +617,31 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* articleDownloader) { warn("File \"%s\" seems to be duplicate, cancelling download and deleting file from queue", fileInfo->GetFilename()); fileCompleted = false; - fileInfo->SetAutoDeleted(true); + fileInfo->SetDupeDeleted(true); DeleteQueueEntry(downloadQueue, fileInfo); } } nzbInfo->SetDownloadedSize(nzbInfo->GetDownloadedSize() + articleDownloader->GetDownloadedSize()); + + CheckHealth(downloadQueue, fileInfo); } bool deleteFileObj = false; - if (fileCompleted && !fileInfo->GetDeleted()) + if ((fileCompleted && !fileInfo->GetDeleted()) || + (fileInfo->GetNzbInfo()->GetParking() && fileInfo->GetActiveDownloads() == 1)) { // all jobs done articleDownloader->CompleteFileParts(); + fileInfo->SetPartialChanged(false); deleteFileObj = true; } { GuardedDownloadQueue downloadQueue = DownloadQueue::Guard(); - CheckHealth(downloadQueue, fileInfo); - - bool hasOtherDownloaders = false; - for (ArticleDownloader* downloader : m_activeDownloads) - { - if (downloader != articleDownloader && downloader->GetFileInfo() == fileInfo) - { - hasOtherDownloaders = true; - break; - } - } + bool hasOtherDownloaders = fileInfo->GetActiveDownloads() > 1; deleteFileObj |= fileInfo->GetDeleted() && !hasOtherDownloaders; // remove downloader from downloader list @@ -721,6 +717,7 @@ void QueueCoordinator::DeleteFileInfo(DownloadQueue* downloadQueue, FileInfo* fi usleep(5*1000); } + bool parking = fileInfo->GetNzbInfo()->GetParking(); bool fileDeleted = fileInfo->GetDeleted(); fileInfo->SetDeleted(true); @@ -729,23 +726,29 @@ void QueueCoordinator::DeleteFileInfo(DownloadQueue* downloadQueue, FileInfo* fi if (g_Options->GetSaveQueue() && g_Options->GetServerMode() && (!completed || (fileInfo->GetMissedArticles() == 0 && fileInfo->GetFailedArticles() == 0))) { - g_DiskState->DiscardFile(fileInfo, true, true, false); + if (parking && fileInfo->GetForceDirectWrite()) + { + g_DiskState->SaveFileState(fileInfo, true); + } + g_DiskState->DiscardFile(fileInfo, !parking, true, false); } - if (!completed) + if (!completed && !parking) { DiscardDiskFile(fileInfo); } NzbInfo* nzbInfo = fileInfo->GetNzbInfo(); + fileInfo->SetParking(parking); + DownloadQueue::Aspect aspect = { completed && !fileDeleted ? DownloadQueue::eaFileCompleted : DownloadQueue::eaFileDeleted, downloadQueue, nzbInfo, fileInfo }; downloadQueue->Notify(&aspect); // nzb-file could be deleted from queue in "Notify", check if it is still in queue. - if (downloadQueue->GetQueue()->Find(nzbInfo) != downloadQueue->GetQueue()->end()) + if (downloadQueue->GetQueue()->Find(nzbInfo) != downloadQueue->GetQueue()->end() && !nzbInfo->GetParking()) { nzbInfo->GetFileList()->Remove(fileInfo); } diff --git a/daemon/queue/QueueEditor.cpp b/daemon/queue/QueueEditor.cpp index 72d82d75..51024562 100644 --- a/daemon/queue/QueueEditor.cpp +++ b/daemon/queue/QueueEditor.cpp @@ -768,6 +768,7 @@ bool QueueEditor::EditGroup(NzbInfo* nzbInfo, DownloadQueue::EEditAction action, if (action == DownloadQueue::eaGroupDelete || action == DownloadQueue::eaGroupDupeDelete || action == DownloadQueue::eaGroupFinalDelete) { nzbInfo->SetDeleting(true); + nzbInfo->SetParking(action == DownloadQueue::eaGroupDelete && CanPark(nzbInfo)); nzbInfo->SetAvoidHistory(action == DownloadQueue::eaGroupFinalDelete); nzbInfo->SetDeletePaused(allPaused); if (action == DownloadQueue::eaGroupDupeDelete) @@ -1051,6 +1052,15 @@ bool QueueEditor::CanCleanupDisk(NzbInfo* nzbInfo) return false; } +bool QueueEditor::CanPark(NzbInfo* nzbInfo) +{ + bool park = !g_Options->GetDeleteCleanupDisk() && g_Options->GetKeepHistory() > 0 && + !nzbInfo->GetUnpackCleanedUpDisk() && + (nzbInfo->GetSuccessArticles() > 0 || nzbInfo->GetFailedArticles() > 0); + + return park; +} + bool QueueEditor::MergeGroups(ItemList* itemList) { if (itemList->size() == 0) diff --git a/daemon/queue/QueueEditor.h b/daemon/queue/QueueEditor.h index 50513215..01ade339 100644 --- a/daemon/queue/QueueEditor.h +++ b/daemon/queue/QueueEditor.h @@ -56,6 +56,7 @@ private: void SetNzbCategory(NzbInfo* nzbInfo, const char* category, bool applyParams); void SetNzbName(NzbInfo* nzbInfo, const char* name); bool CanCleanupDisk(NzbInfo* nzbInfo); + bool CanPark(NzbInfo* nzbInfo); bool MergeGroups(ItemList* itemList); bool SortGroups(ItemList* itemList, const char* sort); bool SplitGroup(ItemList* itemList, const char* name); diff --git a/nzbget.conf b/nzbget.conf index 51bd7221..6cbfd901 100644 --- a/nzbget.conf +++ b/nzbget.conf @@ -905,6 +905,10 @@ DiskSpace=250 # 1) download of nzb-file is cancelled (deleted from queue); # 2) history record with failure-status (par-failure or unpack-failure) # is deleted from history. +# +# If the option is disabled the files remain on disk, including +# partially downloaded files. For such downloads commands "Download +# remaining files" and "Retry failed articles" are available in history. DeleteCleanupDisk=yes # Delete source nzb-file when it is not needed anymore (yes, no). diff --git a/webui/edit.js b/webui/edit.js index 1afa1eb0..1cb33a05 100644 --- a/webui/edit.js +++ b/webui/edit.js @@ -1740,7 +1740,8 @@ var HistoryEditDialog = (new function() Util.show('#HistoryEdit_Return', hist.RemainingFileCount > 0); Util.show('#HistoryEdit_ReturnURL', hist.Kind === 'URL'); Util.show('#HistoryEdit_Redownload', hist.Kind === 'NZB'); - Util.show('#HistoryEdit_RetryFailed', hist.Kind === 'NZB' && hist.FailedArticles > 0 && hist.ParStatus !== 'SUCCESS' && hist.DeleteStatus === 'NONE'); + Util.show('#HistoryEdit_RetryFailed', hist.Kind === 'NZB' && hist.FailedArticles > 0 && hist.ParStatus !== 'SUCCESS' && + (hist.DeleteStatus === 'NONE' || hist.RemainingFileCount > 0)); Util.show('#HistoryEdit_PathGroup, #HistoryEdit_StatisticsGroup, #HistoryEdit_Reprocess', hist.Kind === 'NZB'); Util.show('#HistoryEdit_CategoryGroup', hist.Kind !== 'DUP'); Util.show('#HistoryEdit_DupGroup', hist.Kind === 'DUP');