diff --git a/BinRpc.cpp b/BinRpc.cpp index 975c4af4..a77025ce 100644 --- a/BinRpc.cpp +++ b/BinRpc.cpp @@ -358,7 +358,7 @@ void DownloadBinCommand::Execute() bool bAddTop = ntohl(DownloadRequest.m_bAddFirst); bool bOK = g_pScanner->AddExternalFile(DownloadRequest.m_szFilename, DownloadRequest.m_szCategory, - iPriority, NULL, 0, false, NULL, bAddTop, bAddPaused, NULL, pRecvBuffer, iBufLen) != Scanner::asFailed; + iPriority, NULL, 0, dmScore, NULL, bAddTop, bAddPaused, NULL, pRecvBuffer, iBufLen) != Scanner::asFailed; char tmp[1024]; snprintf(tmp, 1024, bOK ? "Collection %s added to queue" : "Download Request failed for %s", diff --git a/DiskState.cpp b/DiskState.cpp index 271797d7..4ab700da 100644 --- a/DiskState.cpp +++ b/DiskState.cpp @@ -138,7 +138,7 @@ bool DiskState::SaveDownloadQueue(DownloadQueue* pDownloadQueue) return false; } - fprintf(outfile, "%s%i\n", FORMATVERSION_SIGNATURE, 37); + fprintf(outfile, "%s%i\n", FORMATVERSION_SIGNATURE, 38); // save nzb-infos SaveNZBList(pDownloadQueue, outfile); @@ -192,7 +192,7 @@ bool DiskState::LoadDownloadQueue(DownloadQueue* pDownloadQueue) char FileSignatur[128]; fgets(FileSignatur, sizeof(FileSignatur), infile); int iFormatVersion = ParseFormatVersion(FileSignatur); - if (iFormatVersion < 3 || iFormatVersion > 37) + if (iFormatVersion < 3 || iFormatVersion > 38) { error("Could not load diskstate due to file version mismatch"); fclose(infile); @@ -289,7 +289,7 @@ void DiskState::SaveNZBList(DownloadQueue* pDownloadQueue, FILE* outfile) fprintf(outfile, "%i,%i,%i\n", pNZBInfo->GetTotalArticles(), pNZBInfo->GetSuccessArticles(), pNZBInfo->GetFailedArticles()); fprintf(outfile, "%s\n", pNZBInfo->GetDupeKey()); - fprintf(outfile, "%i,%i,%i\n", (bool)pNZBInfo->GetDupe(), (bool)pNZBInfo->GetNoDupeCheck(), pNZBInfo->GetDupeScore()); + fprintf(outfile, "%i,%i,%i\n", (int)pNZBInfo->GetDupe(), (int)pNZBInfo->GetDupeMode(), pNZBInfo->GetDupeScore()); char DestDirSlash[1024]; snprintf(DestDirSlash, 1023, "%s%c", pNZBInfo->GetDestDir(), PATH_SEPARATOR); @@ -557,10 +557,10 @@ bool DiskState::LoadNZBList(DownloadQueue* pDownloadQueue, FILE* infile, int iFo if (iFormatVersion < 36) ConvertDupeKey(buf, sizeof(buf)); pNZBInfo->SetDupeKey(buf); - int iDupe, iNoDupeCheck, iDupeScore; - if (fscanf(infile, "%i,%i,%i\n", &iDupe, &iNoDupeCheck, &iDupeScore) != 3) goto error; + int iDupe, iDupeMode, iDupeScore; + if (fscanf(infile, "%i,%i,%i\n", &iDupe, &iDupeMode, &iDupeScore) != 3) goto error; pNZBInfo->SetDupe((bool)iDupe); - pNZBInfo->SetNoDupeCheck((bool)iNoDupeCheck); + pNZBInfo->SetDupeMode((EDupeMode)iDupeMode); pNZBInfo->SetDupeScore(iDupeScore); } @@ -1225,7 +1225,7 @@ void DiskState::SaveUrlInfo(UrlInfo* pUrlInfo, FILE* outfile) fprintf(outfile, "%s\n", pUrlInfo->GetNZBFilename()); fprintf(outfile, "%s\n", pUrlInfo->GetCategory()); fprintf(outfile, "%s\n", pUrlInfo->GetDupeKey()); - fprintf(outfile, "%i,%i\n", (int)pUrlInfo->GetNoDupeCheck(), pUrlInfo->GetDupeScore()); + fprintf(outfile, "%i,%i\n", (int)pUrlInfo->GetDupeMode(), pUrlInfo->GetDupeScore()); } bool DiskState::LoadUrlInfo(UrlInfo* pUrlInfo, FILE* infile, int iFormatVersion) @@ -1271,9 +1271,9 @@ bool DiskState::LoadUrlInfo(UrlInfo* pUrlInfo, FILE* infile, int iFormatVersion) if (iFormatVersion < 36) ConvertDupeKey(buf, sizeof(buf)); pUrlInfo->SetDupeKey(buf); - int iNoDupeCheck, iDupeScore; - if (fscanf(infile, "%i,%i\n", &iNoDupeCheck, &iDupeScore) != 2) goto error; - pUrlInfo->SetNoDupeCheck(iNoDupeCheck); + int iDupeMode, iDupeScore; + if (fscanf(infile, "%i,%i\n", &iDupeMode, &iDupeScore) != 2) goto error; + pUrlInfo->SetDupeMode((EDupeMode)iDupeMode); pUrlInfo->SetDupeScore(iDupeScore); } @@ -1287,9 +1287,9 @@ void DiskState::SaveDupInfo(DupInfo* pDupInfo, FILE* outfile) { unsigned long High, Low; Util::SplitInt64(pDupInfo->GetSize(), &High, &Low); - fprintf(outfile, "%i,%lu,%lu,%u,%u,%i,%i\n", (int)pDupInfo->GetStatus(), High, Low, + fprintf(outfile, "%i,%lu,%lu,%u,%u,%i,%i,%i\n", (int)pDupInfo->GetStatus(), High, Low, pDupInfo->GetFullContentHash(), pDupInfo->GetFilteredContentHash(), - pDupInfo->GetDupeScore(), (int)pDupInfo->GetDupe()); + pDupInfo->GetDupeScore(), (int)pDupInfo->GetDupe(), (int)pDupInfo->GetDupeMode()); fprintf(outfile, "%s\n", pDupInfo->GetName()); fprintf(outfile, "%s\n", pDupInfo->GetDupeKey()); } @@ -1301,9 +1301,13 @@ bool DiskState::LoadDupInfo(DupInfo* pDupInfo, FILE* infile, int iFormatVersion) int iStatus; unsigned long High, Low; unsigned int iFullContentHash, iFilteredContentHash = 0; - int iDupeScore, iDupe = 0; + int iDupeScore, iDupe = 0, iDupeMode = 0; - if (iFormatVersion >= 37) + if (iFormatVersion >= 38) + { + if (fscanf(infile, "%i,%lu,%lu,%u,%u,%i,%i,%i\n", &iStatus, &High, &Low, &iFullContentHash, &iFilteredContentHash, &iDupeScore, &iDupe, &iDupeMode) != 8) goto error; + } + else if (iFormatVersion >= 37) { if (fscanf(infile, "%i,%lu,%lu,%u,%u,%i,%i\n", &iStatus, &High, &Low, &iFullContentHash, &iFilteredContentHash, &iDupeScore, &iDupe) != 7) goto error; } @@ -1322,6 +1326,7 @@ bool DiskState::LoadDupInfo(DupInfo* pDupInfo, FILE* infile, int iFormatVersion) pDupInfo->SetSize(Util::JoinInt64(High, Low)); pDupInfo->SetDupeScore(iDupeScore); pDupInfo->SetDupe((bool)iDupe); + pDupInfo->SetDupeMode((EDupeMode)iDupeMode); if (!fgets(buf, sizeof(buf), infile)) goto error; if (buf[0] != 0) buf[strlen(buf)-1] = 0; // remove traling '\n' diff --git a/DownloadInfo.cpp b/DownloadInfo.cpp index bc41e8e7..4681577a 100644 --- a/DownloadInfo.cpp +++ b/DownloadInfo.cpp @@ -306,7 +306,7 @@ NZBInfo::NZBInfo() m_szDupeKey = strdup(""); m_iDupeScore = 0; m_bDupe = false; - m_bNoDupeCheck = false; + m_eDupeMode = dmScore; m_iFullContentHash = 0; m_iFilteredContentHash = 0; m_Owner = NULL; @@ -1052,7 +1052,7 @@ UrlInfo::UrlInfo() m_iPriority = 0; m_iDupeScore = 0; m_szDupeKey = strdup(""); - m_bNoDupeCheck = false; + m_eDupeMode = dmScore; m_bAddTop = false; m_bAddPaused = false; m_bForce = false; @@ -1156,6 +1156,7 @@ DupInfo::DupInfo() m_bDupe = false; m_szDupeKey = NULL; m_iDupeScore = 0; + m_eDupeMode = dmScore; m_lSize = 0; m_iFullContentHash = 0; m_iFilteredContentHash = 0; @@ -1347,7 +1348,7 @@ FeedItemInfo::FeedItemInfo() m_iMatchRule = 0; m_szDupeKey = NULL; m_iDupeScore = 0; - m_bNoDupeCheck = false; + m_eDupeMode = dmScore; } FeedItemInfo::~FeedItemInfo() diff --git a/DownloadInfo.h b/DownloadInfo.h index 669d7da0..37dd21c5 100644 --- a/DownloadInfo.h +++ b/DownloadInfo.h @@ -308,6 +308,13 @@ public: void Clear(); }; +enum EDupeMode +{ + dmScore, + dmAll, + dmForce +}; + class NZBInfoList; class NZBInfo @@ -412,7 +419,7 @@ private: bool m_bUnpackCleanedUpDisk; char* m_szDupeKey; int m_iDupeScore; - bool m_bNoDupeCheck; + EDupeMode m_eDupeMode; bool m_bDupe; unsigned int m_iFullContentHash; unsigned int m_iFilteredContentHash; @@ -517,8 +524,8 @@ public: void SetDupeKey(const char* szDupeKey); // needs locking (for shared objects) int GetDupeScore() { return m_iDupeScore; } void SetDupeScore(int iDupeScore) { m_iDupeScore = iDupeScore; } - bool GetNoDupeCheck() { return m_bNoDupeCheck; } - void SetNoDupeCheck(bool bNoDupeCheck) { m_bNoDupeCheck = bNoDupeCheck; } + EDupeMode GetDupeMode() { return m_eDupeMode; } + void SetDupeMode(EDupeMode eDupeMode) { m_eDupeMode = eDupeMode; } int GetDupe() { return m_bDupe; } void SetDupe(bool bDupe) { m_bDupe = bDupe; } unsigned int GetFullContentHash() { return m_iFullContentHash; } @@ -644,7 +651,7 @@ private: int m_iPriority; char* m_szDupeKey; int m_iDupeScore; - bool m_bNoDupeCheck; + EDupeMode m_eDupeMode; bool m_bAddTop; bool m_bAddPaused; bool m_bForce; @@ -669,8 +676,8 @@ public: void SetDupeKey(const char* szDupeKey); int GetDupeScore() { return m_iDupeScore; } void SetDupeScore(int iDupeScore) { m_iDupeScore = iDupeScore; } - bool GetNoDupeCheck() { return m_bNoDupeCheck; } - void SetNoDupeCheck(bool bNoDupeCheck) { m_bNoDupeCheck = bNoDupeCheck; } + EDupeMode GetDupeMode() { return m_eDupeMode; } + void SetDupeMode(EDupeMode eDupeMode) { m_eDupeMode = eDupeMode; } bool GetAddTop() { return m_bAddTop; } void SetAddTop(bool bAddTop) { m_bAddTop = bAddTop; } bool GetAddPaused() { return m_bAddPaused; } @@ -704,6 +711,7 @@ private: bool m_bDupe; char* m_szDupeKey; int m_iDupeScore; + EDupeMode m_eDupeMode; long long m_lSize; unsigned int m_iFullContentHash; unsigned int m_iFilteredContentHash; @@ -720,6 +728,8 @@ public: void SetDupeKey(const char* szDupeKey); // needs locking (for shared objects) int GetDupeScore() { return m_iDupeScore; } void SetDupeScore(int iDupeScore) { m_iDupeScore = iDupeScore; } + EDupeMode GetDupeMode() { return m_eDupeMode; } + void SetDupeMode(EDupeMode eDupeMode) { m_eDupeMode = eDupeMode; } long long GetSize() { return m_lSize; } void SetSize(long long lSize) { m_lSize = lSize; } unsigned int GetFullContentHash() { return m_iFullContentHash; } @@ -894,7 +904,7 @@ private: int m_iMatchRule; char* m_szDupeKey; int m_iDupeScore; - bool m_bNoDupeCheck; + EDupeMode m_eDupeMode; int ParsePrefixedInt(const char *szValue); @@ -945,8 +955,8 @@ public: void BuildDupeKey(); int GetDupeScore() { return m_iDupeScore; } void SetDupeScore(int iDupeScore) { m_iDupeScore = iDupeScore; } - bool GetNoDupeCheck() { return m_bNoDupeCheck; } - void SetNoDupeCheck(bool bNoDupeCheck) { m_bNoDupeCheck = bNoDupeCheck; } + EDupeMode GetDupeMode() { return m_eDupeMode; } + void SetDupeMode(EDupeMode eDupeMode) { m_eDupeMode = eDupeMode; } }; typedef std::deque FeedItemInfosBase; diff --git a/DupeCoordinator.cpp b/DupeCoordinator.cpp new file mode 100644 index 00000000..81d19436 --- /dev/null +++ b/DupeCoordinator.cpp @@ -0,0 +1,602 @@ +/* + * This file is part of nzbget + * + * Copyright (C) 2007-2013 Andrey Prygunkov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * $Revision$ + * $Date$ + * + */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#ifdef WIN32 +#include "win32.h" +#endif + +#include +#include +#include +#ifdef WIN32 +#include +#else +#include +#endif +#include +#include + +#include "nzbget.h" +#include "Options.h" +#include "Log.h" +#include "Util.h" +#include "DiskState.h" +#include "NZBFile.h" +#include "QueueCoordinator.h" +#include "DupeCoordinator.h" + +extern QueueCoordinator* g_pQueueCoordinator; +extern Options* g_pOptions; +extern DiskState* g_pDiskState; + +bool DupeCoordinator::IsDupeSuccess(NZBInfo* pNZBInfo) +{ + bool bFailure = + pNZBInfo->GetDeleteStatus() != NZBInfo::dsNone || + pNZBInfo->GetMarkStatus() == NZBInfo::ksBad || + pNZBInfo->GetParStatus() == NZBInfo::psFailure || + pNZBInfo->GetUnpackStatus() == NZBInfo::usFailure || + (pNZBInfo->GetParStatus() == NZBInfo::psSkipped && + pNZBInfo->GetUnpackStatus() == NZBInfo::usSkipped && + pNZBInfo->CalcHealth() < pNZBInfo->CalcCriticalHealth()); + return !bFailure; +} + +/** + Check if the title was already downloaded or is already queued. +*/ +void DupeCoordinator::NZBFound(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo) +{ + debug("Checking duplicates for %s", pNZBInfo->GetName()); + + bool bHasDupeKey = !Util::EmptyStr(pNZBInfo->GetDupeKey()); + + // find duplicates in download queue having exactly same content + GroupQueue groupQueue; + pDownloadQueue->BuildGroups(&groupQueue); + + for (GroupQueue::iterator it = groupQueue.begin(); it != groupQueue.end(); it++) + { + GroupInfo* pGroupInfo = *it; + NZBInfo* pGroupNZBInfo = pGroupInfo->GetNZBInfo(); + bool bSameContent = (pNZBInfo->GetFullContentHash() > 0 && + pNZBInfo->GetFullContentHash() == pGroupNZBInfo->GetFullContentHash()) || + (pNZBInfo->GetFilteredContentHash() > 0 && + pNZBInfo->GetFilteredContentHash() == pGroupNZBInfo->GetFilteredContentHash()); + if (pGroupNZBInfo != pNZBInfo && bSameContent) + { + if (!strcmp(pNZBInfo->GetName(), pGroupNZBInfo->GetName())) + { + warn("Skipping duplicate %s, already queued", pNZBInfo->GetName()); + } + else + { + warn("Skipping duplicate %s, already queued as %s", + pNZBInfo->GetName(), pGroupNZBInfo->GetName()); + } + pNZBInfo->SetDeleteStatus(NZBInfo::dsManual); // Flag saying QueueCoordinator to skip nzb-file + DeleteQueuedFile(pNZBInfo->GetQueuedFilename()); + return; + } + } + + // find duplicates in post queue having exactly same content + for (PostQueue::iterator it = pDownloadQueue->GetPostQueue()->begin(); it != pDownloadQueue->GetPostQueue()->end(); it++) + { + PostInfo* pPostInfo = *it; + bool bSameContent = (pNZBInfo->GetFullContentHash() > 0 && + pNZBInfo->GetFullContentHash() == pPostInfo->GetNZBInfo()->GetFullContentHash()) || + (pNZBInfo->GetFilteredContentHash() > 0 && + pNZBInfo->GetFilteredContentHash() == pPostInfo->GetNZBInfo()->GetFilteredContentHash()); + if (bSameContent) + { + if (!strcmp(pNZBInfo->GetName(), pPostInfo->GetNZBInfo()->GetName())) + { + warn("Skipping duplicate %s, already queued", pNZBInfo->GetName()); + } + else + { + warn("Skipping duplicate %s, already queued as %s", + pNZBInfo->GetName(), pPostInfo->GetNZBInfo()->GetName()); + } + pNZBInfo->SetDeleteStatus(NZBInfo::dsManual); // Flag saying QueueCoordinator to skip nzb-file + DeleteQueuedFile(pNZBInfo->GetQueuedFilename()); + return; + } + } + + // find duplicates in history + + bool bSkip = false; + bool bGood = false; + bool bSameContent = false; + const char* szDupeName = NULL; + + // find duplicates in queue having exactly same content + // also: nzb-files having duplicates marked as good are skipped + // also: nzb-files having success-duplicates in dup-history but don't having duplicates in recent history are skipped + for (HistoryList::iterator it = pDownloadQueue->GetHistoryList()->begin(); it != pDownloadQueue->GetHistoryList()->end(); it++) + { + HistoryInfo* pHistoryInfo = *it; + + if (pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo && + ((pNZBInfo->GetFullContentHash() > 0 && + pNZBInfo->GetFullContentHash() == pHistoryInfo->GetNZBInfo()->GetFullContentHash()) || + (pNZBInfo->GetFilteredContentHash() > 0 && + pNZBInfo->GetFilteredContentHash() == pHistoryInfo->GetNZBInfo()->GetFilteredContentHash()))) + { + bSkip = true; + bSameContent = true; + szDupeName = pHistoryInfo->GetNZBInfo()->GetName(); + break; + } + + if (pHistoryInfo->GetKind() == HistoryInfo::hkDupInfo && + ((pNZBInfo->GetFullContentHash() > 0 && + pNZBInfo->GetFullContentHash() == pHistoryInfo->GetDupInfo()->GetFullContentHash()) || + (pNZBInfo->GetFilteredContentHash() > 0 && + pNZBInfo->GetFilteredContentHash() == pHistoryInfo->GetDupInfo()->GetFilteredContentHash()))) + { + bSkip = true; + bSameContent = true; + szDupeName = pHistoryInfo->GetDupInfo()->GetName(); + break; + } + + if (pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo && + pHistoryInfo->GetNZBInfo()->GetDupeMode() != dmForce && + pHistoryInfo->GetNZBInfo()->GetMarkStatus() == NZBInfo::ksGood && + (!strcmp(pHistoryInfo->GetNZBInfo()->GetName(), pNZBInfo->GetName()) || + (bHasDupeKey && !strcmp(pHistoryInfo->GetNZBInfo()->GetDupeKey(), pNZBInfo->GetDupeKey())))) + { + bSkip = true; + bGood = true; + szDupeName = pHistoryInfo->GetNZBInfo()->GetName(); + break; + } + + if (pHistoryInfo->GetKind() == HistoryInfo::hkDupInfo && + pHistoryInfo->GetDupInfo()->GetDupeMode() != dmForce && + (!strcmp(pHistoryInfo->GetDupInfo()->GetName(), pNZBInfo->GetName()) || + (bHasDupeKey && !strcmp(pHistoryInfo->GetDupInfo()->GetDupeKey(), pNZBInfo->GetDupeKey()))) && + (pHistoryInfo->GetDupInfo()->GetStatus() == DupInfo::dsGood || + (pHistoryInfo->GetDupInfo()->GetStatus() == DupInfo::dsSuccess && + pNZBInfo->GetDupeScore() <= pHistoryInfo->GetDupInfo()->GetDupeScore()))) + { + bSkip = true; + bGood = pHistoryInfo->GetDupInfo()->GetStatus() == DupInfo::dsGood; + szDupeName = pHistoryInfo->GetDupInfo()->GetName(); + break; + } + } + + if (!bSameContent && !bGood && pNZBInfo->GetDupeMode() == dmScore) + { + // nzb-files having success-duplicates in recent history (with different content) are added to history for backup + for (HistoryList::iterator it = pDownloadQueue->GetHistoryList()->begin(); it != pDownloadQueue->GetHistoryList()->end(); it++) + { + HistoryInfo* pHistoryInfo = *it; + if (pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo && + (!strcmp(pHistoryInfo->GetNZBInfo()->GetName(), pNZBInfo->GetName()) || + (bHasDupeKey && !strcmp(pHistoryInfo->GetNZBInfo()->GetDupeKey(), pNZBInfo->GetDupeKey()))) && + pNZBInfo->GetDupeScore() <= pHistoryInfo->GetNZBInfo()->GetDupeScore() && + IsDupeSuccess(pHistoryInfo->GetNZBInfo())) + { + // Flag saying QueueCoordinator to skip nzb-file, we also use this flag later in "PrePostProcessor::NZBAdded" + pNZBInfo->SetDeleteStatus(NZBInfo::dsDupe); + MarkDupe(pNZBInfo, pHistoryInfo->GetNZBInfo()); + return; + } + } + } + + if (bSkip) + { + if (!strcmp(pNZBInfo->GetName(), szDupeName)) + { + warn("Skipping duplicate %s, found in history with %s", pNZBInfo->GetName(), + bSameContent ? "exactly same content" : bGood ? "good status" : "success status"); + } + else + { + warn("Skipping duplicate %s, found in history %s with %s", + pNZBInfo->GetName(), szDupeName, + bSameContent ? "exactly same content" : bGood ? "good status" : "success status"); + } + + // Flag saying QueueCoordinator to skip nzb-file + pNZBInfo->SetDeleteStatus(NZBInfo::dsManual); + DeleteQueuedFile(pNZBInfo->GetQueuedFilename()); + return; + } +} + +/** + - If download queue or post-queue contain a duplicate the existing item + and the newly added item are marked as duplicates to each other. + The newly added item is paused (if dupemode=score). +*/ +void DupeCoordinator::NZBAdded(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo) +{ + debug("Checking duplicates for %s", pNZBInfo->GetName()); + + bool bHasDupeKey = !Util::EmptyStr(pNZBInfo->GetDupeKey()); + bool bHigherScore = true; + NZBInfo* pDupeNZBInfo = NULL; + + // find all duplicates in post queue + std::set postDupes; + + for (PostQueue::iterator it = pDownloadQueue->GetPostQueue()->begin(); it != pDownloadQueue->GetPostQueue()->end(); it++) + { + PostInfo* pPostInfo = *it; + if (pPostInfo->GetNZBInfo()->GetDupeMode() != dmForce && + (!strcmp(pPostInfo->GetNZBInfo()->GetName(), pNZBInfo->GetName()) || + (bHasDupeKey && !strcmp(pPostInfo->GetNZBInfo()->GetDupeKey(), pNZBInfo->GetDupeKey())))) + { + postDupes.insert(pPostInfo->GetNZBInfo()); + if (!pDupeNZBInfo) + { + pDupeNZBInfo = pPostInfo->GetNZBInfo(); + } + bHigherScore = bHigherScore && pPostInfo->GetNZBInfo()->GetDupeScore() < pNZBInfo->GetDupeScore(); + } + } + + // find all duplicates in download queue + GroupQueue groupQueue; + pDownloadQueue->BuildGroups(&groupQueue); + std::list queueDupes; + GroupInfo* pNewGroupInfo = NULL; + + for (GroupQueue::iterator it = groupQueue.begin(); it != groupQueue.end(); it++) + { + GroupInfo* pGroupInfo = *it; + NZBInfo* pGroupNZBInfo = pGroupInfo->GetNZBInfo(); + if (pGroupNZBInfo != pNZBInfo && + pGroupNZBInfo->GetDupeMode() != dmForce && + (!strcmp(pGroupNZBInfo->GetName(), pNZBInfo->GetName()) || + (bHasDupeKey && !strcmp(pGroupNZBInfo->GetDupeKey(), pNZBInfo->GetDupeKey())))) + { + queueDupes.push_back(pGroupInfo); + if (!pDupeNZBInfo) + { + pDupeNZBInfo = pGroupNZBInfo; + } + bHigherScore = bHigherScore && pGroupNZBInfo->GetDupeScore() < pNZBInfo->GetDupeScore(); + } + if (pGroupNZBInfo == pNZBInfo) + { + pNewGroupInfo = pGroupInfo; + } + } + + if (pDupeNZBInfo) + { + MarkDupe(pNZBInfo, pDupeNZBInfo); + + // pause all duplicates with lower DupeScore, which are not in post-processing (only for dupemode=score) + for (std::list::iterator it = queueDupes.begin(); it != queueDupes.end(); it++) + { + GroupInfo* pGroupInfo = *it; + NZBInfo* pDupeNZB = pGroupInfo->GetNZBInfo(); + if (pDupeNZB->GetDupeMode() == dmScore && + pDupeNZB->GetDupeScore() < pNZBInfo->GetDupeScore() && + postDupes.find(pDupeNZB) == postDupes.end() && + pGroupInfo->GetPausedFileCount() < pGroupInfo->GetRemainingFileCount()) + { + info("Pausing collection %s with lower duplicate score", pDupeNZB->GetName()); + g_pQueueCoordinator->GetQueueEditor()->LockedEditEntry(pDownloadQueue, pGroupInfo->GetLastID(), false, QueueEditor::eaGroupPause, 0, NULL); + } + } + + if (!bHigherScore && pNZBInfo->GetDupeMode() == dmScore) + { + g_pQueueCoordinator->GetQueueEditor()->LockedEditEntry(pDownloadQueue, pNewGroupInfo->GetLastID(), false, QueueEditor::eaGroupPause, 0, NULL); + } + } +} + +void DupeCoordinator::MarkDupe(NZBInfo* pNZBInfo, NZBInfo* pDupeNZBInfo) +{ + info("Marking collection %s as duplicate to %s", pNZBInfo->GetName(), pDupeNZBInfo->GetName()); + + pNZBInfo->SetDupe(true); + pDupeNZBInfo->SetDupe(true); + + if (Util::EmptyStr(pNZBInfo->GetDupeKey()) && !Util::EmptyStr(pDupeNZBInfo->GetDupeKey())) + { + pNZBInfo->SetDupeKey(pDupeNZBInfo->GetDupeKey()); + } +} + +/** + - if download of an item completes successfully and there are + (paused) duplicates to this item in the queue, they all are deleted + from queue; + - if download of an item fails and there are (paused) duplicates to + this item in the queue iten with highest DupeScore is unpaused; +*/ +void DupeCoordinator::NZBCompleted(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo) +{ + debug("Processing duplicates for %s", pNZBInfo->GetName()); + + if (pNZBInfo->GetDupeMode() == dmScore && pNZBInfo->GetDupe()) + { + if (IsDupeSuccess(pNZBInfo)) + { + RemoveDupes(pDownloadQueue, pNZBInfo); + } + else + { + UnpauseBestDupe(pDownloadQueue, pNZBInfo, pNZBInfo->GetName(), pNZBInfo->GetDupeKey()); + } + } +} + +void DupeCoordinator::RemoveDupes(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo) +{ + bool bHasDupeKey = !Util::EmptyStr(pNZBInfo->GetDupeKey()); + IDList groupIDList; + std::set groupNZBs; + + for (FileQueue::iterator it = pDownloadQueue->GetFileQueue()->begin(); it != pDownloadQueue->GetFileQueue()->end(); it++) + { + FileInfo* pFileInfo = *it; + if (pFileInfo->GetNZBInfo() != pNZBInfo && + pFileInfo->GetNZBInfo()->GetDupeMode() == dmScore && + pFileInfo->GetNZBInfo()->GetDupe() && + groupNZBs.find(pFileInfo->GetNZBInfo()) == groupNZBs.end() && + (!strcmp(pFileInfo->GetNZBInfo()->GetName(), pNZBInfo->GetName()) || + (bHasDupeKey && !strcmp(pFileInfo->GetNZBInfo()->GetDupeKey(), pNZBInfo->GetDupeKey())))) + { + groupIDList.push_back(pFileInfo->GetID()); + groupNZBs.insert(pFileInfo->GetNZBInfo()); + pFileInfo->GetNZBInfo()->SetDeleteStatus(NZBInfo::dsDupe); + } + } + + if (!groupIDList.empty()) + { + info("Removing duplicates for %s from queue", pNZBInfo->GetName()); + g_pQueueCoordinator->GetQueueEditor()->LockedEditList(pDownloadQueue, &groupIDList, false, QueueEditor::eaGroupDelete, 0, NULL); + } +} + +void DupeCoordinator::UnpauseBestDupe(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo, const char* szNZBName, const char* szDupeKey) +{ + bool bHasDupeKey = !Util::EmptyStr(szDupeKey); + std::set groupNZBs; + FileInfo* pDupeFileInfo = NULL; + + for (FileQueue::iterator it = pDownloadQueue->GetFileQueue()->begin(); it != pDownloadQueue->GetFileQueue()->end(); it++) + { + FileInfo* pFileInfo = *it; + if (pFileInfo->GetNZBInfo() != pNZBInfo && + pFileInfo->GetNZBInfo()->GetDupe() && + groupNZBs.find(pFileInfo->GetNZBInfo()) == groupNZBs.end() && + (!strcmp(pFileInfo->GetNZBInfo()->GetName(), szNZBName) || + (bHasDupeKey && !strcmp(pFileInfo->GetNZBInfo()->GetDupeKey(), szDupeKey)))) + { + // find nzb with highest DupeScore + if (!pDupeFileInfo || pFileInfo->GetNZBInfo()->GetDupeScore() > pDupeFileInfo->GetNZBInfo()->GetDupeScore()) + { + pDupeFileInfo = pFileInfo; + } + groupNZBs.insert(pFileInfo->GetNZBInfo()); + } + } + + if (pDupeFileInfo) + { + info("Unpausing duplicate %s", pDupeFileInfo->GetNZBInfo()->GetName()); + g_pQueueCoordinator->GetQueueEditor()->LockedEditEntry(pDownloadQueue, pDupeFileInfo->GetID(), false, QueueEditor::eaGroupResume, 0, NULL); + g_pQueueCoordinator->GetQueueEditor()->LockedEditEntry(pDownloadQueue, pDupeFileInfo->GetID(), false, QueueEditor::eaGroupPauseExtraPars, 0, NULL); + } +} + +void DupeCoordinator::HistoryMark(DownloadQueue* pDownloadQueue, HistoryInfo* pHistoryInfo, bool bGood) +{ + char szNZBName[1024]; + pHistoryInfo->GetName(szNZBName, 1024); + + info("Marking %s as %s", szNZBName, (bGood ? "good" : "bad")); + + if (pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo) + { + pHistoryInfo->GetNZBInfo()->SetMarkStatus(bGood ? NZBInfo::ksGood : NZBInfo::ksBad); + } + else if (pHistoryInfo->GetKind() == HistoryInfo::hkDupInfo) + { + pHistoryInfo->GetDupInfo()->SetStatus(bGood ? DupInfo::dsGood : DupInfo::dsBad); + } + else + { + error("Could not mark %s as bad: history item has wrong type", szNZBName); + return; + } + + if (!g_pOptions->GetDupeCheck() || + (pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo && + pHistoryInfo->GetNZBInfo()->GetDupeMode() == dmForce) || + (pHistoryInfo->GetKind() == HistoryInfo::hkDupInfo && + pHistoryInfo->GetDupInfo()->GetDupeMode() == dmForce)) + { + return; + } + + if (bGood) + { + // mark as good + // moving all duplicates from history to dup-history + HistoryCleanup(pDownloadQueue, pHistoryInfo); + return; + } + + // mark as bad continues + + const char* szDupeKey = pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo ? pHistoryInfo->GetNZBInfo()->GetDupeKey() : + pHistoryInfo->GetKind() == HistoryInfo::hkDupInfo ? pHistoryInfo->GetDupInfo()->GetDupeKey() : + NULL; + bool bHasDupeKey = !Util::EmptyStr(szDupeKey); + + // move existing dupe-backups from history to download queue + HistoryList dupeList; + for (HistoryList::iterator it = pDownloadQueue->GetHistoryList()->begin(); it != pDownloadQueue->GetHistoryList()->end(); it++) + { + HistoryInfo* pDupeHistoryInfo = *it; + if (pDupeHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo && + pDupeHistoryInfo->GetNZBInfo()->GetDupe() && + pDupeHistoryInfo->GetNZBInfo()->GetDeleteStatus() == NZBInfo::dsDupe && + (!strcmp(pDupeHistoryInfo->GetNZBInfo()->GetName(), szNZBName) || + (bHasDupeKey && !strcmp(pDupeHistoryInfo->GetNZBInfo()->GetDupeKey(), szDupeKey)))) + { + dupeList.push_back(pDupeHistoryInfo); + } + } + + for (HistoryList::iterator it = dupeList.begin(); it != dupeList.end(); it++) + { + HistoryInfo* pDupeHistoryInfo = *it; + HistoryReturnDupe(pDownloadQueue, pDupeHistoryInfo); + } + + UnpauseBestDupe(pDownloadQueue, NULL, szNZBName, szDupeKey); +} + +void DupeCoordinator::HistoryReturnDupe(DownloadQueue* pDownloadQueue, HistoryInfo* pHistoryInfo) +{ + NZBInfo* pNZBInfo = pHistoryInfo->GetNZBInfo(); + + if (!Util::FileExists(pNZBInfo->GetQueuedFilename())) + { + error("Could not return duplicate %s from history back to queue: could not find source nzb-file %s", + pNZBInfo->GetName(), pNZBInfo->GetQueuedFilename()); + return; + } + + NZBFile* pNZBFile = NZBFile::Create(pNZBInfo->GetQueuedFilename(), ""); + if (pNZBFile == NULL) + { + error("Could not return duplicate %s from history back to queue: could not parse nzb-file", + pNZBInfo->GetName()); + return; + } + + info("Returning duplicate %s from history back to queue", pNZBInfo->GetName()); + + for (NZBFile::FileInfos::iterator it = pNZBFile->GetFileInfos()->begin(); it != pNZBFile->GetFileInfos()->end(); it++) + { + FileInfo* pFileInfo = *it; + pFileInfo->SetNZBInfo(pNZBInfo); + pFileInfo->SetPaused(true); + } + + g_pQueueCoordinator->AddFileInfosToFileQueue(pNZBFile, pDownloadQueue->GetParkedFiles(), false); + + delete pNZBFile; + + HistoryList::iterator it = std::find(pDownloadQueue->GetHistoryList()->begin(), pDownloadQueue->GetHistoryList()->end(), pHistoryInfo); + HistoryReturn(pDownloadQueue, it, pHistoryInfo, false); +} + +void DupeCoordinator::HistoryCleanup(DownloadQueue* pDownloadQueue, HistoryInfo* pMarkHistoryInfo) +{ + const char* szDupeKey = pMarkHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo ? pMarkHistoryInfo->GetNZBInfo()->GetDupeKey() : + pMarkHistoryInfo->GetKind() == HistoryInfo::hkDupInfo ? pMarkHistoryInfo->GetDupInfo()->GetDupeKey() : + NULL; + const char* szNZBName = pMarkHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo ? pMarkHistoryInfo->GetNZBInfo()->GetName() : + pMarkHistoryInfo->GetKind() == HistoryInfo::hkDupInfo ? pMarkHistoryInfo->GetDupInfo()->GetName() : + NULL; + bool bHasDupeKey = !Util::EmptyStr(szDupeKey); + bool bChanged = false; + int index = 0; + + // traversing in a reverse order to delete items in order they were added to history + // (just to produce the log-messages in a more logical order) + for (HistoryList::reverse_iterator it = pDownloadQueue->GetHistoryList()->rbegin(); it != pDownloadQueue->GetHistoryList()->rend(); ) + { + HistoryInfo* pHistoryInfo = *it; + + if (pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo && + pHistoryInfo != pMarkHistoryInfo && pHistoryInfo->GetNZBInfo()->GetDupe() && + (!strcmp(pHistoryInfo->GetNZBInfo()->GetName(), szNZBName) || + (bHasDupeKey && !strcmp(pHistoryInfo->GetNZBInfo()->GetDupeKey(), szDupeKey)))) + { + HistoryTransformToDup(pDownloadQueue, pHistoryInfo, index); + index++; + it = pDownloadQueue->GetHistoryList()->rbegin() + index; + bChanged = true; + } + else + { + it++; + index++; + } + } + + if (bChanged && g_pOptions->GetSaveQueue() && g_pOptions->GetServerMode()) + { + g_pDiskState->SaveDownloadQueue(pDownloadQueue); + } +} + +void DupeCoordinator::HistoryTransformToDup(DownloadQueue* pDownloadQueue, HistoryInfo* pHistoryInfo, int rindex) +{ + char szNiceName[1024]; + pHistoryInfo->GetName(szNiceName, 1024); + + // replace history element + DupInfo* pDupInfo = new DupInfo(); + pDupInfo->SetName(pHistoryInfo->GetNZBInfo()->GetName()); + pDupInfo->SetDupe(pHistoryInfo->GetNZBInfo()->GetDupe()); + pDupInfo->SetDupeKey(pHistoryInfo->GetNZBInfo()->GetDupeKey()); + pDupInfo->SetDupeScore(pHistoryInfo->GetNZBInfo()->GetDupeScore()); + pDupInfo->SetDupeMode(pHistoryInfo->GetNZBInfo()->GetDupeMode()); + pDupInfo->SetSize(pHistoryInfo->GetNZBInfo()->GetSize()); + pDupInfo->SetFullContentHash(pHistoryInfo->GetNZBInfo()->GetFullContentHash()); + pDupInfo->SetFilteredContentHash(pHistoryInfo->GetNZBInfo()->GetFilteredContentHash()); + + pDupInfo->SetStatus( + pHistoryInfo->GetNZBInfo()->GetMarkStatus() == NZBInfo::ksGood ? DupInfo::dsGood : + pHistoryInfo->GetNZBInfo()->GetMarkStatus() == NZBInfo::ksBad ? DupInfo::dsBad : + pHistoryInfo->GetNZBInfo()->GetDeleteStatus() == NZBInfo::dsDupe ? DupInfo::dsDupe : + pHistoryInfo->GetNZBInfo()->GetDeleteStatus() == NZBInfo::dsManual ? DupInfo::dsDeleted : + IsDupeSuccess(pHistoryInfo->GetNZBInfo()) ? DupInfo::dsSuccess : + DupInfo::dsFailed); + + HistoryInfo* pNewHistoryInfo = new HistoryInfo(pDupInfo); + pNewHistoryInfo->SetTime(pHistoryInfo->GetTime()); + (*pDownloadQueue->GetHistoryList())[pDownloadQueue->GetHistoryList()->size() - 1 - rindex] = pNewHistoryInfo; + + DeleteQueuedFile(pHistoryInfo->GetNZBInfo()->GetQueuedFilename()); + + delete pHistoryInfo; + info("Collection %s removed from recent history", szNiceName); +} diff --git a/DupeCoordinator.h b/DupeCoordinator.h new file mode 100644 index 00000000..9c08ffad --- /dev/null +++ b/DupeCoordinator.h @@ -0,0 +1,55 @@ +/* + * This file is part of nzbget + * + * Copyright (C) 2007-2013 Andrey Prygunkov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * $Revision$ + * $Date$ + * + */ + + +#ifndef DUPECOORDINATOR_H +#define DUPECOORDINATOR_H + +#include + +#include "DownloadInfo.h" + +class DupeCoordinator +{ +private: + bool IsDupeSuccess(NZBInfo* pNZBInfo); + void UnpauseBestDupe(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo, const char* szNZBName, const char* szDupeKey); + void RemoveDupes(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo); + void HistoryReturnDupe(DownloadQueue* pDownloadQueue, HistoryInfo* pHistoryInfo); + void HistoryCleanup(DownloadQueue* pDownloadQueue, HistoryInfo* pMarkHistoryInfo); + void MarkDupe(NZBInfo* pNZBInfo, NZBInfo* pDupeNZBInfo); + +protected: + virtual void HistoryReturn(DownloadQueue* pDownloadQueue, HistoryList::iterator itHistory, HistoryInfo* pHistoryInfo, bool bReprocess) = 0; + virtual void DeleteQueuedFile(const char* szQueuedFile) = 0; + +public: + void NZBCompleted(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo); + void NZBFound(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo); + void NZBAdded(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo); + void HistoryMark(DownloadQueue* pDownloadQueue, HistoryInfo* pHistoryInfo, bool bGood); + void HistoryTransformToDup(DownloadQueue* pDownloadQueue, HistoryInfo* pHistoryInfo, int rindex); +}; + +#endif diff --git a/FeedCoordinator.cpp b/FeedCoordinator.cpp index 10257efe..56212607 100644 --- a/FeedCoordinator.cpp +++ b/FeedCoordinator.cpp @@ -406,7 +406,7 @@ void FeedCoordinator::FilterFeed(FeedInfo* pFeedInfo, FeedItemInfos* pFeedItemIn pFeedItemInfo->SetPriority(pFeedInfo->GetPriority()); pFeedItemInfo->SetAddCategory(pFeedInfo->GetCategory()); pFeedItemInfo->SetDupeScore(0); - pFeedItemInfo->SetNoDupeCheck(false); + pFeedItemInfo->SetDupeMode(dmScore); pFeedItemInfo->BuildDupeKey(); if (pFeedFilter) { @@ -494,7 +494,7 @@ void FeedCoordinator::DownloadItem(FeedInfo* pFeedInfo, FeedItemInfo* pFeedItemI pUrlInfo->SetAddPaused(pFeedItemInfo->GetPauseNzb()); pUrlInfo->SetDupeKey(pFeedItemInfo->GetDupeKey()); pUrlInfo->SetDupeScore(pFeedItemInfo->GetDupeScore()); - pUrlInfo->SetNoDupeCheck(pFeedItemInfo->GetNoDupeCheck()); + pUrlInfo->SetDupeMode(pFeedItemInfo->GetDupeMode()); pUrlInfo->SetForce(pFeedInfo->GetForce()); g_pUrlCoordinator->AddUrlToQueue(pUrlInfo, false); } diff --git a/FeedFilter.cpp b/FeedFilter.cpp index 83e45ba3..17d210b0 100644 --- a/FeedFilter.cpp +++ b/FeedFilter.cpp @@ -551,7 +551,7 @@ FeedFilter::Rule::Rule() m_szAddDupeKey = NULL; m_iDupeScore = 0; m_iAddDupeScore = 0; - m_bNoDupeCheck = false; + m_eDupeMode = dmScore; m_bHasCategory = false; m_bHasPriority = false; m_bHasAddPriority = false; @@ -560,7 +560,7 @@ FeedFilter::Rule::Rule() m_bHasAddDupeScore = false; m_bHasDupeKey = false; m_bHasAddDupeKey = false; - m_bHasNoDupeCheck = false; + m_bHasDupeMode = false; m_bPatCategory = false; m_bPatDupeKey = false; m_bPatAddDupeKey = false; @@ -723,7 +723,7 @@ char* FeedFilter::Rule::CompileOptions(char* szRule) if (*szToken) { char* szOption = szToken; - char* szValue = NULL; + const char* szValue = ""; char* szColon = strchr(szToken, ':'); if (szColon) { @@ -744,7 +744,7 @@ char* FeedFilter::Rule::CompileOptions(char* szRule) else if (!strcasecmp(szOption, "pause") || !strcasecmp(szOption, "p")) { m_bHasPause = true; - m_bPause = !szValue || !strcasecmp(szValue, "yes") || !strcasecmp(szValue, "y"); + m_bPause = !*szValue || !strcasecmp(szValue, "yes") || !strcasecmp(szValue, "y"); if (!m_bPause && !(!strcasecmp(szValue, "no") || !strcasecmp(szValue, "n"))) { // error @@ -811,11 +811,22 @@ char* FeedFilter::Rule::CompileOptions(char* szRule) m_szAddDupeKey = strdup(szValue); m_bPatAddDupeKey = strstr(szValue, "${"); } - else if (!strcasecmp(szOption, "nodupe") || !strcasecmp(szOption, "nd") || !strcasecmp(szOption, "n")) + else if (!strcasecmp(szOption, "dupemode") || !strcasecmp(szOption, "dm") || !strcasecmp(szOption, "m")) { - m_bHasNoDupeCheck = true; - m_bNoDupeCheck = !szValue || !strcasecmp(szValue, "yes") || !strcasecmp(szValue, "y"); - if (!m_bNoDupeCheck && !(!strcasecmp(szValue, "no") || !strcasecmp(szValue, "n"))) + m_bHasDupeMode = true; + if (!strcasecmp(szValue, "score") || !strcasecmp(szValue, "s")) + { + m_eDupeMode = dmScore; + } + else if (!strcasecmp(szValue, "all") || !strcasecmp(szValue, "a")) + { + m_eDupeMode = dmAll; + } + else if (!strcasecmp(szValue, "force") || !strcasecmp(szValue, "f")) + { + m_eDupeMode = dmForce; + } + else { // error return NULL; @@ -1085,8 +1096,8 @@ void FeedFilter::ApplyOptions(Rule* pRule, FeedItemInfo* pFeedItemInfo) { pFeedItemInfo->AppendDupeKey(pRule->GetAddDupeKey()); } - if (pRule->HasNoDupeCheck()) + if (pRule->HasDupeMode()) { - pFeedItemInfo->SetNoDupeCheck(pRule->GetNoDupeCheck()); + pFeedItemInfo->SetDupeMode(pRule->GetDupeMode()); } } diff --git a/FeedFilter.h b/FeedFilter.h index d209dfc4..11354744 100644 --- a/FeedFilter.h +++ b/FeedFilter.h @@ -106,7 +106,7 @@ private: int m_iAddDupeScore; char* m_szDupeKey; char* m_szAddDupeKey; - bool m_bNoDupeCheck; + EDupeMode m_eDupeMode; bool m_bHasCategory; bool m_bHasPriority; bool m_bHasAddPriority; @@ -115,7 +115,7 @@ private: bool m_bHasAddDupeScore; bool m_bHasDupeKey; bool m_bHasAddDupeKey; - bool m_bHasNoDupeCheck; + bool m_bHasDupeMode; bool m_bPatCategory; bool m_bPatDupeKey; bool m_bPatAddDupeKey; @@ -143,7 +143,7 @@ private: const char* GetAddDupeKey() { return m_szAddDupeKey; } int GetDupeScore() { return m_iDupeScore; } int GetAddDupeScore() { return m_iAddDupeScore; } - bool GetNoDupeCheck() { return m_bNoDupeCheck; } + EDupeMode GetDupeMode() { return m_eDupeMode; } bool HasCategory() { return m_bHasCategory; } bool HasPriority() { return m_bHasPriority; } bool HasAddPriority() { return m_bHasAddPriority; } @@ -152,7 +152,7 @@ private: bool HasAddDupeScore() { return m_bHasAddDupeScore; } bool HasDupeKey() { return m_bHasDupeKey; } bool HasAddDupeKey() { return m_bHasAddDupeKey; } - bool HasNoDupeCheck() { return m_bHasNoDupeCheck; } + bool HasDupeMode() { return m_bHasDupeMode; } bool Match(FeedItemInfo* pFeedItemInfo); RefValues* GetRefValues() { return &m_RefValues; } void ExpandRefValues(char** pDestStr, char* pPatStr); diff --git a/Makefile.am b/Makefile.am index e2a79d28..ccf302c0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -24,9 +24,9 @@ bin_PROGRAMS = nzbget nzbget_SOURCES = \ ArticleDownloader.cpp ArticleDownloader.h BinRpc.cpp BinRpc.h \ ColoredFrontend.cpp ColoredFrontend.h Connection.cpp Connection.h Decoder.cpp Decoder.h \ - DiskState.cpp DiskState.h DownloadInfo.cpp DownloadInfo.h Frontend.cpp Frontend.h \ - FeedCoordinator.cpp FeedCoordinator.h FeedFile.cpp FeedFile.h FeedFilter.cpp FeedFilter.h \ - Log.cpp Log.h LoggableFrontend.cpp LoggableFrontend.h MessageBase.h \ + DiskState.cpp DiskState.h DownloadInfo.cpp DownloadInfo.h DupeCoordinator.cpp DupeCoordinator.h \ + Frontend.cpp Frontend.h FeedCoordinator.cpp FeedCoordinator.h FeedFile.cpp FeedFile.h \ + FeedFilter.cpp FeedFilter.h Log.cpp Log.h LoggableFrontend.cpp LoggableFrontend.h MessageBase.h \ NCursesFrontend.cpp NCursesFrontend.h NNTPConnection.cpp NNTPConnection.h NZBFile.cpp \ NZBFile.h NewsServer.cpp NewsServer.h Observer.cpp \ Observer.h Options.cpp Options.h ParChecker.cpp ParChecker.h ParRenamer.cpp ParRenamer.h \ diff --git a/Makefile.in b/Makefile.in index 6dfb658e..9d59e76e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -85,12 +85,12 @@ PROGRAMS = $(bin_PROGRAMS) am_nzbget_OBJECTS = ArticleDownloader.$(OBJEXT) BinRpc.$(OBJEXT) \ ColoredFrontend.$(OBJEXT) Connection.$(OBJEXT) \ Decoder.$(OBJEXT) DiskState.$(OBJEXT) DownloadInfo.$(OBJEXT) \ - Frontend.$(OBJEXT) FeedCoordinator.$(OBJEXT) \ - FeedFile.$(OBJEXT) FeedFilter.$(OBJEXT) Log.$(OBJEXT) \ - LoggableFrontend.$(OBJEXT) NCursesFrontend.$(OBJEXT) \ - NNTPConnection.$(OBJEXT) NZBFile.$(OBJEXT) \ - NewsServer.$(OBJEXT) Observer.$(OBJEXT) Options.$(OBJEXT) \ - ParChecker.$(OBJEXT) ParRenamer.$(OBJEXT) \ + DupeCoordinator.$(OBJEXT) Frontend.$(OBJEXT) \ + FeedCoordinator.$(OBJEXT) FeedFile.$(OBJEXT) \ + FeedFilter.$(OBJEXT) Log.$(OBJEXT) LoggableFrontend.$(OBJEXT) \ + NCursesFrontend.$(OBJEXT) NNTPConnection.$(OBJEXT) \ + NZBFile.$(OBJEXT) NewsServer.$(OBJEXT) Observer.$(OBJEXT) \ + Options.$(OBJEXT) ParChecker.$(OBJEXT) ParRenamer.$(OBJEXT) \ ParCoordinator.$(OBJEXT) PrePostProcessor.$(OBJEXT) \ QueueCoordinator.$(OBJEXT) QueueEditor.$(OBJEXT) \ RemoteClient.$(OBJEXT) RemoteServer.$(OBJEXT) \ @@ -244,9 +244,9 @@ target_vendor = @target_vendor@ nzbget_SOURCES = \ ArticleDownloader.cpp ArticleDownloader.h BinRpc.cpp BinRpc.h \ ColoredFrontend.cpp ColoredFrontend.h Connection.cpp Connection.h Decoder.cpp Decoder.h \ - DiskState.cpp DiskState.h DownloadInfo.cpp DownloadInfo.h Frontend.cpp Frontend.h \ - FeedCoordinator.cpp FeedCoordinator.h FeedFile.cpp FeedFile.h FeedFilter.cpp FeedFilter.h \ - Log.cpp Log.h LoggableFrontend.cpp LoggableFrontend.h MessageBase.h \ + DiskState.cpp DiskState.h DownloadInfo.cpp DownloadInfo.h DupeCoordinator.cpp DupeCoordinator.h \ + Frontend.cpp Frontend.h FeedCoordinator.cpp FeedCoordinator.h FeedFile.cpp FeedFile.h \ + FeedFilter.cpp FeedFilter.h Log.cpp Log.h LoggableFrontend.cpp LoggableFrontend.h MessageBase.h \ NCursesFrontend.cpp NCursesFrontend.h NNTPConnection.cpp NNTPConnection.h NZBFile.cpp \ NZBFile.h NewsServer.cpp NewsServer.h Observer.cpp \ Observer.h Options.cpp Options.h ParChecker.cpp ParChecker.h ParRenamer.cpp ParRenamer.h \ @@ -458,6 +458,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Decoder.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DiskState.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadInfo.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DupeCoordinator.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeedCoordinator.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeedFile.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeedFilter.Po@am__quote@ diff --git a/PrePostProcessor.cpp b/PrePostProcessor.cpp index be72e68c..c7c091e4 100644 --- a/PrePostProcessor.cpp +++ b/PrePostProcessor.cpp @@ -70,10 +70,10 @@ PrePostProcessor::PrePostProcessor() m_QueueCoordinatorObserver.m_pOwner = this; g_pQueueCoordinator->Attach(&m_QueueCoordinatorObserver); - #ifndef DISABLE_PARCHECK m_ParCoordinator.m_pOwner = this; #endif + m_DupeCoordinator.m_pOwner = this; } PrePostProcessor::~PrePostProcessor() @@ -247,9 +247,9 @@ void PrePostProcessor::QueueCoordinatorUpdate(Subject * Caller, void * Aspect) void PrePostProcessor::NZBFound(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo) { - if (g_pOptions->GetDupeCheck() && !pNZBInfo->GetNoDupeCheck()) + if (g_pOptions->GetDupeCheck() && pNZBInfo->GetDupeMode() != dmForce) { - CheckDupeFound(pDownloadQueue, pNZBInfo); + m_DupeCoordinator.NZBFound(pDownloadQueue, pNZBInfo); } } @@ -260,9 +260,16 @@ void PrePostProcessor::NZBAdded(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo m_ParCoordinator.PausePars(pDownloadQueue, pNZBInfo); } - if (g_pOptions->GetDupeCheck() && !pNZBInfo->GetNoDupeCheck()) + if (g_pOptions->GetDupeCheck() && pNZBInfo->GetDupeMode() != dmForce) { - CheckDupeAdded(pDownloadQueue, pNZBInfo); + if (pNZBInfo->GetDeleteStatus() == NZBInfo::dsDupe) + { + NZBCompleted(pDownloadQueue, pNZBInfo, false); + } + else + { + m_DupeCoordinator.NZBAdded(pDownloadQueue, pNZBInfo); + } } if (strlen(g_pOptions->GetNZBAddedProcess()) > 0) @@ -413,11 +420,11 @@ void PrePostProcessor::NZBCompleted(DownloadQueue* pDownloadQueue, NZBInfo* pNZB bNeedSave = true; } - if (pNZBInfo->GetDupe() && + if (g_pOptions->GetDupeCheck() && pNZBInfo->GetDupeMode() != dmForce && (pNZBInfo->GetDeleteStatus() == NZBInfo::dsNone || pNZBInfo->GetDeleteStatus() == NZBInfo::dsHealth)) { - DupeCompleted(pDownloadQueue, pNZBInfo); + m_DupeCoordinator.NZBCompleted(pDownloadQueue, pNZBInfo); bNeedSave = true; } @@ -427,384 +434,14 @@ void PrePostProcessor::NZBCompleted(DownloadQueue* pDownloadQueue, NZBInfo* pNZB } } -bool PrePostProcessor::IsDupeSuccess(NZBInfo* pNZBInfo) -{ - bool bFailure = - pNZBInfo->GetDeleteStatus() != NZBInfo::dsNone || - pNZBInfo->GetMarkStatus() == NZBInfo::ksBad || - pNZBInfo->GetParStatus() == NZBInfo::psFailure || - pNZBInfo->GetUnpackStatus() == NZBInfo::usFailure || - (pNZBInfo->GetParStatus() == NZBInfo::psSkipped && - pNZBInfo->GetUnpackStatus() == NZBInfo::usSkipped && - pNZBInfo->CalcHealth() < pNZBInfo->CalcCriticalHealth()); - return !bFailure; -} - /** - - if download of an item completes successfully and there are - (paused) duplicates to this item in the queue, they all are deleted - from queue; - - if download of an item fails and there are (paused) duplicates to - this item in the queue iten with highest DupeScore is unpaused; -*/ -void PrePostProcessor::DupeCompleted(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo) + * Removes old entries from (recent) history + */ +void PrePostProcessor::CheckHistory() { - debug("Processing duplicates for %s", pNZBInfo->GetName()); + DownloadQueue* pDownloadQueue = g_pQueueCoordinator->LockQueue(); - if (IsDupeSuccess(pNZBInfo)) - { - RemoveDupes(pDownloadQueue, pNZBInfo); - } - else - { - UnpauseBestDupe(pDownloadQueue, pNZBInfo, pNZBInfo->GetName(), pNZBInfo->GetDupeKey()); - } -} - -void PrePostProcessor::RemoveDupes(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo) -{ - bool bHasDupeKey = !Util::EmptyStr(pNZBInfo->GetDupeKey()); - IDList groupIDList; - std::set groupNZBs; - - for (FileQueue::iterator it = pDownloadQueue->GetFileQueue()->begin(); it != pDownloadQueue->GetFileQueue()->end(); it++) - { - FileInfo* pFileInfo = *it; - if (pFileInfo->GetNZBInfo() != pNZBInfo && - pFileInfo->GetNZBInfo()->GetDupe() && - groupNZBs.find(pFileInfo->GetNZBInfo()) == groupNZBs.end() && - (!strcmp(pFileInfo->GetNZBInfo()->GetName(), pNZBInfo->GetName()) || - (bHasDupeKey && !strcmp(pFileInfo->GetNZBInfo()->GetDupeKey(), pNZBInfo->GetDupeKey())))) - { - groupIDList.push_back(pFileInfo->GetID()); - groupNZBs.insert(pFileInfo->GetNZBInfo()); - pFileInfo->GetNZBInfo()->SetDeleteStatus(NZBInfo::dsDupe); - } - } - - if (!groupIDList.empty()) - { - info("Removing duplicates for %s from queue", pNZBInfo->GetName()); - g_pQueueCoordinator->GetQueueEditor()->LockedEditList(pDownloadQueue, &groupIDList, false, QueueEditor::eaGroupDelete, 0, NULL); - } -} - -void PrePostProcessor::UnpauseBestDupe(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo, const char* szNZBName, const char* szDupeKey) -{ - bool bHasDupeKey = !Util::EmptyStr(szDupeKey); - std::set groupNZBs; - FileInfo* pDupeFileInfo = NULL; - - for (FileQueue::iterator it = pDownloadQueue->GetFileQueue()->begin(); it != pDownloadQueue->GetFileQueue()->end(); it++) - { - FileInfo* pFileInfo = *it; - if (pFileInfo->GetNZBInfo() != pNZBInfo && - pFileInfo->GetNZBInfo()->GetDupe() && - groupNZBs.find(pFileInfo->GetNZBInfo()) == groupNZBs.end() && - (!strcmp(pFileInfo->GetNZBInfo()->GetName(), szNZBName) || - (bHasDupeKey && !strcmp(pFileInfo->GetNZBInfo()->GetDupeKey(), szDupeKey)))) - { - // find nzb with highest DupeScore - if (!pDupeFileInfo || pFileInfo->GetNZBInfo()->GetDupeScore() > pDupeFileInfo->GetNZBInfo()->GetDupeScore()) - { - pDupeFileInfo = pFileInfo; - } - groupNZBs.insert(pFileInfo->GetNZBInfo()); - } - } - - if (pDupeFileInfo) - { - info("Unpausing duplicate %s", pDupeFileInfo->GetNZBInfo()->GetName()); - g_pQueueCoordinator->GetQueueEditor()->LockedEditEntry(pDownloadQueue, pDupeFileInfo->GetID(), false, QueueEditor::eaGroupResume, 0, NULL); - g_pQueueCoordinator->GetQueueEditor()->LockedEditEntry(pDownloadQueue, pDupeFileInfo->GetID(), false, QueueEditor::eaGroupPauseExtraPars, 0, NULL); - } -} - -/** - Check if the title was already downloaded or is already queued. -*/ -void PrePostProcessor::CheckDupeFound(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo) -{ - debug("Checking duplicates for %s", pNZBInfo->GetName()); - - bool bHasDupeKey = !Util::EmptyStr(pNZBInfo->GetDupeKey()); - - // find duplicates in download queue having exactly same content - GroupQueue groupQueue; - pDownloadQueue->BuildGroups(&groupQueue); - - for (GroupQueue::iterator it = groupQueue.begin(); it != groupQueue.end(); it++) - { - GroupInfo* pGroupInfo = *it; - NZBInfo* pGroupNZBInfo = pGroupInfo->GetNZBInfo(); - bool bSameContent = (pNZBInfo->GetFullContentHash() > 0 && - pNZBInfo->GetFullContentHash() == pGroupNZBInfo->GetFullContentHash()) || - (pNZBInfo->GetFilteredContentHash() > 0 && - pNZBInfo->GetFilteredContentHash() == pGroupNZBInfo->GetFilteredContentHash()); - if (pGroupNZBInfo != pNZBInfo && bSameContent) - { - if (!strcmp(pNZBInfo->GetName(), pGroupNZBInfo->GetName())) - { - warn("Skipping duplicate %s, already queued", pNZBInfo->GetName()); - } - else - { - warn("Skipping duplicate %s, already queued as %s", - pNZBInfo->GetName(), pGroupNZBInfo->GetName()); - } - pNZBInfo->SetDeleteStatus(NZBInfo::dsManual); // Flag saying QueueCoordinator to skip nzb-file - DeleteQueuedFile(pNZBInfo->GetQueuedFilename()); - return; - } - } - - // find duplicates in post queue having exactly same content - for (PostQueue::iterator it = pDownloadQueue->GetPostQueue()->begin(); it != pDownloadQueue->GetPostQueue()->end(); it++) - { - PostInfo* pPostInfo = *it; - bool bSameContent = (pNZBInfo->GetFullContentHash() > 0 && - pNZBInfo->GetFullContentHash() == pPostInfo->GetNZBInfo()->GetFullContentHash()) || - (pNZBInfo->GetFilteredContentHash() > 0 && - pNZBInfo->GetFilteredContentHash() == pPostInfo->GetNZBInfo()->GetFilteredContentHash()); - if (bSameContent) - { - if (!strcmp(pNZBInfo->GetName(), pPostInfo->GetNZBInfo()->GetName())) - { - warn("Skipping duplicate %s, already queued", pNZBInfo->GetName()); - } - else - { - warn("Skipping duplicate %s, already queued as %s", - pNZBInfo->GetName(), pPostInfo->GetNZBInfo()->GetName()); - } - pNZBInfo->SetDeleteStatus(NZBInfo::dsManual); // Flag saying QueueCoordinator to skip nzb-file - DeleteQueuedFile(pNZBInfo->GetQueuedFilename()); - return; - } - } - - // find duplicates in history - - bool bSkip = false; - bool bGood = false; - bool bSameContent = false; - const char* szDupeName = NULL; - - // find duplicates in queue having exactly same content - // also: nzb-files having duplicates marked as good are skipped - // also: nzb-files having success-duplicates in dup-history but don't having duplicates in recent history are skipped - for (HistoryList::iterator it = pDownloadQueue->GetHistoryList()->begin(); it != pDownloadQueue->GetHistoryList()->end(); it++) - { - HistoryInfo* pHistoryInfo = *it; - - if (pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo && - ((pNZBInfo->GetFullContentHash() > 0 && - pNZBInfo->GetFullContentHash() == pHistoryInfo->GetNZBInfo()->GetFullContentHash()) || - (pNZBInfo->GetFilteredContentHash() > 0 && - pNZBInfo->GetFilteredContentHash() == pHistoryInfo->GetNZBInfo()->GetFilteredContentHash()))) - { - bSkip = true; - bSameContent = true; - szDupeName = pHistoryInfo->GetNZBInfo()->GetName(); - break; - } - - if (pHistoryInfo->GetKind() == HistoryInfo::hkDupInfo && - ((pNZBInfo->GetFullContentHash() > 0 && - pNZBInfo->GetFullContentHash() == pHistoryInfo->GetDupInfo()->GetFullContentHash()) || - (pNZBInfo->GetFilteredContentHash() > 0 && - pNZBInfo->GetFilteredContentHash() == pHistoryInfo->GetDupInfo()->GetFilteredContentHash()))) - { - bSkip = true; - bSameContent = true; - szDupeName = pHistoryInfo->GetDupInfo()->GetName(); - break; - } - - if (pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo && - pHistoryInfo->GetNZBInfo()->GetMarkStatus() == NZBInfo::ksGood && - (!strcmp(pHistoryInfo->GetNZBInfo()->GetName(), pNZBInfo->GetName()) || - (bHasDupeKey && !strcmp(pHistoryInfo->GetNZBInfo()->GetDupeKey(), pNZBInfo->GetDupeKey())))) - { - bSkip = true; - bGood = true; - szDupeName = pHistoryInfo->GetNZBInfo()->GetName(); - break; - } - - if (pHistoryInfo->GetKind() == HistoryInfo::hkDupInfo && - (!strcmp(pHistoryInfo->GetDupInfo()->GetName(), pNZBInfo->GetName()) || - (bHasDupeKey && !strcmp(pHistoryInfo->GetDupInfo()->GetDupeKey(), pNZBInfo->GetDupeKey()))) && - (pHistoryInfo->GetDupInfo()->GetStatus() == DupInfo::dsGood || - (pHistoryInfo->GetDupInfo()->GetStatus() == DupInfo::dsSuccess && - pNZBInfo->GetDupeScore() <= pHistoryInfo->GetDupInfo()->GetDupeScore()))) - { - bSkip = true; - bGood = pHistoryInfo->GetDupInfo()->GetStatus() == DupInfo::dsGood; - szDupeName = pHistoryInfo->GetDupInfo()->GetName(); - break; - } - } - - if (!bSameContent && !bGood) - { - // nzb-files having success-duplicates in recent history (with different content) are added to history for backup - for (HistoryList::iterator it = pDownloadQueue->GetHistoryList()->begin(); it != pDownloadQueue->GetHistoryList()->end(); it++) - { - HistoryInfo* pHistoryInfo = *it; - if (pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo && - (!strcmp(pHistoryInfo->GetNZBInfo()->GetName(), pNZBInfo->GetName()) || - (bHasDupeKey && !strcmp(pHistoryInfo->GetNZBInfo()->GetDupeKey(), pNZBInfo->GetDupeKey()))) && - pNZBInfo->GetDupeScore() <= pHistoryInfo->GetNZBInfo()->GetDupeScore() && - IsDupeSuccess(pHistoryInfo->GetNZBInfo())) - { - // Flag saying QueueCoordinator to skip nzb-file, we also use this flag later in "CheckDupeAdded" - pNZBInfo->SetDeleteStatus(NZBInfo::dsDupe); - MarkDupe(pNZBInfo, pHistoryInfo->GetNZBInfo()); - return; - } - } - } - - if (bSkip) - { - if (!strcmp(pNZBInfo->GetName(), szDupeName)) - { - warn("Skipping duplicate %s, found in history with %s", pNZBInfo->GetName(), - bSameContent ? "exactly same content" : bGood ? "good status" : "success status"); - } - else - { - warn("Skipping duplicate %s, found in history %s with %s", - pNZBInfo->GetName(), szDupeName, - bSameContent ? "exactly same content" : bGood ? "good status" : "success status"); - } - - // Flag saying QueueCoordinator to skip nzb-file - pNZBInfo->SetDeleteStatus(NZBInfo::dsManual); - DeleteQueuedFile(pNZBInfo->GetQueuedFilename()); - return; - } -} - -/** - - If download queue or post-queue contain a duplicate the existing item - and the newly added item are marked as duplicates to each other. - The newly added item is paused. -*/ -void PrePostProcessor::CheckDupeAdded(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo) -{ - debug("Checking duplicates for %s", pNZBInfo->GetName()); - - if (pNZBInfo->GetDeleteStatus() == NZBInfo::dsDupe) - { - NZBCompleted(pDownloadQueue, pNZBInfo, false); - return; - } - - bool bHasDupeKey = !Util::EmptyStr(pNZBInfo->GetDupeKey()); - bool bHigherScore = true; - NZBInfo* pDupeNZBInfo = NULL; - - // find all duplicates in post queue - std::set postDupes; - - for (PostQueue::iterator it = pDownloadQueue->GetPostQueue()->begin(); it != pDownloadQueue->GetPostQueue()->end(); it++) - { - PostInfo* pPostInfo = *it; - if (!strcmp(pPostInfo->GetNZBInfo()->GetName(), pNZBInfo->GetName()) || - (bHasDupeKey && !strcmp(pPostInfo->GetNZBInfo()->GetDupeKey(), pNZBInfo->GetDupeKey()))) - { - postDupes.insert(pPostInfo->GetNZBInfo()); - if (!pDupeNZBInfo) - { - pDupeNZBInfo = pPostInfo->GetNZBInfo(); - } - bHigherScore = bHigherScore && pPostInfo->GetNZBInfo()->GetDupeScore() < pNZBInfo->GetDupeScore(); - } - } - - // find all duplicates in download queue - GroupQueue groupQueue; - pDownloadQueue->BuildGroups(&groupQueue); - std::list queueDupes; - GroupInfo* pNewGroupInfo = NULL; - - for (GroupQueue::iterator it = groupQueue.begin(); it != groupQueue.end(); it++) - { - GroupInfo* pGroupInfo = *it; - NZBInfo* pGroupNZBInfo = pGroupInfo->GetNZBInfo(); - if (pGroupNZBInfo != pNZBInfo && - (!strcmp(pGroupNZBInfo->GetName(), pNZBInfo->GetName()) || - (bHasDupeKey && !strcmp(pGroupNZBInfo->GetDupeKey(), pNZBInfo->GetDupeKey())))) - { - queueDupes.push_back(pGroupInfo); - if (!pDupeNZBInfo) - { - pDupeNZBInfo = pGroupNZBInfo; - } - bHigherScore = bHigherScore && pGroupNZBInfo->GetDupeScore() < pNZBInfo->GetDupeScore(); - } - if (pGroupNZBInfo == pNZBInfo) - { - pNewGroupInfo = pGroupInfo; - } - } - - if (pDupeNZBInfo) - { - MarkDupe(pNZBInfo, pDupeNZBInfo); - - // pause all duplicates with lower DupeScore, which are not in post-processing - for (std::list::iterator it = queueDupes.begin(); it != queueDupes.end(); it++) - { - GroupInfo* pGroupInfo = *it; - NZBInfo* pDupeNZB = pGroupInfo->GetNZBInfo(); - if (pDupeNZB->GetDupeScore() < pNZBInfo->GetDupeScore() && - postDupes.find(pDupeNZB) == postDupes.end() && - pGroupInfo->GetPausedFileCount() < pGroupInfo->GetRemainingFileCount()) - { - info("Pausing collection %s with lower duplicate score", pDupeNZB->GetName()); - g_pQueueCoordinator->GetQueueEditor()->LockedEditEntry(pDownloadQueue, pGroupInfo->GetLastID(), false, QueueEditor::eaGroupPause, 0, NULL); - } - } - - if (!bHigherScore) - { - g_pQueueCoordinator->GetQueueEditor()->LockedEditEntry(pDownloadQueue, pNewGroupInfo->GetLastID(), false, QueueEditor::eaGroupPause, 0, NULL); - } - } -} - -void PrePostProcessor::MarkDupe(NZBInfo* pNZBInfo, NZBInfo* pDupeNZBInfo) -{ - info("Marking collection %s as duplicate to %s", pNZBInfo->GetName(), pDupeNZBInfo->GetName()); - - pNZBInfo->SetDupe(true); - pDupeNZBInfo->SetDupe(true); - - if (Util::EmptyStr(pNZBInfo->GetDupeKey()) && !Util::EmptyStr(pDupeNZBInfo->GetDupeKey())) - { - pNZBInfo->SetDupeKey(pDupeNZBInfo->GetDupeKey()); - } -} - -void PrePostProcessor::CleanupHistory(DownloadQueue* pDownloadQueue, EHistoryCleanupMode eMode, HistoryInfo* pMarkHistoryInfo) -{ time_t tMinTime = time(NULL) - g_pOptions->GetKeepHistory() * 60*60*24; - const char* szDupeKey = NULL; - const char* szNZBName = NULL; - if (eMode == cmDupe) - { - szDupeKey = pMarkHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo ? pMarkHistoryInfo->GetNZBInfo()->GetDupeKey() : - pMarkHistoryInfo->GetKind() == HistoryInfo::hkDupInfo ? pMarkHistoryInfo->GetDupInfo()->GetDupeKey() : - NULL; - szNZBName = pMarkHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo ? pMarkHistoryInfo->GetNZBInfo()->GetName() : - pMarkHistoryInfo->GetKind() == HistoryInfo::hkDupInfo ? pMarkHistoryInfo->GetDupInfo()->GetName() : - NULL; - } - bool bHasDupeKey = !Util::EmptyStr(szDupeKey); bool bChanged = false; int index = 0; @@ -813,54 +450,29 @@ void PrePostProcessor::CleanupHistory(DownloadQueue* pDownloadQueue, EHistoryCle for (HistoryList::reverse_iterator it = pDownloadQueue->GetHistoryList()->rbegin(); it != pDownloadQueue->GetHistoryList()->rend(); ) { HistoryInfo* pHistoryInfo = *it; - if ((eMode == cmOld && pHistoryInfo->GetKind() != HistoryInfo::hkDupInfo && pHistoryInfo->GetTime() < tMinTime) - || - (eMode == cmDupe && pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo && - pHistoryInfo != pMarkHistoryInfo && pHistoryInfo->GetNZBInfo()->GetDupe() && - (!strcmp(pHistoryInfo->GetNZBInfo()->GetName(), szNZBName) || - (bHasDupeKey && !strcmp(pHistoryInfo->GetNZBInfo()->GetDupeKey(), szDupeKey))))) + if (pHistoryInfo->GetKind() != HistoryInfo::hkDupInfo && pHistoryInfo->GetTime() < tMinTime) { - char szNiceName[1024]; - pHistoryInfo->GetName(szNiceName, 1024); - if (g_pOptions->GetDupeCheck() && pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo) { // replace history element - DupInfo* pDupInfo = new DupInfo(); - pDupInfo->SetName(pHistoryInfo->GetNZBInfo()->GetName()); - pDupInfo->SetDupe(pHistoryInfo->GetNZBInfo()->GetDupe()); - pDupInfo->SetDupeKey(pHistoryInfo->GetNZBInfo()->GetDupeKey()); - pDupInfo->SetDupeScore(pHistoryInfo->GetNZBInfo()->GetDupeScore()); - pDupInfo->SetSize(pHistoryInfo->GetNZBInfo()->GetSize()); - pDupInfo->SetFullContentHash(pHistoryInfo->GetNZBInfo()->GetFullContentHash()); - pDupInfo->SetFilteredContentHash(pHistoryInfo->GetNZBInfo()->GetFilteredContentHash()); - - pDupInfo->SetStatus( - pHistoryInfo->GetNZBInfo()->GetMarkStatus() == NZBInfo::ksGood ? DupInfo::dsGood : - pHistoryInfo->GetNZBInfo()->GetMarkStatus() == NZBInfo::ksBad ? DupInfo::dsBad : - pHistoryInfo->GetNZBInfo()->GetDeleteStatus() == NZBInfo::dsDupe ? DupInfo::dsDupe : - pHistoryInfo->GetNZBInfo()->GetDeleteStatus() == NZBInfo::dsManual ? DupInfo::dsDeleted : - IsDupeSuccess(pHistoryInfo->GetNZBInfo()) ? DupInfo::dsSuccess : - DupInfo::dsFailed); - - HistoryInfo* pNewHistoryInfo = new HistoryInfo(pDupInfo); - pNewHistoryInfo->SetTime(pHistoryInfo->GetTime()); - (*pDownloadQueue->GetHistoryList())[pDownloadQueue->GetHistoryList()->size() - 1 - index] = pNewHistoryInfo; - info("Collection %s removed from recent history", szNiceName); + m_DupeCoordinator.HistoryTransformToDup(pDownloadQueue, pHistoryInfo, index); index++; } else { + char szNiceName[1024]; + pHistoryInfo->GetName(szNiceName, 1024); + pDownloadQueue->GetHistoryList()->erase(pDownloadQueue->GetHistoryList()->end() - 1 - index); + delete pHistoryInfo; + + if (pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo) + { + DeleteQueuedFile(pHistoryInfo->GetNZBInfo()->GetQueuedFilename()); + } info("Collection %s removed from history", szNiceName); } - if (pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo) - { - DeleteQueuedFile(pHistoryInfo->GetNZBInfo()->GetQueuedFilename()); - } - - delete pHistoryInfo; it = pDownloadQueue->GetHistoryList()->rbegin() + index; bChanged = true; } @@ -875,15 +487,7 @@ void PrePostProcessor::CleanupHistory(DownloadQueue* pDownloadQueue, EHistoryCle { SaveQueue(pDownloadQueue); } -} -/** - * Removes old entries from (recent) history - */ -void PrePostProcessor::CheckHistory() -{ - DownloadQueue* pDownloadQueue = g_pQueueCoordinator->LockQueue(); - CleanupHistory(pDownloadQueue, cmOld, NULL); g_pQueueCoordinator->UnlockQueue(); } @@ -1540,7 +1144,7 @@ bool PrePostProcessor::HistoryEdit(IDList* pIDList, EEditAction eAction, int iOf case eaHistoryMarkBad: case eaHistoryMarkGood: - HistoryMark(pDownloadQueue, pHistoryInfo, eAction == eaHistoryMarkGood); + m_DupeCoordinator.HistoryMark(pDownloadQueue, pHistoryInfo, eAction == eaHistoryMarkGood); break; default: @@ -1720,104 +1324,3 @@ void PrePostProcessor::HistorySetParameter(HistoryInfo* pHistoryInfo, const char free(szStr); } - -void PrePostProcessor::HistoryMark(DownloadQueue* pDownloadQueue, HistoryInfo* pHistoryInfo, bool bGood) -{ - char szNZBName[1024]; - pHistoryInfo->GetName(szNZBName, 1024); - - info("Marking %s as %s", szNZBName, (bGood ? "good" : "bad")); - - if (pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo) - { - pHistoryInfo->GetNZBInfo()->SetMarkStatus(bGood ? NZBInfo::ksGood : NZBInfo::ksBad); - } - else if (pHistoryInfo->GetKind() == HistoryInfo::hkDupInfo) - { - pHistoryInfo->GetDupInfo()->SetStatus(bGood ? DupInfo::dsGood : DupInfo::dsBad); - } - else - { - error("Could not mark %s as bad: history item has wrong type", szNZBName); - return; - } - - if (!g_pOptions->GetDupeCheck()) - { - return; - } - - if (bGood) - { - // mark as good - // moving all duplicates from history to dup-history - CleanupHistory(pDownloadQueue, cmDupe, pHistoryInfo); - return; - } - - // mark as bad continues - - const char* szDupeKey = pHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo ? pHistoryInfo->GetNZBInfo()->GetDupeKey() : - pHistoryInfo->GetKind() == HistoryInfo::hkDupInfo ? pHistoryInfo->GetDupInfo()->GetDupeKey() : - NULL; - bool bHasDupeKey = !Util::EmptyStr(szDupeKey); - - // move existing dupe-backups from history to download queue - HistoryList dupeList; - for (HistoryList::iterator it = pDownloadQueue->GetHistoryList()->begin(); it != pDownloadQueue->GetHistoryList()->end(); it++) - { - HistoryInfo* pDupeHistoryInfo = *it; - if (pDupeHistoryInfo->GetKind() == HistoryInfo::hkNZBInfo && - pDupeHistoryInfo->GetNZBInfo()->GetDupe() && - pDupeHistoryInfo->GetNZBInfo()->GetDeleteStatus() == NZBInfo::dsDupe && - (!strcmp(pDupeHistoryInfo->GetNZBInfo()->GetName(), szNZBName) || - (bHasDupeKey && !strcmp(pDupeHistoryInfo->GetNZBInfo()->GetDupeKey(), szDupeKey)))) - { - dupeList.push_back(pDupeHistoryInfo); - } - } - - for (HistoryList::iterator it = dupeList.begin(); it != dupeList.end(); it++) - { - HistoryInfo* pDupeHistoryInfo = *it; - HistoryReturnDupe(pDownloadQueue, pDupeHistoryInfo); - } - - UnpauseBestDupe(pDownloadQueue, NULL, szNZBName, szDupeKey); -} - -void PrePostProcessor::HistoryReturnDupe(DownloadQueue* pDownloadQueue, HistoryInfo* pHistoryInfo) -{ - NZBInfo* pNZBInfo = pHistoryInfo->GetNZBInfo(); - - if (!Util::FileExists(pNZBInfo->GetQueuedFilename())) - { - error("Could not return duplicate %s from history back to queue: could not find source nzb-file %s", - pNZBInfo->GetName(), pNZBInfo->GetQueuedFilename()); - return; - } - - NZBFile* pNZBFile = NZBFile::Create(pNZBInfo->GetQueuedFilename(), ""); - if (pNZBFile == NULL) - { - error("Could not return duplicate %s from history back to queue: could not parse nzb-file", - pNZBInfo->GetName()); - return; - } - - info("Returning duplicate %s from history back to queue", pNZBInfo->GetName()); - - for (NZBFile::FileInfos::iterator it = pNZBFile->GetFileInfos()->begin(); it != pNZBFile->GetFileInfos()->end(); it++) - { - FileInfo* pFileInfo = *it; - pFileInfo->SetNZBInfo(pNZBInfo); - pFileInfo->SetPaused(true); - } - - g_pQueueCoordinator->AddFileInfosToFileQueue(pNZBFile, pDownloadQueue->GetParkedFiles(), false); - - delete pNZBFile; - - HistoryList::iterator it = std::find(pDownloadQueue->GetHistoryList()->begin(), pDownloadQueue->GetHistoryList()->end(), pHistoryInfo); - HistoryReturn(pDownloadQueue, it, pHistoryInfo, false); -} diff --git a/PrePostProcessor.h b/PrePostProcessor.h index 6d837057..28eaba10 100644 --- a/PrePostProcessor.h +++ b/PrePostProcessor.h @@ -32,6 +32,7 @@ #include "Observer.h" #include "DownloadInfo.h" #include "ParCoordinator.h" +#include "DupeCoordinator.h" class PrePostProcessor : public Thread { @@ -65,18 +66,23 @@ private: protected: virtual bool PauseDownload() { return m_pOwner->PauseDownload(); } virtual bool UnpauseDownload() { return m_pOwner->UnpauseDownload(); } - friend class PrePostProcessor; }; - enum EHistoryCleanupMode + class PostDupeCoordinator: public DupeCoordinator { - cmOld, - cmDupe + private: + PrePostProcessor* m_pOwner; + protected: + virtual void HistoryReturn(DownloadQueue* pDownloadQueue, HistoryList::iterator itHistory, + HistoryInfo* pHistoryInfo, bool bReprocess) { m_pOwner->HistoryReturn(pDownloadQueue, itHistory, pHistoryInfo, bReprocess); } + virtual void DeleteQueuedFile(const char* szQueuedFile) { m_pOwner->DeleteQueuedFile(szQueuedFile); } + friend class PrePostProcessor; }; private: PostParCoordinator m_ParCoordinator; + PostDupeCoordinator m_DupeCoordinator; QueueCoordinatorObserver m_QueueCoordinatorObserver; bool m_bHasMoreJobs; bool m_bSchedulerPauseChanged; @@ -102,12 +108,6 @@ private: void NZBDownloaded(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo); void NZBDeleted(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo); void NZBCompleted(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo, bool bSaveQueue); - void DupeCompleted(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo); - void RemoveDupes(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo); - void UnpauseBestDupe(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo, const char* szNZBName, const char* szDupeKey); - void CheckDupeFound(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo); - void CheckDupeAdded(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo); - void MarkDupe(NZBInfo* pNZBInfo, NZBInfo* pDupeNZBInfo); void DeleteQueuedFile(const char* szQueuedFile); int FindGroupID(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo); bool PostQueueMove(IDList* pIDList, EEditAction eAction, int iOffset); @@ -116,13 +116,10 @@ private: void HistoryDelete(DownloadQueue* pDownloadQueue, HistoryList::iterator itHistory, HistoryInfo* pHistoryInfo); void HistoryReturn(DownloadQueue* pDownloadQueue, HistoryList::iterator itHistory, HistoryInfo* pHistoryInfo, bool bReprocess); void HistorySetParameter(HistoryInfo* pHistoryInfo, const char* szText); - void HistoryMark(DownloadQueue* pDownloadQueue, HistoryInfo* pHistoryInfo, bool bGood); - void HistoryReturnDupe(DownloadQueue* pDownloadQueue, HistoryInfo* pHistoryInfo); - bool IsDupeSuccess(NZBInfo* pNZBInfo); - void CleanupHistory(DownloadQueue* pDownloadQueue, EHistoryCleanupMode eMode, HistoryInfo* pMarkHistoryInfo); + void HistoryTransformToDup(DownloadQueue* pDownloadQueue, HistoryInfo* pHistoryInfo, int rindex); + void CheckHistory(); void Cleanup(); FileInfo* GetQueueGroup(DownloadQueue* pDownloadQueue, NZBInfo* pNZBInfo); - void CheckHistory(); void DeletePostThread(PostInfo* pPostInfo); public: diff --git a/QueueCoordinator.cpp b/QueueCoordinator.cpp index 11541f71..f9a596d9 100644 --- a/QueueCoordinator.cpp +++ b/QueueCoordinator.cpp @@ -310,7 +310,7 @@ void QueueCoordinator::AddFileInfosToFileQueue(NZBFile* pNZBFile, FileQueue* pFi index1++; FileInfo* pFileInfo = *it; - if (g_pOptions->GetDupeCheck() && !pNZBFile->GetNZBInfo()->GetNoDupeCheck()) + if (g_pOptions->GetDupeCheck() && !pNZBFile->GetNZBInfo()->GetDupeMode()) { bool dupe = false; int index2 = 0; diff --git a/Scanner.cpp b/Scanner.cpp index 7d1b6ea6..2c7c811d 100644 --- a/Scanner.cpp +++ b/Scanner.cpp @@ -68,7 +68,7 @@ Scanner::FileData::~FileData() Scanner::QueueData::QueueData(const char* szFilename, const char* szNZBName, const char* szCategory, - int iPriority, const char* szDupeKey, int iDupeScore, bool bNoDupeCheck, + int iPriority, const char* szDupeKey, int iDupeScore, EDupeMode eDupeMode, NZBParameterList* pParameters, bool bAddTop, bool bAddPaused, EAddStatus* pAddStatus) { m_szFilename = strdup(szFilename); @@ -77,7 +77,7 @@ Scanner::QueueData::QueueData(const char* szFilename, const char* szNZBName, con m_iPriority = iPriority; m_szDupeKey = strdup(szDupeKey ? szDupeKey : ""); m_iDupeScore = iDupeScore; - m_bNoDupeCheck = bNoDupeCheck; + m_eDupeMode = eDupeMode; m_bAddTop = bAddTop; m_bAddPaused = bAddPaused; m_pAddStatus = pAddStatus; @@ -340,7 +340,7 @@ void Scanner::ProcessIncomingFile(const char* szDirectory, const char* szBaseFil bool bAddPaused = false; const char* szDupeKey = NULL; int iDupeScore = 0; - bool bNoDupeCheck = false; + EDupeMode eDupeMode = dmScore; EAddStatus eAddStatus = asSkipped; bool bAdded = false; QueueData* pQueueData = NULL; @@ -358,7 +358,7 @@ void Scanner::ProcessIncomingFile(const char* szDirectory, const char* szBaseFil iPriority = pQueueData->GetPriority(); szDupeKey = pQueueData->GetDupeKey(); iDupeScore = pQueueData->GetDupeScore(); - bNoDupeCheck = pQueueData->GetNoDupeCheck(); + eDupeMode = pQueueData->GetDupeMode(); bAddTop = pQueueData->GetAddTop(); bAddPaused = pQueueData->GetAddPaused(); pParameters->CopyFrom(pQueueData->GetParameters()); @@ -393,7 +393,7 @@ void Scanner::ProcessIncomingFile(const char* szDirectory, const char* szBaseFil if (bRenameOK) { bAdded = AddFileToQueue(szRenamedName, szNZBName, szNZBCategory, iPriority, - szDupeKey, iDupeScore, bNoDupeCheck, pParameters, bAddTop, bAddPaused); + szDupeKey, iDupeScore, eDupeMode, pParameters, bAddTop, bAddPaused); } else { @@ -405,7 +405,7 @@ void Scanner::ProcessIncomingFile(const char* szDirectory, const char* szBaseFil else if (bExists && !strcasecmp(szExtension, ".nzb")) { bAdded = AddFileToQueue(szFullFilename, szNZBName, szNZBCategory, iPriority, - szDupeKey, iDupeScore, bNoDupeCheck, pParameters, bAddTop, bAddPaused); + szDupeKey, iDupeScore, eDupeMode, pParameters, bAddTop, bAddPaused); } delete pParameters; @@ -462,7 +462,7 @@ void Scanner::InitPPParameters(const char* szCategory, NZBParameterList* pParame } bool Scanner::AddFileToQueue(const char* szFilename, const char* szNZBName, const char* szCategory, - int iPriority, const char* szDupeKey, int iDupeScore, bool bNoDupeCheck, + int iPriority, const char* szDupeKey, int iDupeScore, EDupeMode eDupeMode, NZBParameterList* pParameters, bool bAddTop, bool bAddPaused) { const char* szBasename = Util::BaseFileName(szFilename); @@ -497,7 +497,7 @@ bool Scanner::AddFileToQueue(const char* szFilename, const char* szNZBName, cons pNZBFile->GetNZBInfo()->SetDupeKey(szDupeKey); pNZBFile->GetNZBInfo()->SetDupeScore(iDupeScore); - pNZBFile->GetNZBInfo()->SetNoDupeCheck(bNoDupeCheck); + pNZBFile->GetNZBInfo()->SetDupeMode(eDupeMode); if (pNZBFile->GetPassword()) { @@ -538,7 +538,7 @@ void Scanner::ScanNZBDir(bool bSyncMode) } Scanner::EAddStatus Scanner::AddExternalFile(const char* szNZBName, const char* szCategory, - int iPriority, const char* szDupeKey, int iDupeScore, bool bNoDupeCheck, + int iPriority, const char* szDupeKey, int iDupeScore, EDupeMode eDupeMode, NZBParameterList* pParameters, bool bAddTop, bool bAddPaused, const char* szFileName, const char* szBuffer, int iBufSize) { @@ -619,7 +619,7 @@ Scanner::EAddStatus Scanner::AddExternalFile(const char* szNZBName, const char* EAddStatus eAddStatus = asSkipped; QueueData* pQueueData = new QueueData(szScanFileName, szNZBName, szUseCategory, - iPriority, szDupeKey, iDupeScore, bNoDupeCheck, pParameters, bAddTop, bAddPaused, &eAddStatus); + iPriority, szDupeKey, iDupeScore, eDupeMode, pParameters, bAddTop, bAddPaused, &eAddStatus); free(szUseCategory); m_QueueList.push_back(pQueueData); diff --git a/Scanner.h b/Scanner.h index 7c0bdaa0..fd6f7166 100644 --- a/Scanner.h +++ b/Scanner.h @@ -70,7 +70,7 @@ private: int m_iPriority; char* m_szDupeKey; int m_iDupeScore; - bool m_bNoDupeCheck; + EDupeMode m_eDupeMode; NZBParameterList m_Parameters; bool m_bAddTop; bool m_bAddPaused; @@ -78,7 +78,7 @@ private: public: QueueData(const char* szFilename, const char* szNZBName, const char* szCategory, - int iPriority, const char* szDupeKey, int iDupeScore, bool bNoDupeCheck, + int iPriority, const char* szDupeKey, int iDupeScore, EDupeMode eDupeMode, NZBParameterList* pParameters, bool bAddTop, bool bAddPaused, EAddStatus* pAddStatus); ~QueueData(); const char* GetFilename() { return m_szFilename; } @@ -87,7 +87,7 @@ private: int GetPriority() { return m_iPriority; } const char* GetDupeKey() { return m_szDupeKey; } int GetDupeScore() { return m_iDupeScore; } - bool GetNoDupeCheck() { return m_bNoDupeCheck; } + EDupeMode GetDupeMode() { return m_eDupeMode; } NZBParameterList* GetParameters() { return &m_Parameters; } bool GetAddTop() { return m_bAddTop; } bool GetAddPaused() { return m_bAddPaused; } @@ -107,7 +107,7 @@ private: void CheckIncomingNZBs(const char* szDirectory, const char* szCategory, bool bCheckStat); bool AddFileToQueue(const char* szFilename, const char* szNZBName, const char* szCategory, - int iPriority, const char* szDupeKey, int iDupeScore, bool bNoDupeCheck, + int iPriority, const char* szDupeKey, int iDupeScore, EDupeMode eDupeMode, NZBParameterList* pParameters, bool bAddTop, bool bAddPaused); void ProcessIncomingFile(const char* szDirectory, const char* szBaseFilename, const char* szFullFilename, const char* szCategory); @@ -122,7 +122,7 @@ public: void ScanNZBDir(bool bSyncMode); void Check(); EAddStatus AddExternalFile(const char* szNZBName, const char* szCategory, int iPriority, - const char* szDupeKey, int iDupeScore, bool bNoDupeCheck, + const char* szDupeKey, int iDupeScore, EDupeMode eDupeMode, NZBParameterList* pParameters, bool bAddTop, bool bAddPaused, const char* szFileName, const char* szBuffer, int iBufSize); }; diff --git a/UrlCoordinator.cpp b/UrlCoordinator.cpp index e222a7c3..0b649d82 100644 --- a/UrlCoordinator.cpp +++ b/UrlCoordinator.cpp @@ -409,7 +409,7 @@ void UrlCoordinator::UrlCompleted(UrlDownloader* pUrlDownloader) Scanner::EAddStatus eAddStatus = g_pScanner->AddExternalFile( pUrlInfo->GetNZBFilename() && strlen(pUrlInfo->GetNZBFilename()) > 0 ? pUrlInfo->GetNZBFilename() : filename, strlen(pUrlInfo->GetCategory()) > 0 ? pUrlInfo->GetCategory() : pUrlDownloader->GetCategory(), - pUrlInfo->GetPriority(), pUrlInfo->GetDupeKey(), pUrlInfo->GetDupeScore(), pUrlInfo->GetNoDupeCheck(), + pUrlInfo->GetPriority(), pUrlInfo->GetDupeKey(), pUrlInfo->GetDupeScore(), pUrlInfo->GetDupeMode(), pUrlDownloader->GetParameters(), pUrlInfo->GetAddTop(), pUrlInfo->GetAddPaused(), pUrlDownloader->GetOutputFilename(), NULL, 0); diff --git a/XmlRpc.cpp b/XmlRpc.cpp index 81d29eed..49958cd0 100644 --- a/XmlRpc.cpp +++ b/XmlRpc.cpp @@ -1672,7 +1672,7 @@ void EditQueueXmlCommand::Execute() BuildBoolResponse(bOK); } -// bool append(string NZBFilename, string Category, int Priority, bool AddToTop, string Content, bool AddPaused, string DupeKey, int DupeScore, bool NoDupeCheck) +// bool append(string NZBFilename, string Category, int Priority, bool AddToTop, string Content, bool AddPaused, string DupeKey, int DupeScore, string DupeMode) // For backward compatibility with 0.8 parameter "Priority" is optional // Parameters starting from "AddPaused" are optional (added in v12) void DownloadXmlCommand::Execute() @@ -1722,7 +1722,7 @@ void DownloadXmlCommand::Execute() bool bAddPaused = false; char* szDupeKey = NULL; int iDupeScore = 0; - bool bNoDupeCheck = false; + EDupeMode eDupeMode = dmScore; if (NextParamAsBool(&bAddPaused)) { if (!NextParamAsStr(&szDupeKey)) @@ -1735,11 +1735,15 @@ void DownloadXmlCommand::Execute() BuildErrorResponse(2, "Invalid parameter (DupeScore)"); return; } - if (!NextParamAsBool(&bNoDupeCheck)) + char* szDupeMode = NULL; + if (!NextParamAsStr(&szDupeMode) || + (strcasecmp(szDupeMode, "score") && strcasecmp(szDupeMode, "all") && strcasecmp(szDupeMode, "force"))) { - BuildErrorResponse(2, "Invalid parameter (NoDupeCheck)"); + BuildErrorResponse(2, "Invalid parameter (DupeMode)"); return; } + eDupeMode = !strcasecmp(szDupeMode, "all") ? dmAll : + !strcasecmp(szDupeMode, "force") ? dmForce : dmScore; } if (IsJson()) @@ -1753,7 +1757,7 @@ void DownloadXmlCommand::Execute() //debug("FileContent=%s", szFileContent); bool bOK = g_pScanner->AddExternalFile(szFileName, szCategory, iPriority, - szDupeKey, iDupeScore, bNoDupeCheck, NULL, bAddTop, bAddPaused, NULL, szFileContent, iLen) != Scanner::asFailed; + szDupeKey, iDupeScore, eDupeMode, NULL, bAddTop, bAddPaused, NULL, szFileContent, iLen) != Scanner::asFailed; BuildBoolResponse(bOK); } @@ -2181,7 +2185,7 @@ void HistoryXmlCommand::Execute() g_pQueueCoordinator->UnlockQueue(); } -// bool appendurl(string NZBFilename, string Category, int Priority, bool AddToTop, string URL, bool AddPaused, string DupeKey, int DupeScore, bool NoDupeCheck) +// bool appendurl(string NZBFilename, string Category, int Priority, bool AddToTop, string URL, bool AddPaused, string DupeKey, int DupeScore, string DupeMode) // Parameters starting from "AddPaused" are optional (added in v12) void DownloadUrlXmlCommand::Execute() { @@ -2228,7 +2232,7 @@ void DownloadUrlXmlCommand::Execute() bool bAddPaused = false; char* szDupeKey = NULL; int iDupeScore = 0; - bool bNoDupeCheck = false; + EDupeMode eDupeMode = dmScore; if (NextParamAsBool(&bAddPaused)) { if (!NextParamAsStr(&szDupeKey)) @@ -2241,11 +2245,15 @@ void DownloadUrlXmlCommand::Execute() BuildErrorResponse(2, "Invalid parameter (DupeScore)"); return; } - if (!NextParamAsBool(&bNoDupeCheck)) + char* szDupeMode = NULL; + if (!NextParamAsStr(&szDupeMode) || + (strcasecmp(szDupeMode, "score") && strcasecmp(szDupeMode, "all") && strcasecmp(szDupeMode, "force"))) { - BuildErrorResponse(2, "Invalid parameter (NoDupeCheck)"); + BuildErrorResponse(2, "Invalid parameter (DupeMode)"); return; } + eDupeMode = !strcasecmp(szDupeMode, "all") ? dmAll : + !strcasecmp(szDupeMode, "force") ? dmForce : dmScore; } DecodeStr(szNZBFileName); @@ -2267,7 +2275,7 @@ void DownloadUrlXmlCommand::Execute() pUrlInfo->SetAddPaused(bAddPaused); pUrlInfo->SetDupeKey(szDupeKey ? szDupeKey : ""); pUrlInfo->SetDupeScore(iDupeScore); - pUrlInfo->SetNoDupeCheck(bNoDupeCheck); + pUrlInfo->SetDupeMode(eDupeMode); char szNicename[1024]; pUrlInfo->GetName(szNicename, sizeof(szNicename)); @@ -2631,7 +2639,7 @@ void ViewFeedXmlCommand::Execute() "Rule%i\n" "DupeKey%s\n" "DupeScore%i\n" - "NoDupeCheck%s\n" + "DupeMode%s\n" "Status%s\n" "\n"; @@ -2652,12 +2660,13 @@ void ViewFeedXmlCommand::Execute() "\"Rule\" : %i,\n" "\"DupeKey\" : \"%s\",\n" "\"DupeScore\" : %i,\n" - "\"NoDupeCheck\" : %s,\n" + "\"DupeMode\" : \"%s\",\n" "\"Status\" : \"%s\"\n" "}"; const char* szStatusType[] = { "UNKNOWN", "BACKLOG", "FETCHED", "NEW" }; const char* szMatchStatusType[] = { "IGNORED", "ACCEPTED", "REJECTED" }; + const char* szDupeModeType[] = { "SCORE", "ALL", "FORCE" }; int iItemBufSize = 10240; char* szItemBuf = (char*)malloc(iItemBufSize); @@ -2686,7 +2695,7 @@ void ViewFeedXmlCommand::Execute() xmltitle, xmlfilename, xmlurl, iSizeLo, iSizeHi, iSizeMB, xmlcategory, xmladdcategory, BoolToStr(pFeedItemInfo->GetPauseNzb()), pFeedItemInfo->GetPriority(), pFeedItemInfo->GetTime(), szMatchStatusType[pFeedItemInfo->GetMatchStatus()], pFeedItemInfo->GetMatchRule(), - xmldupekey, pFeedItemInfo->GetDupeScore(), BoolToStr(pFeedItemInfo->GetNoDupeCheck()), + xmldupekey, pFeedItemInfo->GetDupeScore(), szDupeModeType[pFeedItemInfo->GetDupeMode()], szStatusType[pFeedItemInfo->GetStatus()]); szItemBuf[iItemBufSize-1] = '\0'; diff --git a/nzbget.conf b/nzbget.conf index 573e9754..b8bcf09c 100644 --- a/nzbget.conf +++ b/nzbget.conf @@ -822,22 +822,22 @@ Category4.Name=Software # Options can be defined using long option names or short names: # category (cat, c) - set category name, value is a string; # pause (p) - add nzb in paused or unpaused state, possible -# values are: "yes", "y", "no", "n"; +# values are: yes (y), no (n); # priority (pr, r) - set priority, value is a signed integer number; # priority+ (pr+, r+) - increase priority, value is a signed integer number; # dupescore (ds, s) - set duplicate score, value is a signed integer number; # dupescore+ (ds+, s+) - increase duplicate score, value is a signed integer number; # dupekey (dk, k) - set duplicate key, value is a string; # dupekey+ (dk+, k+) - add to duplicate key, value is a string; -# nodupe (nd, n) - disable duplicate check for the item, possible -# values are: "yes", "y", "no", "n". +# dupemode (dm, m) - set duplicate check mode, possible values +# are: score (s), all (a), force (f). # # Examples of option definitions: # Accept(category:my series, pause:yes, priority:100): my show 1080p; # Options(c:my series, p:y, r:100): 1080p; # Options(s:1000): 1080p; # Options(k+:1080p): 1080p; -# Options(nodupe:yes): BluRay. +# Options(dupemode:force): BluRay. # # Rule-options override values set in feed-options. # diff --git a/nzbget.vcproj b/nzbget.vcproj index d881a978..2444d4c8 100644 --- a/nzbget.vcproj +++ b/nzbget.vcproj @@ -299,6 +299,14 @@ RelativePath=".\DownloadInfo.h" > + + + + diff --git a/webui/feed.js b/webui/feed.js index a616b643..45de00d7 100644 --- a/webui/feed.js +++ b/webui/feed.js @@ -351,7 +351,7 @@ var FeedDialog = (new function($) name += '.nzb'; } RPC.call('appendurl', [name, fetchItems[0].AddCategory, fetchItems[0].Priority, false, - fetchItems[0].URL, false, fetchItems[0].DupeKey, fetchItems[0].DupeScore, fetchItems[0].NoDupeCheck], + fetchItems[0].URL, false, fetchItems[0].DupeKey, fetchItems[0].DupeScore, fetchItems[0].DupeMode], function() { fetchItems.shift(); @@ -624,7 +624,7 @@ var FeedFilterDialog = (new function($) item.PauseNzb !== feedPauseNzb ? (item.PauseNzb ? 'paused' : 'unpaused') : null, item.DupeScore != 0 ? 'dupe-score: ' + item.DupeScore : null, item.DupeKey !== '' ? 'dupe-key: ' + item.DupeKey : null, - item.NoDupeCheck ? 'dupe-check: no' : null]. + item.DupeMode != '' ? 'dupe-mode: ' + item.DupeMode.toLowerCase() : null]. filter(function(e){return e}).join('; '); status = 'ACCEPTED'; countAccepted += 1; diff --git a/webui/index.html b/webui/index.html index b814375c..a2967bef 100644 --- a/webui/index.html +++ b/webui/index.html @@ -1399,14 +1399,18 @@ - + - +
category (cat, c)set category name, value is a string;
pause (p)add nzb in paused or unpaused state, possible values are: "yes", "y", "no", "n";
pause (p)add nzb in paused or unpaused state, possible values are: yes (y), no (n);
priority (pr, r)set priority, value is a signed integer number;
priority+ (pr+, r+)increase priority, value is a signed integer number;
dupescore (ds, s)set duplicate score, value is a signed integer number;
dupescore+ (ds+, s+)increase duplicate score, value is a signed integer number;
dupekey (dk, k)set duplicate key, value is a string;
dupekey+ (dk+, k+)add to duplicate key, value is a string;
nodupe (nd, n)disable duplicate check for the item, possible values are: "yes", "y", "no", "n".
dupemode (dm, m)set duplicate check mode, possible values are: score (s), all (a), force (f). + Score - download duplicates with higher scores; + All - download all duplicates regardless of scores; + Force - treat item as unique, ignore all duplicate checks. + For Score and All item is skipped if a duplicate marked as "good" exists in history.

@@ -1434,13 +1438,11 @@ + - declares a positive term. Terms are positive by default, - the "+" can be omitted; + declares a positive term. Terms are positive by default, the "+" can be omitted; - - declares a negative term. If the term succeed the feed - item is ignored; + declares a negative term. If the term succeeds the rule does not match; field @@ -1449,8 +1451,7 @@ command - a special character defining how to interpret the - parameter (followed after the command): + a special character defining how to interpret the parameter (followed after the command): @@ -1734,7 +1735,7 @@
- +
diff --git a/webui/upload.js b/webui/upload.js index 2dccfadd..36d8944d 100644 --- a/webui/upload.js +++ b/webui/upload.js @@ -223,7 +223,7 @@ var Upload = (new function($) $('#AddDialog_URLLabel img').hide(); $('#AddDialog_URLLabel i').hide(); $('#AddDialog_Paused').prop('checked', false); - $('#AddDialog_NoDupeCheck').prop('checked', false); + $('#AddDialog_DupeForce').prop('checked', false); enableAllButtons(); var v = $('#AddDialog_Priority'); @@ -333,9 +333,9 @@ var Upload = (new function($) var category = $('#AddDialog_Category').val(); var priority = parseInt($('#AddDialog_Priority').val()); var filename = file.name.replace(/\.queued$/g, ''); - var AddPaused = $('#AddDialog_Paused').is(':checked'); - var NoDupeCheck = $('#AddDialog_NoDupeCheck').is(':checked'); - RPC.call('append', [filename, category, priority, false, base64str, AddPaused, '', 0, NoDupeCheck], fileCompleted, fileFailure); + var addPaused = $('#AddDialog_Paused').is(':checked'); + var dupeMode = $('#AddDialog_DupeForce').is(':checked') ? "FORCE" : "SCORE"; + RPC.call('append', [filename, category, priority, false, base64str, addPaused, '', 0, dupeMode], fileCompleted, fileFailure); }; if (reader.readAsBinaryString) @@ -376,10 +376,10 @@ var Upload = (new function($) var category = $('#AddDialog_Category').val(); var priority = parseInt($('#AddDialog_Priority').val()); - var AddPaused = $('#AddDialog_Paused').is(':checked'); - var NoDupeCheck = $('#AddDialog_NoDupeCheck').is(':checked'); + var addPaused = $('#AddDialog_Paused').is(':checked'); + var dupeMode = $('#AddDialog_DupeForce').is(':checked') ? "FORCE" : "SCORE"; - RPC.call('appendurl', ['', category, priority, false, url, AddPaused, '', 0, NoDupeCheck], urlCompleted, urlFailure); + RPC.call('appendurl', ['', category, priority, false, url, addPaused, '', 0, dupeMode], urlCompleted, urlFailure); } function urlCompleted(result)
@ - search for string "param". This is default command, the "@" can be omitted;