From 5e15677218efbc05fc50c8638d41e27d0bdb376b Mon Sep 17 00:00:00 2001 From: Andrey Prygunkov Date: Thu, 12 Jul 2018 21:19:09 +0200 Subject: [PATCH] #541: better duplicate handling of urls --- daemon/postprocess/PrePostProcessor.cpp | 4 +++- daemon/queue/DiskState.cpp | 19 ++++++++++++++----- daemon/queue/DownloadInfo.h | 12 +++++++++++- daemon/queue/DupeCoordinator.cpp | 9 ++++++++- daemon/queue/HistoryCoordinator.cpp | 5 ++++- daemon/queue/Scanner.cpp | 3 ++- 6 files changed, 42 insertions(+), 10 deletions(-) diff --git a/daemon/postprocess/PrePostProcessor.cpp b/daemon/postprocess/PrePostProcessor.cpp index 7c75ee2e..67316da3 100644 --- a/daemon/postprocess/PrePostProcessor.cpp +++ b/daemon/postprocess/PrePostProcessor.cpp @@ -381,6 +381,7 @@ void PrePostProcessor::NzbDeleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo void PrePostProcessor::NzbCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbInfo, bool saveQueue) { + bool downloadDupe = nzbInfo->GetDupeHint() == NzbInfo::dhRedownloadAuto; bool addToHistory = g_Options->GetKeepHistory() > 0 && !nzbInfo->GetAvoidHistory(); if (addToHistory) { @@ -394,7 +395,8 @@ void PrePostProcessor::NzbCompleted(DownloadQueue* downloadQueue, NzbInfo* nzbIn (nzbInfo->GetDeleteStatus() == NzbInfo::dsNone || nzbInfo->GetDeleteStatus() == NzbInfo::dsHealth || nzbInfo->GetDeleteStatus() == NzbInfo::dsBad || - nzbInfo->GetDeleteStatus() == NzbInfo::dsScan)) + nzbInfo->GetDeleteStatus() == NzbInfo::dsScan || + (nzbInfo->GetDeleteStatus() == NzbInfo::dsCopy && downloadDupe))) { g_DupeCoordinator->NzbCompleted(downloadQueue, nzbInfo); needSave = true; diff --git a/daemon/queue/DiskState.cpp b/daemon/queue/DiskState.cpp index 57c9a980..d2c734b5 100644 --- a/daemon/queue/DiskState.cpp +++ b/daemon/queue/DiskState.cpp @@ -1,7 +1,7 @@ /* * This file is part of nzbget. See . * - * Copyright (C) 2007-2017 Andrey Prygunkov + * Copyright (C) 2007-2018 Andrey Prygunkov * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -27,7 +27,7 @@ #include "FileSystem.h" static const char* FORMATVERSION_SIGNATURE = "nzbget diskstate file version "; -const int DISKSTATE_QUEUE_VERSION = 60; +const int DISKSTATE_QUEUE_VERSION = 61; const int DISKSTATE_FILE_VERSION = 5; const int DISKSTATE_STATS_VERSION = 3; const int DISKSTATE_FEEDS_VERSION = 3; @@ -464,7 +464,7 @@ void DiskState::SaveNzbInfo(NzbInfo* nzbInfo, StateDiskFile& outfile) outfile.PrintLine("%i,%i,%i", nzbInfo->GetTotalArticles(), nzbInfo->GetSuccessArticles(), nzbInfo->GetFailedArticles()); outfile.PrintLine("%s", nzbInfo->GetDupeKey()); - outfile.PrintLine("%i,%i", (int)nzbInfo->GetDupeMode(), nzbInfo->GetDupeScore()); + outfile.PrintLine("%i,%i,%i", (int)nzbInfo->GetDupeMode(), nzbInfo->GetDupeScore(), (int)nzbInfo->GetDupeHint()); Util::SplitInt64(nzbInfo->GetDownloadedSize(), &High1, &Low1); outfile.PrintLine("%u,%u,%i,%i,%i,%i,%i", High1, Low1, nzbInfo->GetDownloadSec(), nzbInfo->GetPostTotalSec(), @@ -690,10 +690,19 @@ bool DiskState::LoadNzbInfo(NzbInfo* nzbInfo, Servers* servers, StateDiskFile& i if (!infile.ReadLine(buf, sizeof(buf))) goto error; nzbInfo->SetDupeKey(buf); - int dupeMode, dupeScore; - if (infile.ScanLine("%i,%i", &dupeMode, &dupeScore) != 2) goto error; + int dupeMode, dupeScore, dupeHint; + dupeHint = 0; //clang requires initialization in a separate line (due to goto statements) + if (formatVersion >= 61) + { + if (infile.ScanLine("%i,%i,%i", &dupeMode, &dupeScore, &dupeHint) != 3) goto error; + } + else + { + if (infile.ScanLine("%i,%i", &dupeMode, &dupeScore) != 2) goto error; + } nzbInfo->SetDupeMode((EDupeMode)dupeMode); nzbInfo->SetDupeScore(dupeScore); + nzbInfo->SetDupeMode((EDupeMode)dupeHint); if (formatVersion >= 48) { diff --git a/daemon/queue/DownloadInfo.h b/daemon/queue/DownloadInfo.h index 55130dcc..c650c05a 100644 --- a/daemon/queue/DownloadInfo.h +++ b/daemon/queue/DownloadInfo.h @@ -2,7 +2,7 @@ * This file is part of nzbget. See . * * Copyright (C) 2004 Sven Henkel - * Copyright (C) 2007-2017 Andrey Prygunkov + * Copyright (C) 2007-2018 Andrey Prygunkov * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -446,6 +446,13 @@ public: nkUrl }; + enum EDupeHint + { + dhNone, + dhRedownloadManual, + dhRedownloadAuto + }; + int GetId() { return m_id; } void SetId(int id); static void ResetGenId(bool max); @@ -580,6 +587,8 @@ public: void SetDupeScore(int dupeScore) { m_dupeScore = dupeScore; } EDupeMode GetDupeMode() { return m_dupeMode; } void SetDupeMode(EDupeMode dupeMode) { m_dupeMode = dupeMode; } + EDupeHint GetDupeHint() { return m_dupeHint; } + void SetDupeHint(EDupeHint dupeHint) { m_dupeHint = dupeHint; } uint32 GetFullContentHash() { return m_fullContentHash; } void SetFullContentHash(uint32 fullContentHash) { m_fullContentHash = fullContentHash; } uint32 GetFilteredContentHash() { return m_filteredContentHash; } @@ -698,6 +707,7 @@ private: CString m_dupeKey = ""; int m_dupeScore = 0; EDupeMode m_dupeMode = dmScore; + EDupeHint m_dupeHint = dhNone; uint32 m_fullContentHash = 0; uint32 m_filteredContentHash = 0; FileList m_fileList; diff --git a/daemon/queue/DupeCoordinator.cpp b/daemon/queue/DupeCoordinator.cpp index d119b75a..8c36298d 100644 --- a/daemon/queue/DupeCoordinator.cpp +++ b/daemon/queue/DupeCoordinator.cpp @@ -202,6 +202,12 @@ void DupeCoordinator::NzbFound(DownloadQueue* downloadQueue, NzbInfo* nzbInfo) } } + if (!sameContent && nzbInfo->GetDupeHint() != NzbInfo::dhNone) + { + // dupe check when "download again" URLs: checking same content only + return; + } + if (!sameContent && !good && nzbInfo->GetDupeMode() == dmScore) { // nzb-files having success-duplicates in recent history (with different content) are added to history for backup @@ -238,7 +244,7 @@ void DupeCoordinator::NzbFound(DownloadQueue* downloadQueue, NzbInfo* nzbInfo) sameContent ? "exactly same content" : good ? "good status" : "success status"); } - if (nzbInfo->GetFeedId()) + if (nzbInfo->GetFeedId() && nzbInfo->GetDupeHint() == NzbInfo::dhNone) { warn("%s", *message); // Flag saying QueueCoordinator to skip nzb-file @@ -398,6 +404,7 @@ void DupeCoordinator::ReturnBestDupe(DownloadQueue* downloadQueue, NzbInfo* nzbI if (historyDupe) { info("Found duplicate %s for %s", historyDupe->GetNzbInfo()->GetName(), nzbName); + historyDupe->GetNzbInfo()->SetDupeHint(NzbInfo::dhRedownloadAuto); g_HistoryCoordinator->Redownload(downloadQueue, historyDupe); } } diff --git a/daemon/queue/HistoryCoordinator.cpp b/daemon/queue/HistoryCoordinator.cpp index c72c5cea..3e4ee73a 100644 --- a/daemon/queue/HistoryCoordinator.cpp +++ b/daemon/queue/HistoryCoordinator.cpp @@ -1,7 +1,7 @@ /* * This file is part of nzbget. See . * - * Copyright (C) 2007-2017 Andrey Prygunkov + * Copyright (C) 2007-2018 Andrey Prygunkov * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -183,6 +183,8 @@ void HistoryCoordinator::AddToHistory(DownloadQueue* downloadQueue, NzbInfo* nzb nzbInfo->SetDirectRenameStatus(NzbInfo::tsFailure); } + nzbInfo->SetDupeHint(NzbInfo::dhNone); + nzbInfo->PrintMessage(Message::mkInfo, "Collection %s added to history", nzbInfo->GetName()); } @@ -433,6 +435,7 @@ void HistoryCoordinator::HistoryRedownload(DownloadQueue* downloadQueue, History historyInfo->DiscardNzbInfo(); nzbInfo->SetUrlStatus(NzbInfo::lsNone); nzbInfo->SetDeleteStatus(NzbInfo::dsNone); + nzbInfo->SetDupeHint(nzbInfo->GetDupeHint() == NzbInfo::dhNone ? NzbInfo::dhRedownloadManual : nzbInfo->GetDupeHint()); downloadQueue->GetQueue()->Add(std::unique_ptr(nzbInfo), true); downloadQueue->GetHistory()->erase(itHistory); return; diff --git a/daemon/queue/Scanner.cpp b/daemon/queue/Scanner.cpp index 5f1bc2f6..99fadb6d 100644 --- a/daemon/queue/Scanner.cpp +++ b/daemon/queue/Scanner.cpp @@ -1,7 +1,7 @@ /* * This file is part of nzbget. See . * - * Copyright (C) 2007-2016 Andrey Prygunkov + * Copyright (C) 2007-2018 Andrey Prygunkov * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -438,6 +438,7 @@ bool Scanner::AddFileToQueue(const char* filename, const char* nzbName, const ch nzbInfo->SetUrl(urlInfo->GetUrl()); nzbInfo->SetUrlStatus(urlInfo->GetUrlStatus()); nzbInfo->SetFeedId(urlInfo->GetFeedId()); + nzbInfo->SetDupeHint(urlInfo->GetDupeHint()); } if (nzbFile.GetPassword())