diff --git a/ArticleDownloader.cpp b/ArticleDownloader.cpp index 437419d1..41f27f89 100644 --- a/ArticleDownloader.cpp +++ b/ArticleDownloader.cpp @@ -68,6 +68,7 @@ ArticleDownloader::ArticleDownloader() m_pConnection = NULL; m_eStatus = adUndefined; m_iBytes = 0; + m_bDuplicate = false; memset(&m_tStartTime, 0, sizeof(m_tStartTime)); SetLastUpdateTimeNow(); } @@ -253,6 +254,11 @@ void ArticleDownloader::Run() free(LevelStatus); + if (m_bDuplicate) + { + Status = adFinished; + } + if (Status != adFinished) { Status = adFailed; @@ -290,7 +296,7 @@ ArticleDownloader::EStatus ArticleDownloader::Download() if (!grpchanged) { - if (!m_pConnection->GetAuthError()) + if (!m_pConnection->GetAuthError() && !IsStopped()) { warn("Article %s @ %s failed: Could not join group", m_szInfoName, m_pConnection->GetServer()->GetHost()); } @@ -317,7 +323,7 @@ ArticleDownloader::EStatus ArticleDownloader::Download() if (!answer) { - if (!m_pConnection->GetAuthError()) + if (!m_pConnection->GetAuthError() && !IsStopped()) { warn("Article %s @ %s failed: Connection closed by remote host", m_szInfoName, m_pConnection->GetServer()->GetHost()); } @@ -427,7 +433,100 @@ ArticleDownloader::EStatus ArticleDownloader::Download() bool ArticleDownloader::Write(char* szLine, int iLen) { - if (!m_pOutFile && !g_pOptions->GetDirectWrite()) + if (!m_pOutFile && !PrepareFile(szLine)) + { + return false; + } + + m_iBytes += iLen; + + if (g_pOptions->GetDecoder() == Options::dcYenc) + { + return m_YDecoder.Write(szLine, m_pOutFile); + } + else + { + return fwrite(szLine, 1, iLen, m_pOutFile) > 0; + } +} + +bool ArticleDownloader::PrepareFile(char* szLine) +{ + // prepare file for writing + if (g_pOptions->GetDecoder() == Options::dcYenc) + { + if (!strncmp(szLine, "=ybegin part=", 13)) + { + if (g_pOptions->GetDupeCheck()) + { + m_pFileInfo->LockOutputFile(); + if (!m_pFileInfo->GetOutputInitialized()) + { + char* pb = strstr(szLine, "name="); + if (pb) + { + pb += 5; //=strlen("name=") + char* pe; + for (pe = pb; *pe != '\0' && *pe != '\n' && *pe != '\r'; pe++) ; + if (!m_szArticleFilename) + { + m_szArticleFilename = (char*)malloc(pe - pb + 1); + strncpy(m_szArticleFilename, pb, pe - pb); + m_szArticleFilename[pe - pb] = '\0'; + } + if (m_pFileInfo->IsDupe(m_szArticleFilename)) + { + m_bDuplicate = true; + return false; + } + } + } + if (!g_pOptions->GetDirectWrite()) + { + m_pFileInfo->SetOutputInitialized(true); + } + m_pFileInfo->UnlockOutputFile(); + } + + if (g_pOptions->GetDirectWrite()) + { + char* pb = strstr(szLine, "size="); + if (pb) + { + m_pFileInfo->LockOutputFile(); + if (!m_pFileInfo->GetOutputInitialized()) + { + pb += 5; //=strlen("size=") + long iArticleFilesize = atol(pb); + if (!SetFileSize(m_szOutputFilename, iArticleFilesize)) + { + error("Could not create file %s!", m_szOutputFilename); + return false; + } + m_pFileInfo->SetOutputInitialized(true); + } + m_pFileInfo->UnlockOutputFile(); + + m_pOutFile = fopen(m_szOutputFilename, "r+"); + if (!m_pOutFile) + { + error("Could not open file %s", m_szOutputFilename); + return false; + } + } + } + else + { + m_pOutFile = fopen(m_szTempFilename, "w"); + if (!m_pOutFile) + { + error("Could not create file %s", m_szTempFilename); + return false; + } + } + } + } + else { m_pOutFile = fopen(m_szTempFilename, "w"); if (!m_pOutFile) @@ -436,51 +535,7 @@ bool ArticleDownloader::Write(char* szLine, int iLen) return false; } } - - if (!m_pOutFile && g_pOptions->GetDirectWrite()) - { - if (!strncmp(szLine, "=ybegin part=", 13)) - { - char* pb = strstr(szLine, "size="); - if (pb) - { - m_pFileInfo->LockOutputFile(); - if (!m_pFileInfo->GetOutputInitialized()) - { - pb += 5; //=strlen("size=") - long iArticleFilesize = atol(pb); - if (!SetFileSize(m_szOutputFilename, iArticleFilesize)) - { - error("Could not create file %s!", m_szOutputFilename); - return false; - } - m_pFileInfo->SetOutputInitialized(true); - } - m_pFileInfo->UnlockOutputFile(); - - m_pOutFile = fopen(m_szOutputFilename, "r+"); - if (!m_pOutFile) - { - error("Could not open file %s", m_szOutputFilename); - return false; - } - } - } - } - - bool bOK = false; - - if (g_pOptions->GetDecoder() == Options::dcYenc) - { - bOK = m_YDecoder.Write(szLine, m_pOutFile); - } - else - { - bOK = fwrite(szLine, 1, iLen, m_pOutFile) > 0; - } - - m_iBytes += iLen; - return bOK; + return true; } ArticleDownloader::EStatus ArticleDownloader::Decode() @@ -489,8 +544,6 @@ ArticleDownloader::EStatus ArticleDownloader::Decode() (g_pOptions->GetDecoder() == Options::dcYenc)) { SetStatus(adDecoding); - struct _timeval StartTime, EndTime; - gettimeofday(&StartTime, 0); char tmpdestfile[1024]; char* szDecoderTempFilename = NULL; @@ -530,13 +583,7 @@ ArticleDownloader::EStatus ArticleDownloader::Decode() m_szArticleFilename = strdup(pDecoder->GetArticleFilename()); } - gettimeofday(&EndTime, 0); remove(m_szTempFilename); -#ifdef WIN32 - float fDeltaTime = (float)((EndTime.time - StartTime.time) * 1000 + (EndTime.millitm - StartTime.millitm)); -#else - float fDeltaTime = ((EndTime.tv_sec - StartTime.tv_sec) * 1000000 + (EndTime.tv_usec - StartTime.tv_usec)) / 1000.0; -#endif bool bCrcError = pDecoder->GetCrcError(); if (pDecoder != &m_YDecoder) { @@ -546,7 +593,6 @@ ArticleDownloader::EStatus ArticleDownloader::Decode() if (bOK) { info("Successfully downloaded %s", m_szInfoName); - debug("Decode time %.1f ms", fDeltaTime); if (g_pOptions->GetDirectWrite() && g_pOptions->GetContinuePartial()) { @@ -617,9 +663,23 @@ void ArticleDownloader::Stop() debug("ArticleDownloader stopped successfully"); } +bool ArticleDownloader::Terminate() +{ + NNTPConnection* pConnection = m_pConnection; + bool terminated = Kill(); + if (terminated && pConnection) + { + debug("Terminating connection"); + pConnection->Cancel(); + pConnection->Disconnect(); + g_pServerPool->FreeConnection(pConnection, true); + } + return terminated; +} + void ArticleDownloader::FreeConnection(bool bKeepConnected) { - if (m_pConnection) + if (m_pConnection) { debug("Releasing connection"); m_mutexConnection.Lock(); @@ -827,17 +887,3 @@ void ArticleDownloader::CompleteFileParts() SetStatus(adFinished); } - -bool ArticleDownloader::Terminate() -{ - NNTPConnection* pConnection = m_pConnection; - bool terminated = Kill(); - if (terminated && pConnection) - { - debug("Terminating connection"); - pConnection->Cancel(); - pConnection->Disconnect(); - g_pServerPool->FreeConnection(pConnection, true); - } - return terminated; -} diff --git a/ArticleDownloader.h b/ArticleDownloader.h index 1fd757eb..ee7c8019 100644 --- a/ArticleDownloader.h +++ b/ArticleDownloader.h @@ -72,9 +72,11 @@ private: int m_iBytes; YDecoder m_YDecoder; FILE* m_pOutFile; + bool m_bDuplicate; EStatus Download(); bool Write(char* szLine, int iLen); + bool PrepareFile(char* szLine); EStatus Decode(); void FreeConnection(bool bKeepConnected); diff --git a/Connection.cpp b/Connection.cpp index 33c68268..d51b2dec 100644 --- a/Connection.cpp +++ b/Connection.cpp @@ -88,7 +88,6 @@ Connection::Connection(NetAddress* pNetAddress) m_eStatus = csDisconnected; m_iSocket = INVALID_SOCKET; m_iBufAvail = 0; - m_bCanceling = false; m_iTimeout = 60; m_szReadBuf = (char*)malloc(CONNECTION_READBUFFER_SIZE + 1); } @@ -131,7 +130,6 @@ int Connection::Disconnect() int iRes = DoDisconnect(); m_eStatus = csDisconnected; - m_bCanceling = false; m_iSocket = INVALID_SOCKET; m_iBufAvail = 0; @@ -165,9 +163,6 @@ int Connection::WriteLine(char* line) int iRes = DoWriteLine(line); - if (iRes == EOF) - Connection::DoDisconnect(); - return iRes; } @@ -194,9 +189,6 @@ char* Connection::ReadLine(char* pBuffer, int iSize, int* pBytesRead) char* res = DoReadLine(pBuffer, iSize, pBytesRead); - if (res == NULL) - Connection::DoDisconnect(); - return res; } @@ -476,7 +468,7 @@ SOCKET Connection::DoAccept() SOCKET iSocket = accept(GetSocket(), (struct sockaddr *) & ClientAddress, &SockLen); - if (iSocket == INVALID_SOCKET && !m_bCanceling) + if (iSocket == INVALID_SOCKET && m_eStatus != csCancelled) { ReportError("Could not accept connection", NULL, 0); } @@ -487,15 +479,14 @@ SOCKET Connection::DoAccept() void Connection::Cancel() { debug("Cancelling connection"); - m_bCanceling = true; if (m_iSocket != INVALID_SOCKET) { + m_eStatus = csCancelled; int r = shutdown(m_iSocket, SHUT_RDWR); if (r == -1) { ReportError("Could not shutdown connection", NULL, 0); } - m_eStatus = csCancelled; } } diff --git a/Connection.h b/Connection.h index 8264da3d..07e880e2 100644 --- a/Connection.h +++ b/Connection.h @@ -47,7 +47,6 @@ protected: int m_iBufAvail; char* m_szBufPtr; EStatus m_eStatus; - bool m_bCanceling; int m_iTimeout; unsigned int ResolveHostAddr(const char* szHost); void ReportError(const char* szMsgPrefix, const char* szMsgArg, int ErrCode); diff --git a/Decoder.cpp b/Decoder.cpp index 32f1ffec..f19fcb86 100644 --- a/Decoder.cpp +++ b/Decoder.cpp @@ -191,6 +191,11 @@ void YDecoder::Clear() m_bAutoSeek = false; m_bNeedSetPos = false; m_bCrcCheck = false; + if (m_szArticleFilename) + { + free(m_szArticleFilename); + } + m_szArticleFilename = NULL; } /* from crc32.c (http://www.koders.com/c/fid699AFE0A656F0022C9D6B9D1743E697B69CE5815.aspx) @@ -320,6 +325,10 @@ BreakLoop: pb += 5; //=strlen("name=") char* pe; for (pe = pb; *pe != '\0' && *pe != '\n' && *pe != '\r'; pe++) ; + if (m_szArticleFilename) + { + free(m_szArticleFilename); + } m_szArticleFilename = (char*)malloc(pe - pb + 1); strncpy(m_szArticleFilename, pb, pe - pb); m_szArticleFilename[pe - pb] = '\0'; diff --git a/DownloadInfo.cpp b/DownloadInfo.cpp index 50c0ad80..fd8585af 100644 --- a/DownloadInfo.cpp +++ b/DownloadInfo.cpp @@ -34,6 +34,7 @@ #include #include +#include #include "nzbget.h" #include "DownloadInfo.h" @@ -211,3 +212,25 @@ void FileInfo::UnlockOutputFile() { m_mutexOutputFile.Unlock(); } + +bool FileInfo::IsDupe(const char* szFilename) +{ + struct stat buffer; + char fileName[1024]; + snprintf(fileName, 1024, "%s%c%s", m_szDestDir, (int)PATH_SEPARATOR, szFilename); + fileName[1024-1] = '\0'; + bool exists = !stat(fileName, &buffer); + if (exists) + { + return true; + } + snprintf(fileName, 1024, "%s%c%s_broken", m_szDestDir, (int)PATH_SEPARATOR, szFilename); + fileName[1024-1] = '\0'; + exists = !stat(fileName, &buffer); + if (exists) + { + return true; + } + + return false; +} diff --git a/DownloadInfo.h b/DownloadInfo.h index 8a033cd9..d2cf208f 100644 --- a/DownloadInfo.h +++ b/DownloadInfo.h @@ -125,6 +125,7 @@ public: void UnlockOutputFile(); bool GetOutputInitialized() { return m_bOutputInitialized; } void SetOutputInitialized(bool bOutputInitialized) { m_bOutputInitialized = bOutputInitialized; } + bool IsDupe(const char* szFilename); }; typedef std::deque DownloadQueue; diff --git a/NNTPConnection.cpp b/NNTPConnection.cpp index 1d386792..35971955 100644 --- a/NNTPConnection.cpp +++ b/NNTPConnection.cpp @@ -145,7 +145,10 @@ bool NNTPConnection::AuthInfoUser(int iRecur) if (char* p = strrchr(answer, '\r')) *p = '\0'; // remove last CRLF from error message - error("authorization for %s failed (Answer: %s)", m_pNetAddress->GetHost(), answer); + if (GetStatus() != csCancelled) + { + error("authorization for %s failed (Answer: %s)", m_pNetAddress->GetHost(), answer); + } return false; } @@ -180,7 +183,10 @@ bool NNTPConnection::AuthInfoPass(int iRecur) if (char* p = strrchr(answer, '\r')) *p = '\0'; // remove last CRLF from error message - error("authorization for %s failed (Answer: %s)", m_pNetAddress->GetHost(), answer); + if (GetStatus() != csCancelled) + { + error("authorization for %s failed (Answer: %s)", m_pNetAddress->GetHost(), answer); + } return false; } @@ -214,15 +220,18 @@ bool NNTPConnection::JoinGroup(char* grp) return true; } - if (!answer) + if (GetStatus() != csCancelled) { - warn("Error changing group on %s: Connection closed by remote host.", - GetServer()->GetHost()); - } - else - { - warn("Error changing group on %s to %s: Answer was \"%s\".", - GetServer()->GetHost(), grp, answer); + if (!answer) + { + warn("Error changing group on %s: Connection closed by remote host.", + GetServer()->GetHost()); + } + else + { + warn("Error changing group on %s to %s: Answer was \"%s\".", + GetServer()->GetHost(), grp, answer); + } } return false; diff --git a/QueueCoordinator.cpp b/QueueCoordinator.cpp index 4b60abf5..9b6e1099 100644 --- a/QueueCoordinator.cpp +++ b/QueueCoordinator.cpp @@ -483,11 +483,17 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* pArticleDownloader) bool fileCompleted = (int)pFileInfo->GetArticles()->size() == pFileInfo->GetCompleted(); if (!pFileInfo->GetFilenameConfirmed() && - pArticleDownloader->GetStatus() == ArticleDownloader::adFinished && - pArticleDownloader->GetArticleFilename()) + pArticleDownloader->GetStatus() == ArticleDownloader::adFinished && + pArticleDownloader->GetArticleFilename()) { pFileInfo->SetFilename(pArticleDownloader->GetArticleFilename()); pFileInfo->SetFilenameConfirmed(true); + if (g_pOptions->GetDupeCheck() && IsDupe(pFileInfo)) + { + warn("File \"%s\" seems to be duplicate, cancelling download and deleting file from queue", pFileInfo->GetFilename()); + fileCompleted = false; + DeleteQueueEntry(pFileInfo); + } } bool deleteFileObj = false; @@ -567,19 +573,7 @@ bool QueueCoordinator::IsDupe(FileInfo* pFileInfo) debug("Checking if the file is already queued"); // checking on disk - struct stat buffer; - char fileName[1024]; - snprintf(fileName, 1024, "%s%c%s", pFileInfo->GetDestDir(), (int)PATH_SEPARATOR, pFileInfo->GetFilename()); - fileName[1024-1] = '\0'; - bool exists = !stat(fileName, &buffer); - if (exists) - { - return true; - } - snprintf(fileName, 1024, "%s%c%s_broken", pFileInfo->GetDestDir(), (int)PATH_SEPARATOR, pFileInfo->GetFilename()); - fileName[1024-1] = '\0'; - exists = !stat(fileName, &buffer); - if (exists) + if (pFileInfo->IsDupe(pFileInfo->GetFilename())) { return true; } @@ -589,7 +583,8 @@ bool QueueCoordinator::IsDupe(FileInfo* pFileInfo) { FileInfo* pQueueEntry = *it; if (!strcmp(pFileInfo->GetDestDir(), pQueueEntry->GetDestDir()) && - !strcmp(pFileInfo->GetFilename(), pQueueEntry->GetFilename())) + !strcmp(pFileInfo->GetFilename(), pQueueEntry->GetFilename()) && + pFileInfo != pQueueEntry) { return true; }