From b85a36944d55faaff335e1b6fb847d0774c05bd0 Mon Sep 17 00:00:00 2001 From: Andrey Prygunkov Date: Mon, 31 Dec 2007 16:30:47 +0000 Subject: [PATCH] implemented decode-on-the-fly-technique to reduce disk-io; intermediate files with articles' source text are not created anymore, but only intermediate files with decoded data; futher, decoder can write decoded data directly to the destination file (without any intermediate files at all), this eliminates the necessity of joining of articles later (option ) --- ArticleDownloader.cpp | 250 ++++++++++++++++++++++++++++++---------- ArticleDownloader.h | 8 ++ Decoder.cpp | 261 +++++++++++++++++++++++------------------- Decoder.h | 56 ++++++--- DiskState.cpp | 9 +- DownloadInfo.cpp | 11 ++ DownloadInfo.h | 8 ++ Options.cpp | 87 ++++++++------ Options.h | 2 + QueueCoordinator.cpp | 17 ++- Util.cpp | 45 ++++++++ Util.h | 1 + nzbget.conf.example | 19 ++- 13 files changed, 531 insertions(+), 243 deletions(-) diff --git a/ArticleDownloader.cpp b/ArticleDownloader.cpp index cb748066..e91de8db 100644 --- a/ArticleDownloader.cpp +++ b/ArticleDownloader.cpp @@ -64,6 +64,7 @@ ArticleDownloader::ArticleDownloader() m_szTempFilename = NULL; m_szArticleFilename = NULL; m_szInfoName = NULL; + m_szOutputFilename = NULL; m_pConnection = NULL; m_eStatus = adUndefined; m_iBytes = 0; @@ -87,6 +88,10 @@ ArticleDownloader::~ArticleDownloader() { free(m_szInfoName); } + if (m_szOutputFilename) + { + free(m_szOutputFilename); + } } void ArticleDownloader::SetTempFilename(const char* v) @@ -94,6 +99,11 @@ void ArticleDownloader::SetTempFilename(const char* v) m_szTempFilename = strdup(v); } +void ArticleDownloader::SetOutputFilename(const char* v) +{ + m_szOutputFilename = strdup(v); +} + void ArticleDownloader::SetInfoName(const char * v) { m_szInfoName = strdup(v); @@ -316,18 +326,15 @@ ArticleDownloader::EStatus ArticleDownloader::Download() // positive answer! - const char* dnfilename = m_szTempFilename; - FILE* outfile = fopen(dnfilename, "w"); - - if (!outfile) + if (g_pOptions->GetDecoder() == Options::dcYenc) { - error("Could not create file %s", dnfilename); - return adFatalError; + m_YDecoder.Clear(); + m_YDecoder.SetAutoSeek(g_pOptions->GetDirectWrite()); } gettimeofday(&m_tStartTime, 0); m_iBytes = 0; - + m_pOutFile = NULL; EStatus Status = adRunning; const int LineBufSize = 1024*10; char* szLineBuf = (char*)malloc(LineBufSize); @@ -374,32 +381,106 @@ ArticleDownloader::EStatus ArticleDownloader::Download() line++; } - int wrcnt = (int)fwrite(line, 1, strlen(line), outfile); - if (wrcnt > 0) + if (!Write(line)) { - m_iBytes += wrcnt; + Status = adFatalError; + break; } } free(szLineBuf); - fflush(outfile); - fclose(outfile); + + if (m_pOutFile) + { + fflush(m_pOutFile); + fclose(m_pOutFile); + } if (IsStopped()) { - remove(dnfilename); + remove(m_szTempFilename); return adFailed; } if (Status == adFailed) { warn("Unexpected end of %s", m_szInfoName); - remove(dnfilename); + remove(m_szTempFilename); + return adFailed; + } + + if (Status == adDecodeError) + { + warn("Decoding failed for %s", m_szInfoName); return adFailed; } FreeConnection(); + return Decode(); +} + +bool ArticleDownloader::Write(char* line) +{ + if (!m_pOutFile && !g_pOptions->GetDirectWrite()) + { + m_pOutFile = fopen(m_szTempFilename, "w"); + if (!m_pOutFile) + { + error("Could not create file %s", m_szTempFilename); + return false; + } + } + + if (!m_pOutFile && g_pOptions->GetDirectWrite()) + { + if (strstr(line, "=ybegin part=")) + { + char* pb = strstr(line, "size="); + if (pb) + { + m_pFileInfo->LockOutputFile(); + if (!m_pFileInfo->GetOutputInitialized()) + { + pb += 5; //=strlen("size=") + long iArticleFilesize = atol(pb); + if (!SetFileSize(m_szOutputFilename, iArticleFilesize)) + { + error("Could not create file %s!", m_szOutputFilename); + return false; + } + m_pFileInfo->SetOutputInitialized(true); + } + m_pFileInfo->UnlockOutputFile(); + + m_pOutFile = fopen(m_szOutputFilename, "r+"); + if (!m_pOutFile) + { + error("Could not open file %s", m_szOutputFilename); + return false; + } + } + } + } + + int len = strlen(line); + bool bOK = false; + + if (g_pOptions->GetDecoder() == Options::dcYenc) + { + bOK = m_YDecoder.Write(line, m_pOutFile); + } + else + { + bOK = fwrite(line, 1, len, m_pOutFile) > 0; + } + + m_iBytes += len; + return bOK; +} + +ArticleDownloader::EStatus ArticleDownloader::Decode() +{ if ((g_pOptions->GetDecoder() == Options::dcUulib) || (g_pOptions->GetDecoder() == Options::dcYenc)) { @@ -410,52 +491,80 @@ ArticleDownloader::EStatus ArticleDownloader::Download() struct _timeval StartTime, EndTime; gettimeofday(&StartTime, 0); - Decoder decoder; - if (g_pOptions->GetDecoder() == Options::dcUulib) - { - decoder.SetKind(Decoder::dcUulib); - } - else if (g_pOptions->GetDecoder() == Options::dcYenc) - { - decoder.SetKind(Decoder::dcYenc); - } - decoder.SetSrcFilename(dnfilename); char tmpdestfile[1024]; - snprintf(tmpdestfile, 1024, "%s.dec", m_szResultFilename); - tmpdestfile[1024-1] = '\0'; - decoder.SetDestFilename(tmpdestfile); + char* szDecoderTempFilename = NULL; + Decoder* pDecoder = NULL; - bool bOK = decoder.Execute(); - if (bOK) + if (g_pOptions->GetDecoder() == Options::dcYenc) { - rename(tmpdestfile, m_szResultFilename); + pDecoder = &m_YDecoder; + szDecoderTempFilename = m_szTempFilename; } - else + else if (g_pOptions->GetDecoder() == Options::dcUulib) { - remove(tmpdestfile); + pDecoder = new UULibDecoder(); + pDecoder->SetSrcFilename(m_szTempFilename); + snprintf(tmpdestfile, 1024, "%s.dec", m_szResultFilename); + tmpdestfile[1024-1] = '\0'; + szDecoderTempFilename = tmpdestfile; + pDecoder->SetDestFilename(szDecoderTempFilename); } - if (decoder.GetArticleFilename()) + + bool bOK = pDecoder->Execute(); + + if (!g_pOptions->GetDirectWrite()) { - m_szArticleFilename = strdup(decoder.GetArticleFilename()); + if (bOK) + { + rename(szDecoderTempFilename, m_szResultFilename); + } + else if (g_pOptions->GetDecoder() == Options::dcUulib) + { + remove(szDecoderTempFilename); + } + } + + if (pDecoder->GetArticleFilename()) + { + m_szArticleFilename = strdup(pDecoder->GetArticleFilename()); } gettimeofday(&EndTime, 0); - remove(dnfilename); + remove(m_szTempFilename); #ifdef WIN32 float fDeltaTime = (float)((EndTime.time - StartTime.time) * 1000 + (EndTime.millitm - StartTime.millitm)); #else float fDeltaTime = ((EndTime.tv_sec - StartTime.tv_sec) * 1000000 + (EndTime.tv_usec - StartTime.tv_usec)) / 1000.0; #endif + bool bCrcError = pDecoder->GetCrcError(); + if (pDecoder != &m_YDecoder) + { + delete pDecoder; + } + if (bOK) { info("Successfully downloaded %s", m_szInfoName); debug("Decode time %.1f ms", fDeltaTime); + + if (g_pOptions->GetDirectWrite() && g_pOptions->GetContinuePartial()) + { + // create empty flag-file to indicate that the artcile was downloaded + FILE* flagfile = fopen(m_szResultFilename, "w"); + if (!flagfile) + { + error("Could not create file %s", m_szResultFilename); + // this error can be ignored + } + fclose(flagfile); + } + return adFinished; } else { remove(m_szResultFilename); - if (decoder.GetCrcError()) + if (bCrcError) { warn("Decoding %s failed: CRC-Error", m_szInfoName); return adCrcError; @@ -470,7 +579,7 @@ ArticleDownloader::EStatus ArticleDownloader::Download() else if (g_pOptions->GetDecoder() == Options::dcNone) { // rawmode - rename(dnfilename, m_szResultFilename); + rename(m_szTempFilename, m_szResultFilename); info("Article %s successfully downloaded", m_szInfoName); return adFinished; } @@ -504,7 +613,7 @@ void ArticleDownloader::Stop() m_pConnection->Cancel(); } m_mutexConnection.Unlock(); - debug("ArticleDownloader stopped successfuly"); + debug("ArticleDownloader stopped successfully"); } void ArticleDownloader::FreeConnection() @@ -524,11 +633,8 @@ void ArticleDownloader::CompleteFileParts() { debug("Completing file parts"); debug("ArticleFilename: %s", m_pFileInfo->GetFilename()); - SetStatus(adJoining); - char ofn[1024]; - snprintf(ofn, 1024, "%s%c%s", m_pFileInfo->GetDestDir(), (int)PATH_SEPARATOR, m_pFileInfo->GetFilename()); - ofn[1024-1] = '\0'; + SetStatus(adJoining); char szNZBNiceName[1024]; m_pFileInfo->GetNiceNZBName(szNZBNiceName, 1024); @@ -537,18 +643,26 @@ void ArticleDownloader::CompleteFileParts() snprintf(InfoFilename, 1024, "%s%c%s", szNZBNiceName, (int)PATH_SEPARATOR, m_pFileInfo->GetFilename()); InfoFilename[1024-1] = '\0'; - // Ensure the DstDir is created - mkdir(m_pFileInfo->GetDestDir(), S_DIRMODE); - if (g_pOptions->GetDecoder() == Options::dcNone) { info("Moving articles for %s", InfoFilename); } + else if (g_pOptions->GetDirectWrite()) + { + info("Checking articles for %s", InfoFilename); + } else { info("Joining articles for %s", InfoFilename); } + char ofn[1024]; + snprintf(ofn, 1024, "%s%c%s", m_pFileInfo->GetDestDir(), (int)PATH_SEPARATOR, m_pFileInfo->GetFilename()); + ofn[1024-1] = '\0'; + + // Ensure the DstDir is created + mkdir(m_pFileInfo->GetDestDir(), S_DIRMODE); + // prevent overwriting existing files struct stat statbuf; int dupcount = 0; @@ -560,15 +674,15 @@ void ArticleDownloader::CompleteFileParts() } FILE* outfile = NULL; - char tmpdestfile[1024]; snprintf(tmpdestfile, 1024, "%s.tmp", ofn); tmpdestfile[1024-1] = '\0'; - remove(tmpdestfile); - if ((g_pOptions->GetDecoder() == Options::dcUulib) || - (g_pOptions->GetDecoder() == Options::dcYenc)) + if (((g_pOptions->GetDecoder() == Options::dcUulib) || + (g_pOptions->GetDecoder() == Options::dcYenc)) && + !g_pOptions->GetDirectWrite()) { + remove(tmpdestfile); outfile = fopen(tmpdestfile, "w+"); if (!outfile) { @@ -579,13 +693,21 @@ void ArticleDownloader::CompleteFileParts() } else if (g_pOptions->GetDecoder() == Options::dcNone) { + remove(tmpdestfile); mkdir(ofn, S_DIRMODE); } bool complete = true; int iBrokenCount = 0; static const int BUFFER_SIZE = 1024 * 50; - char* buffer = (char*)malloc(BUFFER_SIZE); + char* buffer = NULL; + + if (((g_pOptions->GetDecoder() == Options::dcUulib) || + (g_pOptions->GetDecoder() == Options::dcYenc)) && + !g_pOptions->GetDirectWrite()) + { + buffer = (char*)malloc(BUFFER_SIZE); + } for (FileInfo::Articles::iterator it = m_pFileInfo->GetArticles()->begin(); it != m_pFileInfo->GetArticles()->end(); it++) { @@ -595,8 +717,9 @@ void ArticleDownloader::CompleteFileParts() iBrokenCount++; complete = false; } - else if ((g_pOptions->GetDecoder() == Options::dcUulib) || - (g_pOptions->GetDecoder() == Options::dcYenc)) + else if (((g_pOptions->GetDecoder() == Options::dcUulib) || + (g_pOptions->GetDecoder() == Options::dcYenc)) && + !g_pOptions->GetDirectWrite()) { FILE* infile; const char* fn = pa->GetResultFilename(); @@ -632,19 +755,30 @@ void ArticleDownloader::CompleteFileParts() rename(fn, dstFileName); } } - free(buffer); - if ((g_pOptions->GetDecoder() == Options::dcUulib) || - (g_pOptions->GetDecoder() == Options::dcYenc)) + if (buffer) + { + free(buffer); + } + + if (outfile) { fclose(outfile); rename(tmpdestfile, ofn); } - for (FileInfo::Articles::iterator it = m_pFileInfo->GetArticles()->begin(); it != m_pFileInfo->GetArticles()->end(); it++) + if (g_pOptions->GetDirectWrite()) { - ArticleInfo* pa = *it; - remove(pa->GetResultFilename()); + rename(m_szOutputFilename, ofn); + } + + if (!g_pOptions->GetDirectWrite() || g_pOptions->GetContinuePartial()) + { + for (FileInfo::Articles::iterator it = m_pFileInfo->GetArticles()->begin(); it != m_pFileInfo->GetArticles()->end(); it++) + { + ArticleInfo* pa = *it; + remove(pa->GetResultFilename()); + } } if (complete) diff --git a/ArticleDownloader.h b/ArticleDownloader.h index 2bbf5f18..c4f90b06 100644 --- a/ArticleDownloader.h +++ b/ArticleDownloader.h @@ -36,6 +36,7 @@ #include "DownloadInfo.h" #include "Thread.h" #include "NNTPConnection.h" +#include "Decoder.h" class ArticleDownloader : public Thread, public Subject { @@ -46,6 +47,7 @@ public: adRunning, adFinished, adFailed, + adDecodeError, adCrcError, adDecoding, adJoining, @@ -63,6 +65,7 @@ private: char* m_szTempFilename; char* m_szArticleFilename; char* m_szInfoName; + char* m_szOutputFilename; time_t m_tLastUpdateTime; Semaphore m_semInitialized; Semaphore m_semWaited; @@ -73,8 +76,12 @@ private: struct timeval m_tStartTime; #endif int m_iBytes; + YDecoder m_YDecoder; + FILE* m_pOutFile; EStatus Download(); + bool Write(char* line); + EStatus Decode(); void FreeConnection(); public: @@ -94,6 +101,7 @@ public: void SetLastUpdateTimeNow() { m_tLastUpdateTime = ::time(NULL); } const char* GetTempFilename() { return m_szTempFilename; } void SetTempFilename(const char* v); + void SetOutputFilename(const char* v); const char* GetArticleFilename() { return m_szArticleFilename; } void SetInfoName(const char* v); const char* GetInfoName() { return m_szInfoName; } diff --git a/Decoder.cpp b/Decoder.cpp index 2d247307..e6dfef52 100644 --- a/Decoder.cpp +++ b/Decoder.cpp @@ -52,19 +52,8 @@ #include "Log.h" #include "Util.h" -Mutex Decoder::m_mutexDecoder; -unsigned int Decoder::crc_tab[256]; - -void Decoder::Init() -{ - debug("Initializing global decoder"); - crc32gentab(); -} - -void Decoder::Final() -{ - debug("Finalizing global Decoder"); -} +Mutex UULibDecoder::m_mutexDecoder; +unsigned int YDecoder::crc_tab[256]; Decoder::Decoder() { @@ -73,7 +62,6 @@ Decoder::Decoder() m_szSrcFilename = NULL; m_szDestFilename = NULL; m_szArticleFilename = NULL; - m_eKind = dcYenc; m_bCrcError = false; } @@ -87,19 +75,12 @@ Decoder::~ Decoder() } } -bool Decoder::Execute() -{ - if (m_eKind == dcUulib) - { - return DecodeUulib(); - } - else - { - return DecodeYenc(); - } -} -bool Decoder::DecodeUulib() +/* + * UULibDecoder + */ + +bool UULibDecoder::Execute() { bool res = false; @@ -179,104 +160,36 @@ bool Decoder::DecodeUulib() } /** + * YDecoder * Very primitive (but fast) implementation of yEnc-Decoder */ -bool Decoder::DecodeYenc() + +void YDecoder::Init() { - FILE* infile = fopen(m_szSrcFilename, "r"); - if (!infile) - { - error("Could not open file \"%s\"", m_szSrcFilename); - return false; - } + debug("Initializing global decoder"); + crc32gentab(); +} - FILE* outfile = fopen(m_szDestFilename, "w"); - if (!outfile) - { - error("Could not create file \"%s\"", m_szDestFilename); - fclose(infile); - return false; - } +void YDecoder::Final() +{ + debug("Finalizing global Decoder"); +} - static const int MAX_LINE_LEN = 1024; - char buffer[MAX_LINE_LEN]; - bool body = false; - bool end = false; - unsigned long expectedCRC = 0; - unsigned long calculatedCRC = 0xFFFFFFFF; - bool eof = !fgets(buffer, sizeof(buffer), infile); - while (!eof) - { - if (body) - { - if (strstr(buffer, "=yend size=")) - { - end = true; - char* pc = strstr(buffer, "pcrc32="); - if (pc) - { - pc += 7; //=strlen("pcrc32=") - expectedCRC = strtoul(pc, NULL, 16); - } - break; - } - char* iptr = buffer; - char* optr = buffer; - while (*iptr) - { - switch (*iptr) - { - case '=': //escape-sequence - iptr++; - *optr = *iptr - 64 - 42; - *optr++; - break; - case '\n': // ignored char - case '\r': // ignored char - break; - default: // normal char - *optr = *iptr - 42; - *optr++; - break; - } - iptr++; - } - calculatedCRC = crc32m(calculatedCRC, (unsigned char *)buffer, optr - buffer); - fwrite(buffer, 1, optr - buffer, outfile); - } - else - { - if (strstr(buffer, "=ypart begin=")) - { - body = true; - } - else if (strstr(buffer, "=ybegin part=")) - { - char* pb = strstr(buffer, "name="); - if (pb) - { - pb += 5; //=strlen("name=") - char* pe; - for (pe = pb; *pe != '\0' && *pe != '\n' && *pe != '\r'; pe++) ; - m_szArticleFilename = (char*)malloc(pe - pb + 1); - strncpy(m_szArticleFilename, pb, pe - pb); - m_szArticleFilename[pe - pb] = '\0'; - } - } - } - eof = !fgets(buffer, sizeof(buffer), infile); - } +YDecoder::YDecoder() +{ + Clear(); +} - calculatedCRC ^= 0xFFFFFFFF; - - debug("Expected pcrc32=%x", expectedCRC); - debug("Calculated pcrc32=%x", calculatedCRC); - m_bCrcError = expectedCRC != calculatedCRC; - - fclose(infile); - fclose(outfile); - - return body && end && !m_bCrcError; +void YDecoder::Clear() +{ + m_bBody = false; + m_bEnd = false; + m_lExpectedCRC = 0; + m_lCalculatedCRC = 0xFFFFFFFF; + m_iBegin = 0; + m_iEnd = 0; + m_bAutoSeek = false; + m_bNeedSetPos = false; } /* from crc32.c (http://www.koders.com/c/fid699AFE0A656F0022C9D6B9D1743E697B69CE5815.aspx) @@ -289,7 +202,7 @@ bool Decoder::DecodeYenc() * calculate the crcTable for crc32-checksums. * it is generated to the polynom [..] */ -void Decoder::crc32gentab() +void YDecoder::crc32gentab() { unsigned long crc, poly; int i, j; @@ -323,7 +236,7 @@ void Decoder::crc32gentab() * reached. the crc32-checksum will be * the result. */ -unsigned long Decoder::crc32m(unsigned long startCrc, unsigned char *block, unsigned int length) +unsigned long YDecoder::crc32m(unsigned long startCrc, unsigned char *block, unsigned int length) { register unsigned long crc; unsigned long i; @@ -335,3 +248,111 @@ unsigned long Decoder::crc32m(unsigned long startCrc, unsigned char *block, unsi } return crc; } + +unsigned int YDecoder::DecodeBuffer(char* buffer) +{ + if (m_bBody) + { + if (strstr(buffer, "=yend size=")) + { + m_bEnd = true; + char* pc = strstr(buffer, "pcrc32="); + if (pc) + { + pc += 7; //=strlen("pcrc32=") + m_lExpectedCRC = strtoul(pc, NULL, 16); + } + return 0; + } + char* iptr = buffer; + char* optr = buffer; + while (*iptr) + { + switch (*iptr) + { + case '=': //escape-sequence + iptr++; + *optr = *iptr - 64 - 42; + *optr++; + break; + case '\n': // ignored char + case '\r': // ignored char + break; + default: // normal char + *optr = *iptr - 42; + *optr++; + break; + } + iptr++; + } + m_lCalculatedCRC = crc32m(m_lCalculatedCRC, (unsigned char *)buffer, optr - buffer); + return optr - buffer; + } + else + { + if (strstr(buffer, "=ypart begin=")) + { + m_bBody = true; + char* pb = strstr(buffer, "begin="); + if (pb) + { + pb += 6; //=strlen("begin=") + m_iBegin = (int)atoi(pb); + } + pb = strstr(buffer, "end="); + if (pb) + { + pb += 4; //=strlen("end=") + m_iEnd = (int)atoi(pb); + } + } + else if (strstr(buffer, "=ybegin part=")) + { + char* pb = strstr(buffer, "name="); + if (pb) + { + pb += 5; //=strlen("name=") + char* pe; + for (pe = pb; *pe != '\0' && *pe != '\n' && *pe != '\r'; pe++) ; + m_szArticleFilename = (char*)malloc(pe - pb + 1); + strncpy(m_szArticleFilename, pb, pe - pb); + m_szArticleFilename[pe - pb] = '\0'; + } + } + } + + return 0; +} + +bool YDecoder::Write(char* buffer, FILE* outfile) +{ + unsigned int wcnt = DecodeBuffer(buffer); + if (wcnt > 0) + { + if (m_bNeedSetPos) + { + if (m_iBegin == 0 || m_iEnd == 0 || !outfile) + { + return false; + } + if (fseek(outfile, m_iBegin - 1, SEEK_SET)) + { + return false; + } + m_bNeedSetPos = false; + } + fwrite(buffer, 1, wcnt, outfile); + } + return true; +} + +bool YDecoder::Execute() +{ + m_lCalculatedCRC ^= 0xFFFFFFFF; + + debug("Expected pcrc32=%x", m_lExpectedCRC); + debug("Calculated pcrc32=%x", m_lCalculatedCRC); + m_bCrcError = m_lExpectedCRC != m_lCalculatedCRC; + + return m_bBody && m_bEnd && !m_bCrcError; +} diff --git a/Decoder.h b/Decoder.h index 7d79ad3e..ffdc64d7 100644 --- a/Decoder.h +++ b/Decoder.h @@ -31,36 +31,54 @@ class Decoder { -public: - enum EKind - { - dcUulib, - dcYenc - }; - -private: - static Mutex m_mutexDecoder; - static unsigned int crc_tab[256]; - EKind m_eKind; +protected: const char* m_szSrcFilename; const char* m_szDestFilename; char* m_szArticleFilename; bool m_bCrcError; - bool DecodeUulib(); - bool DecodeYenc(); - static void crc32gentab(); - unsigned long crc32m(unsigned long startCrc, unsigned char *block, unsigned int length); - public: Decoder(); - ~Decoder(); - bool Execute(); - void SetKind(EKind eKind) { m_eKind = eKind; } + virtual ~Decoder(); + virtual bool Execute() = 0; void SetSrcFilename(const char* szSrcFilename) { m_szSrcFilename = szSrcFilename; } void SetDestFilename(const char* szDestFilename) { m_szDestFilename = szDestFilename; } const char* GetArticleFilename() { return m_szArticleFilename; } bool GetCrcError() { return m_bCrcError; } +}; + +class UULibDecoder: public Decoder +{ +private: + static Mutex m_mutexDecoder; + +public: + virtual bool Execute(); +}; + +class YDecoder: public Decoder +{ +protected: + static unsigned int crc_tab[256]; + bool m_bBody; + bool m_bEnd; + unsigned long m_lExpectedCRC; + unsigned long m_lCalculatedCRC; + unsigned long m_iBegin; + unsigned long m_iEnd; + bool m_bAutoSeek; + bool m_bNeedSetPos; + + unsigned int DecodeBuffer(char* buffer); + static void crc32gentab(); + unsigned long crc32m(unsigned long startCrc, unsigned char *block, unsigned int length); + +public: + YDecoder(); + virtual bool Execute(); + void Clear(); + bool Write(char* buffer, FILE* outfile); + void SetAutoSeek(bool bAutoSeek) { m_bAutoSeek = m_bNeedSetPos = bAutoSeek; } static void Init(); static void Final(); diff --git a/DiskState.cpp b/DiskState.cpp index def00f05..1f2e2e2f 100644 --- a/DiskState.cpp +++ b/DiskState.cpp @@ -361,11 +361,14 @@ void DiskState::CleanupTempDir(DownloadQueue* pDownloadQueue) DirBrowser dir(g_pOptions->GetTempDir()); while (const char* filename = dir.Next()) { - bool del = strstr(filename, ".tmp") || strstr(filename, ".dec"); + int id, part; + bool del = strstr(filename, ".tmp") || strstr(filename, ".dec") || + ((sscanf(filename, "%i.out", &id) == 1) && + !(g_pOptions->GetContinuePartial() && g_pOptions->GetDirectWrite())); if (!del) { - int id, part; - if (sscanf(filename, "%i.%i", &id, &part) == 2) + if ((sscanf(filename, "%i.%i", &id, &part) == 2) || + (sscanf(filename, "%i.out", &id) == 1)) { del = true; ptr = ids; diff --git a/DownloadInfo.cpp b/DownloadInfo.cpp index c4711c00..41081c4b 100644 --- a/DownloadInfo.cpp +++ b/DownloadInfo.cpp @@ -99,6 +99,7 @@ FileInfo::FileInfo() m_bPaused = false; m_bDeleted = false; m_iCompleted = 0; + m_bOutputInitialized = false; m_iIDGen++; m_iID = m_iIDGen; } @@ -277,3 +278,13 @@ bool FileInfo::IsDupe() return exists; } + +void FileInfo::LockOutputFile() +{ + m_mutexOutputFile.Lock(); +} + +void FileInfo::UnlockOutputFile() +{ + m_mutexOutputFile.Unlock(); +} diff --git a/DownloadInfo.h b/DownloadInfo.h index 6b43b0d5..d9b49024 100644 --- a/DownloadInfo.h +++ b/DownloadInfo.h @@ -30,6 +30,8 @@ #include #include +#include "Thread.h" + class ArticleInfo { public: @@ -83,6 +85,8 @@ private: bool m_bDeleted; bool m_bFilenameConfirmed; int m_iCompleted; + bool m_bOutputInitialized; + Mutex m_mutexOutputFile; static int m_iIDGen; @@ -119,6 +123,10 @@ public: void ParseSubject(); bool IsDupe(); void ClearArticles(); + void LockOutputFile(); + void UnlockOutputFile(); + bool GetOutputInitialized() { return m_bOutputInitialized; } + void SetOutputInitialized(bool bOutputInitialized) { m_bOutputInitialized = bOutputInitialized; } }; typedef std::deque DownloadQueue; diff --git a/Options.cpp b/Options.cpp index c9e682ba..d952e631 100644 --- a/Options.cpp +++ b/Options.cpp @@ -129,6 +129,7 @@ static const char* OPTION_CURSESTIME = "cursestime"; static const char* OPTION_CURSESGROUP = "cursesgroup"; static const char* OPTION_RETRYONCRCERROR = "retryoncrcerror"; static const char* OPTION_THREADLIMIT = "threadlimit"; +static const char* OPTION_DIRECTWRITE = "directwrite"; #ifndef WIN32 const char* PossibleConfigLocations[] = @@ -203,6 +204,7 @@ Options::Options(int argc, char* argv[]) m_bCursesTime = false; m_bCursesGroup = false; m_bRetryOnCrcError = false; + m_bDirectWrite = false; m_iThreadLimit = 0; char szFilename[MAX_PATH + 1]; @@ -388,8 +390,9 @@ void Options::InitDefault() SetOption(OPTION_CURSESNZBNAME, "yes"); SetOption(OPTION_CURSESTIME, "no"); SetOption(OPTION_CURSESGROUP, "no"); - SetOption(OPTION_RETRYONCRCERROR, "no"); - SetOption(OPTION_THREADLIMIT, "100"); + SetOption(OPTION_RETRYONCRCERROR, "no"); + SetOption(OPTION_THREADLIMIT, "100"); + SetOption(OPTION_DIRECTWRITE, "no"); } void Options::InitOptFile() @@ -520,6 +523,7 @@ void Options::InitOptions() m_bCursesTime = (bool)ParseOptionValue(OPTION_CURSESTIME, BoolCount, BoolNames, BoolValues); m_bCursesGroup = (bool)ParseOptionValue(OPTION_CURSESGROUP, BoolCount, BoolNames, BoolValues); m_bRetryOnCrcError = (bool)ParseOptionValue(OPTION_RETRYONCRCERROR, BoolCount, BoolNames, BoolValues); + m_bDirectWrite = (bool)ParseOptionValue(OPTION_DIRECTWRITE, BoolCount, BoolNames, BoolValues); const char* OutputModeNames[] = { "loggable", "logable", "log", "colored", "color", "ncurses", "curses" }; const int OutputModeValues[] = { omLoggable, omLoggable, omLoggable, omColored, omColored, omNCurses, omNCurses }; @@ -715,41 +719,45 @@ void Options::InitCommandLine(int argc, char* argv[]) void Options::PrintUsage(char* com) { - printf("Usage: %s [switches] []\n" - "Switches:\n" - " -h, --help Print this help-message\n" - " -v, --version Print version and exit\n" - " -c, --configfile Filename of configuration-file\n" - " -n, --noconfigfile Prevent loading of configuration-file\n" - " (required options must be passed with --option)\n" - " -p, --printconfig Print configuration and exit\n" - " -o, --option Set or override option in configuration-file\n" - " -s, --server Start nzbget as a server in console-mode\n" - " -D, --daemon Start nzbget as a server in daemon-mode\n" - " -Q, --quit Shutdown the server\n" - " -A, --append Send file to the server's download queue\n" - " -C, --connect Attach client to server\n" - " -L, --list Request list of downloads from the server\n" - " -P, --pause Pause downloading on the server\n" - " -U, --unpause Unpause downloading on the server\n" - " -R, --rate Set the download rate on the server\n" - " -T, --top Add file to the top (begining) of queue\n" - " (should be used with switch --append)\n" - " -G, --log Request last lines from server's screen-log\n" - " -E, --edit [G] Edit queue on the server\n" - " where is one of:\n" - " <+offset|-offset> Move file(s) in queue relative to current position\n" - " offset is an integer number\n" - " T Move file(s) to the top of queue\n" - " B Move file(s) to the bottom of queue\n" - " P Pause file(s)\n" - " U Resume (unpause) file(s)\n" - " R Pause pars (only for groups)\n" - " D Delete file(s)\n" - " where is a comma-separated list of file-ids or ranges of file-ids,\n" - " for example: 1-5,3,10-22" - "", - com); + printf("Usage:\n" + " %s [switches] []\n\n" + "Switches:\n" + " -h, --help Print this help-message\n" + " -v, --version Print version and exit\n" + " -c, --configfile Filename of configuration-file\n" + " -n, --noconfigfile Prevent loading of configuration-file\n" + " (required options must be passed with --option)\n" + " -p, --printconfig Print configuration and exit\n" + " -o, --option Set or override option in configuration-file\n" + " -s, --server Start nzbget as a server in console-mode\n" +#ifndef WIN32 + " -D, --daemon Start nzbget as a server in daemon-mode\n" +#endif + " -Q, --quit Shutdown the server\n" + " -A, --append Send file to the server's download queue\n" + " -C, --connect Attach client to server\n" + " -L, --list Request list of downloads from the server\n" + " -P, --pause Pause downloading on the server\n" + " -U, --unpause Unpause downloading on the server\n" + " -R, --rate Set the download rate on the server\n" + " -T, --top Add file to the top (begining) of queue\n" + " (should be used with switch --append)\n" + " -G, --log Request last lines from server's screen-log\n" + " -E, --edit [G] Edit queue on the server\n" + " Affect all files in the group (same nzb-file)\n" + " is one of:\n" + " <+offset|-offset> Move file(s) in queue relative to current position,\n" + " offset is an integer value\n" + " T Move file(s) to the top of queue\n" + " B Move file(s) to the bottom of queue\n" + " P Pause file(s)\n" + " U Resume (unpause) file(s)\n" + // Pause-pars command is not yet implemented + //" R Pause pars (only for groups)\n" + " D Delete file(s)\n" + " Comma-separated list of file-ids or ranges\n" + " of file-ids, e. g.: 1-5,3,10-22\n", + BaseFileName(com)); } void Options::InitFileArg(int argc, char* argv[]) @@ -1086,6 +1094,11 @@ void Options::CheckOptions() abort("FATAL ERROR: Program was compiled without curses-support. Can not use \"curses\" frontend (option \"%s\")\n", OPTION_OUTPUTMODE); } #endif + + if (m_eDecoder != dcYenc) + { + m_bDirectWrite = false; + } } void Options::ParseFileIDList(int argc, char* argv[], int optind) diff --git a/Options.h b/Options.h index 4876de0d..556322bd 100644 --- a/Options.h +++ b/Options.h @@ -127,6 +127,7 @@ private: bool m_bCursesGroup; bool m_bRetryOnCrcError; int m_iThreadLimit; + bool m_bDirectWrite; // Parsed command-line parameters bool m_bServerMode; @@ -213,6 +214,7 @@ public: bool GetCursesGroup() { return m_bCursesGroup; } bool GetRetryOnCrcError() { return m_bRetryOnCrcError; } int GetThreadLimit() { return m_iThreadLimit; } + bool GetDirectWrite() { return m_bDirectWrite; } // Parsed command-line parameters bool GetServerMode() { return m_bServerMode; } diff --git a/QueueCoordinator.cpp b/QueueCoordinator.cpp index f65dbe2f..f881c110 100644 --- a/QueueCoordinator.cpp +++ b/QueueCoordinator.cpp @@ -61,7 +61,7 @@ QueueCoordinator::QueueCoordinator() m_DownloadQueue.clear(); m_ActiveDownloads.clear(); - Decoder::Init(); + YDecoder::Init(); } QueueCoordinator::~QueueCoordinator() @@ -83,7 +83,7 @@ QueueCoordinator::~QueueCoordinator() } m_ActiveDownloads.clear(); - Decoder::Final(); + YDecoder::Final(); debug("QueueCoordinator destroyed"); } @@ -94,8 +94,6 @@ void QueueCoordinator::Run() m_mutexDownloadQueue.Lock(); - g_pDiskState->CleanupTempDir(&m_DownloadQueue); - if (g_pOptions->GetServerMode() && g_pOptions->GetSaveQueue() && g_pDiskState->Exists()) { if (g_pOptions->GetReloadQueue()) @@ -108,6 +106,8 @@ void QueueCoordinator::Run() } } + g_pDiskState->CleanupTempDir(&m_DownloadQueue); + m_mutexDownloadQueue.Unlock(); while (!IsStopped()) @@ -384,7 +384,7 @@ void QueueCoordinator::BuildArticleFilename(ArticleDownloader* pArticleDownloade snprintf(name, 1024, "%s%i.%03i", g_pOptions->GetTempDir(), pFileInfo->GetID(), pArticleInfo->GetPartNumber()); name[1024-1] = '\0'; pArticleInfo->SetResultFilename(name); - + char tmpname[1024]; snprintf(tmpname, 1024, "%s.tmp", name); tmpname[1024-1] = '\0'; @@ -396,6 +396,13 @@ void QueueCoordinator::BuildArticleFilename(ArticleDownloader* pArticleDownloade snprintf(name, 1024, "%s%c%s [%i/%i]", szNZBNiceName, (int)PATH_SEPARATOR, pFileInfo->GetFilename(), pArticleInfo->GetPartNumber(), pFileInfo->GetArticles()->size()); name[1024-1] = '\0'; pArticleDownloader->SetInfoName(name); + + if (g_pOptions->GetDirectWrite()) + { + snprintf(name, 1024, "%s%i.out", g_pOptions->GetTempDir(), pFileInfo->GetID()); + name[1024-1] = '\0'; + pArticleDownloader->SetOutputFilename(name); + } } DownloadQueue* QueueCoordinator::LockQueue() diff --git a/Util.cpp b/Util.cpp index a6aa3c67..3ddfb7cf 100644 --- a/Util.cpp +++ b/Util.cpp @@ -37,6 +37,8 @@ #include #ifdef WIN32 #include +#else +#include #endif #include "nzbget.h" @@ -311,6 +313,49 @@ bool LoadFileIntoBuffer(const char* szFileName, char** pBuffer, int* pBufferLeng return true; } +bool SetFileSize(const char* szFilename, int iSize) +{ + bool bOK = false; +#ifdef WIN32 + FILE* pFile = fopen(szFilename, "a"); + if (pFile) + { + bOK = _chsize_s(pFile->_file, iSize) == 0; + fclose(pFile); + } +#else + // create file + FILE* pFile = fopen(szFilename, "a"); + if (pFile) + { + fclose(pFile); + } + // there no reliable function to expand file on POSIX, so we must try different approaches, + // starting with the fastest one and hoping it will work + // 1) set file size using function "truncate" (it is fast, if works) + truncate(szFilename, iSize); + // check if it worked + pFile = fopen(szFilename, "a"); + if (pFile) + { + fseek(pFile, 0, SEEK_END); + bOK = ftell(pFile) == iSize; + if (!bOK) + { + // 2) truncate did not work, expanding the file by writing in it (it is slow) + fclose(pFile); + truncate(szFilename, 0); + pFile = fopen(szFilename, "a"); + char c = '0'; + fwrite(&c, 1, iSize, pFile); + bOK = ftell(pFile) == iSize; + } + fclose(pFile); + } +#endif + return bOK; +} + long long JoinInt64(unsigned int Hi, unsigned int Lo) { return (((long long)Hi) << 32) + Lo; diff --git a/Util.h b/Util.h index 6c8f9672..7681679d 100644 --- a/Util.h +++ b/Util.h @@ -61,6 +61,7 @@ char* BaseFileName(const char* filename); void NormalizePathSeparators(char* szPath); bool ForceDirectories(const char* szPath); bool LoadFileIntoBuffer(const char* szFileName, char** pBuffer, int* pBufferLength); +bool SetFileSize(const char* szFilename, int iSize); long long JoinInt64(unsigned int Hi, unsigned int Lo); void SplitInt64(long long Int64, unsigned int* Hi, unsigned int* Lo); diff --git a/nzbget.conf.example b/nzbget.conf.example index d951d1d3..ab30817a 100644 --- a/nzbget.conf.example +++ b/nzbget.conf.example @@ -136,14 +136,31 @@ serverport=6789 serverpassword=tegbzn6789 # Determine how the articles should be decoded (uulib, yenc, none) +# yenc - use internal yEnc-Decoder. Supports only yEnc-format, but is +# very fast and does not create temporary files with articles' text, +# decoding them on the fly. # uulib - use uulib to decode files. Supports many encoding formats, # but is slow. -# yenc - use internal yEnc-Decoder. Supports only yEnc-format and is very fast. # none - the articles will not be decoded and joined. External programs # ("uudeview" is one of them) could be used to decode an join downloaded # articles decoder=yEnc +# Write decoded articles directly into destination output file (yes, no) +# With this option enabled the program firstly creates the output +# destination file with required size (total size of all articles), +# then writes on the fly decoded articles directly to this file +# without creating of any temporary files, even for decoded articles. +# This may results in major performance improvement, but this higly depends +# on OS and filesystem used. +# Can improve performance on a very fast internet connections, +# but you need to test if it works in your case. +# The option works only with internal decoder ("decoder=yenc") +# NOTE: for testing download few big files (with total size 500-1000MB) +# and measure required time. Do not rely on the program's speed indicator, +# it is not accurate. +directwrite=no + # How much retries should be attempted if a download error occurs retries=4