diff --git a/Makefile.am b/Makefile.am index 7a58bd9c..6664ac53 100644 --- a/Makefile.am +++ b/Makefile.am @@ -108,6 +108,8 @@ nzbget_SOURCES = \ daemon/postprocess/Repair.h \ daemon/postprocess/Unpack.cpp \ daemon/postprocess/Unpack.h \ + daemon/queue/DirectRenamer.cpp \ + daemon/queue/DirectRenamer.h \ daemon/queue/DiskState.cpp \ daemon/queue/DiskState.h \ daemon/queue/DownloadInfo.cpp \ diff --git a/Makefile.in b/Makefile.in index 84ed6d05..c57d3f1a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -208,7 +208,8 @@ am__nzbget_SOURCES_DIST = daemon/connect/Connection.cpp \ daemon/postprocess/RarReader.h daemon/postprocess/Rename.cpp \ daemon/postprocess/Rename.h daemon/postprocess/Repair.cpp \ daemon/postprocess/Repair.h daemon/postprocess/Unpack.cpp \ - daemon/postprocess/Unpack.h daemon/queue/DiskState.cpp \ + daemon/postprocess/Unpack.h daemon/queue/DirectRenamer.cpp \ + daemon/queue/DirectRenamer.h daemon/queue/DiskState.cpp \ daemon/queue/DiskState.h daemon/queue/DownloadInfo.cpp \ daemon/queue/DownloadInfo.h daemon/queue/DupeCoordinator.cpp \ daemon/queue/DupeCoordinator.h \ @@ -313,11 +314,11 @@ am_nzbget_OBJECTS = Connection.$(OBJEXT) TlsSocket.$(OBJEXT) \ ParChecker.$(OBJEXT) ParParser.$(OBJEXT) ParRenamer.$(OBJEXT) \ PrePostProcessor.$(OBJEXT) RarRenamer.$(OBJEXT) \ RarReader.$(OBJEXT) Rename.$(OBJEXT) Repair.$(OBJEXT) \ - Unpack.$(OBJEXT) DiskState.$(OBJEXT) DownloadInfo.$(OBJEXT) \ - DupeCoordinator.$(OBJEXT) HistoryCoordinator.$(OBJEXT) \ - NzbFile.$(OBJEXT) QueueCoordinator.$(OBJEXT) \ - QueueEditor.$(OBJEXT) Scanner.$(OBJEXT) \ - UrlCoordinator.$(OBJEXT) BinRpc.$(OBJEXT) \ + Unpack.$(OBJEXT) DirectRenamer.$(OBJEXT) DiskState.$(OBJEXT) \ + DownloadInfo.$(OBJEXT) DupeCoordinator.$(OBJEXT) \ + HistoryCoordinator.$(OBJEXT) NzbFile.$(OBJEXT) \ + QueueCoordinator.$(OBJEXT) QueueEditor.$(OBJEXT) \ + Scanner.$(OBJEXT) UrlCoordinator.$(OBJEXT) BinRpc.$(OBJEXT) \ RemoteClient.$(OBJEXT) RemoteServer.$(OBJEXT) \ WebServer.$(OBJEXT) XmlRpc.$(OBJEXT) Log.$(OBJEXT) \ NString.$(OBJEXT) Observer.$(OBJEXT) Script.$(OBJEXT) \ @@ -539,7 +540,8 @@ nzbget_SOURCES = daemon/connect/Connection.cpp \ daemon/postprocess/RarReader.h daemon/postprocess/Rename.cpp \ daemon/postprocess/Rename.h daemon/postprocess/Repair.cpp \ daemon/postprocess/Repair.h daemon/postprocess/Unpack.cpp \ - daemon/postprocess/Unpack.h daemon/queue/DiskState.cpp \ + daemon/postprocess/Unpack.h daemon/queue/DirectRenamer.cpp \ + daemon/queue/DirectRenamer.h daemon/queue/DiskState.cpp \ daemon/queue/DiskState.h daemon/queue/DownloadInfo.cpp \ daemon/queue/DownloadInfo.h daemon/queue/DupeCoordinator.cpp \ daemon/queue/DupeCoordinator.h \ @@ -868,6 +870,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/CommandScript.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Connection.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Decoder.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DirectRenamer.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DiskService.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DiskState.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadInfo.Po@am__quote@ @@ -1595,6 +1598,20 @@ Unpack.obj: daemon/postprocess/Unpack.cpp @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o Unpack.obj `if test -f 'daemon/postprocess/Unpack.cpp'; then $(CYGPATH_W) 'daemon/postprocess/Unpack.cpp'; else $(CYGPATH_W) '$(srcdir)/daemon/postprocess/Unpack.cpp'; fi` +DirectRenamer.o: daemon/queue/DirectRenamer.cpp +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT DirectRenamer.o -MD -MP -MF "$(DEPDIR)/DirectRenamer.Tpo" -c -o DirectRenamer.o `test -f 'daemon/queue/DirectRenamer.cpp' || echo '$(srcdir)/'`daemon/queue/DirectRenamer.cpp; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/DirectRenamer.Tpo" "$(DEPDIR)/DirectRenamer.Po"; else rm -f "$(DEPDIR)/DirectRenamer.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='daemon/queue/DirectRenamer.cpp' object='DirectRenamer.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o DirectRenamer.o `test -f 'daemon/queue/DirectRenamer.cpp' || echo '$(srcdir)/'`daemon/queue/DirectRenamer.cpp + +DirectRenamer.obj: daemon/queue/DirectRenamer.cpp +@am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT DirectRenamer.obj -MD -MP -MF "$(DEPDIR)/DirectRenamer.Tpo" -c -o DirectRenamer.obj `if test -f 'daemon/queue/DirectRenamer.cpp'; then $(CYGPATH_W) 'daemon/queue/DirectRenamer.cpp'; else $(CYGPATH_W) '$(srcdir)/daemon/queue/DirectRenamer.cpp'; fi`; \ +@am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/DirectRenamer.Tpo" "$(DEPDIR)/DirectRenamer.Po"; else rm -f "$(DEPDIR)/DirectRenamer.Tpo"; exit 1; fi +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='daemon/queue/DirectRenamer.cpp' object='DirectRenamer.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o DirectRenamer.obj `if test -f 'daemon/queue/DirectRenamer.cpp'; then $(CYGPATH_W) 'daemon/queue/DirectRenamer.cpp'; else $(CYGPATH_W) '$(srcdir)/daemon/queue/DirectRenamer.cpp'; fi` + DiskState.o: daemon/queue/DiskState.cpp @am__fastdepCXX_TRUE@ if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT DiskState.o -MD -MP -MF "$(DEPDIR)/DiskState.Tpo" -c -o DiskState.o `test -f 'daemon/queue/DiskState.cpp' || echo '$(srcdir)/'`daemon/queue/DiskState.cpp; \ @am__fastdepCXX_TRUE@ then mv -f "$(DEPDIR)/DiskState.Tpo" "$(DEPDIR)/DiskState.Po"; else rm -f "$(DEPDIR)/DiskState.Tpo"; exit 1; fi diff --git a/daemon/nntp/ArticleDownloader.cpp b/daemon/nntp/ArticleDownloader.cpp index d3c36086..4042f08a 100644 --- a/daemon/nntp/ArticleDownloader.cpp +++ b/daemon/nntp/ArticleDownloader.cpp @@ -296,6 +296,11 @@ ArticleDownloader::EStatus ArticleDownloader::Download() m_writingStarted = false; m_articleInfo->SetCrc(0); + if (m_contentAnalyzer) + { + m_contentAnalyzer->Reset(); + } + if (m_connection->GetNewsServer()->GetJoinGroup()) { // change group @@ -541,6 +546,11 @@ bool ArticleDownloader::Write(char* line, int len) bool ok = len == 0 || m_articleWriter.Write(line, len); + if (m_contentAnalyzer) + { + m_contentAnalyzer->Append(line, len); + } + return ok; } diff --git a/daemon/nntp/ArticleDownloader.h b/daemon/nntp/ArticleDownloader.h index 5aedc889..06166d89 100644 --- a/daemon/nntp/ArticleDownloader.h +++ b/daemon/nntp/ArticleDownloader.h @@ -29,6 +29,15 @@ #include "NntpConnection.h" #include "Decoder.h" #include "ArticleWriter.h" +#include "Util.h" + +class ArticleContentAnalyzer +{ +public: + virtual ~ArticleContentAnalyzer() {}; + virtual void Reset() = 0; + virtual void Append(const void* buffer, int len) = 0; +}; class ArticleDownloader : public Thread, public Subject { @@ -77,6 +86,8 @@ public: void SetConnection(NntpConnection* connection) { m_connection = connection; } void CompleteFileParts() { m_articleWriter.CompleteFileParts(); } int GetDownloadedSize() { return m_downloadedSize; } + void SetContentAnalyzer(std::unique_ptr contentAnalyzer) { m_contentAnalyzer = std::move(contentAnalyzer); } + ArticleContentAnalyzer* GetContentAnalyzer() { return m_contentAnalyzer.get(); } void LogDebugInfo(); @@ -97,6 +108,7 @@ private: ServerStatList m_serverStats; bool m_writingStarted; int m_downloadedSize = 0; + std::unique_ptr m_contentAnalyzer; EStatus Download(); EStatus DecodeCheck(); diff --git a/daemon/queue/DirectRenamer.cpp b/daemon/queue/DirectRenamer.cpp new file mode 100644 index 00000000..c363d71f --- /dev/null +++ b/daemon/queue/DirectRenamer.cpp @@ -0,0 +1,79 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2017 Andrey Prygunkov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#include "nzbget.h" +#include "DirectRenamer.h" +#include "Log.h" +#include "Options.h" + +#ifndef DISABLE_PARCHECK +#include "par2cmdline.h" +#include "md5.h" +#endif + +RenameContentAnalyzer::~RenameContentAnalyzer() +{ + Reset(); +} + +void RenameContentAnalyzer::Reset() +{ +#ifndef DISABLE_PARCHECK + delete (Par2::MD5Context*)m_md5Context; +#endif + m_md5Context = nullptr; + m_dataSize = 0; +} + +void RenameContentAnalyzer::Append(const void* buffer, int len) +{ +#ifndef DISABLE_PARCHECK + if (!m_md5Context) + { + m_md5Context = new Par2::MD5Context(); + } + + int rem16kSize = std::min(len, 16 * 1024 - m_dataSize); + + if (rem16kSize > 0) + { + ((Par2::MD5Context*)m_md5Context)->Update(buffer, rem16kSize); + } + + m_dataSize += len; +#endif +} + +// Must be called with locked DownloadQueue +void RenameContentAnalyzer::Finish(FileInfo* fileInfo, ArticleInfo* articleInfo) +{ +#ifndef DISABLE_PARCHECK + Par2::MD5Hash hash; + ((Par2::MD5Context*)m_md5Context)->Final(hash); + + // we don't support analyzing of files split into articles smaller than 16KB + if (articleInfo->GetSize() >= 16 * 1024 || fileInfo->GetArticles()->size() == 1) + { + fileInfo->SetHash16k(hash.print().c_str()); + } +#endif + + debug("file: %s; article-hash16k: %s", fileInfo->GetFilename(), fileInfo->GetHash16k()); +} diff --git a/daemon/queue/DirectRenamer.h b/daemon/queue/DirectRenamer.h new file mode 100644 index 00000000..0b1b92be --- /dev/null +++ b/daemon/queue/DirectRenamer.h @@ -0,0 +1,40 @@ +/* + * This file is part of nzbget. See . + * + * Copyright (C) 2017 Andrey Prygunkov + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + + +#ifndef DIRECTRENAMER_H +#define DIRECTRENAMER_H + +#include "ArticleDownloader.h" + +class RenameContentAnalyzer : public ArticleContentAnalyzer +{ +public: + virtual ~RenameContentAnalyzer(); + virtual void Reset(); + virtual void Append(const void* buffer, int len); + void Finish(FileInfo* fileInfo, ArticleInfo* articleInfo); + +private: + // declared as void* to prevent inclusion if par2-modules into this header file + void* m_md5Context = nullptr; + int m_dataSize = 0; +}; + +#endif diff --git a/daemon/queue/DownloadInfo.h b/daemon/queue/DownloadInfo.h index bca9790b..a56e7334 100644 --- a/daemon/queue/DownloadInfo.h +++ b/daemon/queue/DownloadInfo.h @@ -192,6 +192,9 @@ public: void SetPartialState(EPartialState partialState) { m_partialState = partialState; } uint32 GetCrc() { return m_crc; } void SetCrc(uint32 crc) { m_crc = crc; } + const char* GetHash16k() { return m_hash16k; } + void SetHash16k(const char* hash16k) { m_hash16k = hash16k; } + ServerStatList* GetServerStats() { return &m_serverStats; } private: @@ -228,6 +231,7 @@ private: bool m_forceDirectWrite = false; EPartialState m_partialState = psNone; uint32 m_crc = 0; + CString m_hash16k; static int m_idGen; static int m_idMax; diff --git a/daemon/queue/QueueCoordinator.cpp b/daemon/queue/QueueCoordinator.cpp index 2a5fa84e..9259737c 100644 --- a/daemon/queue/QueueCoordinator.cpp +++ b/daemon/queue/QueueCoordinator.cpp @@ -30,6 +30,7 @@ #include "FileSystem.h" #include "Decoder.h" #include "StatMeter.h" +#include "DirectRenamer.h" bool QueueCoordinator::CoordinatorDownloadQueue::EditEntry( int ID, EEditAction action, const char* args) @@ -589,6 +590,11 @@ void QueueCoordinator::StartArticleDownload(FileInfo* fileInfo, ArticleInfo* art articleDownloader->SetArticleInfo(articleInfo); articleDownloader->SetConnection(connection); + if (articleInfo->GetPartNumber() == 1 && g_Options->GetDirectRename()) + { + articleDownloader->SetContentAnalyzer(std::make_unique()); + } + BString<1024> infoName("%s%c%s [%i/%i]", fileInfo->GetNzbInfo()->GetName(), (int)PATH_SEPARATOR, fileInfo->GetFilename(), articleInfo->GetPartNumber(), (int)fileInfo->GetArticles()->size()); articleDownloader->SetInfoName(infoName); @@ -692,6 +698,11 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* articleDownloader) } } + if (articleDownloader->GetContentAnalyzer() && articleDownloader->GetStatus() == ArticleDownloader::adFinished) + { + ((RenameContentAnalyzer*)articleDownloader->GetContentAnalyzer())->Finish(fileInfo, articleInfo); + } + nzbInfo->SetDownloadedSize(nzbInfo->GetDownloadedSize() + articleDownloader->GetDownloadedSize()); CheckHealth(downloadQueue, fileInfo); diff --git a/nzbget.vcxproj b/nzbget.vcxproj index 85f729d1..cf92b1ec 100644 --- a/nzbget.vcxproj +++ b/nzbget.vcxproj @@ -141,6 +141,7 @@ + @@ -239,6 +240,7 @@ +