mirror/nzbget
1
0
Fork 0
mirror of https://github.com/nzbget/nzbget.git synced 2026-05-19 18:44:29 -04:00
Code Issues Packages Projects Releases Wiki Activity

#362: computing 16k-hashes for downloaded files

Browse Source
This commit is contained in:
Andrey Prygunkov
2017-04-17 15:32:32 +02:00
parent 3ac91a4bb6
commit f7be22893d
9 changed files with 184 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
daemon/queue/DirectRenamer.cpp Normal file
View File

@@ -0,0 +1,79 @@
/*
* This file is part of nzbget. See <http://nzbget.net>.
*
* Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "nzbget.h"
#include "DirectRenamer.h"
#include "Log.h"
#include "Options.h"
#ifndef DISABLE_PARCHECK
#include "par2cmdline.h"
#include "md5.h"
#endif
RenameContentAnalyzer::~RenameContentAnalyzer()
{
Reset();
}
void RenameContentAnalyzer::Reset()
{
#ifndef DISABLE_PARCHECK
delete (Par2::MD5Context*)m_md5Context;
#endif
m_md5Context = nullptr;
m_dataSize = 0;
}
void RenameContentAnalyzer::Append(const void* buffer, int len)
{
#ifndef DISABLE_PARCHECK
if (!m_md5Context)
{
m_md5Context = new Par2::MD5Context();
}
int rem16kSize = std::min(len, 16 * 1024 - m_dataSize);
if (rem16kSize > 0)
{
((Par2::MD5Context*)m_md5Context)->Update(buffer, rem16kSize);
}
m_dataSize += len;
#endif
}
// Must be called with locked DownloadQueue
void RenameContentAnalyzer::Finish(FileInfo* fileInfo, ArticleInfo* articleInfo)
{
#ifndef DISABLE_PARCHECK
Par2::MD5Hash hash;
((Par2::MD5Context*)m_md5Context)->Final(hash);
// we don't support analyzing of files split into articles smaller than 16KB
if (articleInfo->GetSize() >= 16 * 1024 || fileInfo->GetArticles()->size() == 1)
{
fileInfo->SetHash16k(hash.print().c_str());
}
#endif
debug("file: %s; article-hash16k: %s", fileInfo->GetFilename(), fileInfo->GetHash16k());
}

40
daemon/queue/DirectRenamer.h Normal file
View File

@@ -0,0 +1,40 @@
/*
* This file is part of nzbget. See <http://nzbget.net>.
*
* Copyright (C) 2017 Andrey Prygunkov <hugbug@users.sourceforge.net>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef DIRECTRENAMER_H
#define DIRECTRENAMER_H
#include "ArticleDownloader.h"
class RenameContentAnalyzer : public ArticleContentAnalyzer
{
public:
virtual ~RenameContentAnalyzer();
virtual void Reset();
virtual void Append(const void* buffer, int len);
void Finish(FileInfo* fileInfo, ArticleInfo* articleInfo);
private:
// declared as void* to prevent inclusion if par2-modules into this header file
void* m_md5Context = nullptr;
int m_dataSize = 0;
};
#endif

4
daemon/queue/DownloadInfo.h
View File

@@ -192,6 +192,9 @@ public:
void SetPartialState(EPartialState partialState) { m_partialState = partialState; }
uint32 GetCrc() { return m_crc; }
void SetCrc(uint32 crc) { m_crc = crc; }
const char* GetHash16k() { return m_hash16k; }
void SetHash16k(const char* hash16k) { m_hash16k = hash16k; }
ServerStatList* GetServerStats() { return &m_serverStats; }
private:
@@ -228,6 +231,7 @@ private:
bool m_forceDirectWrite = false;
EPartialState m_partialState = psNone;
uint32 m_crc = 0;
CString m_hash16k;
static int m_idGen;
static int m_idMax;

11
daemon/queue/QueueCoordinator.cpp
View File

@@ -30,6 +30,7 @@
#include "FileSystem.h"
#include "Decoder.h"
#include "StatMeter.h"
#include "DirectRenamer.h"
bool QueueCoordinator::CoordinatorDownloadQueue::EditEntry(
int ID, EEditAction action, const char* args)
@@ -589,6 +590,11 @@ void QueueCoordinator::StartArticleDownload(FileInfo* fileInfo, ArticleInfo* art
articleDownloader->SetArticleInfo(articleInfo);
articleDownloader->SetConnection(connection);
if (articleInfo->GetPartNumber() == 1 && g_Options->GetDirectRename())
{
articleDownloader->SetContentAnalyzer(std::make_unique<RenameContentAnalyzer>());
}
BString<1024> infoName("%s%c%s [%i/%i]", fileInfo->GetNzbInfo()->GetName(), (int)PATH_SEPARATOR, fileInfo->GetFilename(), articleInfo->GetPartNumber(), (int)fileInfo->GetArticles()->size());
articleDownloader->SetInfoName(infoName);
@@ -692,6 +698,11 @@ void QueueCoordinator::ArticleCompleted(ArticleDownloader* articleDownloader)
}
}
if (articleDownloader->GetContentAnalyzer() && articleDownloader->GetStatus() == ArticleDownloader::adFinished)
{
((RenameContentAnalyzer*)articleDownloader->GetContentAnalyzer())->Finish(fileInfo, articleInfo);
}
nzbInfo->SetDownloadedSize(nzbInfo->GetDownloadedSize() + articleDownloader->GetDownloadedSize());
CheckHealth(downloadQueue, fileInfo);