From 2a9f9f8e9876dc92e147bfa6fe688455a69a67b4 Mon Sep 17 00:00:00 2001
From: Andrey Prygunkov <hugbug@users.sourceforge.net>
Date: Fri, 7 Aug 2015 18:38:51 +0200
Subject: [PATCH] #64: implemented dupe matcher

Before scanning of dupe directories the directories are quickly checked
by dupe matcher. It determines if they contain or may contain the same
content. If the dupe checker detects a dupe containing different
content as the download being currently processed by par-checker, such
extra dupe is skipped to save time during dupe par scan.
---
 Makefile.am                                   |   6 +
 Makefile.in                                   |  40 ++-
 daemon/main/nzbget.cpp                        |   8 +-
 daemon/postprocess/DupeMatcher.cpp            | 260 ++++++++++++++++++
 daemon/postprocess/DupeMatcher.h              |  55 ++++
 daemon/postprocess/ParChecker.cpp             |  31 ++-
 daemon/postprocess/ParCoordinator.cpp         |  37 ++-
 daemon/postprocess/ParCoordinator.h           |  16 +-
 daemon/queue/DownloadInfo.cpp                 |   5 +
 daemon/util/Script.cpp                        |   2 +-
 nzbget.vcproj                                 |   8 +
 tests/postprocess/DupeMatcherTest.cpp         | 114 ++++++++
 tests/suite/TestUtil.cpp                      |   7 +-
 tests/suite/TestUtil.h                        |   1 +
 .../testdata/dupematcher1/testfile.part01.rar | Bin 0 -> 4096 bytes
 .../testdata/dupematcher1/testfile.part24.rar | Bin 0 -> 4096 bytes
 .../testdata/dupematcher2/testfile.part04.rar | Bin 0 -> 3072 bytes
 .../testdata/dupematcher2/testfile.part43.rar | Bin 0 -> 151 bytes
 18 files changed, 565 insertions(+), 25 deletions(-)
 create mode 100644 daemon/postprocess/DupeMatcher.cpp
 create mode 100644 daemon/postprocess/DupeMatcher.h
 create mode 100644 tests/postprocess/DupeMatcherTest.cpp
 create mode 100644 tests/testdata/dupematcher1/testfile.part01.rar
 create mode 100644 tests/testdata/dupematcher1/testfile.part24.rar
 create mode 100644 tests/testdata/dupematcher2/testfile.part04.rar
 create mode 100644 tests/testdata/dupematcher2/testfile.part43.rar

diff --git a/Makefile.am b/Makefile.am
index 5536fa4c..5e911d53 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -88,6 +88,8 @@ nzbget_SOURCES = \
 	daemon/nntp/StatMeter.h \
 	daemon/postprocess/Cleanup.cpp \
 	daemon/postprocess/Cleanup.h \
+	daemon/postprocess/DupeMatcher.cpp \
+	daemon/postprocess/DupeMatcher.h \
 	daemon/postprocess/ParChecker.cpp \
 	daemon/postprocess/ParChecker.h \
 	daemon/postprocess/ParCoordinator.cpp \
@@ -335,6 +337,10 @@ scripts_FILES = \
 	scripts/Logger.py
 
 testdata_FILES = \
+	tests/testdata/dupematcher1/testfile.part01.rar \
+	tests/testdata/dupematcher1/testfile.part24.rar \
+	tests/testdata/dupematcher2/testfile.part04.rar \
+	tests/testdata/dupematcher2/testfile.part43.rar \
 	tests/testdata/nzbfile/dotless.nzb \
 	tests/testdata/nzbfile/dotless.txt \
 	tests/testdata/nzbfile/plain.nzb \
diff --git a/Makefile.in b/Makefile.in
index fe3d7551..06f63aa6 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -180,7 +180,10 @@ am__nzbget_SOURCES_DIST = daemon/connect/Connection.cpp \
 	daemon/nntp/NNTPConnection.h daemon/nntp/ServerPool.cpp \
 	daemon/nntp/ServerPool.h daemon/nntp/StatMeter.cpp \
 	daemon/nntp/StatMeter.h daemon/postprocess/Cleanup.cpp \
-	daemon/postprocess/Cleanup.h daemon/postprocess/ParChecker.cpp \
+	daemon/postprocess/Cleanup.h \
+	daemon/postprocess/DupeMatcher.cpp \
+	daemon/postprocess/DupeMatcher.h \
+	daemon/postprocess/ParChecker.cpp \
 	daemon/postprocess/ParChecker.h \
 	daemon/postprocess/ParCoordinator.cpp \
 	daemon/postprocess/ParCoordinator.h \
@@ -275,10 +278,11 @@ am_nzbget_OBJECTS = Connection.$(OBJEXT) TLS.$(OBJEXT) \
 	ArticleDownloader.$(OBJEXT) ArticleWriter.$(OBJEXT) \
 	Decoder.$(OBJEXT) NewsServer.$(OBJEXT) \
 	NNTPConnection.$(OBJEXT) ServerPool.$(OBJEXT) \
-	StatMeter.$(OBJEXT) Cleanup.$(OBJEXT) ParChecker.$(OBJEXT) \
-	ParCoordinator.$(OBJEXT) ParParser.$(OBJEXT) \
-	ParRenamer.$(OBJEXT) PrePostProcessor.$(OBJEXT) \
-	Unpack.$(OBJEXT) DiskState.$(OBJEXT) DownloadInfo.$(OBJEXT) \
+	StatMeter.$(OBJEXT) Cleanup.$(OBJEXT) DupeMatcher.$(OBJEXT) \
+	ParChecker.$(OBJEXT) ParCoordinator.$(OBJEXT) \
+	ParParser.$(OBJEXT) ParRenamer.$(OBJEXT) \
+	PrePostProcessor.$(OBJEXT) Unpack.$(OBJEXT) \
+	DiskState.$(OBJEXT) DownloadInfo.$(OBJEXT) \
 	DupeCoordinator.$(OBJEXT) HistoryCoordinator.$(OBJEXT) \
 	NZBFile.$(OBJEXT) QueueCoordinator.$(OBJEXT) \
 	QueueEditor.$(OBJEXT) Scanner.$(OBJEXT) \
@@ -457,7 +461,10 @@ nzbget_SOURCES = daemon/connect/Connection.cpp \
 	daemon/nntp/NNTPConnection.h daemon/nntp/ServerPool.cpp \
 	daemon/nntp/ServerPool.h daemon/nntp/StatMeter.cpp \
 	daemon/nntp/StatMeter.h daemon/postprocess/Cleanup.cpp \
-	daemon/postprocess/Cleanup.h daemon/postprocess/ParChecker.cpp \
+	daemon/postprocess/Cleanup.h \
+	daemon/postprocess/DupeMatcher.cpp \
+	daemon/postprocess/DupeMatcher.h \
+	daemon/postprocess/ParChecker.cpp \
 	daemon/postprocess/ParChecker.h \
 	daemon/postprocess/ParCoordinator.cpp \
 	daemon/postprocess/ParCoordinator.h \
@@ -609,6 +616,10 @@ scripts_FILES = \
 	scripts/Logger.py
 
 testdata_FILES = \
+	tests/testdata/dupematcher1/testfile.part01.rar \
+	tests/testdata/dupematcher1/testfile.part24.rar \
+	tests/testdata/dupematcher2/testfile.part04.rar \
+	tests/testdata/dupematcher2/testfile.part43.rar \
 	tests/testdata/nzbfile/dotless.nzb \
 	tests/testdata/nzbfile/dotless.txt \
 	tests/testdata/nzbfile/plain.nzb \
@@ -762,6 +773,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DiskState.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DownloadInfo.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DupeCoordinator.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/DupeMatcher.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeedCoordinator.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeedFile.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/FeedFilter.Po@am__quote@
@@ -1315,6 +1327,20 @@ Cleanup.obj: daemon/postprocess/Cleanup.cpp
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
 @am__fastdepCXX_FALSE@	$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o Cleanup.obj `if test -f 'daemon/postprocess/Cleanup.cpp'; then $(CYGPATH_W) 'daemon/postprocess/Cleanup.cpp'; else $(CYGPATH_W) '$(srcdir)/daemon/postprocess/Cleanup.cpp'; fi`
 
+DupeMatcher.o: daemon/postprocess/DupeMatcher.cpp
+@am__fastdepCXX_TRUE@	if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT DupeMatcher.o -MD -MP -MF "$(DEPDIR)/DupeMatcher.Tpo" -c -o DupeMatcher.o `test -f 'daemon/postprocess/DupeMatcher.cpp' || echo '$(srcdir)/'`daemon/postprocess/DupeMatcher.cpp; \
+@am__fastdepCXX_TRUE@	then mv -f "$(DEPDIR)/DupeMatcher.Tpo" "$(DEPDIR)/DupeMatcher.Po"; else rm -f "$(DEPDIR)/DupeMatcher.Tpo"; exit 1; fi
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	source='daemon/postprocess/DupeMatcher.cpp' object='DupeMatcher.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o DupeMatcher.o `test -f 'daemon/postprocess/DupeMatcher.cpp' || echo '$(srcdir)/'`daemon/postprocess/DupeMatcher.cpp
+
+DupeMatcher.obj: daemon/postprocess/DupeMatcher.cpp
+@am__fastdepCXX_TRUE@	if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT DupeMatcher.obj -MD -MP -MF "$(DEPDIR)/DupeMatcher.Tpo" -c -o DupeMatcher.obj `if test -f 'daemon/postprocess/DupeMatcher.cpp'; then $(CYGPATH_W) 'daemon/postprocess/DupeMatcher.cpp'; else $(CYGPATH_W) '$(srcdir)/daemon/postprocess/DupeMatcher.cpp'; fi`; \
+@am__fastdepCXX_TRUE@	then mv -f "$(DEPDIR)/DupeMatcher.Tpo" "$(DEPDIR)/DupeMatcher.Po"; else rm -f "$(DEPDIR)/DupeMatcher.Tpo"; exit 1; fi
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	source='daemon/postprocess/DupeMatcher.cpp' object='DupeMatcher.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o DupeMatcher.obj `if test -f 'daemon/postprocess/DupeMatcher.cpp'; then $(CYGPATH_W) 'daemon/postprocess/DupeMatcher.cpp'; else $(CYGPATH_W) '$(srcdir)/daemon/postprocess/DupeMatcher.cpp'; fi`
+
 ParChecker.o: daemon/postprocess/ParChecker.cpp
 @am__fastdepCXX_TRUE@	if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT ParChecker.o -MD -MP -MF "$(DEPDIR)/ParChecker.Tpo" -c -o ParChecker.o `test -f 'daemon/postprocess/ParChecker.cpp' || echo '$(srcdir)/'`daemon/postprocess/ParChecker.cpp; \
 @am__fastdepCXX_TRUE@	then mv -f "$(DEPDIR)/ParChecker.Tpo" "$(DEPDIR)/ParChecker.Po"; else rm -f "$(DEPDIR)/ParChecker.Tpo"; exit 1; fi
@@ -2190,7 +2216,7 @@ distclean-tags:
 distdir: $(DISTFILES)
 	$(am__remove_distdir)
 	mkdir $(distdir)
-	$(mkdir_p) $(distdir)/daemon/windows $(distdir)/lib/par2 $(distdir)/linux $(distdir)/osx $(distdir)/osx/NZBGet.xcodeproj $(distdir)/osx/Resources $(distdir)/osx/Resources/Images $(distdir)/osx/Resources/licenses $(distdir)/posix $(distdir)/scripts $(distdir)/tests/testdata/nzbfile $(distdir)/tests/testdata/parchecker $(distdir)/webui $(distdir)/webui/img $(distdir)/webui/lib $(distdir)/windows $(distdir)/windows/resources $(distdir)/windows/setup
+	$(mkdir_p) $(distdir)/daemon/windows $(distdir)/lib/par2 $(distdir)/linux $(distdir)/osx $(distdir)/osx/NZBGet.xcodeproj $(distdir)/osx/Resources $(distdir)/osx/Resources/Images $(distdir)/osx/Resources/licenses $(distdir)/posix $(distdir)/scripts $(distdir)/tests/testdata/dupematcher1 $(distdir)/tests/testdata/dupematcher2 $(distdir)/tests/testdata/nzbfile $(distdir)/tests/testdata/parchecker $(distdir)/webui $(distdir)/webui/img $(distdir)/webui/lib $(distdir)/windows $(distdir)/windows/resources $(distdir)/windows/setup
 	@srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; \
 	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's|.|.|g'`; \
 	list='$(DISTFILES)'; for file in $$list; do \
diff --git a/daemon/main/nzbget.cpp b/daemon/main/nzbget.cpp
index 5f436307..89773f90 100644
--- a/daemon/main/nzbget.cpp
+++ b/daemon/main/nzbget.cpp
@@ -156,6 +156,10 @@ int main(int argc, char *argv[], char *argp[])
 
 	Util::InitVersionRevision();
 
+	g_iArgumentCount = argc;
+	g_szArguments = (char*(*)[])argv;
+	g_szEnvironmentVariables = (char*(*)[])argp;
+
 	if (argc > 1 && (!strcmp(argv[1], "-tests") || !strcmp(argv[1], "--tests")))
 	{
 #ifdef ENABLE_TESTS
@@ -180,10 +184,6 @@ int main(int argc, char *argv[], char *argp[])
 
 	srand(time(NULL));
 
-	g_iArgumentCount = argc;
-	g_szArguments = (char*(*)[])argv;
-	g_szEnvironmentVariables = (char*(*)[])argp;
-
 #ifdef WIN32
 	for (int i=0; i < argc; i++)
 	{
diff --git a/daemon/postprocess/DupeMatcher.cpp b/daemon/postprocess/DupeMatcher.cpp
new file mode 100644
index 00000000..ee9fd682
--- /dev/null
+++ b/daemon/postprocess/DupeMatcher.cpp
@@ -0,0 +1,260 @@
+/*
+ *  This file is part of nzbget
+ *
+ *  Copyright (C) 2015 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ * $Revision$
+ * $Date$
+ *
+ */
+
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#ifdef WIN32
+#include "win32.h"
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <ctype.h>
+#ifndef WIN32
+#include <unistd.h>
+#endif
+#include <errno.h>
+
+#include "nzbget.h"
+#include "DupeMatcher.h"
+#include "Log.h"
+#include "Util.h"
+#include "Options.h"
+#include "Script.h"
+#include "Thread.h"
+
+class RarLister : public Thread, public ScriptController
+{
+private:
+	DupeMatcher*		m_pOwner;
+	long long			m_lMaxSize;
+	bool				m_bCompressed;
+	bool				m_bLastSizeMax;
+	long long			m_lExpectedSize;
+	char*				m_szFilenameBuf;
+	int					m_iFilenameBufLen;
+	char				m_szLastFilename[1024];
+
+protected:
+	virtual void		AddMessage(Message::EKind eKind, const char* szText);
+
+public:
+	virtual void		Run();
+	static bool			FindLargestFile(DupeMatcher* pOwner, const char* szDirectory,
+							char* szFilenameBuf, int iFilenameBufLen, long long lExpectedSize,
+							int iTimeoutSec, long long* pMaxSize, bool* pCompressed);
+};
+
+bool RarLister::FindLargestFile(DupeMatcher* pOwner, const char* szDirectory,
+	char* szFilenameBuf, int iFilenameBufLen, long long lExpectedSize,
+	int iTimeoutSec, long long* pMaxSize, bool* pCompressed)
+{
+	RarLister unrar;
+	unrar.m_pOwner = pOwner;
+	unrar.m_lExpectedSize = lExpectedSize;
+	unrar.m_lMaxSize = -1;
+	unrar.m_bCompressed = false;
+	unrar.m_bLastSizeMax = false;
+	unrar.m_szFilenameBuf = szFilenameBuf;
+	unrar.m_iFilenameBufLen = iFilenameBufLen;
+
+	char** pCmdArgs = NULL;
+	if (!Util::SplitCommandLine(g_pOptions->GetUnrarCmd(), &pCmdArgs))
+	{
+		return false;
+	}
+	const char* szUnrarPath = *pCmdArgs;
+	unrar.SetScript(szUnrarPath);
+
+	const char* szArgs[4];
+	szArgs[0] = szUnrarPath;
+	szArgs[1] = "lt";
+	szArgs[2] = "*.rar";
+	szArgs[3] = NULL;
+	unrar.SetArgs(szArgs, false);
+	unrar.SetWorkingDir(szDirectory);
+
+	time_t curTime = time(NULL);
+
+	unrar.Start();
+
+	// wait up to iTimeoutSec for unrar output
+	while (unrar.IsRunning() &&
+		curTime + iTimeoutSec > time(NULL) &&
+		curTime >= time(NULL))					// in a case clock was changed
+	{
+		usleep(200 * 1000);
+	}
+
+	if (unrar.IsRunning())
+	{
+		unrar.Terminate();
+	}
+
+	// wait until terminated or killed
+	while (unrar.IsRunning())
+	{
+		usleep(200 * 1000);
+	}
+
+	for (char** szArgPtr = pCmdArgs; *szArgPtr; szArgPtr++)
+	{
+		free(*szArgPtr);
+	}
+	free(pCmdArgs);
+
+	*pMaxSize = unrar.m_lMaxSize;
+	*pCompressed = unrar.m_bCompressed;
+
+	return true;
+}
+
+void RarLister::Run()
+{
+	Execute();
+}
+
+void RarLister::AddMessage(Message::EKind eKind, const char* szText)
+{
+	if (!strncasecmp(szText, "Archive: ", 9))
+	{
+		m_pOwner->PrintMessage(Message::mkDetail, "Reading file %s", szText + 9);
+	}
+	else if (!strncasecmp(szText, "        Name: ", 14))
+	{
+		strncpy(m_szLastFilename, szText + 14, sizeof(m_szLastFilename));
+		m_szLastFilename[sizeof(m_szLastFilename)-1] = '\0';
+	}
+	else if (!strncasecmp(szText, "        Size: ", 14))
+	{
+		m_bLastSizeMax = false;
+		long long lSize = atoll(szText + 14);
+		if (lSize > m_lMaxSize)
+		{
+			m_lMaxSize = lSize;
+			m_bLastSizeMax = true;
+			strncpy(m_szFilenameBuf, m_szLastFilename, m_iFilenameBufLen);
+			m_szFilenameBuf[m_iFilenameBufLen-1] = '\0';
+		}
+		return;
+	}
+
+	if (m_bLastSizeMax && !strncasecmp(szText, " Compression: ", 14))
+	{
+		m_bCompressed = !strstr(szText, " -m0");
+		if (m_lMaxSize > m_lExpectedSize ||
+			DupeMatcher::SizeDiffOK(m_lMaxSize, m_lExpectedSize, 20))
+		{
+			// alread found the largest file, aborting unrar
+			Terminate();
+		}
+	}
+}
+
+
+DupeMatcher::DupeMatcher(const char* szDestDir, long long lExpectedSize)
+{
+	m_szDestDir = strdup(szDestDir);
+	m_lExpectedSize = lExpectedSize;
+	m_lMaxSize = -1;
+	m_bCompressed = false;
+}
+
+DupeMatcher::~DupeMatcher()
+{
+	free(m_szDestDir);
+}
+
+bool DupeMatcher::SizeDiffOK(long long lSize1, long long lSize2, int iMaxDiffPercent)
+{
+	if (lSize1 == 0 || lSize2 == 0)
+	{
+		return false;
+	}
+
+	long long lDiff = lSize1 - lSize2;
+	lDiff = lDiff > 0 ? lDiff : -lDiff;
+	long long lMax = lSize1 > lSize2 ? lSize1 : lSize2;
+	int lDiffPercent = (int)(lDiff * 100 / lMax);
+	return lDiffPercent < iMaxDiffPercent;
+}
+
+bool DupeMatcher::Prepare()
+{
+	char szFilename[1024];
+	FindLargestFile(m_szDestDir, szFilename, sizeof(szFilename), &m_lMaxSize, &m_bCompressed);
+	bool bSizeOK = SizeDiffOK(m_lMaxSize, m_lExpectedSize, 20);
+	PrintMessage(Message::mkDetail, "Found main file %s with size %lli bytes%s",
+		szFilename, m_lMaxSize, bSizeOK ? "" : ", size mismatch");
+	return bSizeOK;
+}
+
+bool DupeMatcher::MatchDupeContent(const char* szDupeDir)
+{
+	long long lDupeMaxSize = 0;
+	bool lDupeCompressed = false;
+	char szFilename[1024];
+	FindLargestFile(szDupeDir, szFilename, sizeof(szFilename), &lDupeMaxSize, &lDupeCompressed);
+	bool bOK = lDupeMaxSize == m_lMaxSize && lDupeCompressed == m_bCompressed;
+	PrintMessage(Message::mkDetail, "Found main file %s with size %lli bytes%s",
+		szFilename, m_lMaxSize, bOK ? "" : ", size mismatch");
+	return bOK;
+}
+
+void DupeMatcher::FindLargestFile(const char* szDirectory, char* szFilenameBuf, int iBufLen,
+	long long* pMaxSize, bool* pCompressed)
+{
+	*pMaxSize = 0;
+	*pCompressed = false;
+
+	DirBrowser dir(szDirectory);
+	while (const char* filename = dir.Next())
+	{
+		if (strcmp(filename, ".") && strcmp(filename, ".."))
+		{
+			char szFullFilename[1024];
+			snprintf(szFullFilename, 1024, "%s%c%s", szDirectory, PATH_SEPARATOR, filename);
+			szFullFilename[1024-1] = '\0';
+
+			long long lFileSize = Util::FileSize(szFullFilename);
+			if (lFileSize > *pMaxSize)
+			{
+				*pMaxSize = lFileSize;
+				strncpy(szFilenameBuf, filename, iBufLen);
+				szFilenameBuf[iBufLen-1] = '\0';
+			}
+
+			if (Util::MatchFileExt(filename, ".rar", ","))
+			{
+				RarLister::FindLargestFile(this, szDirectory, szFilenameBuf, iBufLen,
+					m_lMaxSize, 60, pMaxSize, pCompressed);
+				return;
+			}
+		}
+	}
+}
diff --git a/daemon/postprocess/DupeMatcher.h b/daemon/postprocess/DupeMatcher.h
new file mode 100644
index 00000000..c7051fb0
--- /dev/null
+++ b/daemon/postprocess/DupeMatcher.h
@@ -0,0 +1,55 @@
+/*
+ *  This file is part of nzbget
+ *
+ *  Copyright (C) 2015 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ * $Revision$
+ * $Date$
+ *
+ */
+
+
+#ifndef DUPEMATCHER_H
+#define DUPEMATCHER_H
+
+#include "Log.h"
+
+class DupeMatcher
+{
+private:
+	char*				m_szDestDir;
+	long long			m_lExpectedSize;
+	long long			m_lMaxSize;
+	bool				m_bCompressed;
+
+	void				FindLargestFile(const char* szDirectory, char* szFilenameBuf, int iBufLen,
+							long long* pMaxSize, bool* pCompressed);
+
+	friend class RarLister;
+
+protected:
+	virtual void		PrintMessage(Message::EKind eKind, const char* szFormat, ...) {}
+
+public:
+						DupeMatcher(const char* szDestDir, long long lExpectedSize);
+						~DupeMatcher();
+	bool				Prepare();
+	bool				MatchDupeContent(const char* szDupeDir);
+	static bool			SizeDiffOK(long long lSize1, long long lSize2, int iMaxDiffPercent);
+};
+
+#endif
diff --git a/daemon/postprocess/ParChecker.cpp b/daemon/postprocess/ParChecker.cpp
index 137f3a40..8a6102cc 100644
--- a/daemon/postprocess/ParChecker.cpp
+++ b/daemon/postprocess/ParChecker.cpp
@@ -1004,14 +1004,29 @@ bool ParChecker::AddDupeFiles()
 		FileList extraDirs;
 		RequestExtraDirectories(&extraDirs);
 
-		for (FileList::iterator it = extraDirs.begin(); it != extraDirs.end(); it++)
+		if (!extraDirs.empty())
 		{
-			char* szExtraDir = *it;
-			if (((Repairer*)m_pRepairer)->missingblockcount > 0 && Util::DirectoryExists(szExtraDir))
+			int iWasBlocksMissing = ((Repairer*)m_pRepairer)->missingblockcount;
+
+			for (FileList::iterator it = extraDirs.begin(); it != extraDirs.end(); it++)
 			{
-				bAdded |= AddExtraFiles(false, true, szExtraDir);
+				char* szExtraDir = *it;
+				if (((Repairer*)m_pRepairer)->missingblockcount > 0 && Util::DirectoryExists(szExtraDir))
+				{
+					bAdded |= AddExtraFiles(false, true, szExtraDir);
+				}
+				free(szExtraDir);
+			}
+
+			int iBlocksMissing = ((Repairer*)m_pRepairer)->missingblockcount;
+			if (iBlocksMissing < iWasBlocksMissing)
+			{
+				PrintMessage(Message::mkInfo, "Found extra %i blocks in dupe sources", iWasBlocksMissing - iBlocksMissing);
+			}
+			else
+			{
+				PrintMessage(Message::mkInfo, "No extra blocks found in dupe sources");
 			}
-			free(szExtraDir);
 		}
 	}
 
@@ -1022,7 +1037,7 @@ bool ParChecker::AddExtraFiles(bool bOnlyMissing, bool bExternalDir, const char*
 {
 	if (bExternalDir)
 	{
-		PrintMessage(Message::mkInfo, "Performing dupe par-scan for %s in %s", m_szInfoName, szDirectory);
+		PrintMessage(Message::mkInfo, "Performing dupe par-scan for %s in %s", m_szInfoName, Util::BaseFileName(szDirectory));
 	}
 	else
 	{
@@ -1095,13 +1110,13 @@ bool ParChecker::AddExtraFiles(bool bOnlyMissing, bool bExternalDir, const char*
 
 			if (bOnlyMissing && ((Repairer*)m_pRepairer)->missingfilecount == 0)
 			{
-				PrintMessage(Message::mkInfo, "All missing files found");
+				PrintMessage(Message::mkInfo, "All missing files found, aborting par-scan");
 				break;
 			}
 
 			if (!bOnlyMissing && ((Repairer*)m_pRepairer)->missingblockcount == 0)
 			{
-				PrintMessage(Message::mkInfo, "All missing blocks found");
+				PrintMessage(Message::mkInfo, "All missing blocks found, aborting par-scan");
 				break;
 			}
 		}
diff --git a/daemon/postprocess/ParCoordinator.cpp b/daemon/postprocess/ParCoordinator.cpp
index b9b786da..a6ea92a4 100644
--- a/daemon/postprocess/ParCoordinator.cpp
+++ b/daemon/postprocess/ParCoordinator.cpp
@@ -152,10 +152,29 @@ void ParCoordinator::PostParChecker::RequestExtraDirectories(FileList* pFileList
 	NZBList dupeList;
 	g_pDupeCoordinator->ListHistoryDupes(pDownloadQueue, m_pPostInfo->GetNZBInfo(), &dupeList);
 
-	for (NZBList::iterator it = dupeList.begin(); it != dupeList.end(); it++)
+	if (!dupeList.empty())
 	{
-		NZBInfo* pDupeNZBInfo = *it;
-		pFileList->push_back(strdup(pDupeNZBInfo->GetDestDir()));
+		PostDupeMatcher dupeMatcher(m_pPostInfo);
+		PrintMessage(Message::mkInfo, "Checking %s for dupe scan usability", m_pPostInfo->GetNZBInfo()->GetName());
+		bool bSizeComparisonPossible = dupeMatcher.Prepare();
+		for (NZBList::iterator it = dupeList.begin(); it != dupeList.end(); it++)
+		{
+			NZBInfo* pDupeNZBInfo = *it;
+			if (bSizeComparisonPossible)
+			{
+				PrintMessage(Message::mkInfo, "Checking %s for dupe scan usability", Util::BaseFileName(pDupeNZBInfo->GetDestDir()));
+			}
+			bool bUseDupe = !bSizeComparisonPossible || dupeMatcher.MatchDupeContent(pDupeNZBInfo->GetDestDir());
+			if (bUseDupe)
+			{
+				PrintMessage(Message::mkInfo, "Adding %s to dupe scan sources", Util::BaseFileName(pDupeNZBInfo->GetDestDir()));
+				pFileList->push_back(strdup(pDupeNZBInfo->GetDestDir()));
+			}
+		}
+		if (pFileList->empty())
+		{
+			PrintMessage(Message::mkInfo, "No usable dupe scan sources found");
+		}
 	}
 
 	DownloadQueue::Unlock();
@@ -199,6 +218,18 @@ void ParCoordinator::PostParRenamer::RegisterRenamedFile(const char* szOldFilena
 	}
 }
 
+void ParCoordinator::PostDupeMatcher::PrintMessage(Message::EKind eKind, const char* szFormat, ...)
+{
+	char szText[1024];
+	va_list args;
+	va_start(args, szFormat);
+	vsnprintf(szText, 1024, szFormat, args);
+	va_end(args);
+	szText[1024-1] = '\0';
+
+	m_pPostInfo->GetNZBInfo()->AddMessage(eKind, szText);
+}
+
 #endif
 
 ParCoordinator::ParCoordinator()
diff --git a/daemon/postprocess/ParCoordinator.h b/daemon/postprocess/ParCoordinator.h
index 85dd309d..3242f14b 100644
--- a/daemon/postprocess/ParCoordinator.h
+++ b/daemon/postprocess/ParCoordinator.h
@@ -34,6 +34,7 @@
 #ifndef DISABLE_PARCHECK
 #include "ParChecker.h"
 #include "ParRenamer.h"
+#include "DupeMatcher.h"
 #endif
 
 class ParCoordinator 
@@ -87,7 +88,20 @@ private:
 		
 		friend class ParCoordinator;
 	};
-	
+
+	class PostDupeMatcher: public DupeMatcher
+	{
+	private:
+		PostInfo*		m_pPostInfo;
+	protected:
+		virtual void	PrintMessage(Message::EKind eKind, const char* szFormat, ...);
+	public:
+		PostDupeMatcher(PostInfo* pPostInfo):
+			DupeMatcher(pPostInfo->GetNZBInfo()->GetDestDir(),
+				pPostInfo->GetNZBInfo()->GetSize() - pPostInfo->GetNZBInfo()->GetParSize()),
+				m_pPostInfo(pPostInfo) {}
+	};
+
 	struct BlockInfo
 	{
 		FileInfo*		m_pFileInfo;
diff --git a/daemon/queue/DownloadInfo.cpp b/daemon/queue/DownloadInfo.cpp
index ec98deec..59d8b337 100644
--- a/daemon/queue/DownloadInfo.cpp
+++ b/daemon/queue/DownloadInfo.cpp
@@ -601,6 +601,11 @@ int NZBInfo::CalcCriticalHealth(bool bAllowEstimation)
 		return 1000;
 	}
 
+	if (m_lSize == m_lParSize)
+	{
+		return 0;
+	}
+
 	long long lGoodParSize = m_lParSize - m_lParCurrentFailedSize;
 	int iCriticalHealth = (int)((m_lSize - lGoodParSize*2) * 1000 / (m_lSize - lGoodParSize));
 
diff --git a/daemon/util/Script.cpp b/daemon/util/Script.cpp
index 300fc493..fabab014 100644
--- a/daemon/util/Script.cpp
+++ b/daemon/util/Script.cpp
@@ -599,7 +599,7 @@ int ScriptController::Execute()
 		fclose(m_pReadpipe);
 	}
 
-	if (m_bTerminated)
+	if (m_bTerminated && m_szInfoName)
 	{
 		warn("Interrupted %s", m_szInfoName);
 	}
diff --git a/nzbget.vcproj b/nzbget.vcproj
index 47327e55..7206fa83 100644
--- a/nzbget.vcproj
+++ b/nzbget.vcproj
@@ -576,6 +576,14 @@
 					RelativePath=".\daemon\postprocess\Cleanup.h"
 					>
 				</File>
+				<File
+					RelativePath=".\daemon\postprocess\DupeMatcher.cpp"
+					>
+				</File>
+				<File
+					RelativePath=".\daemon\postprocess\DupeMatcher.h"
+					>
+				</File>
 				<File
 					RelativePath=".\daemon\postprocess\ParChecker.cpp"
 					>
diff --git a/tests/postprocess/DupeMatcherTest.cpp b/tests/postprocess/DupeMatcherTest.cpp
new file mode 100644
index 00000000..a1e67f2c
--- /dev/null
+++ b/tests/postprocess/DupeMatcherTest.cpp
@@ -0,0 +1,114 @@
+/*
+ *  This file is part of nzbget
+ *
+ *  Copyright (C) 2015 Andrey Prygunkov <hugbug@users.sourceforge.net>
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ *
+ * $Revision$
+ * $Date$
+ *
+ */
+
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#ifdef WIN32
+#include "win32.h"
+#endif
+
+#include <iostream>
+#include <fstream>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#ifndef WIN32
+#include <unistd.h>
+#endif
+
+#include "catch.h"
+
+#include "nzbget.h"
+#include "Options.h"
+#include "DupeMatcher.h"
+#include "TestUtil.h"
+
+TEST_CASE("Disk matcher", "[Par][DupeMatcher][Slow][TestData]")
+{
+	Options options(NULL, NULL);
+
+	TestUtil::PrepareWorkingDir("DupeMatcher");
+
+	char szErrBuf[256];
+
+	// prepare directories
+
+	std::string dupe1(TestUtil::WorkingDir() + "/dupe1");
+	REQUIRE(Util::ForceDirectories(dupe1.c_str(), szErrBuf, sizeof(szErrBuf)));
+	TestUtil::CopyAllFiles(dupe1, TestUtil::TestDataDir() + "/parchecker");
+
+	std::string dupe2(TestUtil::WorkingDir() + "/dupe2");
+	REQUIRE(Util::ForceDirectories(dupe2.c_str(), szErrBuf, sizeof(szErrBuf)));
+	TestUtil::CopyAllFiles(dupe2, TestUtil::TestDataDir() + "/parchecker");
+	remove((dupe2 + "/testfile.nfo").c_str());
+
+	std::string rardupe1(TestUtil::TestDataDir() + "/dupematcher1");
+	std::string rardupe2(TestUtil::TestDataDir() + "/dupematcher2");
+
+	std::string nondupe(TestUtil::WorkingDir() + "/nondupe");
+	REQUIRE(Util::ForceDirectories(nondupe.c_str(), szErrBuf, sizeof(szErrBuf)));
+	TestUtil::CopyAllFiles(nondupe, TestUtil::TestDataDir() + "/parchecker");
+	remove((nondupe + "/testfile.dat").c_str());
+
+	// now test
+	long long lExpectedSize = Util::FileSize((dupe1 + "/testfile.dat").c_str());
+
+	DupeMatcher dupe1Matcher(dupe1.c_str(), lExpectedSize);
+	CHECK(dupe1Matcher.Prepare());
+	CHECK(dupe1Matcher.MatchDupeContent(dupe2.c_str()));
+	CHECK(dupe1Matcher.MatchDupeContent(rardupe1.c_str()));
+	CHECK(dupe1Matcher.MatchDupeContent(rardupe2.c_str()));
+	CHECK_FALSE(dupe1Matcher.MatchDupeContent(nondupe.c_str()));
+	
+	DupeMatcher dupe2Matcher(dupe2.c_str(), lExpectedSize);
+	CHECK(dupe2Matcher.Prepare());
+	CHECK(dupe2Matcher.MatchDupeContent(dupe1.c_str()));
+	CHECK(dupe2Matcher.MatchDupeContent(rardupe1.c_str()));
+	CHECK(dupe2Matcher.MatchDupeContent(rardupe2.c_str()));
+	CHECK_FALSE(dupe2Matcher.MatchDupeContent(nondupe.c_str()));
+
+	DupeMatcher nonDupeMatcher(nondupe.c_str(), lExpectedSize);
+	CHECK_FALSE(nonDupeMatcher.Prepare());
+	CHECK_FALSE(nonDupeMatcher.MatchDupeContent(dupe1.c_str()));
+	CHECK_FALSE(nonDupeMatcher.MatchDupeContent(dupe2.c_str()));
+	CHECK_FALSE(nonDupeMatcher.MatchDupeContent(rardupe1.c_str()));
+	CHECK_FALSE(nonDupeMatcher.MatchDupeContent(rardupe2.c_str()));
+
+	DupeMatcher rardupe1matcher(rardupe1.c_str(), lExpectedSize);
+	CHECK(rardupe1matcher.Prepare());
+	CHECK(rardupe1matcher.MatchDupeContent(dupe1.c_str()));
+	CHECK(rardupe1matcher.MatchDupeContent(dupe2.c_str()));		    
+	CHECK(rardupe1matcher.MatchDupeContent(rardupe2.c_str()));
+	CHECK_FALSE(rardupe1matcher.MatchDupeContent(nondupe.c_str()));
+
+	DupeMatcher rardupe2matcher(rardupe2.c_str(), lExpectedSize);
+	CHECK(rardupe2matcher.Prepare());
+	CHECK(rardupe2matcher.MatchDupeContent(rardupe1.c_str()));
+	CHECK(rardupe2matcher.MatchDupeContent(dupe1.c_str()));
+	CHECK(rardupe2matcher.MatchDupeContent(dupe2.c_str()));
+	CHECK_FALSE(rardupe2matcher.MatchDupeContent(nondupe.c_str()));
+}
diff --git a/tests/suite/TestUtil.cpp b/tests/suite/TestUtil.cpp
index 434a7b52..9ee93fe4 100644
--- a/tests/suite/TestUtil.cpp
+++ b/tests/suite/TestUtil.cpp
@@ -123,13 +123,18 @@ void TestUtil::PrepareWorkingDir(const std::string templateDir)
 	Util::CreateDirectory(workDir.c_str());
 	REQUIRE(Util::DirEmpty(workDir.c_str()));
 
+	CopyAllFiles(workDir, srcDir);
+}
+
+void TestUtil::CopyAllFiles(const std::string destDir, const std::string srcDir)
+{
 	DirBrowser dir(srcDir.c_str());
 	while (const char* filename = dir.Next())
 	{
 		if (strcmp(filename, ".") && strcmp(filename, ".."))
 		{
 			std::string srcFile(srcDir + "/" + filename);
-			std::string dstFile(workDir + "/" + filename);
+			std::string dstFile(destDir + "/" + filename);
 			REQUIRE(Util::CopyFile(srcFile.c_str(), dstFile.c_str()));
 		}
 	}
diff --git a/tests/suite/TestUtil.h b/tests/suite/TestUtil.h
index 8189d475..dd0c2405 100644
--- a/tests/suite/TestUtil.h
+++ b/tests/suite/TestUtil.h
@@ -39,6 +39,7 @@ public:
 	static void			CleanupWorkingDir();
 	static void			DisableCout();
 	static void			EnableCout();
+	static void			CopyAllFiles(const std::string destDir, const std::string srcDir);
 };
 
 #endif
diff --git a/tests/testdata/dupematcher1/testfile.part01.rar b/tests/testdata/dupematcher1/testfile.part01.rar
new file mode 100644
index 0000000000000000000000000000000000000000..f96639e1767a9ebe1c6769f37c5c95c0b019df51
GIT binary patch
literal 4096
zcmai1L8}}^5Z;I&1SJRo!Be}J?2@<H_d;?>2%49u5sb#D2)Xr4_wF=1(>>|#d7Dix
zIr|U%1D?Ei)U*FX6ufvAy!uu3%x=$1z=ikN?dh(r`s%B%Y934f!TWDbp4jm2g?A?D
z^YfiJeR(wb>C$BK;_nNSH-7);&ELNN_1{k}AHO{j*b;S!D^u%hN=Ey<SnIerxq0pQ
z)<q!}qOG+IS_F;1t~k>^7-vQ8)<Q;X5Tc5)y|r8h*ZESfoL}o}R>x(#Os1BlQ+oN}
z<Gc5lIJEq*m9e_BjJ~s*ix-F6ZVb0crL+>#M^n~Xl(MWexsg5#Vf9wHHj)R^yJ^!s
z+A|eOuZ-0}G`ewqCu-A}D54V`e1}uX)a}gFw$e7O3KN7ZqdC(eR(iM`YhH`3H!)(P
zv9QJNh?Vgn3Tf5en4oi3$hsD*x(gK|z;0#2CY>Xz3K$zJ*>%D?cV!f=@LF{xBH{C&
zeMvWoM(+A`Tn}JYUalK$BiqaTFiq>-nXzlpx)2xm0ZV~?hgQR_S!Y{WZsz&zz-jCJ
zCWx(xl_(tkQPgy(LF$5-DdTmCU0Bru(33}QwHjGrIEU2S=L6r-{wbl0P-IaPEI=Zm
z7%JD*N)%cYW-YXJ-MY#GnNiLPU?y7a@x76@)N_JWmZfXpn1nBccn@BHNAL~M0H54Y
zaE*?pK^$F6=WiY#i-)IkWJWlzv}c=V&ZvFVdyACx%v97(>%0WJ=#FqC0G0Mz1BRV@
z#MA-v6jN_+Z(Er%4^lZ{U4*rm@heXagUWW`Ey6Aa263t~lnGd{6;NA;GA*lA0O#pJ
ziPL99RS_BjFXdu(!k7S$^c2B`Ii6Wc4_~iWD=+{RQtuA_3Bmpya$0Ek2%uLTOjA(!
z+Sh~ZqHEwziCcFAqwwt<#OJGIE||}X$mpGG7*7&`0n>vwNgTY}3N?JFJWzEdN*RF%
z{OpK}5;_E0TCvr|0uF#+vdlSStb(IZ#iR#~>ue-?_1dyeETebcgO$;>3S)Wt+)a<h
zD-X<+of@gw!_#R-H?L#oEjU!{#N$Wz7NBwEl$fSCrgZNDqF`l%JwxSj&7F2gm;c&4
zr|Ki&nr6$>Tu;kN+I3%-3PoaB+yNzE>`fwxK@&m6#?-FZ5iG`rs3nO2oWnwnfkRc3
zPaNKvtUFRv{EY}uI*U#cwaQPL72_S?H(BmeOe`vN4Tp$m$zT$DMurxXr$@9Q!>?cz
z=r}m%D4!^Ck~lgt@JH}%GhjrOxRr>jPxDCKaKuw@OC17QT7pHrGt<QhiyRs7`e_I4
zLkF`v^dWs)sLq4DnOiAz#(6mDbz6ghy?fJ><Lfu3Vs-*Y){^j@i=smY;!u)qHL}-v
zC=dj)<J#J%6jEt-NuzpCYhd6LZ;1A5&zQ8*Utd5_JM@)VZ3_BL@(4`6cyjuwj(@)V
z0UY4=4iQqo!0S<0CO~s5fSd?abf!k593?)%!H>A8k>_+tPJ~#&f~FRZh6%bU-D<0}
zEup*~BA}lk4vPfz-ZItjJ(zWQ^=fXZksFc@K-Z8cVv;dXeCjPIVvqVpQ+G{EEj78p
z3~@jkV@g?#GuPji08E1lvR!UcO9r3Kie!U^lRiv<vcPmwZk9-~G(Imqu$>;K2gK4n
zcgQ{EwzhUCF~XMH&3>up37pJWpuKm9hbk>Of?ez!G#2_nK}sSb*)y1i$f?jZII)&g
z&F5<8@H|A&SW)@!vrO*fAalBabg!2qAduhAD%nU<nG}bt8_XuGA=1X&(6Da?Un2?(
zz7IHPH6M<XnAE|&F3t>)L&OD&e{&_gKIUw76lrD&&Rkz`%&Fg5O*0jZ0fb2Eqmp|B
z&mb5~WJ)Kp$8_beGlN7rS(hD^qj<%;9FYG(cEBs)3fxg2P_W!WJ}^I${wv}uw51<+
z!3;>L?CLmDJ8tWV+6cY}x@R`Y>0wq*>^-Cwxq%F3vF^!;po63d8(ro$Bj+sg>rs-Z
z7oxnerSm*!_B>gk$}>}arSxM-CB51b-FxQGQgc4yu{jAQqsLklAz#!!$MxJ-d_Ren
z+@%VXRgNF3G>yza*3_7V^|P|>Sh|cwGs4)|$`CL>W3u8JoU<A+dYSXUF)0BUjXFtV
zABGZ==sp5Ku*baLi+%*I10%J|ZKa7AsIxfGuyrtxc>9SN%@rH)yV7+_JzJD$xu|x5
z88&Z}!NiVb3pFBnxB)L@VQK1yY46Y|8jQG}^%(L-04+pPzy{<&QzaisUdWMGwT@!8
zS7I;Z^a{bFGwC;cF#v=HwDcB&`PA>gK(8@i^Gj78I%)=7A>AVI5c$1F-y$pA*Vz@k
zu0rj+4)Y_ROs_rMs*}o(^N(H+Tqx~>^yZmAS2^Esh+#@CgXSTE-gZrab^?t{0ASoj
z5>xoNDz`S?-cxxvMKv|*lIazY-y-Q?QWxBlAyGhG{`&F7rH?L8-h2ATFa4YTWbz+6
Ck~f$D

literal 0
HcmV?d00001

diff --git a/tests/testdata/dupematcher1/testfile.part24.rar b/tests/testdata/dupematcher1/testfile.part24.rar
new file mode 100644
index 0000000000000000000000000000000000000000..4f8996091d0dd2eeac0a7d6aa9af32e71bf90285
GIT binary patch
literal 4096
zcmb_fO>Z1U5M8(+As|AEgy7VO1CH=`lOPUBMpj~D2SZ{nwi88MqTZR_-A-n@hwh$T
zZ*t+xAIP6T;vNp1`2ld}z=c1+l~>h0`{9J-f)CbAZ+CUot5>gTj-~tfqqlk=KJoAO
z-tEP|U!R3fe%S2&bfMRK`u7{XH-G-?x92}T|7ZW=<#&1lTY~aoq${;m%3$p5ZGpeW
zX%kc_f)&1SN|}CT#~JOqf!{MxEAQo4c_D|k4Z;q+tyECLHbGm1Wv2vez1EJd8|$XH
zNS(>Lsg!7}4~?@0>}xZo%XJg33t5)f<l6?@gBV$d13@_>D|WZAW~9fhlXSZvH#g5)
zYNV4jTy${Glw$qFR7<uqQ^Sk`cOzM(NMl4IKb8LRgRMbyZJUEDq#-*&Y9m(6&nCK<
zh`RMb3=xp@UX^SeV1|iGF-06>T#alg+P+i1tto5@kdVj{6nOo<Rjs-q*ibxW)J)X2
z)FZ8&$d9%uSx(g!X(()6BT6swEF<HppAEJ-ZY;DWE#2EDCqx35%q`Dx9_VB`M(^w%
z7I4plyLYnw*C$7Z+3{dETA_p($`T0C5GO|Gydh^Q?L$Ai!#1=jbs^y`yMycoys$$!
z`ts-(cO*wePKR2*O}DN2#CB#nOuz<Jl_DP?fB^>-kuN6FjEUjMNN26m2I-k2qi37~
zr+~T-l#%~%iLV(sTO+ubfjqk-@I)B|H$381rkD$M#p1G*SWYJZst|Kz#{&C}JR88o
zCQRNVQ$*K1+r#6f=sJqV+4DuyQRUTDQ{uvTx)7Ajiis*7i;dvsqBTxBpyhl(@!&V7
z8mXPoqwXh(x+;?ok%H&x*8vH4`&p(+9U>C+6j5Zr){~n>YDXw-RoNNG&<+=fkea00
zSLJn)f7Z|JXyjEOkWT5}4*mNw@!2nq(pq=Y3k0uNAdpPHvy3!*1o>i=WfAmL3A_vG
z#Y|OIr1t8;l|8kRkUA!iGzE5dMCpJ<K{JHbad6NTR5vv0MLuCEi&`f(v?;2#q)o*{
zgStrrAhz(EZewF{H)lhLwt;qo%(474ub`=p#XBh`6*nazTVmq{WD7O3R>m3YDwi?c
zlxhL@uFK%5uAu`sA7)m}<lJA6PF=YqbYN|iMWLGDA%j)WB<!3_ct%1=u1;lz+KTvI
zy#)9HIRMgU1pUOe&O^DN^%3M;A)q<0bCR>JBTNl~o@!6JZ`yho!^UP_fo2ho8w)1o
z*|aOD``eol$bf6wMxen7Wl#wTQ@?6jgCK3NHIi4zs+!|cSqd(E?4we#UIz;o;fIpY
z|6z;V#d?jkLPCP`F-vF}<QP^IGjFH@f)^#^0_&~z6Z#prr&KPr5uh)tY@w&C7g4UA
z3az7^r4k$0KEL!rM<MfxaFFv(iX$K`a_9Z1tvc#q^9?Xh2H5pTC{w4XKz}j=?BHl=
zXQr|Mg)55<K{-0w45Zto{cZCt^)YY<fO561dQPBP19ZDFBEQu%JK9ij?{bUZhXkbr
z-j6ws0@Kno5-&wBK0iNrvC&y1DUnksnH++YpqesM?X0PhdT`QdTK8-zKm2azXz=jX
zV6by=NG`<p-J`?(gWLNDclMGaaryq<@yWr_VW-hf8H^jHK;S3bqPQCbiCgmhTR041
z0`UBN?--`2VMyK-TZZD6!H%5p5cE<rbaXSOMpoVmIU1<~`Jv+Na%D$s1&umeFlT~G
zz}B@|lHxpb@@rr82S+ETgX5#!y_1uNhewCW(^UmuQYqrbtxtOB)oh}WjiysJ|5!Ij
zdj(#t%r<m<W*v2*d6TNEZLDFZrN9W{KJ^F&yuw6<NewW;Cg}*zbE#ZYD~b($b((y!
zF~&iO3xsfrgHPGf2Py;>gV9nb=eUidh&Yt>i_=WeO@uR$2`!1UYU~=aD<eo$PM*}L
zlIR{v%x)amL{0+|&94|~sPFU1M8~_@41im-z(c`ZdA3}M69><I(ahM{w(-dk$VOg3
zR6E0YVJquYiOGq_wG8+nV}!?|l6|AVU{vbLB+pri!whDnQWem&3c59h?lVT*HoAl8
z<s`@$wZxUn;+vg53w|cj1j9uK&t9FbZu*b23Wcf8QjM@TfH3FbB|^5~_^en<f}fy6
zi=hk<XF`}O4x<Lbg+7=k!?8kpq)l7clvqWsKL#vTSU1#ustdrev3$Ns^z>UE{i;xK
zZLGG83Q=wu2`=4LjVskq0}dbNZ~<K)bTXV=hy#{j(Ei_>$)DI4sN|U7P|5}g`s(y_
zuu66aH*H48n^>ox#IGvwo8cbu7)X7N6$Ry-y7&^SN2RKVsCKc@J-rU6hSCyYS{=-N
zrBst~mn>Uu<>XhGI2%GYZ4FThGhNc8PjmRy%a?0P82fsKo<T6{$~YLHGi!_z_%P#s
z+Zv6e)jG`M-3t*}W#kL(gy>*wAZcQp7K|%3zDO|{#la$e2XgrYs2+_;%hr>=<d&G-
lhJ&XH4tXXf#km5?%SQh9etCM~=EdH#-~agYgY>E2`xj<?bawy%

literal 0
HcmV?d00001

diff --git a/tests/testdata/dupematcher2/testfile.part04.rar b/tests/testdata/dupematcher2/testfile.part04.rar
new file mode 100644
index 0000000000000000000000000000000000000000..1cbc1f04b185dd68131e778493cb46b6e7a8182b
GIT binary patch
literal 3072
zcma)8&2Ah;5Z=H6iGUDN4uCimgCcFDtZ^bCWamO~_*Eh-Zvqkrq~4j@oz~8DPr7@u
z+T_AHZ@@cn<iwRb55Obv0$lm3dS*RIF4&gso$jjos_Lt+=E8)JK6qpH{?qvG?Cn|p
z`t{HB@h>-Lzg?Zpp8a!W_S*B`ufF&EkAJ_qcKp_ia3yg}YuoTqWs+;WskmC&r~HjA
zZc=g2?<UV4d{NwdlMY{p(}%X@C6{7w?qhoBlWC^UOqhJS;Lg~~2UCBKTxZJdc=ryD
z7Swf%SO(iAD!zPRY;)A@3vzx(vF|z`Qlu77DROwifdV&XA8DRy<F=8Kr{?yC);>_G
zSwnRumGzD6dsnlg(zhMbugE&8c-3#H^%axj9jz@l6%}3ZCC5m%D#%9af^`Y^;T%qJ
zsmY!jpChp;YYIR(w@L1^i5Ads-B#?9UGL#SJu(miD1HStRUA;+HQ||ix}w@d@~+v_
zTEHMpL^UDVvVkR>w`Pw!PgrwkI9cx~)@T{Q@YeD1bxD)7S{cgW@Z{v^<hnv~l`TzT
z0g`Yt5u=X#R9m;99o)pel;qca)9lq-b)ZBpV&8!{09Japwwpfi*e@h3{sKSlZD87z
zVUI3KFHl<)<ZIVIyuT!bivS+g`CMBvTt)4~fnx?IhvJC&h#;b0Xg6Y4JreLz*3N8<
z)<KVG?mPWdoL?T@6QkmW#xvUIm8mwYeICuV(aJI{r(<&&mYiuh=WZ$>Qx{{LM?snI
z`&0;`$cYlB1D`eXM;G_eOfXQ&Yg|Pa%OxIgrD(>JJ1Ck2sc3Y2=AGkGQjzHxy%%>g
z8OCQBlx<OmCosMoopEwbM=q3LSY*T(a1;}27Qw3*Cg-~!9%NK`dT2U1LyQ)D*174s
zCex^_x!j(roltK<!e*_|*frYOR1>#d+M{nBZ+WkkxO@^z55A!LtqEJ?VKO(wA5kGT
zg<8n;88D;<nO^iRClLkIR^<_Km<<haz($Ag!YO#oabX)-sl=!*2F(CW={pM~>)_iw
zDPz1mVFGM{17OBa9~t99D=rQG#pA3dqlsn)nWV;U8N)rMfEq%5C-D4QZ8Z^58BH34
zj4AG+OFH?7Op(CQ0-1sVI;LgfZv~~7M2CA&o~jWZ29AA`VrI{viJ5LVgenGZJrqJ^
zv5#T_&1{9eFh{9OK&ca&P%IWu0U_!qi?c!_3%&`7Q}E5V=ZEPMAq`!Cv2>Y=Tem+u
zI>z@c+*pgkp_)2EUY!+khVKH>>SBUs5^5yEsA?s=0<Sp)G_-|inGH{seu~4op3g-~
zO`~WGh6_R}aLYbm{7`00h0-uWVlm~Aq87wdBz~Q+I#j&9D^UYv9~sHZ8aW^{xD~RX
z9tbL#=Bd2{dz+~p?Z(+PCdefaB5>8ir(iLy?*Ro_RyJ;F7i>bRvK6SdoBH6DVh0f!
zT9{R7=n<-VE;N+I3=tG9&(l-Lwte|PBnu;4^$my2hhBHGG(zOi^vSmni_%~nkZqw+
z7F7}!uds}f9QxL~0J#jD*YnxMS#HUI?nQ_H%f&8ukghW^?tG|dj^9}BBe@amQ(@1J
zn@0Z1ittT^%HuzP#a`7~4uLa0q`AG+=I(O&<M$W$gaq1I6@94oD_$Vyp60>=UC9!4
zT-c-FVlNxlcT@S(whhn6B8m?XM$k<QHY|ATUnf<>SS?gd6_zLry|j?0+Vi+)?7%V5
zT+m}|jdd~^aCLhj3KDOO7+SpNDV-@oWkJ(ZsnH~qURUkgGS>DSKA^?0S5exNY>*X;
z)>X8^$3YQ_4m?{xf`n)jcUa5TW#WmRiloR;sd(>f9QNs9bw#)z-Xr4a5ECI2Z+zIx
zIN<e2Gm?BtDA^{%7;~7|=Q)ZpVz@JfC#A{nmRaOqB{-CklV)K`ETu{;Fd{OU@_sl1
z%9E1EuC)k-YjUpg2y_AJ0y^RaODtvW4$@w8(+LDd%g5KsResGXkl_YAT+~H&zv857
bqUwg>`uneE?|go3_S4TFUU_|Z)t~(b>;MD=

literal 0
HcmV?d00001

diff --git a/tests/testdata/dupematcher2/testfile.part43.rar b/tests/testdata/dupematcher2/testfile.part43.rar
new file mode 100644
index 0000000000000000000000000000000000000000..080fbf18cc3c8c6dd4f97738f42eb349fa9cc795
GIT binary patch
literal 151
zcmWGaEK-zWXOOHa7G&UMfPnvuC5jC?49-Aa^(#gOCda3Fja|FGyNMX^Fem^8OHzwV
z(lT>W^->Z`s*=)EOLPtN4D}2WxfB$16_WFF3o>&u^U`(GGAmMxxw!I>lz>%a=4F;-
gCgvy<rRJn27N>HpSFPrA5Mj8cv&d2l;s6E)0L<JdeE<Le

literal 0
HcmV?d00001