chore: remove weak hashing which does not pull its weight (#10005)

We've had weak/rolling hashing in the code for quite a while. It was a
popular request for a while, based on the belief that rsync does this
and we should too. However, the benefit is quite small; we save on
average about 0.8% of transferred blocks over the population as a whole:

<img width="974" alt="Screenshot 2025-03-28 at 17 09 02"
src="https://github.com/user-attachments/assets/bbe10dea-f85e-4043-9823-7cef1220b4a2"
/>

This would be fine if the cost was comparably low, however the downside
of attempting rolling hash matching is that we (by default) do a
complete file read on the destination in order to look for matches
before we starting pulling blocks for the file. For any larger file this
means a sometimes long, I/O-intensive pause before the file starts
syncing, for usually no benefit.

I propose we simply rip off the bandaid and save the effort.
This commit is contained in:
Jakob Borg
2025-03-29 12:21:10 +00:00
committed by GitHub
parent 1a25ae32ca
commit b1c8f88a44
38 changed files with 288 additions and 1030 deletions

View File

@@ -72,7 +72,7 @@ func main() {
if *standardBlocks || blockSize < protocol.MinBlockSize {
blockSize = protocol.BlockSize(fi.Size())
}
bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true)
bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil)
if err != nil {
log.Fatal(err)
}

View File

@@ -238,7 +238,7 @@ func (c *CLI) decryptFile(encFi *protocol.FileInfo, plainFi *protocol.FileInfo,
}
// Verify the hash against the plaintext block info
if !scanner.Validate(dec, plainBlock.Hash, 0) {
if !scanner.Validate(dec, plainBlock.Hash) {
// The block decrypted correctly but fails the hash check. This
// is odd and unexpected, but it it's still a valid block from
// the source. The file might have changed while we pulled it?

1
go.mod
View File

@@ -9,7 +9,6 @@ require (
github.com/calmh/incontainer v1.0.0
github.com/calmh/xdr v1.2.0
github.com/ccding/go-stun v0.1.5
github.com/chmduquesne/rollinghash v4.0.0+incompatible
github.com/d4l3k/messagediff v1.2.1
github.com/getsentry/raven-go v0.2.0
github.com/go-ldap/ldap/v3 v3.4.10

2
go.sum
View File

@@ -29,8 +29,6 @@ github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chmduquesne/rollinghash v4.0.0+incompatible h1:hnREQO+DXjqIw3rUTzWN7/+Dpw+N5Um8zpKV0JOEgbo=
github.com/chmduquesne/rollinghash v4.0.0+incompatible/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=

View File

@@ -1093,10 +1093,9 @@ type BlockInfo struct {
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
Hash []byte `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"`
Offset int64 `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"`
Size int32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
WeakHash uint32 `protobuf:"varint,4,opt,name=weak_hash,json=weakHash,proto3" json:"weak_hash,omitempty"`
Hash []byte `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"`
Offset int64 `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"`
Size int32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
}
func (x *BlockInfo) Reset() {
@@ -1150,13 +1149,6 @@ func (x *BlockInfo) GetSize() int32 {
return 0
}
func (x *BlockInfo) GetWeakHash() uint32 {
if x != nil {
return x.WeakHash
}
return 0
}
type Vector struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -1578,7 +1570,6 @@ type Request struct {
Size int32 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"`
Hash []byte `protobuf:"bytes,6,opt,name=hash,proto3" json:"hash,omitempty"`
FromTemporary bool `protobuf:"varint,7,opt,name=from_temporary,json=fromTemporary,proto3" json:"from_temporary,omitempty"`
WeakHash uint32 `protobuf:"varint,8,opt,name=weak_hash,json=weakHash,proto3" json:"weak_hash,omitempty"`
BlockNo int32 `protobuf:"varint,9,opt,name=block_no,json=blockNo,proto3" json:"block_no,omitempty"`
}
@@ -1661,13 +1652,6 @@ func (x *Request) GetFromTemporary() bool {
return false
}
func (x *Request) GetWeakHash() uint32 {
if x != nil {
return x.WeakHash
}
return 0
}
func (x *Request) GetBlockNo() int32 {
if x != nil {
return x.BlockNo
@@ -2079,158 +2063,155 @@ var file_bep_bep_proto_rawDesc = []byte{
0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x69, 0x6e, 0x76,
0x61, 0x6c, 0x69, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x6e, 0x6f, 0x5f, 0x70, 0x65, 0x72, 0x6d, 0x69,
0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6e, 0x6f,
0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x68, 0x0a, 0x09, 0x42,
0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x51, 0x0a, 0x09, 0x42,
0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68,
0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x16, 0x0a, 0x06,
0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66,
0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01,
0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x65, 0x61, 0x6b,
0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x77, 0x65, 0x61,
0x6b, 0x48, 0x61, 0x73, 0x68, 0x22, 0x32, 0x0a, 0x06, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12,
0x28, 0x0a, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28,
0x0b, 0x32, 0x0c, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x52,
0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x22, 0x2f, 0x0a, 0x07, 0x43, 0x6f, 0x75,
0x6e, 0x74, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04,
0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xfd, 0x01, 0x0a, 0x0c, 0x50,
0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x12, 0x21, 0x0a, 0x04, 0x75,
0x6e, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x62, 0x65, 0x70, 0x2e,
0x55, 0x6e, 0x69, 0x78, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x75, 0x6e, 0x69, 0x78, 0x12, 0x2a,
0x0a, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
0x10, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x44, 0x61, 0x74,
0x61, 0x52, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x12, 0x24, 0x0a, 0x05, 0x6c, 0x69,
0x6e, 0x75, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e,
0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x75, 0x78,
0x12, 0x26, 0x0a, 0x06, 0x64, 0x61, 0x72, 0x77, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61,
0x52, 0x06, 0x64, 0x61, 0x72, 0x77, 0x69, 0x6e, 0x12, 0x28, 0x0a, 0x07, 0x66, 0x72, 0x65, 0x65,
0x62, 0x73, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e,
0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x07, 0x66, 0x72, 0x65, 0x65, 0x62,
0x73, 0x64, 0x12, 0x26, 0x0a, 0x06, 0x6e, 0x65, 0x74, 0x62, 0x73, 0x64, 0x18, 0x06, 0x20, 0x01,
0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0x32,
0x0a, 0x06, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x28, 0x0a, 0x08, 0x63, 0x6f, 0x75, 0x6e,
0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x62, 0x65, 0x70,
0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x52, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65,
0x72, 0x73, 0x22, 0x2f, 0x0a, 0x07, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x0e, 0x0a,
0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a,
0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61,
0x6c, 0x75, 0x65, 0x22, 0xfd, 0x01, 0x0a, 0x0c, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d,
0x44, 0x61, 0x74, 0x61, 0x12, 0x21, 0x0a, 0x04, 0x75, 0x6e, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x55, 0x6e, 0x69, 0x78, 0x44, 0x61, 0x74,
0x61, 0x52, 0x04, 0x75, 0x6e, 0x69, 0x78, 0x12, 0x2a, 0x0a, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f,
0x77, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x57,
0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x44, 0x61, 0x74, 0x61, 0x52, 0x07, 0x77, 0x69, 0x6e, 0x64,
0x6f, 0x77, 0x73, 0x12, 0x24, 0x0a, 0x05, 0x6c, 0x69, 0x6e, 0x75, 0x78, 0x18, 0x03, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61,
0x74, 0x61, 0x52, 0x06, 0x6e, 0x65, 0x74, 0x62, 0x73, 0x64, 0x22, 0x6c, 0x0a, 0x08, 0x55, 0x6e,
0x69, 0x78, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f,
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e, 0x65,
0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x6e,
0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70,
0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28,
0x05, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x04, 0x20,
0x01, 0x28, 0x05, 0x52, 0x03, 0x67, 0x69, 0x64, 0x22, 0x52, 0x0a, 0x0b, 0x57, 0x69, 0x6e, 0x64,
0x6f, 0x77, 0x73, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72,
0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e,
0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f,
0x69, 0x73, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c,
0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0x2f, 0x0a, 0x09,
0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x12, 0x22, 0x0a, 0x06, 0x78, 0x61, 0x74,
0x74, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x62, 0x65, 0x70, 0x2e,
0x58, 0x61, 0x74, 0x74, 0x72, 0x52, 0x06, 0x78, 0x61, 0x74, 0x74, 0x72, 0x73, 0x22, 0x31, 0x0a,
0x05, 0x58, 0x61, 0x74, 0x74, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61,
0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
0x22, 0xe4, 0x01, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02,
0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06,
0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f,
0x6c, 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01,
0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73,
0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74,
0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04,
0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x06, 0x20, 0x01,
0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x72, 0x6f, 0x6d,
0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x72, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08,
0x52, 0x0d, 0x66, 0x72, 0x6f, 0x6d, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x72, 0x79, 0x12,
0x1b, 0x0a, 0x09, 0x77, 0x65, 0x61, 0x6b, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x08, 0x20, 0x01,
0x28, 0x0d, 0x52, 0x08, 0x77, 0x65, 0x61, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x12, 0x19, 0x0a, 0x08,
0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x6f, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07,
0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x6f, 0x22, 0x52, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52,
0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28,
0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18,
0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x45, 0x72, 0x72, 0x6f,
0x72, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x65, 0x0a, 0x10, 0x44,
0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12,
0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x39, 0x0a, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74,
0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x46,
0x74, 0x61, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x75, 0x78, 0x12, 0x26, 0x0a, 0x06, 0x64, 0x61, 0x72,
0x77, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e,
0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x06, 0x64, 0x61, 0x72, 0x77, 0x69,
0x6e, 0x12, 0x28, 0x0a, 0x07, 0x66, 0x72, 0x65, 0x65, 0x62, 0x73, 0x64, 0x18, 0x05, 0x20, 0x01,
0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61,
0x74, 0x61, 0x52, 0x07, 0x66, 0x72, 0x65, 0x65, 0x62, 0x73, 0x64, 0x12, 0x26, 0x0a, 0x06, 0x6e,
0x65, 0x74, 0x62, 0x73, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65,
0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x06, 0x6e, 0x65, 0x74,
0x62, 0x73, 0x64, 0x22, 0x6c, 0x0a, 0x08, 0x55, 0x6e, 0x69, 0x78, 0x44, 0x61, 0x74, 0x61, 0x12,
0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d,
0x0a, 0x0a, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01,
0x28, 0x09, 0x52, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a,
0x03, 0x75, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12,
0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x67, 0x69,
0x64, 0x22, 0x52, 0x0a, 0x0b, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x44, 0x61, 0x74, 0x61,
0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12,
0x24, 0x0a, 0x0e, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x73, 0x5f, 0x67, 0x72, 0x6f, 0x75,
0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x73,
0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0x2f, 0x0a, 0x09, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61,
0x74, 0x61, 0x12, 0x22, 0x0a, 0x06, 0x78, 0x61, 0x74, 0x74, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03,
0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x52, 0x06,
0x78, 0x61, 0x74, 0x74, 0x72, 0x73, 0x22, 0x31, 0x0a, 0x05, 0x58, 0x61, 0x74, 0x74, 0x72, 0x12,
0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e,
0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01,
0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xcd, 0x01, 0x0a, 0x07, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
0x05, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a,
0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d,
0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28,
0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a,
0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a,
0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73,
0x68, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72,
0x61, 0x72, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x66, 0x72, 0x6f, 0x6d, 0x54,
0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x62, 0x6c, 0x6f, 0x63,
0x6b, 0x5f, 0x6e, 0x6f, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x62, 0x6c, 0x6f, 0x63,
0x6b, 0x4e, 0x6f, 0x4a, 0x04, 0x08, 0x08, 0x10, 0x09, 0x22, 0x52, 0x0a, 0x08, 0x52, 0x65, 0x73,
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
0x05, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20,
0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x0a, 0x04, 0x63, 0x6f, 0x64,
0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x45, 0x72,
0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x65, 0x0a,
0x10, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73,
0x73, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28,
0x09, 0x52, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x39, 0x0a, 0x07, 0x75, 0x70, 0x64,
0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x62, 0x65, 0x70,
0x2e, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f,
0x67, 0x72, 0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x07, 0x75, 0x70, 0x64,
0x61, 0x74, 0x65, 0x73, 0x22, 0xe5, 0x01, 0x0a, 0x1a, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77,
0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x55, 0x70, 0x64,
0x61, 0x74, 0x65, 0x12, 0x44, 0x0a, 0x0b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x79,
0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x46,
0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72,
0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74,
0x65, 0x73, 0x22, 0xe5, 0x01, 0x0a, 0x1a, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c,
0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74,
0x65, 0x12, 0x44, 0x0a, 0x0b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65,
0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x46, 0x69, 0x6c,
0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73,
0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x75, 0x70, 0x64,
0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x07, 0x76,
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x62,
0x65, 0x70, 0x2e, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69,
0x6f, 0x6e, 0x12, 0x27, 0x0a, 0x0d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x69, 0x6e, 0x64, 0x65,
0x78, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x05, 0x42, 0x02, 0x10, 0x00, 0x52, 0x0c, 0x62,
0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x62,
0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52,
0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x06, 0x0a, 0x04, 0x50, 0x69,
0x6e, 0x67, 0x22, 0x1f, 0x0a, 0x05, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72,
0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x61,
0x73, 0x6f, 0x6e, 0x2a, 0xed, 0x01, 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54,
0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x1b, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54,
0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x43, 0x4f, 0x4e, 0x46,
0x49, 0x47, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f,
0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x44, 0x45, 0x58, 0x10, 0x01, 0x12, 0x1d, 0x0a, 0x19,
0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x44,
0x45, 0x58, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, 0x18, 0x0a, 0x14, 0x4d,
0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x51, 0x55,
0x45, 0x53, 0x54, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45,
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x10, 0x04,
0x12, 0x22, 0x0a, 0x1e, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45,
0x53, 0x53, 0x10, 0x05, 0x12, 0x15, 0x0a, 0x11, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f,
0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x4d,
0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x4f, 0x53,
0x45, 0x10, 0x07, 0x2a, 0x4f, 0x0a, 0x12, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f,
0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x18, 0x4d, 0x45, 0x53,
0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x75,
0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d,
0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a,
0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b,
0x2e, 0x62, 0x65, 0x70, 0x2e, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x07, 0x76, 0x65, 0x72,
0x73, 0x69, 0x6f, 0x6e, 0x12, 0x27, 0x0a, 0x0d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x69, 0x6e,
0x64, 0x65, 0x78, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x05, 0x42, 0x02, 0x10, 0x00, 0x52,
0x0c, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x12, 0x1d, 0x0a,
0x0a, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28,
0x05, 0x52, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x06, 0x0a, 0x04,
0x50, 0x69, 0x6e, 0x67, 0x22, 0x1f, 0x0a, 0x05, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x12, 0x16, 0x0a,
0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72,
0x65, 0x61, 0x73, 0x6f, 0x6e, 0x2a, 0xed, 0x01, 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67,
0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x1b, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45,
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x43, 0x4f,
0x4e, 0x46, 0x49, 0x47, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47,
0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x44, 0x45, 0x58, 0x10, 0x01, 0x12, 0x1d,
0x0a, 0x19, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49,
0x4e, 0x44, 0x45, 0x58, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, 0x18, 0x0a,
0x14, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45,
0x51, 0x55, 0x45, 0x53, 0x54, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x4d, 0x45, 0x53, 0x53, 0x41,
0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45,
0x10, 0x04, 0x12, 0x22, 0x0a, 0x1e, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59,
0x50, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47,
0x52, 0x45, 0x53, 0x53, 0x10, 0x05, 0x12, 0x15, 0x0a, 0x11, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47,
0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x16, 0x0a,
0x12, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c,
0x4f, 0x53, 0x45, 0x10, 0x07, 0x2a, 0x4f, 0x0a, 0x12, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x18, 0x4d,
0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49,
0x4f, 0x4e, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x4d, 0x45, 0x53,
0x53, 0x41, 0x47, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e,
0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x4d, 0x45, 0x53, 0x53, 0x41,
0x47, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4c,
0x5a, 0x34, 0x10, 0x01, 0x2a, 0x56, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73,
0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49,
0x4f, 0x4e, 0x5f, 0x4d, 0x45, 0x54, 0x41, 0x44, 0x41, 0x54, 0x41, 0x10, 0x00, 0x12, 0x15, 0x0a,
0x11, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4e, 0x45, 0x56,
0x45, 0x52, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53,
0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x4c, 0x57, 0x41, 0x59, 0x53, 0x10, 0x02, 0x2a, 0xb0, 0x01, 0x0a,
0x0c, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a,
0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
0x46, 0x49, 0x4c, 0x45, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49,
0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x4f,
0x52, 0x59, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1b, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46,
0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x5f, 0x46,
0x49, 0x4c, 0x45, 0x10, 0x02, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x28, 0x0a, 0x20, 0x46, 0x49, 0x4c,
0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c,
0x49, 0x4e, 0x4b, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x03, 0x1a,
0x02, 0x08, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f,
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x10, 0x04, 0x2a,
0x76, 0x0a, 0x09, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x17, 0x0a, 0x13,
0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f, 0x5f, 0x45, 0x52,
0x52, 0x4f, 0x52, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43,
0x4f, 0x44, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x10, 0x01, 0x12, 0x1b, 0x0a,
0x17, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f, 0x5f, 0x53,
0x55, 0x43, 0x48, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x1b, 0x0a, 0x17, 0x45, 0x52,
0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44,
0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x03, 0x2a, 0x7e, 0x0a, 0x1e, 0x46, 0x69, 0x6c, 0x65, 0x44,
0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x55,
0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2d, 0x0a, 0x29, 0x46, 0x49, 0x4c,
0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47, 0x52,
0x45, 0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x00, 0x12, 0x2d, 0x0a, 0x29, 0x46, 0x49, 0x4c, 0x45,
0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45,
0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46,
0x4f, 0x52, 0x47, 0x45, 0x54, 0x10, 0x01, 0x42, 0x70, 0x0a, 0x07, 0x63, 0x6f, 0x6d, 0x2e, 0x62,
0x65, 0x70, 0x42, 0x08, 0x42, 0x65, 0x70, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2f,
0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x79, 0x6e, 0x63, 0x74,
0x68, 0x69, 0x6e, 0x67, 0x2f, 0x73, 0x79, 0x6e, 0x63, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x2f, 0x69,
0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x62, 0x65, 0x70, 0xa2,
0x02, 0x03, 0x42, 0x58, 0x58, 0xaa, 0x02, 0x03, 0x42, 0x65, 0x70, 0xca, 0x02, 0x03, 0x42, 0x65,
0x70, 0xe2, 0x02, 0x0f, 0x42, 0x65, 0x70, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64,
0x61, 0x74, 0x61, 0xea, 0x02, 0x03, 0x42, 0x65, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x33,
0x5f, 0x4c, 0x5a, 0x34, 0x10, 0x01, 0x2a, 0x56, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65,
0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53,
0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4d, 0x45, 0x54, 0x41, 0x44, 0x41, 0x54, 0x41, 0x10, 0x00, 0x12,
0x15, 0x0a, 0x11, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4e,
0x45, 0x56, 0x45, 0x52, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45,
0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x4c, 0x57, 0x41, 0x59, 0x53, 0x10, 0x02, 0x2a, 0xb0,
0x01, 0x0a, 0x0c, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x54, 0x79, 0x70, 0x65, 0x12,
0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50,
0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x46, 0x49, 0x4c, 0x45,
0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43,
0x54, 0x4f, 0x52, 0x59, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1b, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49,
0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b,
0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x02, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x28, 0x0a, 0x20, 0x46,
0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59,
0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10,
0x03, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e,
0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x10,
0x04, 0x2a, 0x76, 0x0a, 0x09, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x17,
0x0a, 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f, 0x5f,
0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x45, 0x52, 0x52, 0x4f, 0x52,
0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x10, 0x01, 0x12,
0x1b, 0x0a, 0x17, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f,
0x5f, 0x53, 0x55, 0x43, 0x48, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x1b, 0x0a, 0x17,
0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c,
0x49, 0x44, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x03, 0x2a, 0x7e, 0x0a, 0x1e, 0x46, 0x69, 0x6c,
0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73,
0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2d, 0x0a, 0x29, 0x46,
0x49, 0x4c, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f,
0x47, 0x52, 0x45, 0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50,
0x45, 0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x00, 0x12, 0x2d, 0x0a, 0x29, 0x46, 0x49,
0x4c, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47,
0x52, 0x45, 0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
0x5f, 0x46, 0x4f, 0x52, 0x47, 0x45, 0x54, 0x10, 0x01, 0x42, 0x70, 0x0a, 0x07, 0x63, 0x6f, 0x6d,
0x2e, 0x62, 0x65, 0x70, 0x42, 0x08, 0x42, 0x65, 0x70, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01,
0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x79, 0x6e,
0x63, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x2f, 0x73, 0x79, 0x6e, 0x63, 0x74, 0x68, 0x69, 0x6e, 0x67,
0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x62, 0x65,
0x70, 0xa2, 0x02, 0x03, 0x42, 0x58, 0x58, 0xaa, 0x02, 0x03, 0x42, 0x65, 0x70, 0xca, 0x02, 0x03,
0x42, 0x65, 0x70, 0xe2, 0x02, 0x0f, 0x42, 0x65, 0x70, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74,
0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x03, 0x42, 0x65, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x33,
}
var (

View File

@@ -18,7 +18,6 @@
<disableSparseFiles>false</disableSparseFiles>
<disableTempIndexes>false</disableTempIndexes>
<paused>false</paused>
<weakHashThresholdPct>25</weakHashThresholdPct>
<markerName>.stfolder</markerName>
<useLargeBlocks>true</useLargeBlocks>
</folder>
@@ -39,7 +38,6 @@
<disableSparseFiles>false</disableSparseFiles>
<disableTempIndexes>false</disableTempIndexes>
<paused>false</paused>
<weakHashThresholdPct>25</weakHashThresholdPct>
<markerName>.stfolder</markerName>
<useLargeBlocks>true</useLargeBlocks>
</folder>

View File

@@ -114,10 +114,9 @@ func TestDefaultValues(t *testing.T) {
CleanupIntervalS: 3600,
Params: map[string]string{},
},
MaxConflicts: 10,
WeakHashThresholdPct: 25,
MarkerName: ".stfolder",
MaxConcurrentWrites: 2,
MaxConflicts: 10,
MarkerName: ".stfolder",
MaxConcurrentWrites: 2,
XattrFilter: XattrFilter{
Entries: []XattrFilterEntry{},
MaxSingleEntrySize: 1024,
@@ -190,10 +189,9 @@ func TestDeviceConfig(t *testing.T) {
Versioning: VersioningConfiguration{
Params: map[string]string{},
},
WeakHashThresholdPct: 25,
MarkerName: DefaultMarkerName,
JunctionsAsDirs: true,
MaxConcurrentWrites: maxConcurrentWritesDefault,
MarkerName: DefaultMarkerName,
JunctionsAsDirs: true,
MaxConcurrentWrites: maxConcurrentWritesDefault,
XattrFilter: XattrFilter{
Entries: []XattrFilterEntry{},
},

View File

@@ -70,7 +70,6 @@ type FolderConfiguration struct {
DisableSparseFiles bool `json:"disableSparseFiles" xml:"disableSparseFiles"`
DisableTempIndexes bool `json:"disableTempIndexes" xml:"disableTempIndexes"`
Paused bool `json:"paused" xml:"paused"`
WeakHashThresholdPct int `json:"weakHashThresholdPct" xml:"weakHashThresholdPct"`
MarkerName string `json:"markerName" xml:"markerName"`
CopyOwnershipFromParent bool `json:"copyOwnershipFromParent" xml:"copyOwnershipFromParent"`
RawModTimeWindowS int `json:"modTimeWindowS" xml:"modTimeWindowS"`
@@ -311,10 +310,6 @@ func (f *FolderConfiguration) prepare(myID protocol.DeviceID, existingDevices ma
f.Versioning.CleanupIntervalS = 0
}
if f.WeakHashThresholdPct == 0 {
f.WeakHashThresholdPct = 25
}
if f.MarkerName == "" {
f.MarkerName = DefaultMarkerName
}

View File

@@ -70,7 +70,6 @@
<disableSparseFiles>false</disableSparseFiles>
<disableTempIndexes>false</disableTempIndexes>
<paused>false</paused>
<weakHashThresholdPct>25</weakHashThresholdPct>
<markerName>.stfolder</markerName>
<copyOwnershipFromParent>false</copyOwnershipFromParent>
<modTimeWindowS>0</modTimeWindowS>

View File

@@ -75,7 +75,7 @@ func (f *fakeConnection) DownloadProgress(_ context.Context, dp *protocol.Downlo
func (f *fakeConnection) addFileLocked(name string, flags uint32, ftype protocol.FileInfoType, data []byte, version protocol.Vector, localFlags uint32) {
blockSize := protocol.BlockSize(int64(len(data)))
blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), blockSize, int64(len(data)), nil, true)
blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), blockSize, int64(len(data)), nil)
file := protocol.FileInfo{
Name: name,

View File

@@ -560,7 +560,7 @@ func setupKnownFiles(t *testing.T, ffs fs.Filesystem, data []byte) []protocol.Fi
if err != nil {
t.Fatal(err)
}
blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), protocol.BlockSize(int64(len(data))), int64(len(data)), nil, true)
blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), protocol.BlockSize(int64(len(data))), int64(len(data)), nil)
knownFiles := []protocol.FileInfo{
{
Name: "knownDir",

View File

@@ -31,7 +31,6 @@ import (
"github.com/syncthing/syncthing/lib/semaphore"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/versioner"
"github.com/syncthing/syncthing/lib/weakhash"
)
var (
@@ -1136,12 +1135,12 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, snap *db.Snapshot
func (f *sendReceiveFolder) reuseBlocks(blocks []protocol.BlockInfo, reused []int, file protocol.FileInfo, tempName string) ([]protocol.BlockInfo, []int) {
// Check for an old temporary file which might have some blocks we could
// reuse.
tempBlocks, err := scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil, false)
tempBlocks, err := scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil)
if err != nil {
var caseErr *fs.ErrCaseConflict
if errors.As(err, &caseErr) {
if rerr := f.mtimefs.Rename(caseErr.Real, tempName); rerr == nil {
tempBlocks, err = scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil, false)
tempBlocks, err = scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil)
}
}
}
@@ -1310,8 +1309,6 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
f.model.progressEmitter.Register(state.sharedPullerState)
}
weakHashFinder, file := f.initWeakHashFinder(state)
blocks:
for _, block := range state.blocks {
select {
@@ -1336,75 +1333,49 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
buf = protocol.BufferPool.Upgrade(buf, int(block.Size))
var found bool
if f.Type != config.FolderTypeReceiveEncrypted {
found, err = weakHashFinder.Iterate(block.WeakHash, buf, func(offset int64) bool {
if f.verifyBuffer(buf, block) != nil {
return true
}
err = f.limitedWriteAt(dstFd, buf, block.Offset)
if err != nil {
state.fail(fmt.Errorf("dst write: %w", err))
}
if offset == block.Offset {
state.copiedFromOrigin(block.Size)
} else {
state.copiedFromOriginShifted(block.Size)
}
return false
})
found := f.model.finder.Iterate(folders, block.Hash, func(folder, path string, index int32) bool {
ffs := folderFilesystems[folder]
fd, err := ffs.Open(path)
if err != nil {
l.Debugln("weak hasher iter", err)
return false
}
}
defer fd.Close()
if !found {
found = f.model.finder.Iterate(folders, block.Hash, func(folder, path string, index int32) bool {
ffs := folderFilesystems[folder]
fd, err := ffs.Open(path)
if err != nil {
srcOffset := int64(state.file.BlockSize()) * int64(index)
_, err = fd.ReadAt(buf, srcOffset)
if err != nil {
return false
}
// Hash is not SHA256 as it's an encrypted hash token. In that
// case we can't verify the block integrity so we'll take it on
// trust. (The other side can and will verify.)
if f.Type != config.FolderTypeReceiveEncrypted {
if err := f.verifyBuffer(buf, block); err != nil {
l.Debugln("Finder failed to verify buffer", err)
return false
}
defer fd.Close()
}
srcOffset := int64(state.file.BlockSize()) * int64(index)
_, err = fd.ReadAt(buf, srcOffset)
if err != nil {
return false
}
// Hash is not SHA256 as it's an encrypted hash token. In that
// case we can't verify the block integrity so we'll take it on
// trust. (The other side can and will verify.)
if f.Type != config.FolderTypeReceiveEncrypted {
if err := f.verifyBuffer(buf, block); err != nil {
l.Debugln("Finder failed to verify buffer", err)
return false
}
}
if f.CopyRangeMethod != config.CopyRangeMethodStandard {
err = f.withLimiter(func() error {
dstFd.mut.Lock()
defer dstFd.mut.Unlock()
return fs.CopyRange(f.CopyRangeMethod.ToFS(), fd, dstFd.fd, srcOffset, block.Offset, int64(block.Size))
})
} else {
err = f.limitedWriteAt(dstFd, buf, block.Offset)
}
if err != nil {
state.fail(fmt.Errorf("dst write: %w", err))
}
if path == state.file.Name {
state.copiedFromOrigin(block.Size)
} else {
state.copiedFromElsewhere(block.Size)
}
return true
})
}
if f.CopyRangeMethod != config.CopyRangeMethodStandard {
err = f.withLimiter(func() error {
dstFd.mut.Lock()
defer dstFd.mut.Unlock()
return fs.CopyRange(f.CopyRangeMethod.ToFS(), fd, dstFd.fd, srcOffset, block.Offset, int64(block.Size))
})
} else {
err = f.limitedWriteAt(dstFd, buf, block.Offset)
}
if err != nil {
state.fail(fmt.Errorf("dst write: %w", err))
}
if path == state.file.Name {
state.copiedFromOrigin(block.Size)
} else {
state.copiedFromElsewhere(block.Size)
}
return true
})
if state.failed() != nil {
break
@@ -1421,58 +1392,11 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
state.copyDone(block)
}
}
if file != nil {
// os.File used to return invalid argument if nil.
// fs.File panics as it's an interface.
file.Close()
}
out <- state.sharedPullerState
}
}
func (f *sendReceiveFolder) initWeakHashFinder(state copyBlocksState) (*weakhash.Finder, fs.File) {
if f.Type == config.FolderTypeReceiveEncrypted {
l.Debugln("not weak hashing due to folder type", f.Type)
return nil, nil
}
blocksPercentChanged := 0
if tot := len(state.file.Blocks); tot > 0 {
blocksPercentChanged = (tot - state.have) * 100 / tot
}
if blocksPercentChanged < f.WeakHashThresholdPct {
l.Debugf("not weak hashing %s. not enough changed %.02f < %d", state.file.Name, blocksPercentChanged, f.WeakHashThresholdPct)
return nil, nil
}
hashesToFind := make([]uint32, 0, len(state.blocks))
for _, block := range state.blocks {
if block.WeakHash != 0 {
hashesToFind = append(hashesToFind, block.WeakHash)
}
}
if len(hashesToFind) == 0 {
l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name)
return nil, nil
}
file, err := f.mtimefs.Open(state.file.Name)
if err != nil {
l.Debugln("weak hasher", err)
return nil, nil
}
weakHashFinder, err := weakhash.NewFinder(f.ctx, file, state.file.BlockSize(), hashesToFind)
if err != nil {
l.Debugln("weak hasher", err)
return nil, file
}
return weakHashFinder, file
}
func (*sendReceiveFolder) verifyBuffer(buf []byte, block protocol.BlockInfo) error {
if len(buf) != int(block.Size) {
return fmt.Errorf("length mismatch %d != %d", len(buf), block.Size)
@@ -1574,7 +1498,7 @@ loop:
activity.using(selected)
var buf []byte
blockNo := int(state.block.Offset / int64(state.file.BlockSize()))
buf, lastError = f.model.RequestGlobal(f.ctx, selected.ID, f.folderID, state.file.Name, blockNo, state.block.Offset, int(state.block.Size), state.block.Hash, state.block.WeakHash, selected.FromTemporary)
buf, lastError = f.model.RequestGlobal(f.ctx, selected.ID, f.folderID, state.file.Name, blockNo, state.block.Offset, int(state.block.Size), state.block.Hash, selected.FromTemporary)
activity.done(selected)
if lastError != nil {
l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, selected.ID.Short(), "returned error:", lastError)

View File

@@ -25,7 +25,6 @@ import (
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/sync"
)
@@ -300,7 +299,7 @@ func TestCopierFinder(t *testing.T) {
}
// Verify that the fetched blocks have actually been written to the temp file
blks, err := scanner.HashFile(context.TODO(), f.ID, f.Filesystem(nil), tempFile, protocol.MinBlockSize, nil, false)
blks, err := scanner.HashFile(context.TODO(), f.ID, f.Filesystem(nil), tempFile, protocol.MinBlockSize, nil)
if err != nil {
t.Log(err)
}
@@ -312,128 +311,6 @@ func TestCopierFinder(t *testing.T) {
}
}
func TestWeakHash(t *testing.T) {
// Setup the model/pull environment
_, fo, wcfgCancel := setupSendReceiveFolder(t)
defer wcfgCancel()
ffs := fo.Filesystem(nil)
tempFile := fs.TempName("weakhash")
var shift int64 = 10
var size int64 = 1 << 20
expectBlocks := int(size / protocol.MinBlockSize)
expectPulls := int(shift / protocol.MinBlockSize)
if shift > 0 {
expectPulls++
}
f, err := ffs.Create("weakhash")
must(t, err)
defer f.Close()
_, err = io.CopyN(f, rand.Reader, size)
if err != nil {
t.Error(err)
}
info, err := f.Stat()
if err != nil {
t.Error(err)
}
// Create two files, second file has `shifted` bytes random prefix, yet
// both are of the same length, for example:
// File 1: abcdefgh
// File 2: xyabcdef
f.Seek(0, io.SeekStart)
existing, err := scanner.Blocks(context.TODO(), f, protocol.MinBlockSize, size, nil, true)
if err != nil {
t.Error(err)
}
f.Seek(0, io.SeekStart)
remainder := io.LimitReader(f, size-shift)
prefix := io.LimitReader(rand.Reader, shift)
nf := io.MultiReader(prefix, remainder)
desired, err := scanner.Blocks(context.TODO(), nf, protocol.MinBlockSize, size, nil, true)
if err != nil {
t.Error(err)
}
existingFile := protocol.FileInfo{
Name: "weakhash",
Blocks: existing,
Size: size,
ModifiedS: info.ModTime().Unix(),
ModifiedNs: int32(info.ModTime().Nanosecond()),
}
desiredFile := protocol.FileInfo{
Name: "weakhash",
Size: size,
Blocks: desired,
ModifiedS: info.ModTime().Unix() + 1,
}
fo.updateLocalsFromScanning([]protocol.FileInfo{existingFile})
copyChan := make(chan copyBlocksState)
pullChan := make(chan pullBlockState, expectBlocks)
finisherChan := make(chan *sharedPullerState, 1)
// Run a single fetcher routine
go fo.copierRoutine(copyChan, pullChan, finisherChan)
defer close(copyChan)
// Test 1 - no weak hashing, file gets fully repulled (`expectBlocks` pulls).
fo.WeakHashThresholdPct = 101
fo.handleFile(desiredFile, fsetSnapshot(t, fo.fset), copyChan)
var pulls []pullBlockState
timeout := time.After(10 * time.Second)
for len(pulls) < expectBlocks {
select {
case pull := <-pullChan:
pulls = append(pulls, pull)
case <-timeout:
t.Fatalf("timed out, got %d pulls expected %d", len(pulls), expectPulls)
}
}
finish := <-finisherChan
select {
case <-pullChan:
t.Fatal("Pull channel has data to be read")
case <-finisherChan:
t.Fatal("Finisher channel has data to be read")
default:
}
cleanupSharedPullerState(finish)
if err := ffs.Remove(tempFile); err != nil {
t.Fatal(err)
}
// Test 2 - using weak hash, expectPulls blocks pulled.
fo.WeakHashThresholdPct = -1
fo.handleFile(desiredFile, fsetSnapshot(t, fo.fset), copyChan)
pulls = pulls[:0]
for len(pulls) < expectPulls {
select {
case pull := <-pullChan:
pulls = append(pulls, pull)
case <-time.After(10 * time.Second):
t.Fatalf("timed out, got %d pulls expected %d", len(pulls), expectPulls)
}
}
finish = <-finisherChan
cleanupSharedPullerState(finish)
expectShifted := expectBlocks - expectPulls
if finish.copyOriginShifted != expectShifted {
t.Errorf("did not copy %d shifted", expectShifted)
}
}
// Test that updating a file removes its old blocks from the blockmap
func TestCopierCleanup(t *testing.T) {
iterFn := func(folder, file string, index int32) bool {
@@ -709,8 +586,8 @@ func TestIssue3164(t *testing.T) {
func TestDiff(t *testing.T) {
for i, test := range diffTestData {
a, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.a), test.s, -1, nil, false)
b, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.b), test.s, -1, nil, false)
a, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.a), test.s, -1, nil)
b, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.b), test.s, -1, nil)
_, d := blockDiff(a, b)
if len(d) != len(test.d) {
t.Fatalf("Incorrect length for diff %d; %d != %d", i, len(d), len(test.d))
@@ -730,8 +607,8 @@ func TestDiff(t *testing.T) {
func BenchmarkDiff(b *testing.B) {
testCases := make([]struct{ a, b []protocol.BlockInfo }, 0, len(diffTestData))
for _, test := range diffTestData {
a, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.a), test.s, -1, nil, false)
b, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.b), test.s, -1, nil, false)
a, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.a), test.s, -1, nil)
b, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.b), test.s, -1, nil)
testCases = append(testCases, struct{ a, b []protocol.BlockInfo }{a, b})
}
b.ReportAllocs()

View File

@@ -55,16 +55,15 @@ var (
Namespace: "syncthing",
Subsystem: "model",
Name: "folder_processed_bytes_total",
Help: "Total amount of data processed during folder syncing, per folder ID and data source (network/local_origin/local_other/local_shifted/skipped)",
Help: "Total amount of data processed during folder syncing, per folder ID and data source (network/local_origin/local_other/skipped)",
}, []string{"folder", "source"})
)
const (
metricSourceNetwork = "network" // from the network
metricSourceLocalOrigin = "local_origin" // from the existing version of the local file
metricSourceLocalOther = "local_other" // from a different local file
metricSourceLocalShifted = "local_shifted" // from the existing version of the local file, rolling hash shifted
metricSourceSkipped = "skipped" // block of all zeroes, invented out of thin air
metricSourceNetwork = "network" // from the network
metricSourceLocalOrigin = "local_origin" // from the existing version of the local file
metricSourceLocalOther = "local_other" // from a different local file
metricSourceSkipped = "skipped" // block of all zeroes, invented out of thin air
metricScopeGlobal = "global"
metricScopeLocal = "local"
@@ -88,6 +87,5 @@ func registerFolderMetrics(folderID string) {
metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceNetwork)
metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceLocalOrigin)
metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceLocalOther)
metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceLocalShifted)
metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceSkipped)
}

View File

@@ -435,19 +435,18 @@ type Model struct {
result1 protocol.RequestResponse
result2 error
}
RequestGlobalStub func(context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, uint32, bool) ([]byte, error)
RequestGlobalStub func(context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, bool) ([]byte, error)
requestGlobalMutex sync.RWMutex
requestGlobalArgsForCall []struct {
arg1 context.Context
arg2 protocol.DeviceID
arg3 string
arg4 string
arg5 int
arg6 int64
arg7 int
arg8 []byte
arg9 uint32
arg10 bool
arg1 context.Context
arg2 protocol.DeviceID
arg3 string
arg4 string
arg5 int
arg6 int64
arg7 int
arg8 []byte
arg9 bool
}
requestGlobalReturns struct {
result1 []byte
@@ -2580,7 +2579,7 @@ func (fake *Model) RequestReturnsOnCall(i int, result1 protocol.RequestResponse,
}{result1, result2}
}
func (fake *Model) RequestGlobal(arg1 context.Context, arg2 protocol.DeviceID, arg3 string, arg4 string, arg5 int, arg6 int64, arg7 int, arg8 []byte, arg9 uint32, arg10 bool) ([]byte, error) {
func (fake *Model) RequestGlobal(arg1 context.Context, arg2 protocol.DeviceID, arg3 string, arg4 string, arg5 int, arg6 int64, arg7 int, arg8 []byte, arg9 bool) ([]byte, error) {
var arg8Copy []byte
if arg8 != nil {
arg8Copy = make([]byte, len(arg8))
@@ -2589,23 +2588,22 @@ func (fake *Model) RequestGlobal(arg1 context.Context, arg2 protocol.DeviceID, a
fake.requestGlobalMutex.Lock()
ret, specificReturn := fake.requestGlobalReturnsOnCall[len(fake.requestGlobalArgsForCall)]
fake.requestGlobalArgsForCall = append(fake.requestGlobalArgsForCall, struct {
arg1 context.Context
arg2 protocol.DeviceID
arg3 string
arg4 string
arg5 int
arg6 int64
arg7 int
arg8 []byte
arg9 uint32
arg10 bool
}{arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8Copy, arg9, arg10})
arg1 context.Context
arg2 protocol.DeviceID
arg3 string
arg4 string
arg5 int
arg6 int64
arg7 int
arg8 []byte
arg9 bool
}{arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8Copy, arg9})
stub := fake.RequestGlobalStub
fakeReturns := fake.requestGlobalReturns
fake.recordInvocation("RequestGlobal", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8Copy, arg9, arg10})
fake.recordInvocation("RequestGlobal", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8Copy, arg9})
fake.requestGlobalMutex.Unlock()
if stub != nil {
return stub(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10)
return stub(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9)
}
if specificReturn {
return ret.result1, ret.result2
@@ -2619,17 +2617,17 @@ func (fake *Model) RequestGlobalCallCount() int {
return len(fake.requestGlobalArgsForCall)
}
func (fake *Model) RequestGlobalCalls(stub func(context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, uint32, bool) ([]byte, error)) {
func (fake *Model) RequestGlobalCalls(stub func(context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, bool) ([]byte, error)) {
fake.requestGlobalMutex.Lock()
defer fake.requestGlobalMutex.Unlock()
fake.RequestGlobalStub = stub
}
func (fake *Model) RequestGlobalArgsForCall(i int) (context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, uint32, bool) {
func (fake *Model) RequestGlobalArgsForCall(i int) (context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, bool) {
fake.requestGlobalMutex.RLock()
defer fake.requestGlobalMutex.RUnlock()
argsForCall := fake.requestGlobalArgsForCall[i]
return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7, argsForCall.arg8, argsForCall.arg9, argsForCall.arg10
return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7, argsForCall.arg8, argsForCall.arg9
}
func (fake *Model) RequestGlobalReturns(result1 []byte, result2 error) {

View File

@@ -118,7 +118,7 @@ type Model interface {
GlobalDirectoryTree(folder, prefix string, levels int, dirsOnly bool) ([]*TreeEntry, error)
RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error)
RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error)
}
type model struct {
@@ -606,8 +606,6 @@ func (m *model) UsageReportingStats(report *contract.Report, version int, previe
report.BlockStats.Pulled = v
case "copyOrigin":
report.BlockStats.CopyOrigin = v
case "copyOriginShifted":
report.BlockStats.CopyOriginShifted = v
case "copyElsewhere":
report.BlockStats.CopyElsewhere = v
}
@@ -2036,7 +2034,7 @@ func (m *model) Request(conn protocol.Connection, req *protocol.Request) (out pr
return nil, protocol.ErrNoSuchFile
}
_, err := readOffsetIntoBuf(folderFs, tempFn, req.Offset, res.data)
if err == nil && scanner.Validate(res.data, req.Hash, req.WeakHash) {
if err == nil && scanner.Validate(res.data, req.Hash) {
return res, nil
}
// Fall through to reading from a non-temp file, just in case the temp
@@ -2065,8 +2063,8 @@ func (m *model) Request(conn protocol.Connection, req *protocol.Request) (out pr
return nil, protocol.ErrGeneric
}
if folderCfg.Type != config.FolderTypeReceiveEncrypted && len(req.Hash) > 0 && !scanner.Validate(res.data[:n], req.Hash, req.WeakHash) {
m.recheckFile(deviceID, req.Folder, req.Name, req.Offset, req.Hash, req.WeakHash)
if folderCfg.Type != config.FolderTypeReceiveEncrypted && len(req.Hash) > 0 && !scanner.Validate(res.data[:n], req.Hash) {
m.recheckFile(deviceID, req.Folder, req.Name, req.Offset, req.Hash)
l.Debugf("%v REQ(in) failed validating data: %s: %q / %q o=%d s=%d", m, deviceID.Short(), req.Folder, req.Name, req.Offset, req.Size)
return nil, protocol.ErrNoSuchFile
}
@@ -2092,7 +2090,7 @@ func newLimitedRequestResponse(size int, limiters ...*semaphore.Semaphore) *requ
return res
}
func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, weakHash uint32) {
func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte) {
cf, ok, err := m.CurrentFolderFile(folder, name)
if err != nil {
l.Debugf("%v recheckFile: %s: %q / %q: current file error: %v", m, deviceID, folder, name, err)
@@ -2121,10 +2119,6 @@ func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, off
l.Debugf("%v recheckFile: %s: %q / %q i=%d: hash mismatch %x != %x", m, deviceID, folder, name, blockIndex, block.Hash, hash)
return
}
if weakHash != 0 && block.WeakHash != weakHash {
l.Debugf("%v recheckFile: %s: %q / %q i=%d: weak hash mismatch %v != %v", m, deviceID, folder, name, blockIndex, block.WeakHash, weakHash)
return
}
// The hashes provided part of the request match what we expect to find according
// to what we have in the database, yet the content we've read off the filesystem doesn't
@@ -2462,14 +2456,14 @@ func (m *model) deviceDidCloseRLocked(deviceID protocol.DeviceID, duration time.
}
}
func (m *model) RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
func (m *model) RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
conn, connOK := m.requestConnectionForDevice(deviceID)
if !connOK {
return nil, fmt.Errorf("requestGlobal: no connection to device: %s", deviceID.Short())
}
l.Debugf("%v REQ(out): %s (%s): %q / %q b=%d o=%d s=%d h=%x wh=%x ft=%t", m, deviceID.Short(), conn, folder, name, blockNo, offset, size, hash, weakHash, fromTemporary)
return conn.Request(ctx, &protocol.Request{Folder: folder, Name: name, BlockNo: blockNo, Offset: offset, Size: size, Hash: hash, WeakHash: weakHash, FromTemporary: fromTemporary})
l.Debugf("%v REQ(out): %s (%s): %q / %q b=%d o=%d s=%d h=%x ft=%t", m, deviceID.Short(), conn, folder, name, blockNo, offset, size, hash, fromTemporary)
return conn.Request(ctx, &protocol.Request{Folder: folder, Name: name, BlockNo: blockNo, Offset: offset, Size: size, Hash: hash, FromTemporary: fromTemporary})
}
// requestConnectionForDevice returns a connection to the given device, to

View File

@@ -222,7 +222,7 @@ func BenchmarkRequestOut(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
data, err := m.RequestGlobal(context.Background(), device1, "default", files[i%n].Name, 0, 0, 32, nil, 0, false)
data, err := m.RequestGlobal(context.Background(), device1, "default", files[i%n].Name, 0, 0, 32, nil, false)
if err != nil {
b.Error(err)
}
@@ -2819,9 +2819,9 @@ func TestIssue5002(t *testing.T) {
}
blockSize := int32(file.BlockSize())
m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size-int64(blockSize), []byte{1, 2, 3, 4}, 0)
m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size, []byte{1, 2, 3, 4}, 0) // panic
m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size+int64(blockSize), []byte{1, 2, 3, 4}, 0)
m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size-int64(blockSize), []byte{1, 2, 3, 4})
m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size, []byte{1, 2, 3, 4}) // panic
m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size+int64(blockSize), []byte{1, 2, 3, 4})
}
func TestParentOfUnignored(t *testing.T) {

View File

@@ -436,11 +436,8 @@ func TestRescanIfHaveInvalidContent(t *testing.T) {
}
f := checkReceived(<-received)
if f.Blocks[0].WeakHash != 103547413 {
t.Fatalf("unexpected weak hash: %d != 103547413", f.Blocks[0].WeakHash)
}
res, err := m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash, WeakHash: f.Blocks[0].WeakHash})
res, err := m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash})
if err != nil {
t.Fatal(err)
}
@@ -454,17 +451,14 @@ func TestRescanIfHaveInvalidContent(t *testing.T) {
writeFile(t, tfs, "foo", payload)
_, err = m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash, WeakHash: f.Blocks[0].WeakHash})
_, err = m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash})
if err == nil {
t.Fatalf("expected failure")
}
select {
case fs := <-received:
f := checkReceived(fs)
if f.Blocks[0].WeakHash != 41943361 {
t.Fatalf("unexpected weak hash: %d != 41943361", f.Blocks[0].WeakHash)
}
checkReceived(fs)
case <-time.After(time.Second):
t.Fatalf("timed out")
}

View File

@@ -285,15 +285,6 @@ func (s *sharedPullerState) skippedSparseBlock(bytes int) {
metricFolderProcessedBytesTotal.WithLabelValues(s.folder, metricSourceSkipped).Add(float64(bytes))
}
func (s *sharedPullerState) copiedFromOriginShifted(bytes int) {
s.mut.Lock()
s.copyOrigin++
s.copyOriginShifted++
s.updated = time.Now()
s.mut.Unlock()
metricFolderProcessedBytesTotal.WithLabelValues(s.folder, metricSourceLocalShifted).Add(float64(bytes))
}
func (s *sharedPullerState) pullStarted() {
s.mut.Lock()
s.copyTotal--

View File

@@ -544,32 +544,29 @@ func (f *FileInfo) setNoContent() {
}
type BlockInfo struct {
Hash []byte
Offset int64
Size int
WeakHash uint32
Hash []byte
Offset int64
Size int
}
func (b BlockInfo) ToWire() *bep.BlockInfo {
return &bep.BlockInfo{
Hash: b.Hash,
Offset: b.Offset,
Size: int32(b.Size),
WeakHash: b.WeakHash,
Hash: b.Hash,
Offset: b.Offset,
Size: int32(b.Size),
}
}
func BlockInfoFromWire(w *bep.BlockInfo) BlockInfo {
return BlockInfo{
Hash: w.Hash,
Offset: w.Offset,
Size: int(w.Size),
WeakHash: w.WeakHash,
Hash: w.Hash,
Offset: w.Offset,
Size: int(w.Size),
}
}
func (b BlockInfo) String() string {
return fmt.Sprintf("Block{%d/%d/%d/%x}", b.Offset, b.Size, b.WeakHash, b.Hash)
return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
}
// For each block size, the hash of a block of all zeroes
@@ -596,7 +593,6 @@ func BlocksHash(bs []BlockInfo) []byte {
h := sha256.New()
for _, b := range bs {
_, _ = h.Write(b.Hash)
_ = binary.Write(h, binary.BigEndian, b.WeakHash)
}
return h.Sum(nil)
}

View File

@@ -25,7 +25,6 @@ type Request struct {
Size int
Hash []byte
FromTemporary bool
WeakHash uint32
BlockNo int
}
@@ -38,7 +37,6 @@ func (r *Request) toWire() *bep.Request {
Size: int32(r.Size),
Hash: r.Hash,
FromTemporary: r.FromTemporary,
WeakHash: r.WeakHash,
BlockNo: int32(r.BlockNo),
}
}
@@ -52,7 +50,6 @@ func requestFromWire(w *bep.Request) *Request {
Size: int(w.Size),
Hash: w.Hash,
FromTemporary: w.FromTemporary,
WeakHash: w.WeakHash,
BlockNo: int(w.BlockNo),
}
}

View File

@@ -17,7 +17,6 @@ type TestModel struct {
offset int64
size int32
hash []byte
weakHash uint32
fromTemporary bool
indexFn func(string, []FileInfo)
ccFn func(*ClusterConfig)
@@ -48,7 +47,6 @@ func (t *TestModel) Request(_ Connection, req *Request) (RequestResponse, error)
t.offset = req.Offset
t.size = int32(req.Size)
t.hash = req.Hash
t.weakHash = req.WeakHash
t.fromTemporary = req.FromTemporary
buf := make([]byte, len(t.data))
copy(buf, t.data)

View File

@@ -16,7 +16,7 @@ import (
)
// HashFile hashes the files and returns a list of blocks representing the file.
func HashFile(ctx context.Context, folderID string, fs fs.Filesystem, path string, blockSize int, counter Counter, useWeakHashes bool) ([]protocol.BlockInfo, error) {
func HashFile(ctx context.Context, folderID string, fs fs.Filesystem, path string, blockSize int, counter Counter) ([]protocol.BlockInfo, error) {
fd, err := fs.Open(path)
if err != nil {
l.Debugln("open:", err)
@@ -36,7 +36,7 @@ func HashFile(ctx context.Context, folderID string, fs fs.Filesystem, path strin
// Hash the file. This may take a while for large files.
blocks, err := Blocks(ctx, fd, blockSize, size, counter, useWeakHashes)
blocks, err := Blocks(ctx, fd, blockSize, size, counter)
if err != nil {
l.Debugln("blocks:", err)
return nil, err
@@ -108,7 +108,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) {
panic("Bug. Asked to hash a directory or a deleted file.")
}
blocks, err := HashFile(ctx, ph.folderID, ph.fs, f.Name, f.BlockSize(), ph.counter, true)
blocks, err := HashFile(ctx, ph.folderID, ph.fs, f.Name, f.BlockSize(), ph.counter)
if err != nil {
handleError(ctx, "hashing", f.Name, err, ph.outbox)
continue

View File

@@ -10,8 +10,6 @@ import (
"bytes"
"context"
"crypto/sha256"
"hash"
"hash/adler32"
"io"
"github.com/syncthing/syncthing/lib/protocol"
@@ -24,7 +22,7 @@ type Counter interface {
}
// Blocks returns the blockwise hash of the reader.
func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, counter Counter, useWeakHashes bool) ([]protocol.BlockInfo, error) {
func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, counter Counter) ([]protocol.BlockInfo, error) {
if counter == nil {
counter = &noopCounter{}
}
@@ -32,15 +30,6 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
hf := sha256.New()
const hashLength = sha256.Size
var weakHf hash.Hash32 = noopHash{}
var multiHf io.Writer = hf
if useWeakHashes {
// Use an actual weak hash function, make the multiHf
// write to both hash functions.
weakHf = adler32.New()
multiHf = io.MultiWriter(hf, weakHf)
}
var blocks []protocol.BlockInfo
var hashes, thisHash []byte
@@ -70,7 +59,7 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
}
lr.N = int64(blocksize)
n, err := io.CopyBuffer(multiHf, lr, buf)
n, err := io.CopyBuffer(hf, lr, buf)
if err != nil {
return nil, err
}
@@ -87,17 +76,15 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
thisHash, hashes = hashes[:hashLength], hashes[hashLength:]
b := protocol.BlockInfo{
Size: int(n),
Offset: offset,
Hash: thisHash,
WeakHash: weakHf.Sum32(),
Size: int(n),
Offset: offset,
Hash: thisHash,
}
blocks = append(blocks, b)
offset += n
hf.Reset()
weakHf.Reset()
}
if len(blocks) == 0 {
@@ -112,14 +99,8 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
return blocks, nil
}
// Validate quickly validates buf against the 32-bit weakHash, if not zero,
// else against the cryptohash hash, if len(hash)>0. It is satisfied if
// either hash matches or neither hash is given.
func Validate(buf, hash []byte, weakHash uint32) bool {
if weakHash != 0 && adler32.Checksum(buf) == weakHash {
return true
}
// Validate validates the hash, if len(hash)>0.
func Validate(buf, hash []byte) bool {
if len(hash) > 0 {
hbuf := sha256.Sum256(buf)
return bytes.Equal(hbuf[:], hash)

View File

@@ -9,51 +9,40 @@ package scanner
import (
"bytes"
"context"
"crypto/rand"
"crypto/sha256"
"fmt"
origAdler32 "hash/adler32"
mrand "math/rand"
"testing"
"testing/quick"
rollingAdler32 "github.com/chmduquesne/rollinghash/adler32"
"github.com/syncthing/syncthing/lib/protocol"
)
var blocksTestData = []struct {
data []byte
blocksize int
hash []string
weakhash []uint32
}{
{
[]byte(""), 1024,
[]string{
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
},
[]uint32{0},
},
{
[]byte("contents"), 1024,
[]string{
"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8",
},
[]uint32{0x0f3a036f},
},
{
[]byte("contents"), 9,
[]string{
"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8",
},
[]uint32{0x0f3a036f},
},
{
[]byte("contents"), 8,
[]string{
"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8",
},
[]uint32{0x0f3a036f},
},
{
[]byte("contents"), 7,
@@ -61,7 +50,6 @@ var blocksTestData = []struct {
"ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
"043a718774c572bd8a25adbeb1bfcd5c0256ae11cecf9f9c3f925d0e52beaf89",
},
[]uint32{0x0bcb02fc, 0x00740074},
},
{
[]byte("contents"), 3,
@@ -70,7 +58,6 @@ var blocksTestData = []struct {
"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3",
"44ad63f60af0f6db6fdde6d5186ef78176367df261fa06be3079b6c80c8adba4",
},
[]uint32{0x02780141, 0x02970148, 0x015d00e8},
},
{
[]byte("conconts"), 3,
@@ -79,7 +66,6 @@ var blocksTestData = []struct {
"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
"44ad63f60af0f6db6fdde6d5186ef78176367df261fa06be3079b6c80c8adba4",
},
[]uint32{0x02780141, 0x02780141, 0x015d00e8},
},
{
[]byte("contenten"), 3,
@@ -88,14 +74,13 @@ var blocksTestData = []struct {
"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3",
"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3",
},
[]uint32{0x02780141, 0x02970148, 0x02970148},
},
}
func TestBlocks(t *testing.T) {
for testNo, test := range blocksTestData {
buf := bytes.NewBuffer(test.data)
blocks, err := Blocks(context.TODO(), buf, test.blocksize, -1, nil, true)
blocks, err := Blocks(context.TODO(), buf, test.blocksize, -1, nil)
if err != nil {
t.Fatal(err)
}
@@ -119,9 +104,6 @@ func TestBlocks(t *testing.T) {
if h := fmt.Sprintf("%x", blocks[i].Hash); h != test.hash[i] {
t.Errorf("%d/%d: Incorrect block hash %q != %q", testNo, i, h, test.hash[i])
}
if h := blocks[i].WeakHash; h != test.weakhash[i] {
t.Errorf("%d/%d: Incorrect block weakhash 0x%08x != 0x%08x", testNo, i, h, test.weakhash[i])
}
i++
}
@@ -129,85 +111,10 @@ func TestBlocks(t *testing.T) {
}
}
func TestAdler32Variants(t *testing.T) {
// Verify that the two adler32 functions give matching results for a few
// different blocks of data.
hf1 := origAdler32.New()
hf2 := rollingAdler32.New()
checkFn := func(data []byte) bool {
hf1.Write(data)
sum1 := hf1.Sum32()
hf2.Write(data)
sum2 := hf2.Sum32()
hf1.Reset()
hf2.Reset()
// Make sure whatever we use in Validate matches too resp. this
// tests gets adjusted if we ever switch the weak hash algo.
return sum1 == sum2 && Validate(data, nil, sum1)
}
// protocol block sized data
data := make([]byte, protocol.MinBlockSize)
for i := 0; i < 5; i++ {
rand.Read(data)
if !checkFn(data) {
t.Errorf("Hash mismatch on block sized data")
}
}
// random small blocks
if err := quick.Check(checkFn, nil); err != nil {
t.Error(err)
}
// rolling should have the same result as the individual blocks
// themselves.
windowSize := 128
hf3 := rollingAdler32.New()
hf3.Write(data[:windowSize])
for i := windowSize; i < len(data); i++ {
if i%windowSize == 0 {
// let the reference function catch up
window := data[i-windowSize : i]
hf1.Reset()
hf1.Write(window)
hf2.Reset()
hf2.Write(window)
// verify that they are in sync with the rolling function
sum1 := hf1.Sum32()
sum2 := hf2.Sum32()
sum3 := hf3.Sum32()
t.Logf("At i=%d, sum2=%08x, sum3=%08x", i, sum2, sum3)
if sum2 != sum3 {
t.Errorf("Mismatch after roll; i=%d, sum2=%08x, sum3=%08x", i, sum2, sum3)
break
}
if sum1 != sum3 {
t.Errorf("Mismatch after roll; i=%d, sum1=%08x, sum3=%08x", i, sum1, sum3)
break
}
if !Validate(window, nil, sum1) {
t.Errorf("Validation failure after roll; i=%d", i)
}
}
hf3.Roll(data[i])
}
}
func BenchmarkValidate(b *testing.B) {
type block struct {
data []byte
hash [sha256.Size]byte
weakhash uint32
data []byte
hash [sha256.Size]byte
}
var blocks []block
const blocksPerType = 100
@@ -220,16 +127,6 @@ func BenchmarkValidate(b *testing.B) {
b.data = make([]byte, 128<<10)
r.Read(b.data)
b.hash = sha256.Sum256(b.data)
b.weakhash = origAdler32.Checksum(b.data)
blocks = append(blocks, b)
}
// Blocks where the hash matches, but the weakhash doesn't.
for i := 0; i < blocksPerType; i++ {
var b block
b.data = make([]byte, 128<<10)
r.Read(b.data)
b.hash = sha256.Sum256(b.data)
b.weakhash = 1 // Zeros causes Validate to skip the weakhash.
blocks = append(blocks, b)
}
@@ -238,7 +135,7 @@ func BenchmarkValidate(b *testing.B) {
for i := 0; i < b.N; i++ {
for _, b := range blocks {
Validate(b.data, b.hash[:], b.weakhash)
Validate(b.data, b.hash[:])
}
}
}

View File

@@ -162,7 +162,7 @@ func TestVerify(t *testing.T) {
progress := newByteCounter()
defer progress.Close()
blocks, err := Blocks(context.TODO(), buf, blocksize, -1, progress, false)
blocks, err := Blocks(context.TODO(), buf, blocksize, -1, progress)
if err != nil {
t.Fatal(err)
}
@@ -640,7 +640,7 @@ func BenchmarkHashFile(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := HashFile(context.TODO(), "", testFs, testdataName, protocol.MinBlockSize, nil, true); err != nil {
if _, err := HashFile(context.TODO(), "", testFs, testdataName, protocol.MinBlockSize, nil); err != nil {
b.Fatal(err)
}
}

View File

@@ -42,7 +42,7 @@ func (m *Internals) SetIgnores(folderID string, content []string) error {
}
func (m *Internals) DownloadBlock(ctx context.Context, deviceID protocol.DeviceID, folderID string, path string, blockNumber int, blockInfo protocol.BlockInfo, allowFromTemporary bool) ([]byte, error) {
return m.model.RequestGlobal(ctx, deviceID, folderID, path, int(blockNumber), blockInfo.Offset, blockInfo.Size, blockInfo.Hash, blockInfo.WeakHash, allowFromTemporary)
return m.model.RequestGlobal(ctx, deviceID, folderID, path, int(blockNumber), blockInfo.Offset, blockInfo.Size, blockInfo.Hash, allowFromTemporary)
}
func (m *Internals) BlockAvailability(folderID string, file protocol.FileInfo, block protocol.BlockInfo) ([]model.Availability, error) {

View File

@@ -177,7 +177,7 @@ func (a *App) startup() error {
}()
}
perf := ur.CpuBench(context.Background(), 3, 150*time.Millisecond, true)
perf := ur.CpuBench(context.Background(), 3, 150*time.Millisecond)
l.Infof("Hashing performance is %.02f MB/s", perf)
if err := db.UpdateSchema(a.ll); err != nil {

View File

@@ -111,8 +111,6 @@ type Report struct {
ConflictsOther int `json:"conflictsOther,omitempty" metric:"folder_feature{feature=ConflictsOther},summary" since:"3"`
DisableSparseFiles int `json:"disableSparseFiles,omitempty" metric:"folder_feature{feature=DisableSparseFiles},summary" since:"3"`
DisableTempIndexes int `json:"disableTempIndexes,omitempty" metric:"folder_feature{feature=DisableTempIndexes},summary" since:"3"`
AlwaysWeakHash int `json:"alwaysWeakHash,omitempty" metric:"folder_feature{feature=AlwaysWeakhash},summary" since:"3"`
CustomWeakHashThreshold int `json:"customWeakHashThreshold,omitempty" metric:"folder_feature{feature=CustomWeakhashThreshold},summary" since:"3"`
FsWatcherEnabled int `json:"fsWatcherEnabled,omitempty" metric:"folder_feature{feature=FSWatcherEnabled},summary" since:"3"`
PullOrder map[string]int `json:"pullOrder,omitempty" metric:"folder_pull_order,summaryVec:order" since:"3"`
FilesystemType map[string]int `json:"filesystemType,omitempty" metric:"folder_file_system_type,summaryVec:type" since:"3"`
@@ -152,13 +150,12 @@ type Report struct {
} `json:"guiStats,omitempty" since:"3"`
BlockStats struct {
Total int `json:"total,omitempty" metric:"blocks_processed_total,gauge" since:"3"`
Renamed int `json:"renamed,omitempty" metric:"blocks_processed{source=renamed},gauge" since:"3"`
Reused int `json:"reused,omitempty" metric:"blocks_processed{source=reused},gauge" since:"3"`
Pulled int `json:"pulled,omitempty" metric:"blocks_processed{source=pulled},gauge" since:"3"`
CopyOrigin int `json:"copyOrigin,omitempty" metric:"blocks_processed{source=copy_origin},gauge" since:"3"`
CopyOriginShifted int `json:"copyOriginShifted,omitempty" metric:"blocks_processed{source=copy_origin_shifted},gauge" since:"3"`
CopyElsewhere int `json:"copyElsewhere,omitempty" metric:"blocks_processed{source=copy_elsewhere},gauge" since:"3"`
Total int `json:"total,omitempty" metric:"blocks_processed_total,gauge" since:"3"`
Renamed int `json:"renamed,omitempty" metric:"blocks_processed{source=renamed},gauge" since:"3"`
Reused int `json:"reused,omitempty" metric:"blocks_processed{source=reused},gauge" since:"3"`
Pulled int `json:"pulled,omitempty" metric:"blocks_processed{source=pulled},gauge" since:"3"`
CopyOrigin int `json:"copyOrigin,omitempty" metric:"blocks_processed{source=copy_origin},gauge" since:"3"`
CopyElsewhere int `json:"copyElsewhere,omitempty" metric:"blocks_processed{source=copy_elsewhere},gauge" since:"3"`
} `json:"blockStats,omitempty" since:"3"`
TransportStats map[string]int `json:"transportStats,omitempty" since:"3"`
@@ -175,9 +172,6 @@ type Report struct {
Stars int `json:"stars,omitempty" metric:"folder_ignore_lines{kind=stars},summary" since:"3"`
} `json:"ignoreStats,omitempty" since:"3"`
// V3 fields added late in the RC
WeakHashEnabled bool `json:"weakHashEnabled,omitempty" metric:"-" since:"3"` // Deprecated and not provided client-side anymore
// Added in post processing
Received time.Time `json:"received,omitempty"`
Date string `json:"date,omitempty"`

View File

@@ -116,8 +116,8 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (
report.TotMiB = int(totBytes / 1024 / 1024)
report.FolderMaxMiB = int(maxBytes / 1024 / 1024)
report.MemoryUsageMiB = int((mem.Sys - mem.HeapReleased) / 1024 / 1024)
report.SHA256Perf = CpuBench(ctx, 5, 125*time.Millisecond, false)
report.HashPerf = CpuBench(ctx, 5, 125*time.Millisecond, true)
report.SHA256Perf = CpuBench(ctx, 5, 125*time.Millisecond)
report.HashPerf = report.SHA256Perf
report.MemorySize = int(memorySize() / 1024 / 1024)
report.NumCPU = runtime.NumCPU()
@@ -252,11 +252,6 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (
if cfg.DisableTempIndexes {
report.FolderUsesV3.DisableTempIndexes++
}
if cfg.WeakHashThresholdPct < 0 {
report.FolderUsesV3.AlwaysWeakHash++
} else if cfg.WeakHashThresholdPct != 25 {
report.FolderUsesV3.CustomWeakHashThreshold++
}
if cfg.FSWatcherEnabled {
report.FolderUsesV3.FsWatcherEnabled++
}
@@ -443,7 +438,7 @@ var (
)
// CpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s
func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWeakHash bool) float64 {
func CpuBench(ctx context.Context, iterations int, duration time.Duration) float64 {
blocksResultMut.Lock()
defer blocksResultMut.Unlock()
@@ -454,7 +449,7 @@ func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWe
var perf float64
for i := 0; i < iterations; i++ {
if v := cpuBenchOnce(ctx, duration, useWeakHash, bs); v > perf {
if v := cpuBenchOnce(ctx, duration, bs); v > perf {
perf = v
}
}
@@ -467,13 +462,13 @@ func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWe
return perf
}
func cpuBenchOnce(ctx context.Context, duration time.Duration, useWeakHash bool, bs []byte) float64 {
func cpuBenchOnce(ctx context.Context, duration time.Duration, bs []byte) float64 {
t0 := time.Now()
b := 0
var err error
for time.Since(t0) < duration {
r := bytes.NewReader(bs)
blocksResult, err = scanner.Blocks(ctx, r, protocol.MinBlockSize, int64(len(bs)), nil, useWeakHash)
blocksResult, err = scanner.Blocks(ctx, r, protocol.MinBlockSize, int64(len(bs)), nil)
if err != nil {
return 0 // Context done
}

View File

@@ -1,151 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package weakhash
import (
"bytes"
"context"
"fmt"
"hash"
vadler32 "hash/adler32"
"io"
"math/rand"
"os"
"testing"
"github.com/chmduquesne/rollinghash/adler32"
"github.com/chmduquesne/rollinghash/bozo32"
"github.com/chmduquesne/rollinghash/buzhash32"
"github.com/chmduquesne/rollinghash/buzhash64"
)
const (
testFile = "../model/testdata/tmpfile"
size = 128 << 10
)
func BenchmarkFind1MFile(b *testing.B) {
b.ReportAllocs()
b.SetBytes(1 << 20)
for i := 0; i < b.N; i++ {
fd, err := os.Open(testFile)
if err != nil {
b.Fatal(err)
}
_, err = Find(context.Background(), fd, []uint32{0, 1, 2}, size)
if err != nil {
b.Fatal(err)
}
fd.Close()
}
}
type RollingHash interface {
hash.Hash
Roll(byte)
}
func BenchmarkBlock(b *testing.B) {
tests := []struct {
name string
hash hash.Hash
}{
{
"adler32", adler32.New(),
},
{
"bozo32", bozo32.New(),
},
{
"buzhash32", buzhash32.New(),
},
{
"buzhash64", buzhash64.New(),
},
{
"vanilla-adler32", vadler32.New(),
},
}
sizes := []int64{128 << 10, 16 << 20}
buf := make([]byte, 16<<20)
rand.Read(buf)
for _, testSize := range sizes {
for _, test := range tests {
b.Run(test.name+"-"+fmt.Sprint(testSize), func(bb *testing.B) {
bb.Run("", func(bbb *testing.B) {
bbb.ResetTimer()
for i := 0; i < bbb.N; i++ {
lr := io.LimitReader(bytes.NewReader(buf), testSize)
n, err := io.Copy(test.hash, lr)
if err != nil {
bbb.Error(err)
}
if n != testSize {
bbb.Errorf("%d != %d", n, testSize)
}
test.hash.Sum(nil)
test.hash.Reset()
}
bbb.SetBytes(testSize)
bbb.ReportAllocs()
})
})
}
}
}
func BenchmarkRoll(b *testing.B) {
tests := []struct {
name string
hash RollingHash
}{
{
"adler32", adler32.New(),
},
{
"bozo32", bozo32.New(),
},
{
"buzhash32", buzhash32.New(),
},
{
"buzhash64", buzhash64.New(),
},
}
sizes := []int64{128 << 10, 16 << 20}
for _, testSize := range sizes {
for _, test := range tests {
b.Run(test.name+"-"+fmt.Sprint(testSize), func(bb *testing.B) {
bb.Run("", func(bbb *testing.B) {
data := make([]byte, testSize)
if _, err := test.hash.Write(data); err != nil {
bbb.Error(err)
}
bbb.ResetTimer()
for i := 0; i < bbb.N; i++ {
for j := int64(0); j <= testSize; j++ {
test.hash.Roll('a')
}
}
bbb.SetBytes(testSize)
bbb.ReportAllocs()
})
})
}
}
}

View File

@@ -1,118 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package weakhash
import (
"bufio"
"context"
"io"
"github.com/chmduquesne/rollinghash/adler32"
)
const (
Size = 4
// don't track more hits than this for any given weakhash
maxWeakhashFinderHits = 10
)
// Find finds all the blocks of the given size within io.Reader that matches
// the hashes provided, and returns a hash -> slice of offsets within reader
// map, that produces the same weak hash.
func Find(ctx context.Context, ir io.Reader, hashesToFind []uint32, size int) (map[uint32][]int64, error) {
if ir == nil || len(hashesToFind) == 0 {
return nil, nil
}
r := bufio.NewReader(ir)
hf := adler32.New()
n, err := io.CopyN(hf, r, int64(size))
if err == io.EOF {
return nil, nil
}
if err != nil {
return nil, err
}
if n != int64(size) {
return nil, io.ErrShortBuffer
}
offsets := make(map[uint32][]int64)
for _, hashToFind := range hashesToFind {
offsets[hashToFind] = make([]int64, 0, maxWeakhashFinderHits)
}
var i int64
var hash uint32
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
hash = hf.Sum32()
if existing, ok := offsets[hash]; ok && len(existing) < maxWeakhashFinderHits {
offsets[hash] = append(existing, i)
}
i++
bt, err := r.ReadByte()
if err == io.EOF {
break
} else if err != nil {
return offsets, err
}
hf.Roll(bt)
}
return offsets, nil
}
func NewFinder(ctx context.Context, ir io.ReadSeeker, size int, hashesToFind []uint32) (*Finder, error) {
offsets, err := Find(ctx, ir, hashesToFind, size)
if err != nil {
return nil, err
}
return &Finder{
reader: ir,
size: size,
offsets: offsets,
}, nil
}
type Finder struct {
reader io.ReadSeeker
size int
offsets map[uint32][]int64
}
// Iterate iterates all available blocks that matches the provided hash, reads
// them into buf, and calls the iterator function. The iterator function should
// return whether it wishes to continue iterating.
func (h *Finder) Iterate(hash uint32, buf []byte, iterFunc func(int64) bool) (bool, error) {
if h == nil || hash == 0 || len(buf) != h.size {
return false, nil
}
for _, offset := range h.offsets[hash] {
_, err := h.reader.Seek(offset, io.SeekStart)
if err != nil {
return false, err
}
_, err = h.reader.Read(buf)
if err != nil {
return false, err
}
if !iterFunc(offset) {
return true, nil
}
}
return false, nil
}

View File

@@ -1,68 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// The existence of this file means we get 0% test coverage rather than no
// test coverage at all. Remove when implementing an actual test.
package weakhash
import (
"bytes"
"context"
"io"
"os"
"reflect"
"testing"
)
var payload = []byte("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz")
func TestFinder(t *testing.T) {
f, err := os.CreateTemp("", "")
if err != nil {
t.Error(err)
}
defer os.Remove(f.Name())
defer f.Close()
if _, err := f.Write(payload); err != nil {
t.Error(err)
}
if _, err := f.Seek(0, io.SeekStart); err != nil {
t.Error(err)
}
hashes := []uint32{65143183, 65798547}
finder, err := NewFinder(context.Background(), f, 4, hashes)
if err != nil {
t.Error(err)
}
expected := map[uint32][]int64{
65143183: {1, 27, 53, 79},
65798547: {2, 28, 54, 80},
}
actual := make(map[uint32][]int64)
b := make([]byte, Size)
for _, hash := range hashes {
_, err := finder.Iterate(hash, b[:4], func(offset int64) bool {
if !bytes.Equal(b, payload[offset:offset+4]) {
t.Errorf("Not equal at %d: %s != %s", offset, string(b), string(payload[offset:offset+4]))
}
actual[hash] = append(actual[hash], offset)
return true
})
if err != nil {
t.Error(err)
}
}
if !reflect.DeepEqual(actual, expected) {
t.Errorf("Not equal: %#v != %#v", actual, expected)
}
}

View File

@@ -145,7 +145,7 @@ message BlockInfo {
bytes hash = 3;
int64 offset = 1;
int32 size = 2;
uint32 weak_hash = 4;
reserved 4;
}
message Vector {
@@ -203,8 +203,8 @@ message Request {
int32 size = 5;
bytes hash = 6;
bool from_temporary = 7;
uint32 weak_hash = 8;
int32 block_no = 9;
reserved 8;
}
// Response

View File

@@ -24,7 +24,6 @@
<disableSparseFiles>false</disableSparseFiles>
<disableTempIndexes>false</disableTempIndexes>
<paused>false</paused>
<weakHashThresholdPct>25</weakHashThresholdPct>
<markerName>.stfolder</markerName>
<copyOwnershipFromParent>false</copyOwnershipFromParent>
<modTimeWindowS>0</modTimeWindowS>
@@ -153,7 +152,6 @@
<disableSparseFiles>false</disableSparseFiles>
<disableTempIndexes>false</disableTempIndexes>
<paused>false</paused>
<weakHashThresholdPct>25</weakHashThresholdPct>
<markerName>.stfolder</markerName>
<copyOwnershipFromParent>false</copyOwnershipFromParent>
<modTimeWindowS>0</modTimeWindowS>

View File

@@ -24,7 +24,6 @@
<disableSparseFiles>false</disableSparseFiles>
<disableTempIndexes>false</disableTempIndexes>
<paused>false</paused>
<weakHashThresholdPct>25</weakHashThresholdPct>
<markerName>.stfolder</markerName>
<copyOwnershipFromParent>false</copyOwnershipFromParent>
<modTimeWindowS>0</modTimeWindowS>
@@ -151,7 +150,6 @@
<disableSparseFiles>false</disableSparseFiles>
<disableTempIndexes>false</disableTempIndexes>
<paused>false</paused>
<weakHashThresholdPct>25</weakHashThresholdPct>
<markerName>.stfolder</markerName>
<copyOwnershipFromParent>false</copyOwnershipFromParent>
<modTimeWindowS>0</modTimeWindowS>

View File

@@ -20,7 +20,6 @@
<disableSparseFiles>false</disableSparseFiles>
<disableTempIndexes>false</disableTempIndexes>
<paused>false</paused>
<weakHashThresholdPct>25</weakHashThresholdPct>
<markerName>.stfolder</markerName>
<copyOwnershipFromParent>false</copyOwnershipFromParent>
<modTimeWindowS>0</modTimeWindowS>
@@ -50,7 +49,6 @@
<disableSparseFiles>false</disableSparseFiles>
<disableTempIndexes>false</disableTempIndexes>
<paused>false</paused>
<weakHashThresholdPct>25</weakHashThresholdPct>
<markerName>.stfolder</markerName>
<copyOwnershipFromParent>false</copyOwnershipFromParent>
<modTimeWindowS>0</modTimeWindowS>

View File

@@ -17,7 +17,6 @@
<disableSparseFiles>false</disableSparseFiles>
<disableTempIndexes>false</disableTempIndexes>
<paused>false</paused>
<weakHashThresholdPct>25</weakHashThresholdPct>
<markerName>.stfolder</markerName>
<copyOwnershipFromParent>false</copyOwnershipFromParent>
<modTimeWindowS>0</modTimeWindowS>