mirror of
https://github.com/syncthing/syncthing.git
synced 2025-12-23 14:08:06 -05:00
chore: remove weak hashing which does not pull its weight (#10005)
We've had weak/rolling hashing in the code for quite a while. It was a popular request for a while, based on the belief that rsync does this and we should too. However, the benefit is quite small; we save on average about 0.8% of transferred blocks over the population as a whole: <img width="974" alt="Screenshot 2025-03-28 at 17 09 02" src="https://github.com/user-attachments/assets/bbe10dea-f85e-4043-9823-7cef1220b4a2" /> This would be fine if the cost was comparably low, however the downside of attempting rolling hash matching is that we (by default) do a complete file read on the destination in order to look for matches before we starting pulling blocks for the file. For any larger file this means a sometimes long, I/O-intensive pause before the file starts syncing, for usually no benefit. I propose we simply rip off the bandaid and save the effort.
This commit is contained in:
@@ -72,7 +72,7 @@ func main() {
|
||||
if *standardBlocks || blockSize < protocol.MinBlockSize {
|
||||
blockSize = protocol.BlockSize(fi.Size())
|
||||
}
|
||||
bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true)
|
||||
bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -238,7 +238,7 @@ func (c *CLI) decryptFile(encFi *protocol.FileInfo, plainFi *protocol.FileInfo,
|
||||
}
|
||||
|
||||
// Verify the hash against the plaintext block info
|
||||
if !scanner.Validate(dec, plainBlock.Hash, 0) {
|
||||
if !scanner.Validate(dec, plainBlock.Hash) {
|
||||
// The block decrypted correctly but fails the hash check. This
|
||||
// is odd and unexpected, but it it's still a valid block from
|
||||
// the source. The file might have changed while we pulled it?
|
||||
|
||||
1
go.mod
1
go.mod
@@ -9,7 +9,6 @@ require (
|
||||
github.com/calmh/incontainer v1.0.0
|
||||
github.com/calmh/xdr v1.2.0
|
||||
github.com/ccding/go-stun v0.1.5
|
||||
github.com/chmduquesne/rollinghash v4.0.0+incompatible
|
||||
github.com/d4l3k/messagediff v1.2.1
|
||||
github.com/getsentry/raven-go v0.2.0
|
||||
github.com/go-ldap/ldap/v3 v3.4.10
|
||||
|
||||
2
go.sum
2
go.sum
@@ -29,8 +29,6 @@ github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV
|
||||
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chmduquesne/rollinghash v4.0.0+incompatible h1:hnREQO+DXjqIw3rUTzWN7/+Dpw+N5Um8zpKV0JOEgbo=
|
||||
github.com/chmduquesne/rollinghash v4.0.0+incompatible/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
|
||||
@@ -1093,10 +1093,9 @@ type BlockInfo struct {
|
||||
sizeCache protoimpl.SizeCache
|
||||
unknownFields protoimpl.UnknownFields
|
||||
|
||||
Hash []byte `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"`
|
||||
Offset int64 `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"`
|
||||
Size int32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
|
||||
WeakHash uint32 `protobuf:"varint,4,opt,name=weak_hash,json=weakHash,proto3" json:"weak_hash,omitempty"`
|
||||
Hash []byte `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"`
|
||||
Offset int64 `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"`
|
||||
Size int32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`
|
||||
}
|
||||
|
||||
func (x *BlockInfo) Reset() {
|
||||
@@ -1150,13 +1149,6 @@ func (x *BlockInfo) GetSize() int32 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *BlockInfo) GetWeakHash() uint32 {
|
||||
if x != nil {
|
||||
return x.WeakHash
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type Vector struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -1578,7 +1570,6 @@ type Request struct {
|
||||
Size int32 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"`
|
||||
Hash []byte `protobuf:"bytes,6,opt,name=hash,proto3" json:"hash,omitempty"`
|
||||
FromTemporary bool `protobuf:"varint,7,opt,name=from_temporary,json=fromTemporary,proto3" json:"from_temporary,omitempty"`
|
||||
WeakHash uint32 `protobuf:"varint,8,opt,name=weak_hash,json=weakHash,proto3" json:"weak_hash,omitempty"`
|
||||
BlockNo int32 `protobuf:"varint,9,opt,name=block_no,json=blockNo,proto3" json:"block_no,omitempty"`
|
||||
}
|
||||
|
||||
@@ -1661,13 +1652,6 @@ func (x *Request) GetFromTemporary() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *Request) GetWeakHash() uint32 {
|
||||
if x != nil {
|
||||
return x.WeakHash
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *Request) GetBlockNo() int32 {
|
||||
if x != nil {
|
||||
return x.BlockNo
|
||||
@@ -2079,158 +2063,155 @@ var file_bep_bep_proto_rawDesc = []byte{
|
||||
0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x69, 0x6e, 0x76,
|
||||
0x61, 0x6c, 0x69, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x6e, 0x6f, 0x5f, 0x70, 0x65, 0x72, 0x6d, 0x69,
|
||||
0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6e, 0x6f,
|
||||
0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x68, 0x0a, 0x09, 0x42,
|
||||
0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x51, 0x0a, 0x09, 0x42,
|
||||
0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68,
|
||||
0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x16, 0x0a, 0x06,
|
||||
0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66,
|
||||
0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01,
|
||||
0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x65, 0x61, 0x6b,
|
||||
0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x77, 0x65, 0x61,
|
||||
0x6b, 0x48, 0x61, 0x73, 0x68, 0x22, 0x32, 0x0a, 0x06, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12,
|
||||
0x28, 0x0a, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28,
|
||||
0x0b, 0x32, 0x0c, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x52,
|
||||
0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x22, 0x2f, 0x0a, 0x07, 0x43, 0x6f, 0x75,
|
||||
0x6e, 0x74, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04,
|
||||
0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
|
||||
0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xfd, 0x01, 0x0a, 0x0c, 0x50,
|
||||
0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x12, 0x21, 0x0a, 0x04, 0x75,
|
||||
0x6e, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x62, 0x65, 0x70, 0x2e,
|
||||
0x55, 0x6e, 0x69, 0x78, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x75, 0x6e, 0x69, 0x78, 0x12, 0x2a,
|
||||
0x0a, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32,
|
||||
0x10, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x44, 0x61, 0x74,
|
||||
0x61, 0x52, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x12, 0x24, 0x0a, 0x05, 0x6c, 0x69,
|
||||
0x6e, 0x75, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e,
|
||||
0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x75, 0x78,
|
||||
0x12, 0x26, 0x0a, 0x06, 0x64, 0x61, 0x72, 0x77, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b,
|
||||
0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61,
|
||||
0x52, 0x06, 0x64, 0x61, 0x72, 0x77, 0x69, 0x6e, 0x12, 0x28, 0x0a, 0x07, 0x66, 0x72, 0x65, 0x65,
|
||||
0x62, 0x73, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e,
|
||||
0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x07, 0x66, 0x72, 0x65, 0x65, 0x62,
|
||||
0x73, 0x64, 0x12, 0x26, 0x0a, 0x06, 0x6e, 0x65, 0x74, 0x62, 0x73, 0x64, 0x18, 0x06, 0x20, 0x01,
|
||||
0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0x32,
|
||||
0x0a, 0x06, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x28, 0x0a, 0x08, 0x63, 0x6f, 0x75, 0x6e,
|
||||
0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x62, 0x65, 0x70,
|
||||
0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x52, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65,
|
||||
0x72, 0x73, 0x22, 0x2f, 0x0a, 0x07, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x0e, 0x0a,
|
||||
0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a,
|
||||
0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61,
|
||||
0x6c, 0x75, 0x65, 0x22, 0xfd, 0x01, 0x0a, 0x0c, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d,
|
||||
0x44, 0x61, 0x74, 0x61, 0x12, 0x21, 0x0a, 0x04, 0x75, 0x6e, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01,
|
||||
0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x55, 0x6e, 0x69, 0x78, 0x44, 0x61, 0x74,
|
||||
0x61, 0x52, 0x04, 0x75, 0x6e, 0x69, 0x78, 0x12, 0x2a, 0x0a, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f,
|
||||
0x77, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x57,
|
||||
0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x44, 0x61, 0x74, 0x61, 0x52, 0x07, 0x77, 0x69, 0x6e, 0x64,
|
||||
0x6f, 0x77, 0x73, 0x12, 0x24, 0x0a, 0x05, 0x6c, 0x69, 0x6e, 0x75, 0x78, 0x18, 0x03, 0x20, 0x01,
|
||||
0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61,
|
||||
0x74, 0x61, 0x52, 0x06, 0x6e, 0x65, 0x74, 0x62, 0x73, 0x64, 0x22, 0x6c, 0x0a, 0x08, 0x55, 0x6e,
|
||||
0x69, 0x78, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f,
|
||||
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e, 0x65,
|
||||
0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x6e,
|
||||
0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70,
|
||||
0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28,
|
||||
0x05, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x04, 0x20,
|
||||
0x01, 0x28, 0x05, 0x52, 0x03, 0x67, 0x69, 0x64, 0x22, 0x52, 0x0a, 0x0b, 0x57, 0x69, 0x6e, 0x64,
|
||||
0x6f, 0x77, 0x73, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72,
|
||||
0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e,
|
||||
0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f,
|
||||
0x69, 0x73, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c,
|
||||
0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0x2f, 0x0a, 0x09,
|
||||
0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x12, 0x22, 0x0a, 0x06, 0x78, 0x61, 0x74,
|
||||
0x74, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x62, 0x65, 0x70, 0x2e,
|
||||
0x58, 0x61, 0x74, 0x74, 0x72, 0x52, 0x06, 0x78, 0x61, 0x74, 0x74, 0x72, 0x73, 0x22, 0x31, 0x0a,
|
||||
0x05, 0x58, 0x61, 0x74, 0x74, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
|
||||
0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61,
|
||||
0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65,
|
||||
0x22, 0xe4, 0x01, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02,
|
||||
0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06,
|
||||
0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f,
|
||||
0x6c, 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01,
|
||||
0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73,
|
||||
0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74,
|
||||
0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04,
|
||||
0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x06, 0x20, 0x01,
|
||||
0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x72, 0x6f, 0x6d,
|
||||
0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x72, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08,
|
||||
0x52, 0x0d, 0x66, 0x72, 0x6f, 0x6d, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x72, 0x79, 0x12,
|
||||
0x1b, 0x0a, 0x09, 0x77, 0x65, 0x61, 0x6b, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x08, 0x20, 0x01,
|
||||
0x28, 0x0d, 0x52, 0x08, 0x77, 0x65, 0x61, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x12, 0x19, 0x0a, 0x08,
|
||||
0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x6f, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07,
|
||||
0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x6f, 0x22, 0x52, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f,
|
||||
0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52,
|
||||
0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28,
|
||||
0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18,
|
||||
0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x45, 0x72, 0x72, 0x6f,
|
||||
0x72, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x65, 0x0a, 0x10, 0x44,
|
||||
0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12,
|
||||
0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
|
||||
0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x39, 0x0a, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74,
|
||||
0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x46,
|
||||
0x74, 0x61, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x75, 0x78, 0x12, 0x26, 0x0a, 0x06, 0x64, 0x61, 0x72,
|
||||
0x77, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e,
|
||||
0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x06, 0x64, 0x61, 0x72, 0x77, 0x69,
|
||||
0x6e, 0x12, 0x28, 0x0a, 0x07, 0x66, 0x72, 0x65, 0x65, 0x62, 0x73, 0x64, 0x18, 0x05, 0x20, 0x01,
|
||||
0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61,
|
||||
0x74, 0x61, 0x52, 0x07, 0x66, 0x72, 0x65, 0x65, 0x62, 0x73, 0x64, 0x12, 0x26, 0x0a, 0x06, 0x6e,
|
||||
0x65, 0x74, 0x62, 0x73, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65,
|
||||
0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x06, 0x6e, 0x65, 0x74,
|
||||
0x62, 0x73, 0x64, 0x22, 0x6c, 0x0a, 0x08, 0x55, 0x6e, 0x69, 0x78, 0x44, 0x61, 0x74, 0x61, 0x12,
|
||||
0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20,
|
||||
0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d,
|
||||
0x0a, 0x0a, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01,
|
||||
0x28, 0x09, 0x52, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a,
|
||||
0x03, 0x75, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12,
|
||||
0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x67, 0x69,
|
||||
0x64, 0x22, 0x52, 0x0a, 0x0b, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x44, 0x61, 0x74, 0x61,
|
||||
0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01,
|
||||
0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12,
|
||||
0x24, 0x0a, 0x0e, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x73, 0x5f, 0x67, 0x72, 0x6f, 0x75,
|
||||
0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x73,
|
||||
0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0x2f, 0x0a, 0x09, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61,
|
||||
0x74, 0x61, 0x12, 0x22, 0x0a, 0x06, 0x78, 0x61, 0x74, 0x74, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03,
|
||||
0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x52, 0x06,
|
||||
0x78, 0x61, 0x74, 0x74, 0x72, 0x73, 0x22, 0x31, 0x0a, 0x05, 0x58, 0x61, 0x74, 0x74, 0x72, 0x12,
|
||||
0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e,
|
||||
0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01,
|
||||
0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xcd, 0x01, 0x0a, 0x07, 0x52, 0x65,
|
||||
0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
|
||||
0x05, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18,
|
||||
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a,
|
||||
0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d,
|
||||
0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28,
|
||||
0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a,
|
||||
0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a,
|
||||
0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73,
|
||||
0x68, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72,
|
||||
0x61, 0x72, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x66, 0x72, 0x6f, 0x6d, 0x54,
|
||||
0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x62, 0x6c, 0x6f, 0x63,
|
||||
0x6b, 0x5f, 0x6e, 0x6f, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x62, 0x6c, 0x6f, 0x63,
|
||||
0x6b, 0x4e, 0x6f, 0x4a, 0x04, 0x08, 0x08, 0x10, 0x09, 0x22, 0x52, 0x0a, 0x08, 0x52, 0x65, 0x73,
|
||||
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28,
|
||||
0x05, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20,
|
||||
0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x0a, 0x04, 0x63, 0x6f, 0x64,
|
||||
0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x45, 0x72,
|
||||
0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x65, 0x0a,
|
||||
0x10, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73,
|
||||
0x73, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28,
|
||||
0x09, 0x52, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x39, 0x0a, 0x07, 0x75, 0x70, 0x64,
|
||||
0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x62, 0x65, 0x70,
|
||||
0x2e, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f,
|
||||
0x67, 0x72, 0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x07, 0x75, 0x70, 0x64,
|
||||
0x61, 0x74, 0x65, 0x73, 0x22, 0xe5, 0x01, 0x0a, 0x1a, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77,
|
||||
0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x55, 0x70, 0x64,
|
||||
0x61, 0x74, 0x65, 0x12, 0x44, 0x0a, 0x0b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x79,
|
||||
0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x46,
|
||||
0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72,
|
||||
0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74,
|
||||
0x65, 0x73, 0x22, 0xe5, 0x01, 0x0a, 0x1a, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c,
|
||||
0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74,
|
||||
0x65, 0x12, 0x44, 0x0a, 0x0b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65,
|
||||
0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x46, 0x69, 0x6c,
|
||||
0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73,
|
||||
0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x75, 0x70, 0x64,
|
||||
0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
|
||||
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x07, 0x76,
|
||||
0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x62,
|
||||
0x65, 0x70, 0x2e, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69,
|
||||
0x6f, 0x6e, 0x12, 0x27, 0x0a, 0x0d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x69, 0x6e, 0x64, 0x65,
|
||||
0x78, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x05, 0x42, 0x02, 0x10, 0x00, 0x52, 0x0c, 0x62,
|
||||
0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x62,
|
||||
0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52,
|
||||
0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x06, 0x0a, 0x04, 0x50, 0x69,
|
||||
0x6e, 0x67, 0x22, 0x1f, 0x0a, 0x05, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72,
|
||||
0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x61,
|
||||
0x73, 0x6f, 0x6e, 0x2a, 0xed, 0x01, 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54,
|
||||
0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x1b, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54,
|
||||
0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x43, 0x4f, 0x4e, 0x46,
|
||||
0x49, 0x47, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f,
|
||||
0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x44, 0x45, 0x58, 0x10, 0x01, 0x12, 0x1d, 0x0a, 0x19,
|
||||
0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x44,
|
||||
0x45, 0x58, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, 0x18, 0x0a, 0x14, 0x4d,
|
||||
0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x51, 0x55,
|
||||
0x45, 0x53, 0x54, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45,
|
||||
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x10, 0x04,
|
||||
0x12, 0x22, 0x0a, 0x1e, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
|
||||
0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45,
|
||||
0x53, 0x53, 0x10, 0x05, 0x12, 0x15, 0x0a, 0x11, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f,
|
||||
0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x4d,
|
||||
0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x4f, 0x53,
|
||||
0x45, 0x10, 0x07, 0x2a, 0x4f, 0x0a, 0x12, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f,
|
||||
0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x18, 0x4d, 0x45, 0x53,
|
||||
0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x75,
|
||||
0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d,
|
||||
0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a,
|
||||
0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b,
|
||||
0x2e, 0x62, 0x65, 0x70, 0x2e, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x07, 0x76, 0x65, 0x72,
|
||||
0x73, 0x69, 0x6f, 0x6e, 0x12, 0x27, 0x0a, 0x0d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x69, 0x6e,
|
||||
0x64, 0x65, 0x78, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x05, 0x42, 0x02, 0x10, 0x00, 0x52,
|
||||
0x0c, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x12, 0x1d, 0x0a,
|
||||
0x0a, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28,
|
||||
0x05, 0x52, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x06, 0x0a, 0x04,
|
||||
0x50, 0x69, 0x6e, 0x67, 0x22, 0x1f, 0x0a, 0x05, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x12, 0x16, 0x0a,
|
||||
0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72,
|
||||
0x65, 0x61, 0x73, 0x6f, 0x6e, 0x2a, 0xed, 0x01, 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67,
|
||||
0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x1b, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45,
|
||||
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x43, 0x4f,
|
||||
0x4e, 0x46, 0x49, 0x47, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47,
|
||||
0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x44, 0x45, 0x58, 0x10, 0x01, 0x12, 0x1d,
|
||||
0x0a, 0x19, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49,
|
||||
0x4e, 0x44, 0x45, 0x58, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, 0x18, 0x0a,
|
||||
0x14, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45,
|
||||
0x51, 0x55, 0x45, 0x53, 0x54, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x4d, 0x45, 0x53, 0x53, 0x41,
|
||||
0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45,
|
||||
0x10, 0x04, 0x12, 0x22, 0x0a, 0x1e, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59,
|
||||
0x50, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47,
|
||||
0x52, 0x45, 0x53, 0x53, 0x10, 0x05, 0x12, 0x15, 0x0a, 0x11, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47,
|
||||
0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x16, 0x0a,
|
||||
0x12, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c,
|
||||
0x4f, 0x53, 0x45, 0x10, 0x07, 0x2a, 0x4f, 0x0a, 0x12, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
|
||||
0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x18, 0x4d,
|
||||
0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49,
|
||||
0x4f, 0x4e, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x4d, 0x45, 0x53,
|
||||
0x53, 0x41, 0x47, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e,
|
||||
0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x4d, 0x45, 0x53, 0x53, 0x41,
|
||||
0x47, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4c,
|
||||
0x5a, 0x34, 0x10, 0x01, 0x2a, 0x56, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73,
|
||||
0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49,
|
||||
0x4f, 0x4e, 0x5f, 0x4d, 0x45, 0x54, 0x41, 0x44, 0x41, 0x54, 0x41, 0x10, 0x00, 0x12, 0x15, 0x0a,
|
||||
0x11, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4e, 0x45, 0x56,
|
||||
0x45, 0x52, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53,
|
||||
0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x4c, 0x57, 0x41, 0x59, 0x53, 0x10, 0x02, 0x2a, 0xb0, 0x01, 0x0a,
|
||||
0x0c, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a,
|
||||
0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
|
||||
0x46, 0x49, 0x4c, 0x45, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49,
|
||||
0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x4f,
|
||||
0x52, 0x59, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1b, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46,
|
||||
0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x5f, 0x46,
|
||||
0x49, 0x4c, 0x45, 0x10, 0x02, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x28, 0x0a, 0x20, 0x46, 0x49, 0x4c,
|
||||
0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c,
|
||||
0x49, 0x4e, 0x4b, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x03, 0x1a,
|
||||
0x02, 0x08, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f,
|
||||
0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x10, 0x04, 0x2a,
|
||||
0x76, 0x0a, 0x09, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x17, 0x0a, 0x13,
|
||||
0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f, 0x5f, 0x45, 0x52,
|
||||
0x52, 0x4f, 0x52, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43,
|
||||
0x4f, 0x44, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x10, 0x01, 0x12, 0x1b, 0x0a,
|
||||
0x17, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f, 0x5f, 0x53,
|
||||
0x55, 0x43, 0x48, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x1b, 0x0a, 0x17, 0x45, 0x52,
|
||||
0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44,
|
||||
0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x03, 0x2a, 0x7e, 0x0a, 0x1e, 0x46, 0x69, 0x6c, 0x65, 0x44,
|
||||
0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x55,
|
||||
0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2d, 0x0a, 0x29, 0x46, 0x49, 0x4c,
|
||||
0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47, 0x52,
|
||||
0x45, 0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f,
|
||||
0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x00, 0x12, 0x2d, 0x0a, 0x29, 0x46, 0x49, 0x4c, 0x45,
|
||||
0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45,
|
||||
0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46,
|
||||
0x4f, 0x52, 0x47, 0x45, 0x54, 0x10, 0x01, 0x42, 0x70, 0x0a, 0x07, 0x63, 0x6f, 0x6d, 0x2e, 0x62,
|
||||
0x65, 0x70, 0x42, 0x08, 0x42, 0x65, 0x70, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2f,
|
||||
0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x79, 0x6e, 0x63, 0x74,
|
||||
0x68, 0x69, 0x6e, 0x67, 0x2f, 0x73, 0x79, 0x6e, 0x63, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x2f, 0x69,
|
||||
0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x62, 0x65, 0x70, 0xa2,
|
||||
0x02, 0x03, 0x42, 0x58, 0x58, 0xaa, 0x02, 0x03, 0x42, 0x65, 0x70, 0xca, 0x02, 0x03, 0x42, 0x65,
|
||||
0x70, 0xe2, 0x02, 0x0f, 0x42, 0x65, 0x70, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64,
|
||||
0x61, 0x74, 0x61, 0xea, 0x02, 0x03, 0x42, 0x65, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
|
||||
0x33,
|
||||
0x5f, 0x4c, 0x5a, 0x34, 0x10, 0x01, 0x2a, 0x56, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65,
|
||||
0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53,
|
||||
0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4d, 0x45, 0x54, 0x41, 0x44, 0x41, 0x54, 0x41, 0x10, 0x00, 0x12,
|
||||
0x15, 0x0a, 0x11, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4e,
|
||||
0x45, 0x56, 0x45, 0x52, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45,
|
||||
0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x4c, 0x57, 0x41, 0x59, 0x53, 0x10, 0x02, 0x2a, 0xb0,
|
||||
0x01, 0x0a, 0x0c, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x54, 0x79, 0x70, 0x65, 0x12,
|
||||
0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50,
|
||||
0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x46, 0x49, 0x4c, 0x45,
|
||||
0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43,
|
||||
0x54, 0x4f, 0x52, 0x59, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1b, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49,
|
||||
0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b,
|
||||
0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x02, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x28, 0x0a, 0x20, 0x46,
|
||||
0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59,
|
||||
0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10,
|
||||
0x03, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e,
|
||||
0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x10,
|
||||
0x04, 0x2a, 0x76, 0x0a, 0x09, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x17,
|
||||
0x0a, 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f, 0x5f,
|
||||
0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x45, 0x52, 0x52, 0x4f, 0x52,
|
||||
0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x10, 0x01, 0x12,
|
||||
0x1b, 0x0a, 0x17, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f,
|
||||
0x5f, 0x53, 0x55, 0x43, 0x48, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x1b, 0x0a, 0x17,
|
||||
0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c,
|
||||
0x49, 0x44, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x03, 0x2a, 0x7e, 0x0a, 0x1e, 0x46, 0x69, 0x6c,
|
||||
0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73,
|
||||
0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2d, 0x0a, 0x29, 0x46,
|
||||
0x49, 0x4c, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f,
|
||||
0x47, 0x52, 0x45, 0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50,
|
||||
0x45, 0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x00, 0x12, 0x2d, 0x0a, 0x29, 0x46, 0x49,
|
||||
0x4c, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47,
|
||||
0x52, 0x45, 0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45,
|
||||
0x5f, 0x46, 0x4f, 0x52, 0x47, 0x45, 0x54, 0x10, 0x01, 0x42, 0x70, 0x0a, 0x07, 0x63, 0x6f, 0x6d,
|
||||
0x2e, 0x62, 0x65, 0x70, 0x42, 0x08, 0x42, 0x65, 0x70, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01,
|
||||
0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x79, 0x6e,
|
||||
0x63, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x2f, 0x73, 0x79, 0x6e, 0x63, 0x74, 0x68, 0x69, 0x6e, 0x67,
|
||||
0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x62, 0x65,
|
||||
0x70, 0xa2, 0x02, 0x03, 0x42, 0x58, 0x58, 0xaa, 0x02, 0x03, 0x42, 0x65, 0x70, 0xca, 0x02, 0x03,
|
||||
0x42, 0x65, 0x70, 0xe2, 0x02, 0x0f, 0x42, 0x65, 0x70, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74,
|
||||
0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x03, 0x42, 0x65, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f,
|
||||
0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
2
lib/api/testdata/config/config.xml
vendored
2
lib/api/testdata/config/config.xml
vendored
@@ -18,7 +18,6 @@
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
<paused>false</paused>
|
||||
<weakHashThresholdPct>25</weakHashThresholdPct>
|
||||
<markerName>.stfolder</markerName>
|
||||
<useLargeBlocks>true</useLargeBlocks>
|
||||
</folder>
|
||||
@@ -39,7 +38,6 @@
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
<paused>false</paused>
|
||||
<weakHashThresholdPct>25</weakHashThresholdPct>
|
||||
<markerName>.stfolder</markerName>
|
||||
<useLargeBlocks>true</useLargeBlocks>
|
||||
</folder>
|
||||
|
||||
@@ -114,10 +114,9 @@ func TestDefaultValues(t *testing.T) {
|
||||
CleanupIntervalS: 3600,
|
||||
Params: map[string]string{},
|
||||
},
|
||||
MaxConflicts: 10,
|
||||
WeakHashThresholdPct: 25,
|
||||
MarkerName: ".stfolder",
|
||||
MaxConcurrentWrites: 2,
|
||||
MaxConflicts: 10,
|
||||
MarkerName: ".stfolder",
|
||||
MaxConcurrentWrites: 2,
|
||||
XattrFilter: XattrFilter{
|
||||
Entries: []XattrFilterEntry{},
|
||||
MaxSingleEntrySize: 1024,
|
||||
@@ -190,10 +189,9 @@ func TestDeviceConfig(t *testing.T) {
|
||||
Versioning: VersioningConfiguration{
|
||||
Params: map[string]string{},
|
||||
},
|
||||
WeakHashThresholdPct: 25,
|
||||
MarkerName: DefaultMarkerName,
|
||||
JunctionsAsDirs: true,
|
||||
MaxConcurrentWrites: maxConcurrentWritesDefault,
|
||||
MarkerName: DefaultMarkerName,
|
||||
JunctionsAsDirs: true,
|
||||
MaxConcurrentWrites: maxConcurrentWritesDefault,
|
||||
XattrFilter: XattrFilter{
|
||||
Entries: []XattrFilterEntry{},
|
||||
},
|
||||
|
||||
@@ -70,7 +70,6 @@ type FolderConfiguration struct {
|
||||
DisableSparseFiles bool `json:"disableSparseFiles" xml:"disableSparseFiles"`
|
||||
DisableTempIndexes bool `json:"disableTempIndexes" xml:"disableTempIndexes"`
|
||||
Paused bool `json:"paused" xml:"paused"`
|
||||
WeakHashThresholdPct int `json:"weakHashThresholdPct" xml:"weakHashThresholdPct"`
|
||||
MarkerName string `json:"markerName" xml:"markerName"`
|
||||
CopyOwnershipFromParent bool `json:"copyOwnershipFromParent" xml:"copyOwnershipFromParent"`
|
||||
RawModTimeWindowS int `json:"modTimeWindowS" xml:"modTimeWindowS"`
|
||||
@@ -311,10 +310,6 @@ func (f *FolderConfiguration) prepare(myID protocol.DeviceID, existingDevices ma
|
||||
f.Versioning.CleanupIntervalS = 0
|
||||
}
|
||||
|
||||
if f.WeakHashThresholdPct == 0 {
|
||||
f.WeakHashThresholdPct = 25
|
||||
}
|
||||
|
||||
if f.MarkerName == "" {
|
||||
f.MarkerName = DefaultMarkerName
|
||||
}
|
||||
|
||||
1
lib/config/testdata/overridenvalues.xml
vendored
1
lib/config/testdata/overridenvalues.xml
vendored
@@ -70,7 +70,6 @@
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
<paused>false</paused>
|
||||
<weakHashThresholdPct>25</weakHashThresholdPct>
|
||||
<markerName>.stfolder</markerName>
|
||||
<copyOwnershipFromParent>false</copyOwnershipFromParent>
|
||||
<modTimeWindowS>0</modTimeWindowS>
|
||||
|
||||
@@ -75,7 +75,7 @@ func (f *fakeConnection) DownloadProgress(_ context.Context, dp *protocol.Downlo
|
||||
|
||||
func (f *fakeConnection) addFileLocked(name string, flags uint32, ftype protocol.FileInfoType, data []byte, version protocol.Vector, localFlags uint32) {
|
||||
blockSize := protocol.BlockSize(int64(len(data)))
|
||||
blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), blockSize, int64(len(data)), nil, true)
|
||||
blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), blockSize, int64(len(data)), nil)
|
||||
|
||||
file := protocol.FileInfo{
|
||||
Name: name,
|
||||
|
||||
@@ -560,7 +560,7 @@ func setupKnownFiles(t *testing.T, ffs fs.Filesystem, data []byte) []protocol.Fi
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), protocol.BlockSize(int64(len(data))), int64(len(data)), nil, true)
|
||||
blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), protocol.BlockSize(int64(len(data))), int64(len(data)), nil)
|
||||
knownFiles := []protocol.FileInfo{
|
||||
{
|
||||
Name: "knownDir",
|
||||
|
||||
@@ -31,7 +31,6 @@ import (
|
||||
"github.com/syncthing/syncthing/lib/semaphore"
|
||||
"github.com/syncthing/syncthing/lib/sync"
|
||||
"github.com/syncthing/syncthing/lib/versioner"
|
||||
"github.com/syncthing/syncthing/lib/weakhash"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -1136,12 +1135,12 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, snap *db.Snapshot
|
||||
func (f *sendReceiveFolder) reuseBlocks(blocks []protocol.BlockInfo, reused []int, file protocol.FileInfo, tempName string) ([]protocol.BlockInfo, []int) {
|
||||
// Check for an old temporary file which might have some blocks we could
|
||||
// reuse.
|
||||
tempBlocks, err := scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil, false)
|
||||
tempBlocks, err := scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil)
|
||||
if err != nil {
|
||||
var caseErr *fs.ErrCaseConflict
|
||||
if errors.As(err, &caseErr) {
|
||||
if rerr := f.mtimefs.Rename(caseErr.Real, tempName); rerr == nil {
|
||||
tempBlocks, err = scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil, false)
|
||||
tempBlocks, err = scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1310,8 +1309,6 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
|
||||
f.model.progressEmitter.Register(state.sharedPullerState)
|
||||
}
|
||||
|
||||
weakHashFinder, file := f.initWeakHashFinder(state)
|
||||
|
||||
blocks:
|
||||
for _, block := range state.blocks {
|
||||
select {
|
||||
@@ -1336,75 +1333,49 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
|
||||
|
||||
buf = protocol.BufferPool.Upgrade(buf, int(block.Size))
|
||||
|
||||
var found bool
|
||||
if f.Type != config.FolderTypeReceiveEncrypted {
|
||||
found, err = weakHashFinder.Iterate(block.WeakHash, buf, func(offset int64) bool {
|
||||
if f.verifyBuffer(buf, block) != nil {
|
||||
return true
|
||||
}
|
||||
|
||||
err = f.limitedWriteAt(dstFd, buf, block.Offset)
|
||||
if err != nil {
|
||||
state.fail(fmt.Errorf("dst write: %w", err))
|
||||
}
|
||||
if offset == block.Offset {
|
||||
state.copiedFromOrigin(block.Size)
|
||||
} else {
|
||||
state.copiedFromOriginShifted(block.Size)
|
||||
}
|
||||
|
||||
return false
|
||||
})
|
||||
found := f.model.finder.Iterate(folders, block.Hash, func(folder, path string, index int32) bool {
|
||||
ffs := folderFilesystems[folder]
|
||||
fd, err := ffs.Open(path)
|
||||
if err != nil {
|
||||
l.Debugln("weak hasher iter", err)
|
||||
return false
|
||||
}
|
||||
}
|
||||
defer fd.Close()
|
||||
|
||||
if !found {
|
||||
found = f.model.finder.Iterate(folders, block.Hash, func(folder, path string, index int32) bool {
|
||||
ffs := folderFilesystems[folder]
|
||||
fd, err := ffs.Open(path)
|
||||
if err != nil {
|
||||
srcOffset := int64(state.file.BlockSize()) * int64(index)
|
||||
_, err = fd.ReadAt(buf, srcOffset)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// Hash is not SHA256 as it's an encrypted hash token. In that
|
||||
// case we can't verify the block integrity so we'll take it on
|
||||
// trust. (The other side can and will verify.)
|
||||
if f.Type != config.FolderTypeReceiveEncrypted {
|
||||
if err := f.verifyBuffer(buf, block); err != nil {
|
||||
l.Debugln("Finder failed to verify buffer", err)
|
||||
return false
|
||||
}
|
||||
defer fd.Close()
|
||||
}
|
||||
|
||||
srcOffset := int64(state.file.BlockSize()) * int64(index)
|
||||
_, err = fd.ReadAt(buf, srcOffset)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// Hash is not SHA256 as it's an encrypted hash token. In that
|
||||
// case we can't verify the block integrity so we'll take it on
|
||||
// trust. (The other side can and will verify.)
|
||||
if f.Type != config.FolderTypeReceiveEncrypted {
|
||||
if err := f.verifyBuffer(buf, block); err != nil {
|
||||
l.Debugln("Finder failed to verify buffer", err)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if f.CopyRangeMethod != config.CopyRangeMethodStandard {
|
||||
err = f.withLimiter(func() error {
|
||||
dstFd.mut.Lock()
|
||||
defer dstFd.mut.Unlock()
|
||||
return fs.CopyRange(f.CopyRangeMethod.ToFS(), fd, dstFd.fd, srcOffset, block.Offset, int64(block.Size))
|
||||
})
|
||||
} else {
|
||||
err = f.limitedWriteAt(dstFd, buf, block.Offset)
|
||||
}
|
||||
if err != nil {
|
||||
state.fail(fmt.Errorf("dst write: %w", err))
|
||||
}
|
||||
if path == state.file.Name {
|
||||
state.copiedFromOrigin(block.Size)
|
||||
} else {
|
||||
state.copiedFromElsewhere(block.Size)
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
if f.CopyRangeMethod != config.CopyRangeMethodStandard {
|
||||
err = f.withLimiter(func() error {
|
||||
dstFd.mut.Lock()
|
||||
defer dstFd.mut.Unlock()
|
||||
return fs.CopyRange(f.CopyRangeMethod.ToFS(), fd, dstFd.fd, srcOffset, block.Offset, int64(block.Size))
|
||||
})
|
||||
} else {
|
||||
err = f.limitedWriteAt(dstFd, buf, block.Offset)
|
||||
}
|
||||
if err != nil {
|
||||
state.fail(fmt.Errorf("dst write: %w", err))
|
||||
}
|
||||
if path == state.file.Name {
|
||||
state.copiedFromOrigin(block.Size)
|
||||
} else {
|
||||
state.copiedFromElsewhere(block.Size)
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
if state.failed() != nil {
|
||||
break
|
||||
@@ -1421,58 +1392,11 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
|
||||
state.copyDone(block)
|
||||
}
|
||||
}
|
||||
if file != nil {
|
||||
// os.File used to return invalid argument if nil.
|
||||
// fs.File panics as it's an interface.
|
||||
file.Close()
|
||||
}
|
||||
|
||||
out <- state.sharedPullerState
|
||||
}
|
||||
}
|
||||
|
||||
func (f *sendReceiveFolder) initWeakHashFinder(state copyBlocksState) (*weakhash.Finder, fs.File) {
|
||||
if f.Type == config.FolderTypeReceiveEncrypted {
|
||||
l.Debugln("not weak hashing due to folder type", f.Type)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
blocksPercentChanged := 0
|
||||
if tot := len(state.file.Blocks); tot > 0 {
|
||||
blocksPercentChanged = (tot - state.have) * 100 / tot
|
||||
}
|
||||
|
||||
if blocksPercentChanged < f.WeakHashThresholdPct {
|
||||
l.Debugf("not weak hashing %s. not enough changed %.02f < %d", state.file.Name, blocksPercentChanged, f.WeakHashThresholdPct)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
hashesToFind := make([]uint32, 0, len(state.blocks))
|
||||
for _, block := range state.blocks {
|
||||
if block.WeakHash != 0 {
|
||||
hashesToFind = append(hashesToFind, block.WeakHash)
|
||||
}
|
||||
}
|
||||
|
||||
if len(hashesToFind) == 0 {
|
||||
l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
file, err := f.mtimefs.Open(state.file.Name)
|
||||
if err != nil {
|
||||
l.Debugln("weak hasher", err)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
weakHashFinder, err := weakhash.NewFinder(f.ctx, file, state.file.BlockSize(), hashesToFind)
|
||||
if err != nil {
|
||||
l.Debugln("weak hasher", err)
|
||||
return nil, file
|
||||
}
|
||||
return weakHashFinder, file
|
||||
}
|
||||
|
||||
func (*sendReceiveFolder) verifyBuffer(buf []byte, block protocol.BlockInfo) error {
|
||||
if len(buf) != int(block.Size) {
|
||||
return fmt.Errorf("length mismatch %d != %d", len(buf), block.Size)
|
||||
@@ -1574,7 +1498,7 @@ loop:
|
||||
activity.using(selected)
|
||||
var buf []byte
|
||||
blockNo := int(state.block.Offset / int64(state.file.BlockSize()))
|
||||
buf, lastError = f.model.RequestGlobal(f.ctx, selected.ID, f.folderID, state.file.Name, blockNo, state.block.Offset, int(state.block.Size), state.block.Hash, state.block.WeakHash, selected.FromTemporary)
|
||||
buf, lastError = f.model.RequestGlobal(f.ctx, selected.ID, f.folderID, state.file.Name, blockNo, state.block.Offset, int(state.block.Size), state.block.Hash, selected.FromTemporary)
|
||||
activity.done(selected)
|
||||
if lastError != nil {
|
||||
l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, selected.ID.Short(), "returned error:", lastError)
|
||||
|
||||
@@ -25,7 +25,6 @@ import (
|
||||
"github.com/syncthing/syncthing/lib/fs"
|
||||
"github.com/syncthing/syncthing/lib/ignore"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/rand"
|
||||
"github.com/syncthing/syncthing/lib/scanner"
|
||||
"github.com/syncthing/syncthing/lib/sync"
|
||||
)
|
||||
@@ -300,7 +299,7 @@ func TestCopierFinder(t *testing.T) {
|
||||
}
|
||||
|
||||
// Verify that the fetched blocks have actually been written to the temp file
|
||||
blks, err := scanner.HashFile(context.TODO(), f.ID, f.Filesystem(nil), tempFile, protocol.MinBlockSize, nil, false)
|
||||
blks, err := scanner.HashFile(context.TODO(), f.ID, f.Filesystem(nil), tempFile, protocol.MinBlockSize, nil)
|
||||
if err != nil {
|
||||
t.Log(err)
|
||||
}
|
||||
@@ -312,128 +311,6 @@ func TestCopierFinder(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWeakHash(t *testing.T) {
|
||||
// Setup the model/pull environment
|
||||
_, fo, wcfgCancel := setupSendReceiveFolder(t)
|
||||
defer wcfgCancel()
|
||||
ffs := fo.Filesystem(nil)
|
||||
|
||||
tempFile := fs.TempName("weakhash")
|
||||
var shift int64 = 10
|
||||
var size int64 = 1 << 20
|
||||
expectBlocks := int(size / protocol.MinBlockSize)
|
||||
expectPulls := int(shift / protocol.MinBlockSize)
|
||||
if shift > 0 {
|
||||
expectPulls++
|
||||
}
|
||||
|
||||
f, err := ffs.Create("weakhash")
|
||||
must(t, err)
|
||||
defer f.Close()
|
||||
_, err = io.CopyN(f, rand.Reader, size)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
// Create two files, second file has `shifted` bytes random prefix, yet
|
||||
// both are of the same length, for example:
|
||||
// File 1: abcdefgh
|
||||
// File 2: xyabcdef
|
||||
f.Seek(0, io.SeekStart)
|
||||
existing, err := scanner.Blocks(context.TODO(), f, protocol.MinBlockSize, size, nil, true)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
f.Seek(0, io.SeekStart)
|
||||
remainder := io.LimitReader(f, size-shift)
|
||||
prefix := io.LimitReader(rand.Reader, shift)
|
||||
nf := io.MultiReader(prefix, remainder)
|
||||
desired, err := scanner.Blocks(context.TODO(), nf, protocol.MinBlockSize, size, nil, true)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
existingFile := protocol.FileInfo{
|
||||
Name: "weakhash",
|
||||
Blocks: existing,
|
||||
Size: size,
|
||||
ModifiedS: info.ModTime().Unix(),
|
||||
ModifiedNs: int32(info.ModTime().Nanosecond()),
|
||||
}
|
||||
desiredFile := protocol.FileInfo{
|
||||
Name: "weakhash",
|
||||
Size: size,
|
||||
Blocks: desired,
|
||||
ModifiedS: info.ModTime().Unix() + 1,
|
||||
}
|
||||
|
||||
fo.updateLocalsFromScanning([]protocol.FileInfo{existingFile})
|
||||
|
||||
copyChan := make(chan copyBlocksState)
|
||||
pullChan := make(chan pullBlockState, expectBlocks)
|
||||
finisherChan := make(chan *sharedPullerState, 1)
|
||||
|
||||
// Run a single fetcher routine
|
||||
go fo.copierRoutine(copyChan, pullChan, finisherChan)
|
||||
defer close(copyChan)
|
||||
|
||||
// Test 1 - no weak hashing, file gets fully repulled (`expectBlocks` pulls).
|
||||
fo.WeakHashThresholdPct = 101
|
||||
fo.handleFile(desiredFile, fsetSnapshot(t, fo.fset), copyChan)
|
||||
|
||||
var pulls []pullBlockState
|
||||
timeout := time.After(10 * time.Second)
|
||||
for len(pulls) < expectBlocks {
|
||||
select {
|
||||
case pull := <-pullChan:
|
||||
pulls = append(pulls, pull)
|
||||
case <-timeout:
|
||||
t.Fatalf("timed out, got %d pulls expected %d", len(pulls), expectPulls)
|
||||
}
|
||||
}
|
||||
finish := <-finisherChan
|
||||
|
||||
select {
|
||||
case <-pullChan:
|
||||
t.Fatal("Pull channel has data to be read")
|
||||
case <-finisherChan:
|
||||
t.Fatal("Finisher channel has data to be read")
|
||||
default:
|
||||
}
|
||||
|
||||
cleanupSharedPullerState(finish)
|
||||
if err := ffs.Remove(tempFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Test 2 - using weak hash, expectPulls blocks pulled.
|
||||
fo.WeakHashThresholdPct = -1
|
||||
fo.handleFile(desiredFile, fsetSnapshot(t, fo.fset), copyChan)
|
||||
|
||||
pulls = pulls[:0]
|
||||
for len(pulls) < expectPulls {
|
||||
select {
|
||||
case pull := <-pullChan:
|
||||
pulls = append(pulls, pull)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("timed out, got %d pulls expected %d", len(pulls), expectPulls)
|
||||
}
|
||||
}
|
||||
|
||||
finish = <-finisherChan
|
||||
cleanupSharedPullerState(finish)
|
||||
|
||||
expectShifted := expectBlocks - expectPulls
|
||||
if finish.copyOriginShifted != expectShifted {
|
||||
t.Errorf("did not copy %d shifted", expectShifted)
|
||||
}
|
||||
}
|
||||
|
||||
// Test that updating a file removes its old blocks from the blockmap
|
||||
func TestCopierCleanup(t *testing.T) {
|
||||
iterFn := func(folder, file string, index int32) bool {
|
||||
@@ -709,8 +586,8 @@ func TestIssue3164(t *testing.T) {
|
||||
|
||||
func TestDiff(t *testing.T) {
|
||||
for i, test := range diffTestData {
|
||||
a, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.a), test.s, -1, nil, false)
|
||||
b, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.b), test.s, -1, nil, false)
|
||||
a, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.a), test.s, -1, nil)
|
||||
b, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.b), test.s, -1, nil)
|
||||
_, d := blockDiff(a, b)
|
||||
if len(d) != len(test.d) {
|
||||
t.Fatalf("Incorrect length for diff %d; %d != %d", i, len(d), len(test.d))
|
||||
@@ -730,8 +607,8 @@ func TestDiff(t *testing.T) {
|
||||
func BenchmarkDiff(b *testing.B) {
|
||||
testCases := make([]struct{ a, b []protocol.BlockInfo }, 0, len(diffTestData))
|
||||
for _, test := range diffTestData {
|
||||
a, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.a), test.s, -1, nil, false)
|
||||
b, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.b), test.s, -1, nil, false)
|
||||
a, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.a), test.s, -1, nil)
|
||||
b, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.b), test.s, -1, nil)
|
||||
testCases = append(testCases, struct{ a, b []protocol.BlockInfo }{a, b})
|
||||
}
|
||||
b.ReportAllocs()
|
||||
|
||||
@@ -55,16 +55,15 @@ var (
|
||||
Namespace: "syncthing",
|
||||
Subsystem: "model",
|
||||
Name: "folder_processed_bytes_total",
|
||||
Help: "Total amount of data processed during folder syncing, per folder ID and data source (network/local_origin/local_other/local_shifted/skipped)",
|
||||
Help: "Total amount of data processed during folder syncing, per folder ID and data source (network/local_origin/local_other/skipped)",
|
||||
}, []string{"folder", "source"})
|
||||
)
|
||||
|
||||
const (
|
||||
metricSourceNetwork = "network" // from the network
|
||||
metricSourceLocalOrigin = "local_origin" // from the existing version of the local file
|
||||
metricSourceLocalOther = "local_other" // from a different local file
|
||||
metricSourceLocalShifted = "local_shifted" // from the existing version of the local file, rolling hash shifted
|
||||
metricSourceSkipped = "skipped" // block of all zeroes, invented out of thin air
|
||||
metricSourceNetwork = "network" // from the network
|
||||
metricSourceLocalOrigin = "local_origin" // from the existing version of the local file
|
||||
metricSourceLocalOther = "local_other" // from a different local file
|
||||
metricSourceSkipped = "skipped" // block of all zeroes, invented out of thin air
|
||||
|
||||
metricScopeGlobal = "global"
|
||||
metricScopeLocal = "local"
|
||||
@@ -88,6 +87,5 @@ func registerFolderMetrics(folderID string) {
|
||||
metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceNetwork)
|
||||
metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceLocalOrigin)
|
||||
metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceLocalOther)
|
||||
metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceLocalShifted)
|
||||
metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceSkipped)
|
||||
}
|
||||
|
||||
@@ -435,19 +435,18 @@ type Model struct {
|
||||
result1 protocol.RequestResponse
|
||||
result2 error
|
||||
}
|
||||
RequestGlobalStub func(context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, uint32, bool) ([]byte, error)
|
||||
RequestGlobalStub func(context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, bool) ([]byte, error)
|
||||
requestGlobalMutex sync.RWMutex
|
||||
requestGlobalArgsForCall []struct {
|
||||
arg1 context.Context
|
||||
arg2 protocol.DeviceID
|
||||
arg3 string
|
||||
arg4 string
|
||||
arg5 int
|
||||
arg6 int64
|
||||
arg7 int
|
||||
arg8 []byte
|
||||
arg9 uint32
|
||||
arg10 bool
|
||||
arg1 context.Context
|
||||
arg2 protocol.DeviceID
|
||||
arg3 string
|
||||
arg4 string
|
||||
arg5 int
|
||||
arg6 int64
|
||||
arg7 int
|
||||
arg8 []byte
|
||||
arg9 bool
|
||||
}
|
||||
requestGlobalReturns struct {
|
||||
result1 []byte
|
||||
@@ -2580,7 +2579,7 @@ func (fake *Model) RequestReturnsOnCall(i int, result1 protocol.RequestResponse,
|
||||
}{result1, result2}
|
||||
}
|
||||
|
||||
func (fake *Model) RequestGlobal(arg1 context.Context, arg2 protocol.DeviceID, arg3 string, arg4 string, arg5 int, arg6 int64, arg7 int, arg8 []byte, arg9 uint32, arg10 bool) ([]byte, error) {
|
||||
func (fake *Model) RequestGlobal(arg1 context.Context, arg2 protocol.DeviceID, arg3 string, arg4 string, arg5 int, arg6 int64, arg7 int, arg8 []byte, arg9 bool) ([]byte, error) {
|
||||
var arg8Copy []byte
|
||||
if arg8 != nil {
|
||||
arg8Copy = make([]byte, len(arg8))
|
||||
@@ -2589,23 +2588,22 @@ func (fake *Model) RequestGlobal(arg1 context.Context, arg2 protocol.DeviceID, a
|
||||
fake.requestGlobalMutex.Lock()
|
||||
ret, specificReturn := fake.requestGlobalReturnsOnCall[len(fake.requestGlobalArgsForCall)]
|
||||
fake.requestGlobalArgsForCall = append(fake.requestGlobalArgsForCall, struct {
|
||||
arg1 context.Context
|
||||
arg2 protocol.DeviceID
|
||||
arg3 string
|
||||
arg4 string
|
||||
arg5 int
|
||||
arg6 int64
|
||||
arg7 int
|
||||
arg8 []byte
|
||||
arg9 uint32
|
||||
arg10 bool
|
||||
}{arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8Copy, arg9, arg10})
|
||||
arg1 context.Context
|
||||
arg2 protocol.DeviceID
|
||||
arg3 string
|
||||
arg4 string
|
||||
arg5 int
|
||||
arg6 int64
|
||||
arg7 int
|
||||
arg8 []byte
|
||||
arg9 bool
|
||||
}{arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8Copy, arg9})
|
||||
stub := fake.RequestGlobalStub
|
||||
fakeReturns := fake.requestGlobalReturns
|
||||
fake.recordInvocation("RequestGlobal", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8Copy, arg9, arg10})
|
||||
fake.recordInvocation("RequestGlobal", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8Copy, arg9})
|
||||
fake.requestGlobalMutex.Unlock()
|
||||
if stub != nil {
|
||||
return stub(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10)
|
||||
return stub(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9)
|
||||
}
|
||||
if specificReturn {
|
||||
return ret.result1, ret.result2
|
||||
@@ -2619,17 +2617,17 @@ func (fake *Model) RequestGlobalCallCount() int {
|
||||
return len(fake.requestGlobalArgsForCall)
|
||||
}
|
||||
|
||||
func (fake *Model) RequestGlobalCalls(stub func(context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, uint32, bool) ([]byte, error)) {
|
||||
func (fake *Model) RequestGlobalCalls(stub func(context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, bool) ([]byte, error)) {
|
||||
fake.requestGlobalMutex.Lock()
|
||||
defer fake.requestGlobalMutex.Unlock()
|
||||
fake.RequestGlobalStub = stub
|
||||
}
|
||||
|
||||
func (fake *Model) RequestGlobalArgsForCall(i int) (context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, uint32, bool) {
|
||||
func (fake *Model) RequestGlobalArgsForCall(i int) (context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, bool) {
|
||||
fake.requestGlobalMutex.RLock()
|
||||
defer fake.requestGlobalMutex.RUnlock()
|
||||
argsForCall := fake.requestGlobalArgsForCall[i]
|
||||
return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7, argsForCall.arg8, argsForCall.arg9, argsForCall.arg10
|
||||
return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7, argsForCall.arg8, argsForCall.arg9
|
||||
}
|
||||
|
||||
func (fake *Model) RequestGlobalReturns(result1 []byte, result2 error) {
|
||||
|
||||
@@ -118,7 +118,7 @@ type Model interface {
|
||||
|
||||
GlobalDirectoryTree(folder, prefix string, levels int, dirsOnly bool) ([]*TreeEntry, error)
|
||||
|
||||
RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error)
|
||||
RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error)
|
||||
}
|
||||
|
||||
type model struct {
|
||||
@@ -606,8 +606,6 @@ func (m *model) UsageReportingStats(report *contract.Report, version int, previe
|
||||
report.BlockStats.Pulled = v
|
||||
case "copyOrigin":
|
||||
report.BlockStats.CopyOrigin = v
|
||||
case "copyOriginShifted":
|
||||
report.BlockStats.CopyOriginShifted = v
|
||||
case "copyElsewhere":
|
||||
report.BlockStats.CopyElsewhere = v
|
||||
}
|
||||
@@ -2036,7 +2034,7 @@ func (m *model) Request(conn protocol.Connection, req *protocol.Request) (out pr
|
||||
return nil, protocol.ErrNoSuchFile
|
||||
}
|
||||
_, err := readOffsetIntoBuf(folderFs, tempFn, req.Offset, res.data)
|
||||
if err == nil && scanner.Validate(res.data, req.Hash, req.WeakHash) {
|
||||
if err == nil && scanner.Validate(res.data, req.Hash) {
|
||||
return res, nil
|
||||
}
|
||||
// Fall through to reading from a non-temp file, just in case the temp
|
||||
@@ -2065,8 +2063,8 @@ func (m *model) Request(conn protocol.Connection, req *protocol.Request) (out pr
|
||||
return nil, protocol.ErrGeneric
|
||||
}
|
||||
|
||||
if folderCfg.Type != config.FolderTypeReceiveEncrypted && len(req.Hash) > 0 && !scanner.Validate(res.data[:n], req.Hash, req.WeakHash) {
|
||||
m.recheckFile(deviceID, req.Folder, req.Name, req.Offset, req.Hash, req.WeakHash)
|
||||
if folderCfg.Type != config.FolderTypeReceiveEncrypted && len(req.Hash) > 0 && !scanner.Validate(res.data[:n], req.Hash) {
|
||||
m.recheckFile(deviceID, req.Folder, req.Name, req.Offset, req.Hash)
|
||||
l.Debugf("%v REQ(in) failed validating data: %s: %q / %q o=%d s=%d", m, deviceID.Short(), req.Folder, req.Name, req.Offset, req.Size)
|
||||
return nil, protocol.ErrNoSuchFile
|
||||
}
|
||||
@@ -2092,7 +2090,7 @@ func newLimitedRequestResponse(size int, limiters ...*semaphore.Semaphore) *requ
|
||||
return res
|
||||
}
|
||||
|
||||
func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, weakHash uint32) {
|
||||
func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte) {
|
||||
cf, ok, err := m.CurrentFolderFile(folder, name)
|
||||
if err != nil {
|
||||
l.Debugf("%v recheckFile: %s: %q / %q: current file error: %v", m, deviceID, folder, name, err)
|
||||
@@ -2121,10 +2119,6 @@ func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, off
|
||||
l.Debugf("%v recheckFile: %s: %q / %q i=%d: hash mismatch %x != %x", m, deviceID, folder, name, blockIndex, block.Hash, hash)
|
||||
return
|
||||
}
|
||||
if weakHash != 0 && block.WeakHash != weakHash {
|
||||
l.Debugf("%v recheckFile: %s: %q / %q i=%d: weak hash mismatch %v != %v", m, deviceID, folder, name, blockIndex, block.WeakHash, weakHash)
|
||||
return
|
||||
}
|
||||
|
||||
// The hashes provided part of the request match what we expect to find according
|
||||
// to what we have in the database, yet the content we've read off the filesystem doesn't
|
||||
@@ -2462,14 +2456,14 @@ func (m *model) deviceDidCloseRLocked(deviceID protocol.DeviceID, duration time.
|
||||
}
|
||||
}
|
||||
|
||||
func (m *model) RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) {
|
||||
func (m *model) RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) {
|
||||
conn, connOK := m.requestConnectionForDevice(deviceID)
|
||||
if !connOK {
|
||||
return nil, fmt.Errorf("requestGlobal: no connection to device: %s", deviceID.Short())
|
||||
}
|
||||
|
||||
l.Debugf("%v REQ(out): %s (%s): %q / %q b=%d o=%d s=%d h=%x wh=%x ft=%t", m, deviceID.Short(), conn, folder, name, blockNo, offset, size, hash, weakHash, fromTemporary)
|
||||
return conn.Request(ctx, &protocol.Request{Folder: folder, Name: name, BlockNo: blockNo, Offset: offset, Size: size, Hash: hash, WeakHash: weakHash, FromTemporary: fromTemporary})
|
||||
l.Debugf("%v REQ(out): %s (%s): %q / %q b=%d o=%d s=%d h=%x ft=%t", m, deviceID.Short(), conn, folder, name, blockNo, offset, size, hash, fromTemporary)
|
||||
return conn.Request(ctx, &protocol.Request{Folder: folder, Name: name, BlockNo: blockNo, Offset: offset, Size: size, Hash: hash, FromTemporary: fromTemporary})
|
||||
}
|
||||
|
||||
// requestConnectionForDevice returns a connection to the given device, to
|
||||
|
||||
@@ -222,7 +222,7 @@ func BenchmarkRequestOut(b *testing.B) {
|
||||
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
data, err := m.RequestGlobal(context.Background(), device1, "default", files[i%n].Name, 0, 0, 32, nil, 0, false)
|
||||
data, err := m.RequestGlobal(context.Background(), device1, "default", files[i%n].Name, 0, 0, 32, nil, false)
|
||||
if err != nil {
|
||||
b.Error(err)
|
||||
}
|
||||
@@ -2819,9 +2819,9 @@ func TestIssue5002(t *testing.T) {
|
||||
}
|
||||
blockSize := int32(file.BlockSize())
|
||||
|
||||
m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size-int64(blockSize), []byte{1, 2, 3, 4}, 0)
|
||||
m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size, []byte{1, 2, 3, 4}, 0) // panic
|
||||
m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size+int64(blockSize), []byte{1, 2, 3, 4}, 0)
|
||||
m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size-int64(blockSize), []byte{1, 2, 3, 4})
|
||||
m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size, []byte{1, 2, 3, 4}) // panic
|
||||
m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size+int64(blockSize), []byte{1, 2, 3, 4})
|
||||
}
|
||||
|
||||
func TestParentOfUnignored(t *testing.T) {
|
||||
|
||||
@@ -436,11 +436,8 @@ func TestRescanIfHaveInvalidContent(t *testing.T) {
|
||||
}
|
||||
|
||||
f := checkReceived(<-received)
|
||||
if f.Blocks[0].WeakHash != 103547413 {
|
||||
t.Fatalf("unexpected weak hash: %d != 103547413", f.Blocks[0].WeakHash)
|
||||
}
|
||||
|
||||
res, err := m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash, WeakHash: f.Blocks[0].WeakHash})
|
||||
res, err := m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -454,17 +451,14 @@ func TestRescanIfHaveInvalidContent(t *testing.T) {
|
||||
|
||||
writeFile(t, tfs, "foo", payload)
|
||||
|
||||
_, err = m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash, WeakHash: f.Blocks[0].WeakHash})
|
||||
_, err = m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash})
|
||||
if err == nil {
|
||||
t.Fatalf("expected failure")
|
||||
}
|
||||
|
||||
select {
|
||||
case fs := <-received:
|
||||
f := checkReceived(fs)
|
||||
if f.Blocks[0].WeakHash != 41943361 {
|
||||
t.Fatalf("unexpected weak hash: %d != 41943361", f.Blocks[0].WeakHash)
|
||||
}
|
||||
checkReceived(fs)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("timed out")
|
||||
}
|
||||
|
||||
@@ -285,15 +285,6 @@ func (s *sharedPullerState) skippedSparseBlock(bytes int) {
|
||||
metricFolderProcessedBytesTotal.WithLabelValues(s.folder, metricSourceSkipped).Add(float64(bytes))
|
||||
}
|
||||
|
||||
func (s *sharedPullerState) copiedFromOriginShifted(bytes int) {
|
||||
s.mut.Lock()
|
||||
s.copyOrigin++
|
||||
s.copyOriginShifted++
|
||||
s.updated = time.Now()
|
||||
s.mut.Unlock()
|
||||
metricFolderProcessedBytesTotal.WithLabelValues(s.folder, metricSourceLocalShifted).Add(float64(bytes))
|
||||
}
|
||||
|
||||
func (s *sharedPullerState) pullStarted() {
|
||||
s.mut.Lock()
|
||||
s.copyTotal--
|
||||
|
||||
@@ -544,32 +544,29 @@ func (f *FileInfo) setNoContent() {
|
||||
}
|
||||
|
||||
type BlockInfo struct {
|
||||
Hash []byte
|
||||
Offset int64
|
||||
Size int
|
||||
WeakHash uint32
|
||||
Hash []byte
|
||||
Offset int64
|
||||
Size int
|
||||
}
|
||||
|
||||
func (b BlockInfo) ToWire() *bep.BlockInfo {
|
||||
return &bep.BlockInfo{
|
||||
Hash: b.Hash,
|
||||
Offset: b.Offset,
|
||||
Size: int32(b.Size),
|
||||
WeakHash: b.WeakHash,
|
||||
Hash: b.Hash,
|
||||
Offset: b.Offset,
|
||||
Size: int32(b.Size),
|
||||
}
|
||||
}
|
||||
|
||||
func BlockInfoFromWire(w *bep.BlockInfo) BlockInfo {
|
||||
return BlockInfo{
|
||||
Hash: w.Hash,
|
||||
Offset: w.Offset,
|
||||
Size: int(w.Size),
|
||||
WeakHash: w.WeakHash,
|
||||
Hash: w.Hash,
|
||||
Offset: w.Offset,
|
||||
Size: int(w.Size),
|
||||
}
|
||||
}
|
||||
|
||||
func (b BlockInfo) String() string {
|
||||
return fmt.Sprintf("Block{%d/%d/%d/%x}", b.Offset, b.Size, b.WeakHash, b.Hash)
|
||||
return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
|
||||
}
|
||||
|
||||
// For each block size, the hash of a block of all zeroes
|
||||
@@ -596,7 +593,6 @@ func BlocksHash(bs []BlockInfo) []byte {
|
||||
h := sha256.New()
|
||||
for _, b := range bs {
|
||||
_, _ = h.Write(b.Hash)
|
||||
_ = binary.Write(h, binary.BigEndian, b.WeakHash)
|
||||
}
|
||||
return h.Sum(nil)
|
||||
}
|
||||
|
||||
@@ -25,7 +25,6 @@ type Request struct {
|
||||
Size int
|
||||
Hash []byte
|
||||
FromTemporary bool
|
||||
WeakHash uint32
|
||||
BlockNo int
|
||||
}
|
||||
|
||||
@@ -38,7 +37,6 @@ func (r *Request) toWire() *bep.Request {
|
||||
Size: int32(r.Size),
|
||||
Hash: r.Hash,
|
||||
FromTemporary: r.FromTemporary,
|
||||
WeakHash: r.WeakHash,
|
||||
BlockNo: int32(r.BlockNo),
|
||||
}
|
||||
}
|
||||
@@ -52,7 +50,6 @@ func requestFromWire(w *bep.Request) *Request {
|
||||
Size: int(w.Size),
|
||||
Hash: w.Hash,
|
||||
FromTemporary: w.FromTemporary,
|
||||
WeakHash: w.WeakHash,
|
||||
BlockNo: int(w.BlockNo),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,6 @@ type TestModel struct {
|
||||
offset int64
|
||||
size int32
|
||||
hash []byte
|
||||
weakHash uint32
|
||||
fromTemporary bool
|
||||
indexFn func(string, []FileInfo)
|
||||
ccFn func(*ClusterConfig)
|
||||
@@ -48,7 +47,6 @@ func (t *TestModel) Request(_ Connection, req *Request) (RequestResponse, error)
|
||||
t.offset = req.Offset
|
||||
t.size = int32(req.Size)
|
||||
t.hash = req.Hash
|
||||
t.weakHash = req.WeakHash
|
||||
t.fromTemporary = req.FromTemporary
|
||||
buf := make([]byte, len(t.data))
|
||||
copy(buf, t.data)
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
)
|
||||
|
||||
// HashFile hashes the files and returns a list of blocks representing the file.
|
||||
func HashFile(ctx context.Context, folderID string, fs fs.Filesystem, path string, blockSize int, counter Counter, useWeakHashes bool) ([]protocol.BlockInfo, error) {
|
||||
func HashFile(ctx context.Context, folderID string, fs fs.Filesystem, path string, blockSize int, counter Counter) ([]protocol.BlockInfo, error) {
|
||||
fd, err := fs.Open(path)
|
||||
if err != nil {
|
||||
l.Debugln("open:", err)
|
||||
@@ -36,7 +36,7 @@ func HashFile(ctx context.Context, folderID string, fs fs.Filesystem, path strin
|
||||
|
||||
// Hash the file. This may take a while for large files.
|
||||
|
||||
blocks, err := Blocks(ctx, fd, blockSize, size, counter, useWeakHashes)
|
||||
blocks, err := Blocks(ctx, fd, blockSize, size, counter)
|
||||
if err != nil {
|
||||
l.Debugln("blocks:", err)
|
||||
return nil, err
|
||||
@@ -108,7 +108,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) {
|
||||
panic("Bug. Asked to hash a directory or a deleted file.")
|
||||
}
|
||||
|
||||
blocks, err := HashFile(ctx, ph.folderID, ph.fs, f.Name, f.BlockSize(), ph.counter, true)
|
||||
blocks, err := HashFile(ctx, ph.folderID, ph.fs, f.Name, f.BlockSize(), ph.counter)
|
||||
if err != nil {
|
||||
handleError(ctx, "hashing", f.Name, err, ph.outbox)
|
||||
continue
|
||||
|
||||
@@ -10,8 +10,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"hash"
|
||||
"hash/adler32"
|
||||
"io"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
@@ -24,7 +22,7 @@ type Counter interface {
|
||||
}
|
||||
|
||||
// Blocks returns the blockwise hash of the reader.
|
||||
func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, counter Counter, useWeakHashes bool) ([]protocol.BlockInfo, error) {
|
||||
func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, counter Counter) ([]protocol.BlockInfo, error) {
|
||||
if counter == nil {
|
||||
counter = &noopCounter{}
|
||||
}
|
||||
@@ -32,15 +30,6 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
|
||||
hf := sha256.New()
|
||||
const hashLength = sha256.Size
|
||||
|
||||
var weakHf hash.Hash32 = noopHash{}
|
||||
var multiHf io.Writer = hf
|
||||
if useWeakHashes {
|
||||
// Use an actual weak hash function, make the multiHf
|
||||
// write to both hash functions.
|
||||
weakHf = adler32.New()
|
||||
multiHf = io.MultiWriter(hf, weakHf)
|
||||
}
|
||||
|
||||
var blocks []protocol.BlockInfo
|
||||
var hashes, thisHash []byte
|
||||
|
||||
@@ -70,7 +59,7 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
|
||||
}
|
||||
|
||||
lr.N = int64(blocksize)
|
||||
n, err := io.CopyBuffer(multiHf, lr, buf)
|
||||
n, err := io.CopyBuffer(hf, lr, buf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -87,17 +76,15 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
|
||||
thisHash, hashes = hashes[:hashLength], hashes[hashLength:]
|
||||
|
||||
b := protocol.BlockInfo{
|
||||
Size: int(n),
|
||||
Offset: offset,
|
||||
Hash: thisHash,
|
||||
WeakHash: weakHf.Sum32(),
|
||||
Size: int(n),
|
||||
Offset: offset,
|
||||
Hash: thisHash,
|
||||
}
|
||||
|
||||
blocks = append(blocks, b)
|
||||
offset += n
|
||||
|
||||
hf.Reset()
|
||||
weakHf.Reset()
|
||||
}
|
||||
|
||||
if len(blocks) == 0 {
|
||||
@@ -112,14 +99,8 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou
|
||||
return blocks, nil
|
||||
}
|
||||
|
||||
// Validate quickly validates buf against the 32-bit weakHash, if not zero,
|
||||
// else against the cryptohash hash, if len(hash)>0. It is satisfied if
|
||||
// either hash matches or neither hash is given.
|
||||
func Validate(buf, hash []byte, weakHash uint32) bool {
|
||||
if weakHash != 0 && adler32.Checksum(buf) == weakHash {
|
||||
return true
|
||||
}
|
||||
|
||||
// Validate validates the hash, if len(hash)>0.
|
||||
func Validate(buf, hash []byte) bool {
|
||||
if len(hash) > 0 {
|
||||
hbuf := sha256.Sum256(buf)
|
||||
return bytes.Equal(hbuf[:], hash)
|
||||
|
||||
@@ -9,51 +9,40 @@ package scanner
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
origAdler32 "hash/adler32"
|
||||
mrand "math/rand"
|
||||
"testing"
|
||||
"testing/quick"
|
||||
|
||||
rollingAdler32 "github.com/chmduquesne/rollinghash/adler32"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
)
|
||||
|
||||
var blocksTestData = []struct {
|
||||
data []byte
|
||||
blocksize int
|
||||
hash []string
|
||||
weakhash []uint32
|
||||
}{
|
||||
{
|
||||
[]byte(""), 1024,
|
||||
[]string{
|
||||
"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
},
|
||||
[]uint32{0},
|
||||
},
|
||||
{
|
||||
[]byte("contents"), 1024,
|
||||
[]string{
|
||||
"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8",
|
||||
},
|
||||
[]uint32{0x0f3a036f},
|
||||
},
|
||||
{
|
||||
[]byte("contents"), 9,
|
||||
[]string{
|
||||
"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8",
|
||||
},
|
||||
[]uint32{0x0f3a036f},
|
||||
},
|
||||
{
|
||||
[]byte("contents"), 8,
|
||||
[]string{
|
||||
"d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8",
|
||||
},
|
||||
[]uint32{0x0f3a036f},
|
||||
},
|
||||
{
|
||||
[]byte("contents"), 7,
|
||||
@@ -61,7 +50,6 @@ var blocksTestData = []struct {
|
||||
"ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73",
|
||||
"043a718774c572bd8a25adbeb1bfcd5c0256ae11cecf9f9c3f925d0e52beaf89",
|
||||
},
|
||||
[]uint32{0x0bcb02fc, 0x00740074},
|
||||
},
|
||||
{
|
||||
[]byte("contents"), 3,
|
||||
@@ -70,7 +58,6 @@ var blocksTestData = []struct {
|
||||
"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3",
|
||||
"44ad63f60af0f6db6fdde6d5186ef78176367df261fa06be3079b6c80c8adba4",
|
||||
},
|
||||
[]uint32{0x02780141, 0x02970148, 0x015d00e8},
|
||||
},
|
||||
{
|
||||
[]byte("conconts"), 3,
|
||||
@@ -79,7 +66,6 @@ var blocksTestData = []struct {
|
||||
"1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952",
|
||||
"44ad63f60af0f6db6fdde6d5186ef78176367df261fa06be3079b6c80c8adba4",
|
||||
},
|
||||
[]uint32{0x02780141, 0x02780141, 0x015d00e8},
|
||||
},
|
||||
{
|
||||
[]byte("contenten"), 3,
|
||||
@@ -88,14 +74,13 @@ var blocksTestData = []struct {
|
||||
"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3",
|
||||
"e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3",
|
||||
},
|
||||
[]uint32{0x02780141, 0x02970148, 0x02970148},
|
||||
},
|
||||
}
|
||||
|
||||
func TestBlocks(t *testing.T) {
|
||||
for testNo, test := range blocksTestData {
|
||||
buf := bytes.NewBuffer(test.data)
|
||||
blocks, err := Blocks(context.TODO(), buf, test.blocksize, -1, nil, true)
|
||||
blocks, err := Blocks(context.TODO(), buf, test.blocksize, -1, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -119,9 +104,6 @@ func TestBlocks(t *testing.T) {
|
||||
if h := fmt.Sprintf("%x", blocks[i].Hash); h != test.hash[i] {
|
||||
t.Errorf("%d/%d: Incorrect block hash %q != %q", testNo, i, h, test.hash[i])
|
||||
}
|
||||
if h := blocks[i].WeakHash; h != test.weakhash[i] {
|
||||
t.Errorf("%d/%d: Incorrect block weakhash 0x%08x != 0x%08x", testNo, i, h, test.weakhash[i])
|
||||
}
|
||||
|
||||
i++
|
||||
}
|
||||
@@ -129,85 +111,10 @@ func TestBlocks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdler32Variants(t *testing.T) {
|
||||
// Verify that the two adler32 functions give matching results for a few
|
||||
// different blocks of data.
|
||||
|
||||
hf1 := origAdler32.New()
|
||||
hf2 := rollingAdler32.New()
|
||||
|
||||
checkFn := func(data []byte) bool {
|
||||
hf1.Write(data)
|
||||
sum1 := hf1.Sum32()
|
||||
|
||||
hf2.Write(data)
|
||||
sum2 := hf2.Sum32()
|
||||
|
||||
hf1.Reset()
|
||||
hf2.Reset()
|
||||
|
||||
// Make sure whatever we use in Validate matches too resp. this
|
||||
// tests gets adjusted if we ever switch the weak hash algo.
|
||||
return sum1 == sum2 && Validate(data, nil, sum1)
|
||||
}
|
||||
|
||||
// protocol block sized data
|
||||
data := make([]byte, protocol.MinBlockSize)
|
||||
for i := 0; i < 5; i++ {
|
||||
rand.Read(data)
|
||||
if !checkFn(data) {
|
||||
t.Errorf("Hash mismatch on block sized data")
|
||||
}
|
||||
}
|
||||
|
||||
// random small blocks
|
||||
if err := quick.Check(checkFn, nil); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
// rolling should have the same result as the individual blocks
|
||||
// themselves.
|
||||
|
||||
windowSize := 128
|
||||
|
||||
hf3 := rollingAdler32.New()
|
||||
hf3.Write(data[:windowSize])
|
||||
|
||||
for i := windowSize; i < len(data); i++ {
|
||||
if i%windowSize == 0 {
|
||||
// let the reference function catch up
|
||||
window := data[i-windowSize : i]
|
||||
hf1.Reset()
|
||||
hf1.Write(window)
|
||||
hf2.Reset()
|
||||
hf2.Write(window)
|
||||
|
||||
// verify that they are in sync with the rolling function
|
||||
sum1 := hf1.Sum32()
|
||||
sum2 := hf2.Sum32()
|
||||
sum3 := hf3.Sum32()
|
||||
t.Logf("At i=%d, sum2=%08x, sum3=%08x", i, sum2, sum3)
|
||||
if sum2 != sum3 {
|
||||
t.Errorf("Mismatch after roll; i=%d, sum2=%08x, sum3=%08x", i, sum2, sum3)
|
||||
break
|
||||
}
|
||||
if sum1 != sum3 {
|
||||
t.Errorf("Mismatch after roll; i=%d, sum1=%08x, sum3=%08x", i, sum1, sum3)
|
||||
break
|
||||
}
|
||||
if !Validate(window, nil, sum1) {
|
||||
t.Errorf("Validation failure after roll; i=%d", i)
|
||||
}
|
||||
}
|
||||
hf3.Roll(data[i])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkValidate(b *testing.B) {
|
||||
type block struct {
|
||||
data []byte
|
||||
hash [sha256.Size]byte
|
||||
weakhash uint32
|
||||
data []byte
|
||||
hash [sha256.Size]byte
|
||||
}
|
||||
var blocks []block
|
||||
const blocksPerType = 100
|
||||
@@ -220,16 +127,6 @@ func BenchmarkValidate(b *testing.B) {
|
||||
b.data = make([]byte, 128<<10)
|
||||
r.Read(b.data)
|
||||
b.hash = sha256.Sum256(b.data)
|
||||
b.weakhash = origAdler32.Checksum(b.data)
|
||||
blocks = append(blocks, b)
|
||||
}
|
||||
// Blocks where the hash matches, but the weakhash doesn't.
|
||||
for i := 0; i < blocksPerType; i++ {
|
||||
var b block
|
||||
b.data = make([]byte, 128<<10)
|
||||
r.Read(b.data)
|
||||
b.hash = sha256.Sum256(b.data)
|
||||
b.weakhash = 1 // Zeros causes Validate to skip the weakhash.
|
||||
blocks = append(blocks, b)
|
||||
}
|
||||
|
||||
@@ -238,7 +135,7 @@ func BenchmarkValidate(b *testing.B) {
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
for _, b := range blocks {
|
||||
Validate(b.data, b.hash[:], b.weakhash)
|
||||
Validate(b.data, b.hash[:])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,7 +162,7 @@ func TestVerify(t *testing.T) {
|
||||
progress := newByteCounter()
|
||||
defer progress.Close()
|
||||
|
||||
blocks, err := Blocks(context.TODO(), buf, blocksize, -1, progress, false)
|
||||
blocks, err := Blocks(context.TODO(), buf, blocksize, -1, progress)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -640,7 +640,7 @@ func BenchmarkHashFile(b *testing.B) {
|
||||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
if _, err := HashFile(context.TODO(), "", testFs, testdataName, protocol.MinBlockSize, nil, true); err != nil {
|
||||
if _, err := HashFile(context.TODO(), "", testFs, testdataName, protocol.MinBlockSize, nil); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ func (m *Internals) SetIgnores(folderID string, content []string) error {
|
||||
}
|
||||
|
||||
func (m *Internals) DownloadBlock(ctx context.Context, deviceID protocol.DeviceID, folderID string, path string, blockNumber int, blockInfo protocol.BlockInfo, allowFromTemporary bool) ([]byte, error) {
|
||||
return m.model.RequestGlobal(ctx, deviceID, folderID, path, int(blockNumber), blockInfo.Offset, blockInfo.Size, blockInfo.Hash, blockInfo.WeakHash, allowFromTemporary)
|
||||
return m.model.RequestGlobal(ctx, deviceID, folderID, path, int(blockNumber), blockInfo.Offset, blockInfo.Size, blockInfo.Hash, allowFromTemporary)
|
||||
}
|
||||
|
||||
func (m *Internals) BlockAvailability(folderID string, file protocol.FileInfo, block protocol.BlockInfo) ([]model.Availability, error) {
|
||||
|
||||
@@ -177,7 +177,7 @@ func (a *App) startup() error {
|
||||
}()
|
||||
}
|
||||
|
||||
perf := ur.CpuBench(context.Background(), 3, 150*time.Millisecond, true)
|
||||
perf := ur.CpuBench(context.Background(), 3, 150*time.Millisecond)
|
||||
l.Infof("Hashing performance is %.02f MB/s", perf)
|
||||
|
||||
if err := db.UpdateSchema(a.ll); err != nil {
|
||||
|
||||
@@ -111,8 +111,6 @@ type Report struct {
|
||||
ConflictsOther int `json:"conflictsOther,omitempty" metric:"folder_feature{feature=ConflictsOther},summary" since:"3"`
|
||||
DisableSparseFiles int `json:"disableSparseFiles,omitempty" metric:"folder_feature{feature=DisableSparseFiles},summary" since:"3"`
|
||||
DisableTempIndexes int `json:"disableTempIndexes,omitempty" metric:"folder_feature{feature=DisableTempIndexes},summary" since:"3"`
|
||||
AlwaysWeakHash int `json:"alwaysWeakHash,omitempty" metric:"folder_feature{feature=AlwaysWeakhash},summary" since:"3"`
|
||||
CustomWeakHashThreshold int `json:"customWeakHashThreshold,omitempty" metric:"folder_feature{feature=CustomWeakhashThreshold},summary" since:"3"`
|
||||
FsWatcherEnabled int `json:"fsWatcherEnabled,omitempty" metric:"folder_feature{feature=FSWatcherEnabled},summary" since:"3"`
|
||||
PullOrder map[string]int `json:"pullOrder,omitempty" metric:"folder_pull_order,summaryVec:order" since:"3"`
|
||||
FilesystemType map[string]int `json:"filesystemType,omitempty" metric:"folder_file_system_type,summaryVec:type" since:"3"`
|
||||
@@ -152,13 +150,12 @@ type Report struct {
|
||||
} `json:"guiStats,omitempty" since:"3"`
|
||||
|
||||
BlockStats struct {
|
||||
Total int `json:"total,omitempty" metric:"blocks_processed_total,gauge" since:"3"`
|
||||
Renamed int `json:"renamed,omitempty" metric:"blocks_processed{source=renamed},gauge" since:"3"`
|
||||
Reused int `json:"reused,omitempty" metric:"blocks_processed{source=reused},gauge" since:"3"`
|
||||
Pulled int `json:"pulled,omitempty" metric:"blocks_processed{source=pulled},gauge" since:"3"`
|
||||
CopyOrigin int `json:"copyOrigin,omitempty" metric:"blocks_processed{source=copy_origin},gauge" since:"3"`
|
||||
CopyOriginShifted int `json:"copyOriginShifted,omitempty" metric:"blocks_processed{source=copy_origin_shifted},gauge" since:"3"`
|
||||
CopyElsewhere int `json:"copyElsewhere,omitempty" metric:"blocks_processed{source=copy_elsewhere},gauge" since:"3"`
|
||||
Total int `json:"total,omitempty" metric:"blocks_processed_total,gauge" since:"3"`
|
||||
Renamed int `json:"renamed,omitempty" metric:"blocks_processed{source=renamed},gauge" since:"3"`
|
||||
Reused int `json:"reused,omitempty" metric:"blocks_processed{source=reused},gauge" since:"3"`
|
||||
Pulled int `json:"pulled,omitempty" metric:"blocks_processed{source=pulled},gauge" since:"3"`
|
||||
CopyOrigin int `json:"copyOrigin,omitempty" metric:"blocks_processed{source=copy_origin},gauge" since:"3"`
|
||||
CopyElsewhere int `json:"copyElsewhere,omitempty" metric:"blocks_processed{source=copy_elsewhere},gauge" since:"3"`
|
||||
} `json:"blockStats,omitempty" since:"3"`
|
||||
|
||||
TransportStats map[string]int `json:"transportStats,omitempty" since:"3"`
|
||||
@@ -175,9 +172,6 @@ type Report struct {
|
||||
Stars int `json:"stars,omitempty" metric:"folder_ignore_lines{kind=stars},summary" since:"3"`
|
||||
} `json:"ignoreStats,omitempty" since:"3"`
|
||||
|
||||
// V3 fields added late in the RC
|
||||
WeakHashEnabled bool `json:"weakHashEnabled,omitempty" metric:"-" since:"3"` // Deprecated and not provided client-side anymore
|
||||
|
||||
// Added in post processing
|
||||
Received time.Time `json:"received,omitempty"`
|
||||
Date string `json:"date,omitempty"`
|
||||
|
||||
@@ -116,8 +116,8 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (
|
||||
report.TotMiB = int(totBytes / 1024 / 1024)
|
||||
report.FolderMaxMiB = int(maxBytes / 1024 / 1024)
|
||||
report.MemoryUsageMiB = int((mem.Sys - mem.HeapReleased) / 1024 / 1024)
|
||||
report.SHA256Perf = CpuBench(ctx, 5, 125*time.Millisecond, false)
|
||||
report.HashPerf = CpuBench(ctx, 5, 125*time.Millisecond, true)
|
||||
report.SHA256Perf = CpuBench(ctx, 5, 125*time.Millisecond)
|
||||
report.HashPerf = report.SHA256Perf
|
||||
report.MemorySize = int(memorySize() / 1024 / 1024)
|
||||
report.NumCPU = runtime.NumCPU()
|
||||
|
||||
@@ -252,11 +252,6 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (
|
||||
if cfg.DisableTempIndexes {
|
||||
report.FolderUsesV3.DisableTempIndexes++
|
||||
}
|
||||
if cfg.WeakHashThresholdPct < 0 {
|
||||
report.FolderUsesV3.AlwaysWeakHash++
|
||||
} else if cfg.WeakHashThresholdPct != 25 {
|
||||
report.FolderUsesV3.CustomWeakHashThreshold++
|
||||
}
|
||||
if cfg.FSWatcherEnabled {
|
||||
report.FolderUsesV3.FsWatcherEnabled++
|
||||
}
|
||||
@@ -443,7 +438,7 @@ var (
|
||||
)
|
||||
|
||||
// CpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s
|
||||
func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWeakHash bool) float64 {
|
||||
func CpuBench(ctx context.Context, iterations int, duration time.Duration) float64 {
|
||||
blocksResultMut.Lock()
|
||||
defer blocksResultMut.Unlock()
|
||||
|
||||
@@ -454,7 +449,7 @@ func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWe
|
||||
|
||||
var perf float64
|
||||
for i := 0; i < iterations; i++ {
|
||||
if v := cpuBenchOnce(ctx, duration, useWeakHash, bs); v > perf {
|
||||
if v := cpuBenchOnce(ctx, duration, bs); v > perf {
|
||||
perf = v
|
||||
}
|
||||
}
|
||||
@@ -467,13 +462,13 @@ func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWe
|
||||
return perf
|
||||
}
|
||||
|
||||
func cpuBenchOnce(ctx context.Context, duration time.Duration, useWeakHash bool, bs []byte) float64 {
|
||||
func cpuBenchOnce(ctx context.Context, duration time.Duration, bs []byte) float64 {
|
||||
t0 := time.Now()
|
||||
b := 0
|
||||
var err error
|
||||
for time.Since(t0) < duration {
|
||||
r := bytes.NewReader(bs)
|
||||
blocksResult, err = scanner.Blocks(ctx, r, protocol.MinBlockSize, int64(len(bs)), nil, useWeakHash)
|
||||
blocksResult, err = scanner.Blocks(ctx, r, protocol.MinBlockSize, int64(len(bs)), nil)
|
||||
if err != nil {
|
||||
return 0 // Context done
|
||||
}
|
||||
|
||||
@@ -1,151 +0,0 @@
|
||||
// Copyright (C) 2016 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package weakhash
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"hash"
|
||||
vadler32 "hash/adler32"
|
||||
"io"
|
||||
"math/rand"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/chmduquesne/rollinghash/adler32"
|
||||
"github.com/chmduquesne/rollinghash/bozo32"
|
||||
"github.com/chmduquesne/rollinghash/buzhash32"
|
||||
"github.com/chmduquesne/rollinghash/buzhash64"
|
||||
)
|
||||
|
||||
const (
|
||||
testFile = "../model/testdata/tmpfile"
|
||||
size = 128 << 10
|
||||
)
|
||||
|
||||
func BenchmarkFind1MFile(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(1 << 20)
|
||||
for i := 0; i < b.N; i++ {
|
||||
fd, err := os.Open(testFile)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
_, err = Find(context.Background(), fd, []uint32{0, 1, 2}, size)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
fd.Close()
|
||||
}
|
||||
}
|
||||
|
||||
type RollingHash interface {
|
||||
hash.Hash
|
||||
Roll(byte)
|
||||
}
|
||||
|
||||
func BenchmarkBlock(b *testing.B) {
|
||||
tests := []struct {
|
||||
name string
|
||||
hash hash.Hash
|
||||
}{
|
||||
{
|
||||
"adler32", adler32.New(),
|
||||
},
|
||||
{
|
||||
"bozo32", bozo32.New(),
|
||||
},
|
||||
{
|
||||
"buzhash32", buzhash32.New(),
|
||||
},
|
||||
{
|
||||
"buzhash64", buzhash64.New(),
|
||||
},
|
||||
{
|
||||
"vanilla-adler32", vadler32.New(),
|
||||
},
|
||||
}
|
||||
|
||||
sizes := []int64{128 << 10, 16 << 20}
|
||||
|
||||
buf := make([]byte, 16<<20)
|
||||
rand.Read(buf)
|
||||
|
||||
for _, testSize := range sizes {
|
||||
for _, test := range tests {
|
||||
b.Run(test.name+"-"+fmt.Sprint(testSize), func(bb *testing.B) {
|
||||
bb.Run("", func(bbb *testing.B) {
|
||||
bbb.ResetTimer()
|
||||
for i := 0; i < bbb.N; i++ {
|
||||
lr := io.LimitReader(bytes.NewReader(buf), testSize)
|
||||
n, err := io.Copy(test.hash, lr)
|
||||
if err != nil {
|
||||
bbb.Error(err)
|
||||
}
|
||||
if n != testSize {
|
||||
bbb.Errorf("%d != %d", n, testSize)
|
||||
}
|
||||
|
||||
test.hash.Sum(nil)
|
||||
test.hash.Reset()
|
||||
}
|
||||
|
||||
bbb.SetBytes(testSize)
|
||||
bbb.ReportAllocs()
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRoll(b *testing.B) {
|
||||
tests := []struct {
|
||||
name string
|
||||
hash RollingHash
|
||||
}{
|
||||
{
|
||||
"adler32", adler32.New(),
|
||||
},
|
||||
{
|
||||
"bozo32", bozo32.New(),
|
||||
},
|
||||
{
|
||||
"buzhash32", buzhash32.New(),
|
||||
},
|
||||
{
|
||||
"buzhash64", buzhash64.New(),
|
||||
},
|
||||
}
|
||||
|
||||
sizes := []int64{128 << 10, 16 << 20}
|
||||
|
||||
for _, testSize := range sizes {
|
||||
for _, test := range tests {
|
||||
b.Run(test.name+"-"+fmt.Sprint(testSize), func(bb *testing.B) {
|
||||
bb.Run("", func(bbb *testing.B) {
|
||||
data := make([]byte, testSize)
|
||||
|
||||
if _, err := test.hash.Write(data); err != nil {
|
||||
bbb.Error(err)
|
||||
}
|
||||
|
||||
bbb.ResetTimer()
|
||||
|
||||
for i := 0; i < bbb.N; i++ {
|
||||
for j := int64(0); j <= testSize; j++ {
|
||||
test.hash.Roll('a')
|
||||
}
|
||||
}
|
||||
|
||||
bbb.SetBytes(testSize)
|
||||
bbb.ReportAllocs()
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
// Copyright (C) 2016 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package weakhash
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"github.com/chmduquesne/rollinghash/adler32"
|
||||
)
|
||||
|
||||
const (
|
||||
Size = 4
|
||||
|
||||
// don't track more hits than this for any given weakhash
|
||||
maxWeakhashFinderHits = 10
|
||||
)
|
||||
|
||||
// Find finds all the blocks of the given size within io.Reader that matches
|
||||
// the hashes provided, and returns a hash -> slice of offsets within reader
|
||||
// map, that produces the same weak hash.
|
||||
func Find(ctx context.Context, ir io.Reader, hashesToFind []uint32, size int) (map[uint32][]int64, error) {
|
||||
if ir == nil || len(hashesToFind) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
r := bufio.NewReader(ir)
|
||||
hf := adler32.New()
|
||||
|
||||
n, err := io.CopyN(hf, r, int64(size))
|
||||
if err == io.EOF {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if n != int64(size) {
|
||||
return nil, io.ErrShortBuffer
|
||||
}
|
||||
|
||||
offsets := make(map[uint32][]int64)
|
||||
for _, hashToFind := range hashesToFind {
|
||||
offsets[hashToFind] = make([]int64, 0, maxWeakhashFinderHits)
|
||||
}
|
||||
|
||||
var i int64
|
||||
var hash uint32
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
hash = hf.Sum32()
|
||||
if existing, ok := offsets[hash]; ok && len(existing) < maxWeakhashFinderHits {
|
||||
offsets[hash] = append(existing, i)
|
||||
}
|
||||
i++
|
||||
|
||||
bt, err := r.ReadByte()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return offsets, err
|
||||
}
|
||||
hf.Roll(bt)
|
||||
}
|
||||
return offsets, nil
|
||||
}
|
||||
|
||||
func NewFinder(ctx context.Context, ir io.ReadSeeker, size int, hashesToFind []uint32) (*Finder, error) {
|
||||
offsets, err := Find(ctx, ir, hashesToFind, size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Finder{
|
||||
reader: ir,
|
||||
size: size,
|
||||
offsets: offsets,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type Finder struct {
|
||||
reader io.ReadSeeker
|
||||
size int
|
||||
offsets map[uint32][]int64
|
||||
}
|
||||
|
||||
// Iterate iterates all available blocks that matches the provided hash, reads
|
||||
// them into buf, and calls the iterator function. The iterator function should
|
||||
// return whether it wishes to continue iterating.
|
||||
func (h *Finder) Iterate(hash uint32, buf []byte, iterFunc func(int64) bool) (bool, error) {
|
||||
if h == nil || hash == 0 || len(buf) != h.size {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
for _, offset := range h.offsets[hash] {
|
||||
_, err := h.reader.Seek(offset, io.SeekStart)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
_, err = h.reader.Read(buf)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !iterFunc(offset) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
// Copyright (C) 2016 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
// The existence of this file means we get 0% test coverage rather than no
|
||||
// test coverage at all. Remove when implementing an actual test.
|
||||
|
||||
package weakhash
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var payload = []byte("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz")
|
||||
|
||||
func TestFinder(t *testing.T) {
|
||||
f, err := os.CreateTemp("", "")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
defer f.Close()
|
||||
|
||||
if _, err := f.Write(payload); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if _, err := f.Seek(0, io.SeekStart); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
hashes := []uint32{65143183, 65798547}
|
||||
finder, err := NewFinder(context.Background(), f, 4, hashes)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
expected := map[uint32][]int64{
|
||||
65143183: {1, 27, 53, 79},
|
||||
65798547: {2, 28, 54, 80},
|
||||
}
|
||||
actual := make(map[uint32][]int64)
|
||||
|
||||
b := make([]byte, Size)
|
||||
|
||||
for _, hash := range hashes {
|
||||
_, err := finder.Iterate(hash, b[:4], func(offset int64) bool {
|
||||
if !bytes.Equal(b, payload[offset:offset+4]) {
|
||||
t.Errorf("Not equal at %d: %s != %s", offset, string(b), string(payload[offset:offset+4]))
|
||||
}
|
||||
actual[hash] = append(actual[hash], offset)
|
||||
return true
|
||||
})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(actual, expected) {
|
||||
t.Errorf("Not equal: %#v != %#v", actual, expected)
|
||||
}
|
||||
}
|
||||
@@ -145,7 +145,7 @@ message BlockInfo {
|
||||
bytes hash = 3;
|
||||
int64 offset = 1;
|
||||
int32 size = 2;
|
||||
uint32 weak_hash = 4;
|
||||
reserved 4;
|
||||
}
|
||||
|
||||
message Vector {
|
||||
@@ -203,8 +203,8 @@ message Request {
|
||||
int32 size = 5;
|
||||
bytes hash = 6;
|
||||
bool from_temporary = 7;
|
||||
uint32 weak_hash = 8;
|
||||
int32 block_no = 9;
|
||||
reserved 8;
|
||||
}
|
||||
|
||||
// Response
|
||||
|
||||
@@ -24,7 +24,6 @@
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
<paused>false</paused>
|
||||
<weakHashThresholdPct>25</weakHashThresholdPct>
|
||||
<markerName>.stfolder</markerName>
|
||||
<copyOwnershipFromParent>false</copyOwnershipFromParent>
|
||||
<modTimeWindowS>0</modTimeWindowS>
|
||||
@@ -153,7 +152,6 @@
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
<paused>false</paused>
|
||||
<weakHashThresholdPct>25</weakHashThresholdPct>
|
||||
<markerName>.stfolder</markerName>
|
||||
<copyOwnershipFromParent>false</copyOwnershipFromParent>
|
||||
<modTimeWindowS>0</modTimeWindowS>
|
||||
|
||||
@@ -24,7 +24,6 @@
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
<paused>false</paused>
|
||||
<weakHashThresholdPct>25</weakHashThresholdPct>
|
||||
<markerName>.stfolder</markerName>
|
||||
<copyOwnershipFromParent>false</copyOwnershipFromParent>
|
||||
<modTimeWindowS>0</modTimeWindowS>
|
||||
@@ -151,7 +150,6 @@
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
<paused>false</paused>
|
||||
<weakHashThresholdPct>25</weakHashThresholdPct>
|
||||
<markerName>.stfolder</markerName>
|
||||
<copyOwnershipFromParent>false</copyOwnershipFromParent>
|
||||
<modTimeWindowS>0</modTimeWindowS>
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
<paused>false</paused>
|
||||
<weakHashThresholdPct>25</weakHashThresholdPct>
|
||||
<markerName>.stfolder</markerName>
|
||||
<copyOwnershipFromParent>false</copyOwnershipFromParent>
|
||||
<modTimeWindowS>0</modTimeWindowS>
|
||||
@@ -50,7 +49,6 @@
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
<paused>false</paused>
|
||||
<weakHashThresholdPct>25</weakHashThresholdPct>
|
||||
<markerName>.stfolder</markerName>
|
||||
<copyOwnershipFromParent>false</copyOwnershipFromParent>
|
||||
<modTimeWindowS>0</modTimeWindowS>
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
<disableSparseFiles>false</disableSparseFiles>
|
||||
<disableTempIndexes>false</disableTempIndexes>
|
||||
<paused>false</paused>
|
||||
<weakHashThresholdPct>25</weakHashThresholdPct>
|
||||
<markerName>.stfolder</markerName>
|
||||
<copyOwnershipFromParent>false</copyOwnershipFromParent>
|
||||
<modTimeWindowS>0</modTimeWindowS>
|
||||
|
||||
Reference in New Issue
Block a user