From b1c8f88a448166a304a46528c7188d05b8209a6f Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sat, 29 Mar 2025 12:21:10 +0000 Subject: [PATCH] chore: remove weak hashing which does not pull its weight (#10005) We've had weak/rolling hashing in the code for quite a while. It was a popular request for a while, based on the belief that rsync does this and we should too. However, the benefit is quite small; we save on average about 0.8% of transferred blocks over the population as a whole: Screenshot 2025-03-28 at 17 09 02 This would be fine if the cost was comparably low, however the downside of attempting rolling hash matching is that we (by default) do a complete file read on the destination in order to look for matches before we starting pulling blocks for the file. For any larger file this means a sometimes long, I/O-intensive pause before the file starts syncing, for usually no benefit. I propose we simply rip off the bandaid and save the effort. --- cmd/dev/stfileinfo/main.go | 2 +- cmd/syncthing/decrypt/decrypt.go | 2 +- go.mod | 1 - go.sum | 2 - internal/gen/bep/bep.pb.go | 309 +++++++++++------------- lib/api/testdata/config/config.xml | 2 - lib/config/config_test.go | 14 +- lib/config/folderconfiguration.go | 5 - lib/config/testdata/overridenvalues.xml | 1 - lib/model/fakeconns_test.go | 2 +- lib/model/folder_recvonly_test.go | 2 +- lib/model/folder_sendrecv.go | 156 +++--------- lib/model/folder_sendrecv_test.go | 133 +--------- lib/model/metrics.go | 12 +- lib/model/mocks/model.go | 54 ++--- lib/model/model.go | 22 +- lib/model/model_test.go | 8 +- lib/model/requests_test.go | 12 +- lib/model/sharedpullerstate.go | 9 - lib/protocol/bep_fileinfo.go | 24 +- lib/protocol/bep_request_response.go | 3 - lib/protocol/common_test.go | 2 - lib/scanner/blockqueue.go | 6 +- lib/scanner/blocks.go | 33 +-- lib/scanner/blocks_test.go | 111 +-------- lib/scanner/walk_test.go | 4 +- lib/syncthing/internals.go | 2 +- lib/syncthing/syncthing.go | 2 +- lib/ur/contract/contract.go | 18 +- lib/ur/usage_report.go | 17 +- lib/weakhash/benchmark_test.go | 151 ------------ lib/weakhash/weakhash.go | 118 --------- lib/weakhash/weakhash_test.go | 68 ------ proto/bep/bep.proto | 4 +- test/h1/config.xml | 2 - test/h2/config.xml | 2 - test/h3/config.xml | 2 - test/h4/config.xml | 1 - 38 files changed, 288 insertions(+), 1030 deletions(-) delete mode 100644 lib/weakhash/benchmark_test.go delete mode 100644 lib/weakhash/weakhash.go delete mode 100644 lib/weakhash/weakhash_test.go diff --git a/cmd/dev/stfileinfo/main.go b/cmd/dev/stfileinfo/main.go index 805a1294b..4c1e2aedb 100644 --- a/cmd/dev/stfileinfo/main.go +++ b/cmd/dev/stfileinfo/main.go @@ -72,7 +72,7 @@ func main() { if *standardBlocks || blockSize < protocol.MinBlockSize { blockSize = protocol.BlockSize(fi.Size()) } - bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true) + bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil) if err != nil { log.Fatal(err) } diff --git a/cmd/syncthing/decrypt/decrypt.go b/cmd/syncthing/decrypt/decrypt.go index ed162964c..4545e119c 100644 --- a/cmd/syncthing/decrypt/decrypt.go +++ b/cmd/syncthing/decrypt/decrypt.go @@ -238,7 +238,7 @@ func (c *CLI) decryptFile(encFi *protocol.FileInfo, plainFi *protocol.FileInfo, } // Verify the hash against the plaintext block info - if !scanner.Validate(dec, plainBlock.Hash, 0) { + if !scanner.Validate(dec, plainBlock.Hash) { // The block decrypted correctly but fails the hash check. This // is odd and unexpected, but it it's still a valid block from // the source. The file might have changed while we pulled it? diff --git a/go.mod b/go.mod index f2ce0ee4d..4ebd75662 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/calmh/incontainer v1.0.0 github.com/calmh/xdr v1.2.0 github.com/ccding/go-stun v0.1.5 - github.com/chmduquesne/rollinghash v4.0.0+incompatible github.com/d4l3k/messagediff v1.2.1 github.com/getsentry/raven-go v0.2.0 github.com/go-ldap/ldap/v3 v3.4.10 diff --git a/go.sum b/go.sum index bd69955ed..2d7cbc8ae 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,6 @@ github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chmduquesne/rollinghash v4.0.0+incompatible h1:hnREQO+DXjqIw3rUTzWN7/+Dpw+N5Um8zpKV0JOEgbo= -github.com/chmduquesne/rollinghash v4.0.0+incompatible/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= diff --git a/internal/gen/bep/bep.pb.go b/internal/gen/bep/bep.pb.go index 686fc6c04..11c0badc3 100644 --- a/internal/gen/bep/bep.pb.go +++ b/internal/gen/bep/bep.pb.go @@ -1093,10 +1093,9 @@ type BlockInfo struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Hash []byte `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"` - Offset int64 `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"` - Size int32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` - WeakHash uint32 `protobuf:"varint,4,opt,name=weak_hash,json=weakHash,proto3" json:"weak_hash,omitempty"` + Hash []byte `protobuf:"bytes,3,opt,name=hash,proto3" json:"hash,omitempty"` + Offset int64 `protobuf:"varint,1,opt,name=offset,proto3" json:"offset,omitempty"` + Size int32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` } func (x *BlockInfo) Reset() { @@ -1150,13 +1149,6 @@ func (x *BlockInfo) GetSize() int32 { return 0 } -func (x *BlockInfo) GetWeakHash() uint32 { - if x != nil { - return x.WeakHash - } - return 0 -} - type Vector struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1578,7 +1570,6 @@ type Request struct { Size int32 `protobuf:"varint,5,opt,name=size,proto3" json:"size,omitempty"` Hash []byte `protobuf:"bytes,6,opt,name=hash,proto3" json:"hash,omitempty"` FromTemporary bool `protobuf:"varint,7,opt,name=from_temporary,json=fromTemporary,proto3" json:"from_temporary,omitempty"` - WeakHash uint32 `protobuf:"varint,8,opt,name=weak_hash,json=weakHash,proto3" json:"weak_hash,omitempty"` BlockNo int32 `protobuf:"varint,9,opt,name=block_no,json=blockNo,proto3" json:"block_no,omitempty"` } @@ -1661,13 +1652,6 @@ func (x *Request) GetFromTemporary() bool { return false } -func (x *Request) GetWeakHash() uint32 { - if x != nil { - return x.WeakHash - } - return 0 -} - func (x *Request) GetBlockNo() int32 { if x != nil { return x.BlockNo @@ -2079,158 +2063,155 @@ var file_bep_bep_proto_rawDesc = []byte{ 0x76, 0x61, 0x6c, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x69, 0x6e, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x6e, 0x6f, 0x5f, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6e, 0x6f, - 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x68, 0x0a, 0x09, 0x42, + 0x50, 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x51, 0x0a, 0x09, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x65, 0x61, 0x6b, - 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x77, 0x65, 0x61, - 0x6b, 0x48, 0x61, 0x73, 0x68, 0x22, 0x32, 0x0a, 0x06, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, - 0x28, 0x0a, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x0c, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x52, - 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x73, 0x22, 0x2f, 0x0a, 0x07, 0x43, 0x6f, 0x75, - 0x6e, 0x74, 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, - 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xfd, 0x01, 0x0a, 0x0c, 0x50, - 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x44, 0x61, 0x74, 0x61, 0x12, 0x21, 0x0a, 0x04, 0x75, - 0x6e, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x62, 0x65, 0x70, 0x2e, - 0x55, 0x6e, 0x69, 0x78, 0x44, 0x61, 0x74, 0x61, 0x52, 0x04, 0x75, 0x6e, 0x69, 0x78, 0x12, 0x2a, - 0x0a, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x10, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x44, 0x61, 0x74, - 0x61, 0x52, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x12, 0x24, 0x0a, 0x05, 0x6c, 0x69, - 0x6e, 0x75, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, - 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x75, 0x78, - 0x12, 0x26, 0x0a, 0x06, 0x64, 0x61, 0x72, 0x77, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, - 0x52, 0x06, 0x64, 0x61, 0x72, 0x77, 0x69, 0x6e, 0x12, 0x28, 0x0a, 0x07, 0x66, 0x72, 0x65, 0x65, - 0x62, 0x73, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, - 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x07, 0x66, 0x72, 0x65, 0x65, 0x62, - 0x73, 0x64, 0x12, 0x26, 0x0a, 0x06, 0x6e, 0x65, 0x74, 0x62, 0x73, 0x64, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0x32, + 0x0a, 0x06, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x12, 0x28, 0x0a, 0x08, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x62, 0x65, 0x70, + 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x52, 0x08, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, + 0x72, 0x73, 0x22, 0x2f, 0x0a, 0x07, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x22, 0xfd, 0x01, 0x0a, 0x0c, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, + 0x44, 0x61, 0x74, 0x61, 0x12, 0x21, 0x0a, 0x04, 0x75, 0x6e, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x55, 0x6e, 0x69, 0x78, 0x44, 0x61, 0x74, + 0x61, 0x52, 0x04, 0x75, 0x6e, 0x69, 0x78, 0x12, 0x2a, 0x0a, 0x07, 0x77, 0x69, 0x6e, 0x64, 0x6f, + 0x77, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x57, + 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x44, 0x61, 0x74, 0x61, 0x52, 0x07, 0x77, 0x69, 0x6e, 0x64, + 0x6f, 0x77, 0x73, 0x12, 0x24, 0x0a, 0x05, 0x6c, 0x69, 0x6e, 0x75, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, - 0x74, 0x61, 0x52, 0x06, 0x6e, 0x65, 0x74, 0x62, 0x73, 0x64, 0x22, 0x6c, 0x0a, 0x08, 0x55, 0x6e, - 0x69, 0x78, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e, 0x65, - 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70, - 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x03, 0x67, 0x69, 0x64, 0x22, 0x52, 0x0a, 0x0b, 0x57, 0x69, 0x6e, 0x64, - 0x6f, 0x77, 0x73, 0x44, 0x61, 0x74, 0x61, 0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e, - 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x24, 0x0a, 0x0e, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, - 0x69, 0x73, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, - 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0x2f, 0x0a, 0x09, - 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x12, 0x22, 0x0a, 0x06, 0x78, 0x61, 0x74, - 0x74, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x62, 0x65, 0x70, 0x2e, - 0x58, 0x61, 0x74, 0x74, 0x72, 0x52, 0x06, 0x78, 0x61, 0x74, 0x74, 0x72, 0x73, 0x22, 0x31, 0x0a, - 0x05, 0x58, 0x61, 0x74, 0x74, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x22, 0xe4, 0x01, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, - 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f, - 0x6c, 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, - 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, - 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, - 0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x72, 0x6f, 0x6d, - 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x72, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x0d, 0x66, 0x72, 0x6f, 0x6d, 0x54, 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x72, 0x79, 0x12, - 0x1b, 0x0a, 0x09, 0x77, 0x65, 0x61, 0x6b, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x08, 0x20, 0x01, - 0x28, 0x0d, 0x52, 0x08, 0x77, 0x65, 0x61, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x12, 0x19, 0x0a, 0x08, - 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x6e, 0x6f, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, - 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x4e, 0x6f, 0x22, 0x52, 0x0a, 0x08, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x45, 0x72, 0x72, 0x6f, - 0x72, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x65, 0x0a, 0x10, 0x44, - 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x12, - 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x39, 0x0a, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x46, + 0x74, 0x61, 0x52, 0x05, 0x6c, 0x69, 0x6e, 0x75, 0x78, 0x12, 0x26, 0x0a, 0x06, 0x64, 0x61, 0x72, + 0x77, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, + 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x06, 0x64, 0x61, 0x72, 0x77, 0x69, + 0x6e, 0x12, 0x28, 0x0a, 0x07, 0x66, 0x72, 0x65, 0x65, 0x62, 0x73, 0x64, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, + 0x74, 0x61, 0x52, 0x07, 0x66, 0x72, 0x65, 0x65, 0x62, 0x73, 0x64, 0x12, 0x26, 0x0a, 0x06, 0x6e, + 0x65, 0x74, 0x62, 0x73, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x62, 0x65, + 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, 0x74, 0x61, 0x52, 0x06, 0x6e, 0x65, 0x74, + 0x62, 0x73, 0x64, 0x22, 0x6c, 0x0a, 0x08, 0x55, 0x6e, 0x69, 0x78, 0x44, 0x61, 0x74, 0x61, 0x12, + 0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, + 0x0a, 0x0a, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x10, 0x0a, + 0x03, 0x75, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, + 0x10, 0x0a, 0x03, 0x67, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x67, 0x69, + 0x64, 0x22, 0x52, 0x0a, 0x0b, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x73, 0x44, 0x61, 0x74, 0x61, + 0x12, 0x1d, 0x0a, 0x0a, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, + 0x24, 0x0a, 0x0e, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x73, 0x5f, 0x67, 0x72, 0x6f, 0x75, + 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49, 0x73, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x22, 0x2f, 0x0a, 0x09, 0x58, 0x61, 0x74, 0x74, 0x72, 0x44, 0x61, + 0x74, 0x61, 0x12, 0x22, 0x0a, 0x06, 0x78, 0x61, 0x74, 0x74, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x52, 0x06, + 0x78, 0x61, 0x74, 0x74, 0x72, 0x73, 0x22, 0x31, 0x0a, 0x05, 0x58, 0x61, 0x74, 0x74, 0x72, 0x12, + 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xcd, 0x01, 0x0a, 0x07, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, + 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, + 0x68, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x72, 0x6f, 0x6d, 0x5f, 0x74, 0x65, 0x6d, 0x70, 0x6f, 0x72, + 0x61, 0x72, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x66, 0x72, 0x6f, 0x6d, 0x54, + 0x65, 0x6d, 0x70, 0x6f, 0x72, 0x61, 0x72, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x62, 0x6c, 0x6f, 0x63, + 0x6b, 0x5f, 0x6e, 0x6f, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x62, 0x6c, 0x6f, 0x63, + 0x6b, 0x4e, 0x6f, 0x4a, 0x04, 0x08, 0x08, 0x10, 0x09, 0x22, 0x52, 0x0a, 0x08, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x22, 0x0a, 0x04, 0x63, 0x6f, 0x64, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0e, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x45, 0x72, + 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x22, 0x65, 0x0a, + 0x10, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x12, 0x16, 0x0a, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x66, 0x6f, 0x6c, 0x64, 0x65, 0x72, 0x12, 0x39, 0x0a, 0x07, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x62, 0x65, 0x70, + 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, + 0x67, 0x72, 0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x07, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x73, 0x22, 0xe5, 0x01, 0x0a, 0x1a, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77, + 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x12, 0x44, 0x0a, 0x0b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, - 0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x07, 0x75, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x73, 0x22, 0xe5, 0x01, 0x0a, 0x1a, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, - 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, - 0x65, 0x12, 0x44, 0x0a, 0x0b, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x46, 0x69, 0x6c, - 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, - 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x75, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, 0x07, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x62, - 0x65, 0x70, 0x2e, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x27, 0x0a, 0x0d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x69, 0x6e, 0x64, 0x65, - 0x78, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x05, 0x42, 0x02, 0x10, 0x00, 0x52, 0x0c, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x06, 0x0a, 0x04, 0x50, 0x69, - 0x6e, 0x67, 0x22, 0x1f, 0x0a, 0x05, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x72, - 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x61, - 0x73, 0x6f, 0x6e, 0x2a, 0xed, 0x01, 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x54, - 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x1b, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x43, 0x4f, 0x4e, 0x46, - 0x49, 0x47, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, - 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x44, 0x45, 0x58, 0x10, 0x01, 0x12, 0x1d, 0x0a, 0x19, - 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x44, - 0x45, 0x58, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, 0x18, 0x0a, 0x14, 0x4d, - 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x51, 0x55, - 0x45, 0x53, 0x54, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, 0x10, 0x04, - 0x12, 0x22, 0x0a, 0x1e, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, - 0x53, 0x53, 0x10, 0x05, 0x12, 0x15, 0x0a, 0x11, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, - 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x16, 0x0a, 0x12, 0x4d, - 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x4f, 0x53, - 0x45, 0x10, 0x07, 0x2a, 0x4f, 0x0a, 0x12, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, - 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x18, 0x4d, 0x45, 0x53, + 0x65, 0x73, 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0a, 0x75, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x25, 0x0a, + 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, + 0x2e, 0x62, 0x65, 0x70, 0x2e, 0x56, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x52, 0x07, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x27, 0x0a, 0x0d, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x69, 0x6e, + 0x64, 0x65, 0x78, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x05, 0x42, 0x02, 0x10, 0x00, 0x52, + 0x0c, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x65, 0x73, 0x12, 0x1d, 0x0a, + 0x0a, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x06, 0x0a, 0x04, + 0x50, 0x69, 0x6e, 0x67, 0x22, 0x1f, 0x0a, 0x05, 0x43, 0x6c, 0x6f, 0x73, 0x65, 0x12, 0x16, 0x0a, + 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, + 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x2a, 0xed, 0x01, 0x0a, 0x0b, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x1b, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, + 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x5f, 0x43, 0x4f, + 0x4e, 0x46, 0x49, 0x47, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, + 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x44, 0x45, 0x58, 0x10, 0x01, 0x12, 0x1d, + 0x0a, 0x19, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, + 0x4e, 0x44, 0x45, 0x58, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x02, 0x12, 0x18, 0x0a, + 0x14, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, + 0x51, 0x55, 0x45, 0x53, 0x54, 0x10, 0x03, 0x12, 0x19, 0x0a, 0x15, 0x4d, 0x45, 0x53, 0x53, 0x41, + 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x52, 0x45, 0x53, 0x50, 0x4f, 0x4e, 0x53, 0x45, + 0x10, 0x04, 0x12, 0x22, 0x0a, 0x1e, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, + 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47, + 0x52, 0x45, 0x53, 0x53, 0x10, 0x05, 0x12, 0x15, 0x0a, 0x11, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, + 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x50, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x16, 0x0a, + 0x12, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x43, 0x4c, + 0x4f, 0x53, 0x45, 0x10, 0x07, 0x2a, 0x4f, 0x0a, 0x12, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1c, 0x0a, 0x18, 0x4d, + 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, + 0x4f, 0x4e, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, - 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x1b, 0x0a, 0x17, 0x4d, 0x45, 0x53, 0x53, 0x41, - 0x47, 0x45, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4c, - 0x5a, 0x34, 0x10, 0x01, 0x2a, 0x56, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, - 0x4f, 0x4e, 0x5f, 0x4d, 0x45, 0x54, 0x41, 0x44, 0x41, 0x54, 0x41, 0x10, 0x00, 0x12, 0x15, 0x0a, - 0x11, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4e, 0x45, 0x56, - 0x45, 0x52, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, - 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x4c, 0x57, 0x41, 0x59, 0x53, 0x10, 0x02, 0x2a, 0xb0, 0x01, 0x0a, - 0x0c, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a, - 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x46, 0x49, 0x4c, 0x45, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, - 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x4f, - 0x52, 0x59, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1b, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, - 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x5f, 0x46, - 0x49, 0x4c, 0x45, 0x10, 0x02, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x28, 0x0a, 0x20, 0x46, 0x49, 0x4c, - 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, - 0x49, 0x4e, 0x4b, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, 0x03, 0x1a, - 0x02, 0x08, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x10, 0x04, 0x2a, - 0x76, 0x0a, 0x09, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x17, 0x0a, 0x13, - 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f, 0x5f, 0x45, 0x52, - 0x52, 0x4f, 0x52, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, - 0x4f, 0x44, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x10, 0x01, 0x12, 0x1b, 0x0a, - 0x17, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f, 0x5f, 0x53, - 0x55, 0x43, 0x48, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x1b, 0x0a, 0x17, 0x45, 0x52, - 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, - 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x03, 0x2a, 0x7e, 0x0a, 0x1e, 0x46, 0x69, 0x6c, 0x65, 0x44, - 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, 0x73, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2d, 0x0a, 0x29, 0x46, 0x49, 0x4c, - 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47, 0x52, - 0x45, 0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x00, 0x12, 0x2d, 0x0a, 0x29, 0x46, 0x49, 0x4c, 0x45, - 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47, 0x52, 0x45, - 0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, - 0x4f, 0x52, 0x47, 0x45, 0x54, 0x10, 0x01, 0x42, 0x70, 0x0a, 0x07, 0x63, 0x6f, 0x6d, 0x2e, 0x62, - 0x65, 0x70, 0x42, 0x08, 0x42, 0x65, 0x70, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x2f, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x79, 0x6e, 0x63, 0x74, - 0x68, 0x69, 0x6e, 0x67, 0x2f, 0x73, 0x79, 0x6e, 0x63, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x2f, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x62, 0x65, 0x70, 0xa2, - 0x02, 0x03, 0x42, 0x58, 0x58, 0xaa, 0x02, 0x03, 0x42, 0x65, 0x70, 0xca, 0x02, 0x03, 0x42, 0x65, - 0x70, 0xe2, 0x02, 0x0f, 0x42, 0x65, 0x70, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0xea, 0x02, 0x03, 0x42, 0x65, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x5f, 0x4c, 0x5a, 0x34, 0x10, 0x01, 0x2a, 0x56, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, + 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4d, 0x45, 0x54, 0x41, 0x44, 0x41, 0x54, 0x41, 0x10, 0x00, 0x12, + 0x15, 0x0a, 0x11, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4e, + 0x45, 0x56, 0x45, 0x52, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, + 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x4c, 0x57, 0x41, 0x59, 0x53, 0x10, 0x02, 0x2a, 0xb0, + 0x01, 0x0a, 0x0c, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x00, 0x12, 0x1c, 0x0a, 0x18, 0x46, 0x49, 0x4c, 0x45, + 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43, + 0x54, 0x4f, 0x52, 0x59, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1b, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, + 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b, + 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x02, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x28, 0x0a, 0x20, 0x46, + 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, + 0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x5f, 0x44, 0x49, 0x52, 0x45, 0x43, 0x54, 0x4f, 0x52, 0x59, 0x10, + 0x03, 0x1a, 0x02, 0x08, 0x01, 0x12, 0x1a, 0x0a, 0x16, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x49, 0x4e, + 0x46, 0x4f, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x59, 0x4d, 0x4c, 0x49, 0x4e, 0x4b, 0x10, + 0x04, 0x2a, 0x76, 0x0a, 0x09, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x17, + 0x0a, 0x13, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f, 0x5f, + 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x45, 0x52, 0x52, 0x4f, 0x52, + 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x47, 0x45, 0x4e, 0x45, 0x52, 0x49, 0x43, 0x10, 0x01, 0x12, + 0x1b, 0x0a, 0x17, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x4e, 0x4f, + 0x5f, 0x53, 0x55, 0x43, 0x48, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x1b, 0x0a, 0x17, + 0x45, 0x52, 0x52, 0x4f, 0x52, 0x5f, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, + 0x49, 0x44, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x03, 0x2a, 0x7e, 0x0a, 0x1e, 0x46, 0x69, 0x6c, + 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x50, 0x72, 0x6f, 0x67, 0x72, 0x65, 0x73, + 0x73, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2d, 0x0a, 0x29, 0x46, + 0x49, 0x4c, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, + 0x47, 0x52, 0x45, 0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x41, 0x50, 0x50, 0x45, 0x4e, 0x44, 0x10, 0x00, 0x12, 0x2d, 0x0a, 0x29, 0x46, 0x49, + 0x4c, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x50, 0x52, 0x4f, 0x47, + 0x52, 0x45, 0x53, 0x53, 0x5f, 0x55, 0x50, 0x44, 0x41, 0x54, 0x45, 0x5f, 0x54, 0x59, 0x50, 0x45, + 0x5f, 0x46, 0x4f, 0x52, 0x47, 0x45, 0x54, 0x10, 0x01, 0x42, 0x70, 0x0a, 0x07, 0x63, 0x6f, 0x6d, + 0x2e, 0x62, 0x65, 0x70, 0x42, 0x08, 0x42, 0x65, 0x70, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, + 0x5a, 0x2f, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x79, 0x6e, + 0x63, 0x74, 0x68, 0x69, 0x6e, 0x67, 0x2f, 0x73, 0x79, 0x6e, 0x63, 0x74, 0x68, 0x69, 0x6e, 0x67, + 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x62, 0x65, + 0x70, 0xa2, 0x02, 0x03, 0x42, 0x58, 0x58, 0xaa, 0x02, 0x03, 0x42, 0x65, 0x70, 0xca, 0x02, 0x03, + 0x42, 0x65, 0x70, 0xe2, 0x02, 0x0f, 0x42, 0x65, 0x70, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x03, 0x42, 0x65, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, } var ( diff --git a/lib/api/testdata/config/config.xml b/lib/api/testdata/config/config.xml index 5bd47cc87..3f547e0e9 100644 --- a/lib/api/testdata/config/config.xml +++ b/lib/api/testdata/config/config.xml @@ -18,7 +18,6 @@ false false false - 25 .stfolder true @@ -39,7 +38,6 @@ false false false - 25 .stfolder true diff --git a/lib/config/config_test.go b/lib/config/config_test.go index 14ef53308..6359f42ab 100644 --- a/lib/config/config_test.go +++ b/lib/config/config_test.go @@ -114,10 +114,9 @@ func TestDefaultValues(t *testing.T) { CleanupIntervalS: 3600, Params: map[string]string{}, }, - MaxConflicts: 10, - WeakHashThresholdPct: 25, - MarkerName: ".stfolder", - MaxConcurrentWrites: 2, + MaxConflicts: 10, + MarkerName: ".stfolder", + MaxConcurrentWrites: 2, XattrFilter: XattrFilter{ Entries: []XattrFilterEntry{}, MaxSingleEntrySize: 1024, @@ -190,10 +189,9 @@ func TestDeviceConfig(t *testing.T) { Versioning: VersioningConfiguration{ Params: map[string]string{}, }, - WeakHashThresholdPct: 25, - MarkerName: DefaultMarkerName, - JunctionsAsDirs: true, - MaxConcurrentWrites: maxConcurrentWritesDefault, + MarkerName: DefaultMarkerName, + JunctionsAsDirs: true, + MaxConcurrentWrites: maxConcurrentWritesDefault, XattrFilter: XattrFilter{ Entries: []XattrFilterEntry{}, }, diff --git a/lib/config/folderconfiguration.go b/lib/config/folderconfiguration.go index d8ec913e8..76c3b3e39 100644 --- a/lib/config/folderconfiguration.go +++ b/lib/config/folderconfiguration.go @@ -70,7 +70,6 @@ type FolderConfiguration struct { DisableSparseFiles bool `json:"disableSparseFiles" xml:"disableSparseFiles"` DisableTempIndexes bool `json:"disableTempIndexes" xml:"disableTempIndexes"` Paused bool `json:"paused" xml:"paused"` - WeakHashThresholdPct int `json:"weakHashThresholdPct" xml:"weakHashThresholdPct"` MarkerName string `json:"markerName" xml:"markerName"` CopyOwnershipFromParent bool `json:"copyOwnershipFromParent" xml:"copyOwnershipFromParent"` RawModTimeWindowS int `json:"modTimeWindowS" xml:"modTimeWindowS"` @@ -311,10 +310,6 @@ func (f *FolderConfiguration) prepare(myID protocol.DeviceID, existingDevices ma f.Versioning.CleanupIntervalS = 0 } - if f.WeakHashThresholdPct == 0 { - f.WeakHashThresholdPct = 25 - } - if f.MarkerName == "" { f.MarkerName = DefaultMarkerName } diff --git a/lib/config/testdata/overridenvalues.xml b/lib/config/testdata/overridenvalues.xml index 9503607e2..1c142a59f 100644 --- a/lib/config/testdata/overridenvalues.xml +++ b/lib/config/testdata/overridenvalues.xml @@ -70,7 +70,6 @@ false false false - 25 .stfolder false 0 diff --git a/lib/model/fakeconns_test.go b/lib/model/fakeconns_test.go index 0d929bb82..831198276 100644 --- a/lib/model/fakeconns_test.go +++ b/lib/model/fakeconns_test.go @@ -75,7 +75,7 @@ func (f *fakeConnection) DownloadProgress(_ context.Context, dp *protocol.Downlo func (f *fakeConnection) addFileLocked(name string, flags uint32, ftype protocol.FileInfoType, data []byte, version protocol.Vector, localFlags uint32) { blockSize := protocol.BlockSize(int64(len(data))) - blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), blockSize, int64(len(data)), nil, true) + blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), blockSize, int64(len(data)), nil) file := protocol.FileInfo{ Name: name, diff --git a/lib/model/folder_recvonly_test.go b/lib/model/folder_recvonly_test.go index 534677ed2..729b68ebb 100644 --- a/lib/model/folder_recvonly_test.go +++ b/lib/model/folder_recvonly_test.go @@ -560,7 +560,7 @@ func setupKnownFiles(t *testing.T, ffs fs.Filesystem, data []byte) []protocol.Fi if err != nil { t.Fatal(err) } - blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), protocol.BlockSize(int64(len(data))), int64(len(data)), nil, true) + blocks, _ := scanner.Blocks(context.TODO(), bytes.NewReader(data), protocol.BlockSize(int64(len(data))), int64(len(data)), nil) knownFiles := []protocol.FileInfo{ { Name: "knownDir", diff --git a/lib/model/folder_sendrecv.go b/lib/model/folder_sendrecv.go index 73e3ef184..8f6544126 100644 --- a/lib/model/folder_sendrecv.go +++ b/lib/model/folder_sendrecv.go @@ -31,7 +31,6 @@ import ( "github.com/syncthing/syncthing/lib/semaphore" "github.com/syncthing/syncthing/lib/sync" "github.com/syncthing/syncthing/lib/versioner" - "github.com/syncthing/syncthing/lib/weakhash" ) var ( @@ -1136,12 +1135,12 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, snap *db.Snapshot func (f *sendReceiveFolder) reuseBlocks(blocks []protocol.BlockInfo, reused []int, file protocol.FileInfo, tempName string) ([]protocol.BlockInfo, []int) { // Check for an old temporary file which might have some blocks we could // reuse. - tempBlocks, err := scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil, false) + tempBlocks, err := scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil) if err != nil { var caseErr *fs.ErrCaseConflict if errors.As(err, &caseErr) { if rerr := f.mtimefs.Rename(caseErr.Real, tempName); rerr == nil { - tempBlocks, err = scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil, false) + tempBlocks, err = scanner.HashFile(f.ctx, f.ID, f.mtimefs, tempName, file.BlockSize(), nil) } } } @@ -1310,8 +1309,6 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch f.model.progressEmitter.Register(state.sharedPullerState) } - weakHashFinder, file := f.initWeakHashFinder(state) - blocks: for _, block := range state.blocks { select { @@ -1336,75 +1333,49 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch buf = protocol.BufferPool.Upgrade(buf, int(block.Size)) - var found bool - if f.Type != config.FolderTypeReceiveEncrypted { - found, err = weakHashFinder.Iterate(block.WeakHash, buf, func(offset int64) bool { - if f.verifyBuffer(buf, block) != nil { - return true - } - - err = f.limitedWriteAt(dstFd, buf, block.Offset) - if err != nil { - state.fail(fmt.Errorf("dst write: %w", err)) - } - if offset == block.Offset { - state.copiedFromOrigin(block.Size) - } else { - state.copiedFromOriginShifted(block.Size) - } - - return false - }) + found := f.model.finder.Iterate(folders, block.Hash, func(folder, path string, index int32) bool { + ffs := folderFilesystems[folder] + fd, err := ffs.Open(path) if err != nil { - l.Debugln("weak hasher iter", err) + return false } - } + defer fd.Close() - if !found { - found = f.model.finder.Iterate(folders, block.Hash, func(folder, path string, index int32) bool { - ffs := folderFilesystems[folder] - fd, err := ffs.Open(path) - if err != nil { + srcOffset := int64(state.file.BlockSize()) * int64(index) + _, err = fd.ReadAt(buf, srcOffset) + if err != nil { + return false + } + + // Hash is not SHA256 as it's an encrypted hash token. In that + // case we can't verify the block integrity so we'll take it on + // trust. (The other side can and will verify.) + if f.Type != config.FolderTypeReceiveEncrypted { + if err := f.verifyBuffer(buf, block); err != nil { + l.Debugln("Finder failed to verify buffer", err) return false } - defer fd.Close() + } - srcOffset := int64(state.file.BlockSize()) * int64(index) - _, err = fd.ReadAt(buf, srcOffset) - if err != nil { - return false - } - - // Hash is not SHA256 as it's an encrypted hash token. In that - // case we can't verify the block integrity so we'll take it on - // trust. (The other side can and will verify.) - if f.Type != config.FolderTypeReceiveEncrypted { - if err := f.verifyBuffer(buf, block); err != nil { - l.Debugln("Finder failed to verify buffer", err) - return false - } - } - - if f.CopyRangeMethod != config.CopyRangeMethodStandard { - err = f.withLimiter(func() error { - dstFd.mut.Lock() - defer dstFd.mut.Unlock() - return fs.CopyRange(f.CopyRangeMethod.ToFS(), fd, dstFd.fd, srcOffset, block.Offset, int64(block.Size)) - }) - } else { - err = f.limitedWriteAt(dstFd, buf, block.Offset) - } - if err != nil { - state.fail(fmt.Errorf("dst write: %w", err)) - } - if path == state.file.Name { - state.copiedFromOrigin(block.Size) - } else { - state.copiedFromElsewhere(block.Size) - } - return true - }) - } + if f.CopyRangeMethod != config.CopyRangeMethodStandard { + err = f.withLimiter(func() error { + dstFd.mut.Lock() + defer dstFd.mut.Unlock() + return fs.CopyRange(f.CopyRangeMethod.ToFS(), fd, dstFd.fd, srcOffset, block.Offset, int64(block.Size)) + }) + } else { + err = f.limitedWriteAt(dstFd, buf, block.Offset) + } + if err != nil { + state.fail(fmt.Errorf("dst write: %w", err)) + } + if path == state.file.Name { + state.copiedFromOrigin(block.Size) + } else { + state.copiedFromElsewhere(block.Size) + } + return true + }) if state.failed() != nil { break @@ -1421,58 +1392,11 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch state.copyDone(block) } } - if file != nil { - // os.File used to return invalid argument if nil. - // fs.File panics as it's an interface. - file.Close() - } out <- state.sharedPullerState } } -func (f *sendReceiveFolder) initWeakHashFinder(state copyBlocksState) (*weakhash.Finder, fs.File) { - if f.Type == config.FolderTypeReceiveEncrypted { - l.Debugln("not weak hashing due to folder type", f.Type) - return nil, nil - } - - blocksPercentChanged := 0 - if tot := len(state.file.Blocks); tot > 0 { - blocksPercentChanged = (tot - state.have) * 100 / tot - } - - if blocksPercentChanged < f.WeakHashThresholdPct { - l.Debugf("not weak hashing %s. not enough changed %.02f < %d", state.file.Name, blocksPercentChanged, f.WeakHashThresholdPct) - return nil, nil - } - - hashesToFind := make([]uint32, 0, len(state.blocks)) - for _, block := range state.blocks { - if block.WeakHash != 0 { - hashesToFind = append(hashesToFind, block.WeakHash) - } - } - - if len(hashesToFind) == 0 { - l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name) - return nil, nil - } - - file, err := f.mtimefs.Open(state.file.Name) - if err != nil { - l.Debugln("weak hasher", err) - return nil, nil - } - - weakHashFinder, err := weakhash.NewFinder(f.ctx, file, state.file.BlockSize(), hashesToFind) - if err != nil { - l.Debugln("weak hasher", err) - return nil, file - } - return weakHashFinder, file -} - func (*sendReceiveFolder) verifyBuffer(buf []byte, block protocol.BlockInfo) error { if len(buf) != int(block.Size) { return fmt.Errorf("length mismatch %d != %d", len(buf), block.Size) @@ -1574,7 +1498,7 @@ loop: activity.using(selected) var buf []byte blockNo := int(state.block.Offset / int64(state.file.BlockSize())) - buf, lastError = f.model.RequestGlobal(f.ctx, selected.ID, f.folderID, state.file.Name, blockNo, state.block.Offset, int(state.block.Size), state.block.Hash, state.block.WeakHash, selected.FromTemporary) + buf, lastError = f.model.RequestGlobal(f.ctx, selected.ID, f.folderID, state.file.Name, blockNo, state.block.Offset, int(state.block.Size), state.block.Hash, selected.FromTemporary) activity.done(selected) if lastError != nil { l.Debugln("request:", f.folderID, state.file.Name, state.block.Offset, state.block.Size, selected.ID.Short(), "returned error:", lastError) diff --git a/lib/model/folder_sendrecv_test.go b/lib/model/folder_sendrecv_test.go index 69b35c018..ee013467b 100644 --- a/lib/model/folder_sendrecv_test.go +++ b/lib/model/folder_sendrecv_test.go @@ -25,7 +25,6 @@ import ( "github.com/syncthing/syncthing/lib/fs" "github.com/syncthing/syncthing/lib/ignore" "github.com/syncthing/syncthing/lib/protocol" - "github.com/syncthing/syncthing/lib/rand" "github.com/syncthing/syncthing/lib/scanner" "github.com/syncthing/syncthing/lib/sync" ) @@ -300,7 +299,7 @@ func TestCopierFinder(t *testing.T) { } // Verify that the fetched blocks have actually been written to the temp file - blks, err := scanner.HashFile(context.TODO(), f.ID, f.Filesystem(nil), tempFile, protocol.MinBlockSize, nil, false) + blks, err := scanner.HashFile(context.TODO(), f.ID, f.Filesystem(nil), tempFile, protocol.MinBlockSize, nil) if err != nil { t.Log(err) } @@ -312,128 +311,6 @@ func TestCopierFinder(t *testing.T) { } } -func TestWeakHash(t *testing.T) { - // Setup the model/pull environment - _, fo, wcfgCancel := setupSendReceiveFolder(t) - defer wcfgCancel() - ffs := fo.Filesystem(nil) - - tempFile := fs.TempName("weakhash") - var shift int64 = 10 - var size int64 = 1 << 20 - expectBlocks := int(size / protocol.MinBlockSize) - expectPulls := int(shift / protocol.MinBlockSize) - if shift > 0 { - expectPulls++ - } - - f, err := ffs.Create("weakhash") - must(t, err) - defer f.Close() - _, err = io.CopyN(f, rand.Reader, size) - if err != nil { - t.Error(err) - } - info, err := f.Stat() - if err != nil { - t.Error(err) - } - - // Create two files, second file has `shifted` bytes random prefix, yet - // both are of the same length, for example: - // File 1: abcdefgh - // File 2: xyabcdef - f.Seek(0, io.SeekStart) - existing, err := scanner.Blocks(context.TODO(), f, protocol.MinBlockSize, size, nil, true) - if err != nil { - t.Error(err) - } - - f.Seek(0, io.SeekStart) - remainder := io.LimitReader(f, size-shift) - prefix := io.LimitReader(rand.Reader, shift) - nf := io.MultiReader(prefix, remainder) - desired, err := scanner.Blocks(context.TODO(), nf, protocol.MinBlockSize, size, nil, true) - if err != nil { - t.Error(err) - } - - existingFile := protocol.FileInfo{ - Name: "weakhash", - Blocks: existing, - Size: size, - ModifiedS: info.ModTime().Unix(), - ModifiedNs: int32(info.ModTime().Nanosecond()), - } - desiredFile := protocol.FileInfo{ - Name: "weakhash", - Size: size, - Blocks: desired, - ModifiedS: info.ModTime().Unix() + 1, - } - - fo.updateLocalsFromScanning([]protocol.FileInfo{existingFile}) - - copyChan := make(chan copyBlocksState) - pullChan := make(chan pullBlockState, expectBlocks) - finisherChan := make(chan *sharedPullerState, 1) - - // Run a single fetcher routine - go fo.copierRoutine(copyChan, pullChan, finisherChan) - defer close(copyChan) - - // Test 1 - no weak hashing, file gets fully repulled (`expectBlocks` pulls). - fo.WeakHashThresholdPct = 101 - fo.handleFile(desiredFile, fsetSnapshot(t, fo.fset), copyChan) - - var pulls []pullBlockState - timeout := time.After(10 * time.Second) - for len(pulls) < expectBlocks { - select { - case pull := <-pullChan: - pulls = append(pulls, pull) - case <-timeout: - t.Fatalf("timed out, got %d pulls expected %d", len(pulls), expectPulls) - } - } - finish := <-finisherChan - - select { - case <-pullChan: - t.Fatal("Pull channel has data to be read") - case <-finisherChan: - t.Fatal("Finisher channel has data to be read") - default: - } - - cleanupSharedPullerState(finish) - if err := ffs.Remove(tempFile); err != nil { - t.Fatal(err) - } - - // Test 2 - using weak hash, expectPulls blocks pulled. - fo.WeakHashThresholdPct = -1 - fo.handleFile(desiredFile, fsetSnapshot(t, fo.fset), copyChan) - - pulls = pulls[:0] - for len(pulls) < expectPulls { - select { - case pull := <-pullChan: - pulls = append(pulls, pull) - case <-time.After(10 * time.Second): - t.Fatalf("timed out, got %d pulls expected %d", len(pulls), expectPulls) - } - } - - finish = <-finisherChan - cleanupSharedPullerState(finish) - - expectShifted := expectBlocks - expectPulls - if finish.copyOriginShifted != expectShifted { - t.Errorf("did not copy %d shifted", expectShifted) - } -} - // Test that updating a file removes its old blocks from the blockmap func TestCopierCleanup(t *testing.T) { iterFn := func(folder, file string, index int32) bool { @@ -709,8 +586,8 @@ func TestIssue3164(t *testing.T) { func TestDiff(t *testing.T) { for i, test := range diffTestData { - a, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.a), test.s, -1, nil, false) - b, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.b), test.s, -1, nil, false) + a, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.a), test.s, -1, nil) + b, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.b), test.s, -1, nil) _, d := blockDiff(a, b) if len(d) != len(test.d) { t.Fatalf("Incorrect length for diff %d; %d != %d", i, len(d), len(test.d)) @@ -730,8 +607,8 @@ func TestDiff(t *testing.T) { func BenchmarkDiff(b *testing.B) { testCases := make([]struct{ a, b []protocol.BlockInfo }, 0, len(diffTestData)) for _, test := range diffTestData { - a, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.a), test.s, -1, nil, false) - b, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.b), test.s, -1, nil, false) + a, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.a), test.s, -1, nil) + b, _ := scanner.Blocks(context.TODO(), bytes.NewBufferString(test.b), test.s, -1, nil) testCases = append(testCases, struct{ a, b []protocol.BlockInfo }{a, b}) } b.ReportAllocs() diff --git a/lib/model/metrics.go b/lib/model/metrics.go index ad211aecb..faae3238a 100644 --- a/lib/model/metrics.go +++ b/lib/model/metrics.go @@ -55,16 +55,15 @@ var ( Namespace: "syncthing", Subsystem: "model", Name: "folder_processed_bytes_total", - Help: "Total amount of data processed during folder syncing, per folder ID and data source (network/local_origin/local_other/local_shifted/skipped)", + Help: "Total amount of data processed during folder syncing, per folder ID and data source (network/local_origin/local_other/skipped)", }, []string{"folder", "source"}) ) const ( - metricSourceNetwork = "network" // from the network - metricSourceLocalOrigin = "local_origin" // from the existing version of the local file - metricSourceLocalOther = "local_other" // from a different local file - metricSourceLocalShifted = "local_shifted" // from the existing version of the local file, rolling hash shifted - metricSourceSkipped = "skipped" // block of all zeroes, invented out of thin air + metricSourceNetwork = "network" // from the network + metricSourceLocalOrigin = "local_origin" // from the existing version of the local file + metricSourceLocalOther = "local_other" // from a different local file + metricSourceSkipped = "skipped" // block of all zeroes, invented out of thin air metricScopeGlobal = "global" metricScopeLocal = "local" @@ -88,6 +87,5 @@ func registerFolderMetrics(folderID string) { metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceNetwork) metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceLocalOrigin) metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceLocalOther) - metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceLocalShifted) metricFolderProcessedBytesTotal.WithLabelValues(folderID, metricSourceSkipped) } diff --git a/lib/model/mocks/model.go b/lib/model/mocks/model.go index 3855f7e99..6aa17ef60 100644 --- a/lib/model/mocks/model.go +++ b/lib/model/mocks/model.go @@ -435,19 +435,18 @@ type Model struct { result1 protocol.RequestResponse result2 error } - RequestGlobalStub func(context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, uint32, bool) ([]byte, error) + RequestGlobalStub func(context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, bool) ([]byte, error) requestGlobalMutex sync.RWMutex requestGlobalArgsForCall []struct { - arg1 context.Context - arg2 protocol.DeviceID - arg3 string - arg4 string - arg5 int - arg6 int64 - arg7 int - arg8 []byte - arg9 uint32 - arg10 bool + arg1 context.Context + arg2 protocol.DeviceID + arg3 string + arg4 string + arg5 int + arg6 int64 + arg7 int + arg8 []byte + arg9 bool } requestGlobalReturns struct { result1 []byte @@ -2580,7 +2579,7 @@ func (fake *Model) RequestReturnsOnCall(i int, result1 protocol.RequestResponse, }{result1, result2} } -func (fake *Model) RequestGlobal(arg1 context.Context, arg2 protocol.DeviceID, arg3 string, arg4 string, arg5 int, arg6 int64, arg7 int, arg8 []byte, arg9 uint32, arg10 bool) ([]byte, error) { +func (fake *Model) RequestGlobal(arg1 context.Context, arg2 protocol.DeviceID, arg3 string, arg4 string, arg5 int, arg6 int64, arg7 int, arg8 []byte, arg9 bool) ([]byte, error) { var arg8Copy []byte if arg8 != nil { arg8Copy = make([]byte, len(arg8)) @@ -2589,23 +2588,22 @@ func (fake *Model) RequestGlobal(arg1 context.Context, arg2 protocol.DeviceID, a fake.requestGlobalMutex.Lock() ret, specificReturn := fake.requestGlobalReturnsOnCall[len(fake.requestGlobalArgsForCall)] fake.requestGlobalArgsForCall = append(fake.requestGlobalArgsForCall, struct { - arg1 context.Context - arg2 protocol.DeviceID - arg3 string - arg4 string - arg5 int - arg6 int64 - arg7 int - arg8 []byte - arg9 uint32 - arg10 bool - }{arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8Copy, arg9, arg10}) + arg1 context.Context + arg2 protocol.DeviceID + arg3 string + arg4 string + arg5 int + arg6 int64 + arg7 int + arg8 []byte + arg9 bool + }{arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8Copy, arg9}) stub := fake.RequestGlobalStub fakeReturns := fake.requestGlobalReturns - fake.recordInvocation("RequestGlobal", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8Copy, arg9, arg10}) + fake.recordInvocation("RequestGlobal", []interface{}{arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8Copy, arg9}) fake.requestGlobalMutex.Unlock() if stub != nil { - return stub(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10) + return stub(arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9) } if specificReturn { return ret.result1, ret.result2 @@ -2619,17 +2617,17 @@ func (fake *Model) RequestGlobalCallCount() int { return len(fake.requestGlobalArgsForCall) } -func (fake *Model) RequestGlobalCalls(stub func(context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, uint32, bool) ([]byte, error)) { +func (fake *Model) RequestGlobalCalls(stub func(context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, bool) ([]byte, error)) { fake.requestGlobalMutex.Lock() defer fake.requestGlobalMutex.Unlock() fake.RequestGlobalStub = stub } -func (fake *Model) RequestGlobalArgsForCall(i int) (context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, uint32, bool) { +func (fake *Model) RequestGlobalArgsForCall(i int) (context.Context, protocol.DeviceID, string, string, int, int64, int, []byte, bool) { fake.requestGlobalMutex.RLock() defer fake.requestGlobalMutex.RUnlock() argsForCall := fake.requestGlobalArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7, argsForCall.arg8, argsForCall.arg9, argsForCall.arg10 + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5, argsForCall.arg6, argsForCall.arg7, argsForCall.arg8, argsForCall.arg9 } func (fake *Model) RequestGlobalReturns(result1 []byte, result2 error) { diff --git a/lib/model/model.go b/lib/model/model.go index 9e898395d..63bdbea2f 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -118,7 +118,7 @@ type Model interface { GlobalDirectoryTree(folder, prefix string, levels int, dirsOnly bool) ([]*TreeEntry, error) - RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) + RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) } type model struct { @@ -606,8 +606,6 @@ func (m *model) UsageReportingStats(report *contract.Report, version int, previe report.BlockStats.Pulled = v case "copyOrigin": report.BlockStats.CopyOrigin = v - case "copyOriginShifted": - report.BlockStats.CopyOriginShifted = v case "copyElsewhere": report.BlockStats.CopyElsewhere = v } @@ -2036,7 +2034,7 @@ func (m *model) Request(conn protocol.Connection, req *protocol.Request) (out pr return nil, protocol.ErrNoSuchFile } _, err := readOffsetIntoBuf(folderFs, tempFn, req.Offset, res.data) - if err == nil && scanner.Validate(res.data, req.Hash, req.WeakHash) { + if err == nil && scanner.Validate(res.data, req.Hash) { return res, nil } // Fall through to reading from a non-temp file, just in case the temp @@ -2065,8 +2063,8 @@ func (m *model) Request(conn protocol.Connection, req *protocol.Request) (out pr return nil, protocol.ErrGeneric } - if folderCfg.Type != config.FolderTypeReceiveEncrypted && len(req.Hash) > 0 && !scanner.Validate(res.data[:n], req.Hash, req.WeakHash) { - m.recheckFile(deviceID, req.Folder, req.Name, req.Offset, req.Hash, req.WeakHash) + if folderCfg.Type != config.FolderTypeReceiveEncrypted && len(req.Hash) > 0 && !scanner.Validate(res.data[:n], req.Hash) { + m.recheckFile(deviceID, req.Folder, req.Name, req.Offset, req.Hash) l.Debugf("%v REQ(in) failed validating data: %s: %q / %q o=%d s=%d", m, deviceID.Short(), req.Folder, req.Name, req.Offset, req.Size) return nil, protocol.ErrNoSuchFile } @@ -2092,7 +2090,7 @@ func newLimitedRequestResponse(size int, limiters ...*semaphore.Semaphore) *requ return res } -func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, weakHash uint32) { +func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte) { cf, ok, err := m.CurrentFolderFile(folder, name) if err != nil { l.Debugf("%v recheckFile: %s: %q / %q: current file error: %v", m, deviceID, folder, name, err) @@ -2121,10 +2119,6 @@ func (m *model) recheckFile(deviceID protocol.DeviceID, folder, name string, off l.Debugf("%v recheckFile: %s: %q / %q i=%d: hash mismatch %x != %x", m, deviceID, folder, name, blockIndex, block.Hash, hash) return } - if weakHash != 0 && block.WeakHash != weakHash { - l.Debugf("%v recheckFile: %s: %q / %q i=%d: weak hash mismatch %v != %v", m, deviceID, folder, name, blockIndex, block.WeakHash, weakHash) - return - } // The hashes provided part of the request match what we expect to find according // to what we have in the database, yet the content we've read off the filesystem doesn't @@ -2462,14 +2456,14 @@ func (m *model) deviceDidCloseRLocked(deviceID protocol.DeviceID, duration time. } } -func (m *model) RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, weakHash uint32, fromTemporary bool) ([]byte, error) { +func (m *model) RequestGlobal(ctx context.Context, deviceID protocol.DeviceID, folder, name string, blockNo int, offset int64, size int, hash []byte, fromTemporary bool) ([]byte, error) { conn, connOK := m.requestConnectionForDevice(deviceID) if !connOK { return nil, fmt.Errorf("requestGlobal: no connection to device: %s", deviceID.Short()) } - l.Debugf("%v REQ(out): %s (%s): %q / %q b=%d o=%d s=%d h=%x wh=%x ft=%t", m, deviceID.Short(), conn, folder, name, blockNo, offset, size, hash, weakHash, fromTemporary) - return conn.Request(ctx, &protocol.Request{Folder: folder, Name: name, BlockNo: blockNo, Offset: offset, Size: size, Hash: hash, WeakHash: weakHash, FromTemporary: fromTemporary}) + l.Debugf("%v REQ(out): %s (%s): %q / %q b=%d o=%d s=%d h=%x ft=%t", m, deviceID.Short(), conn, folder, name, blockNo, offset, size, hash, fromTemporary) + return conn.Request(ctx, &protocol.Request{Folder: folder, Name: name, BlockNo: blockNo, Offset: offset, Size: size, Hash: hash, FromTemporary: fromTemporary}) } // requestConnectionForDevice returns a connection to the given device, to diff --git a/lib/model/model_test.go b/lib/model/model_test.go index 1c79f0e1e..0c6274248 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -222,7 +222,7 @@ func BenchmarkRequestOut(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - data, err := m.RequestGlobal(context.Background(), device1, "default", files[i%n].Name, 0, 0, 32, nil, 0, false) + data, err := m.RequestGlobal(context.Background(), device1, "default", files[i%n].Name, 0, 0, 32, nil, false) if err != nil { b.Error(err) } @@ -2819,9 +2819,9 @@ func TestIssue5002(t *testing.T) { } blockSize := int32(file.BlockSize()) - m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size-int64(blockSize), []byte{1, 2, 3, 4}, 0) - m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size, []byte{1, 2, 3, 4}, 0) // panic - m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size+int64(blockSize), []byte{1, 2, 3, 4}, 0) + m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size-int64(blockSize), []byte{1, 2, 3, 4}) + m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size, []byte{1, 2, 3, 4}) // panic + m.recheckFile(protocol.LocalDeviceID, "default", "foo", file.Size+int64(blockSize), []byte{1, 2, 3, 4}) } func TestParentOfUnignored(t *testing.T) { diff --git a/lib/model/requests_test.go b/lib/model/requests_test.go index 75a02e33c..991656ed3 100644 --- a/lib/model/requests_test.go +++ b/lib/model/requests_test.go @@ -436,11 +436,8 @@ func TestRescanIfHaveInvalidContent(t *testing.T) { } f := checkReceived(<-received) - if f.Blocks[0].WeakHash != 103547413 { - t.Fatalf("unexpected weak hash: %d != 103547413", f.Blocks[0].WeakHash) - } - res, err := m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash, WeakHash: f.Blocks[0].WeakHash}) + res, err := m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash}) if err != nil { t.Fatal(err) } @@ -454,17 +451,14 @@ func TestRescanIfHaveInvalidContent(t *testing.T) { writeFile(t, tfs, "foo", payload) - _, err = m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash, WeakHash: f.Blocks[0].WeakHash}) + _, err = m.Request(device1Conn, &protocol.Request{Folder: "default", Name: "foo", Size: len(payload), Hash: f.Blocks[0].Hash}) if err == nil { t.Fatalf("expected failure") } select { case fs := <-received: - f := checkReceived(fs) - if f.Blocks[0].WeakHash != 41943361 { - t.Fatalf("unexpected weak hash: %d != 41943361", f.Blocks[0].WeakHash) - } + checkReceived(fs) case <-time.After(time.Second): t.Fatalf("timed out") } diff --git a/lib/model/sharedpullerstate.go b/lib/model/sharedpullerstate.go index 70b82dc31..623011cbc 100644 --- a/lib/model/sharedpullerstate.go +++ b/lib/model/sharedpullerstate.go @@ -285,15 +285,6 @@ func (s *sharedPullerState) skippedSparseBlock(bytes int) { metricFolderProcessedBytesTotal.WithLabelValues(s.folder, metricSourceSkipped).Add(float64(bytes)) } -func (s *sharedPullerState) copiedFromOriginShifted(bytes int) { - s.mut.Lock() - s.copyOrigin++ - s.copyOriginShifted++ - s.updated = time.Now() - s.mut.Unlock() - metricFolderProcessedBytesTotal.WithLabelValues(s.folder, metricSourceLocalShifted).Add(float64(bytes)) -} - func (s *sharedPullerState) pullStarted() { s.mut.Lock() s.copyTotal-- diff --git a/lib/protocol/bep_fileinfo.go b/lib/protocol/bep_fileinfo.go index fa5f78d5d..f29fd1ae7 100644 --- a/lib/protocol/bep_fileinfo.go +++ b/lib/protocol/bep_fileinfo.go @@ -544,32 +544,29 @@ func (f *FileInfo) setNoContent() { } type BlockInfo struct { - Hash []byte - Offset int64 - Size int - WeakHash uint32 + Hash []byte + Offset int64 + Size int } func (b BlockInfo) ToWire() *bep.BlockInfo { return &bep.BlockInfo{ - Hash: b.Hash, - Offset: b.Offset, - Size: int32(b.Size), - WeakHash: b.WeakHash, + Hash: b.Hash, + Offset: b.Offset, + Size: int32(b.Size), } } func BlockInfoFromWire(w *bep.BlockInfo) BlockInfo { return BlockInfo{ - Hash: w.Hash, - Offset: w.Offset, - Size: int(w.Size), - WeakHash: w.WeakHash, + Hash: w.Hash, + Offset: w.Offset, + Size: int(w.Size), } } func (b BlockInfo) String() string { - return fmt.Sprintf("Block{%d/%d/%d/%x}", b.Offset, b.Size, b.WeakHash, b.Hash) + return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash) } // For each block size, the hash of a block of all zeroes @@ -596,7 +593,6 @@ func BlocksHash(bs []BlockInfo) []byte { h := sha256.New() for _, b := range bs { _, _ = h.Write(b.Hash) - _ = binary.Write(h, binary.BigEndian, b.WeakHash) } return h.Sum(nil) } diff --git a/lib/protocol/bep_request_response.go b/lib/protocol/bep_request_response.go index 2affd3724..db0722791 100644 --- a/lib/protocol/bep_request_response.go +++ b/lib/protocol/bep_request_response.go @@ -25,7 +25,6 @@ type Request struct { Size int Hash []byte FromTemporary bool - WeakHash uint32 BlockNo int } @@ -38,7 +37,6 @@ func (r *Request) toWire() *bep.Request { Size: int32(r.Size), Hash: r.Hash, FromTemporary: r.FromTemporary, - WeakHash: r.WeakHash, BlockNo: int32(r.BlockNo), } } @@ -52,7 +50,6 @@ func requestFromWire(w *bep.Request) *Request { Size: int(w.Size), Hash: w.Hash, FromTemporary: w.FromTemporary, - WeakHash: w.WeakHash, BlockNo: int(w.BlockNo), } } diff --git a/lib/protocol/common_test.go b/lib/protocol/common_test.go index 74200df10..b6e25c2b9 100644 --- a/lib/protocol/common_test.go +++ b/lib/protocol/common_test.go @@ -17,7 +17,6 @@ type TestModel struct { offset int64 size int32 hash []byte - weakHash uint32 fromTemporary bool indexFn func(string, []FileInfo) ccFn func(*ClusterConfig) @@ -48,7 +47,6 @@ func (t *TestModel) Request(_ Connection, req *Request) (RequestResponse, error) t.offset = req.Offset t.size = int32(req.Size) t.hash = req.Hash - t.weakHash = req.WeakHash t.fromTemporary = req.FromTemporary buf := make([]byte, len(t.data)) copy(buf, t.data) diff --git a/lib/scanner/blockqueue.go b/lib/scanner/blockqueue.go index 911b482f0..8182ef74d 100644 --- a/lib/scanner/blockqueue.go +++ b/lib/scanner/blockqueue.go @@ -16,7 +16,7 @@ import ( ) // HashFile hashes the files and returns a list of blocks representing the file. -func HashFile(ctx context.Context, folderID string, fs fs.Filesystem, path string, blockSize int, counter Counter, useWeakHashes bool) ([]protocol.BlockInfo, error) { +func HashFile(ctx context.Context, folderID string, fs fs.Filesystem, path string, blockSize int, counter Counter) ([]protocol.BlockInfo, error) { fd, err := fs.Open(path) if err != nil { l.Debugln("open:", err) @@ -36,7 +36,7 @@ func HashFile(ctx context.Context, folderID string, fs fs.Filesystem, path strin // Hash the file. This may take a while for large files. - blocks, err := Blocks(ctx, fd, blockSize, size, counter, useWeakHashes) + blocks, err := Blocks(ctx, fd, blockSize, size, counter) if err != nil { l.Debugln("blocks:", err) return nil, err @@ -108,7 +108,7 @@ func (ph *parallelHasher) hashFiles(ctx context.Context) { panic("Bug. Asked to hash a directory or a deleted file.") } - blocks, err := HashFile(ctx, ph.folderID, ph.fs, f.Name, f.BlockSize(), ph.counter, true) + blocks, err := HashFile(ctx, ph.folderID, ph.fs, f.Name, f.BlockSize(), ph.counter) if err != nil { handleError(ctx, "hashing", f.Name, err, ph.outbox) continue diff --git a/lib/scanner/blocks.go b/lib/scanner/blocks.go index d875d1e19..e299ca359 100644 --- a/lib/scanner/blocks.go +++ b/lib/scanner/blocks.go @@ -10,8 +10,6 @@ import ( "bytes" "context" "crypto/sha256" - "hash" - "hash/adler32" "io" "github.com/syncthing/syncthing/lib/protocol" @@ -24,7 +22,7 @@ type Counter interface { } // Blocks returns the blockwise hash of the reader. -func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, counter Counter, useWeakHashes bool) ([]protocol.BlockInfo, error) { +func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, counter Counter) ([]protocol.BlockInfo, error) { if counter == nil { counter = &noopCounter{} } @@ -32,15 +30,6 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou hf := sha256.New() const hashLength = sha256.Size - var weakHf hash.Hash32 = noopHash{} - var multiHf io.Writer = hf - if useWeakHashes { - // Use an actual weak hash function, make the multiHf - // write to both hash functions. - weakHf = adler32.New() - multiHf = io.MultiWriter(hf, weakHf) - } - var blocks []protocol.BlockInfo var hashes, thisHash []byte @@ -70,7 +59,7 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou } lr.N = int64(blocksize) - n, err := io.CopyBuffer(multiHf, lr, buf) + n, err := io.CopyBuffer(hf, lr, buf) if err != nil { return nil, err } @@ -87,17 +76,15 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou thisHash, hashes = hashes[:hashLength], hashes[hashLength:] b := protocol.BlockInfo{ - Size: int(n), - Offset: offset, - Hash: thisHash, - WeakHash: weakHf.Sum32(), + Size: int(n), + Offset: offset, + Hash: thisHash, } blocks = append(blocks, b) offset += n hf.Reset() - weakHf.Reset() } if len(blocks) == 0 { @@ -112,14 +99,8 @@ func Blocks(ctx context.Context, r io.Reader, blocksize int, sizehint int64, cou return blocks, nil } -// Validate quickly validates buf against the 32-bit weakHash, if not zero, -// else against the cryptohash hash, if len(hash)>0. It is satisfied if -// either hash matches or neither hash is given. -func Validate(buf, hash []byte, weakHash uint32) bool { - if weakHash != 0 && adler32.Checksum(buf) == weakHash { - return true - } - +// Validate validates the hash, if len(hash)>0. +func Validate(buf, hash []byte) bool { if len(hash) > 0 { hbuf := sha256.Sum256(buf) return bytes.Equal(hbuf[:], hash) diff --git a/lib/scanner/blocks_test.go b/lib/scanner/blocks_test.go index fd98e3ff2..b54eb4f74 100644 --- a/lib/scanner/blocks_test.go +++ b/lib/scanner/blocks_test.go @@ -9,51 +9,40 @@ package scanner import ( "bytes" "context" - "crypto/rand" "crypto/sha256" "fmt" - origAdler32 "hash/adler32" mrand "math/rand" "testing" - "testing/quick" - - rollingAdler32 "github.com/chmduquesne/rollinghash/adler32" - "github.com/syncthing/syncthing/lib/protocol" ) var blocksTestData = []struct { data []byte blocksize int hash []string - weakhash []uint32 }{ { []byte(""), 1024, []string{ "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", }, - []uint32{0}, }, { []byte("contents"), 1024, []string{ "d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8", }, - []uint32{0x0f3a036f}, }, { []byte("contents"), 9, []string{ "d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8", }, - []uint32{0x0f3a036f}, }, { []byte("contents"), 8, []string{ "d1b2a59fbea7e20077af9f91b27e95e865061b270be03ff539ab3b73587882e8", }, - []uint32{0x0f3a036f}, }, { []byte("contents"), 7, @@ -61,7 +50,6 @@ var blocksTestData = []struct { "ed7002b439e9ac845f22357d822bac1444730fbdb6016d3ec9432297b9ec9f73", "043a718774c572bd8a25adbeb1bfcd5c0256ae11cecf9f9c3f925d0e52beaf89", }, - []uint32{0x0bcb02fc, 0x00740074}, }, { []byte("contents"), 3, @@ -70,7 +58,6 @@ var blocksTestData = []struct { "e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3", "44ad63f60af0f6db6fdde6d5186ef78176367df261fa06be3079b6c80c8adba4", }, - []uint32{0x02780141, 0x02970148, 0x015d00e8}, }, { []byte("conconts"), 3, @@ -79,7 +66,6 @@ var blocksTestData = []struct { "1143da2bc54c495c4be31d3868785d39ffdfd56df5668f0645d8f14d47647952", "44ad63f60af0f6db6fdde6d5186ef78176367df261fa06be3079b6c80c8adba4", }, - []uint32{0x02780141, 0x02780141, 0x015d00e8}, }, { []byte("contenten"), 3, @@ -88,14 +74,13 @@ var blocksTestData = []struct { "e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3", "e4432baa90819aaef51d2a7f8e148bf7e679610f3173752fabb4dcb2d0f418d3", }, - []uint32{0x02780141, 0x02970148, 0x02970148}, }, } func TestBlocks(t *testing.T) { for testNo, test := range blocksTestData { buf := bytes.NewBuffer(test.data) - blocks, err := Blocks(context.TODO(), buf, test.blocksize, -1, nil, true) + blocks, err := Blocks(context.TODO(), buf, test.blocksize, -1, nil) if err != nil { t.Fatal(err) } @@ -119,9 +104,6 @@ func TestBlocks(t *testing.T) { if h := fmt.Sprintf("%x", blocks[i].Hash); h != test.hash[i] { t.Errorf("%d/%d: Incorrect block hash %q != %q", testNo, i, h, test.hash[i]) } - if h := blocks[i].WeakHash; h != test.weakhash[i] { - t.Errorf("%d/%d: Incorrect block weakhash 0x%08x != 0x%08x", testNo, i, h, test.weakhash[i]) - } i++ } @@ -129,85 +111,10 @@ func TestBlocks(t *testing.T) { } } -func TestAdler32Variants(t *testing.T) { - // Verify that the two adler32 functions give matching results for a few - // different blocks of data. - - hf1 := origAdler32.New() - hf2 := rollingAdler32.New() - - checkFn := func(data []byte) bool { - hf1.Write(data) - sum1 := hf1.Sum32() - - hf2.Write(data) - sum2 := hf2.Sum32() - - hf1.Reset() - hf2.Reset() - - // Make sure whatever we use in Validate matches too resp. this - // tests gets adjusted if we ever switch the weak hash algo. - return sum1 == sum2 && Validate(data, nil, sum1) - } - - // protocol block sized data - data := make([]byte, protocol.MinBlockSize) - for i := 0; i < 5; i++ { - rand.Read(data) - if !checkFn(data) { - t.Errorf("Hash mismatch on block sized data") - } - } - - // random small blocks - if err := quick.Check(checkFn, nil); err != nil { - t.Error(err) - } - - // rolling should have the same result as the individual blocks - // themselves. - - windowSize := 128 - - hf3 := rollingAdler32.New() - hf3.Write(data[:windowSize]) - - for i := windowSize; i < len(data); i++ { - if i%windowSize == 0 { - // let the reference function catch up - window := data[i-windowSize : i] - hf1.Reset() - hf1.Write(window) - hf2.Reset() - hf2.Write(window) - - // verify that they are in sync with the rolling function - sum1 := hf1.Sum32() - sum2 := hf2.Sum32() - sum3 := hf3.Sum32() - t.Logf("At i=%d, sum2=%08x, sum3=%08x", i, sum2, sum3) - if sum2 != sum3 { - t.Errorf("Mismatch after roll; i=%d, sum2=%08x, sum3=%08x", i, sum2, sum3) - break - } - if sum1 != sum3 { - t.Errorf("Mismatch after roll; i=%d, sum1=%08x, sum3=%08x", i, sum1, sum3) - break - } - if !Validate(window, nil, sum1) { - t.Errorf("Validation failure after roll; i=%d", i) - } - } - hf3.Roll(data[i]) - } -} - func BenchmarkValidate(b *testing.B) { type block struct { - data []byte - hash [sha256.Size]byte - weakhash uint32 + data []byte + hash [sha256.Size]byte } var blocks []block const blocksPerType = 100 @@ -220,16 +127,6 @@ func BenchmarkValidate(b *testing.B) { b.data = make([]byte, 128<<10) r.Read(b.data) b.hash = sha256.Sum256(b.data) - b.weakhash = origAdler32.Checksum(b.data) - blocks = append(blocks, b) - } - // Blocks where the hash matches, but the weakhash doesn't. - for i := 0; i < blocksPerType; i++ { - var b block - b.data = make([]byte, 128<<10) - r.Read(b.data) - b.hash = sha256.Sum256(b.data) - b.weakhash = 1 // Zeros causes Validate to skip the weakhash. blocks = append(blocks, b) } @@ -238,7 +135,7 @@ func BenchmarkValidate(b *testing.B) { for i := 0; i < b.N; i++ { for _, b := range blocks { - Validate(b.data, b.hash[:], b.weakhash) + Validate(b.data, b.hash[:]) } } } diff --git a/lib/scanner/walk_test.go b/lib/scanner/walk_test.go index 5c6254d29..ebfedb2e0 100644 --- a/lib/scanner/walk_test.go +++ b/lib/scanner/walk_test.go @@ -162,7 +162,7 @@ func TestVerify(t *testing.T) { progress := newByteCounter() defer progress.Close() - blocks, err := Blocks(context.TODO(), buf, blocksize, -1, progress, false) + blocks, err := Blocks(context.TODO(), buf, blocksize, -1, progress) if err != nil { t.Fatal(err) } @@ -640,7 +640,7 @@ func BenchmarkHashFile(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - if _, err := HashFile(context.TODO(), "", testFs, testdataName, protocol.MinBlockSize, nil, true); err != nil { + if _, err := HashFile(context.TODO(), "", testFs, testdataName, protocol.MinBlockSize, nil); err != nil { b.Fatal(err) } } diff --git a/lib/syncthing/internals.go b/lib/syncthing/internals.go index 704cfba2a..135b29263 100644 --- a/lib/syncthing/internals.go +++ b/lib/syncthing/internals.go @@ -42,7 +42,7 @@ func (m *Internals) SetIgnores(folderID string, content []string) error { } func (m *Internals) DownloadBlock(ctx context.Context, deviceID protocol.DeviceID, folderID string, path string, blockNumber int, blockInfo protocol.BlockInfo, allowFromTemporary bool) ([]byte, error) { - return m.model.RequestGlobal(ctx, deviceID, folderID, path, int(blockNumber), blockInfo.Offset, blockInfo.Size, blockInfo.Hash, blockInfo.WeakHash, allowFromTemporary) + return m.model.RequestGlobal(ctx, deviceID, folderID, path, int(blockNumber), blockInfo.Offset, blockInfo.Size, blockInfo.Hash, allowFromTemporary) } func (m *Internals) BlockAvailability(folderID string, file protocol.FileInfo, block protocol.BlockInfo) ([]model.Availability, error) { diff --git a/lib/syncthing/syncthing.go b/lib/syncthing/syncthing.go index 7d562abf7..60ae452c9 100644 --- a/lib/syncthing/syncthing.go +++ b/lib/syncthing/syncthing.go @@ -177,7 +177,7 @@ func (a *App) startup() error { }() } - perf := ur.CpuBench(context.Background(), 3, 150*time.Millisecond, true) + perf := ur.CpuBench(context.Background(), 3, 150*time.Millisecond) l.Infof("Hashing performance is %.02f MB/s", perf) if err := db.UpdateSchema(a.ll); err != nil { diff --git a/lib/ur/contract/contract.go b/lib/ur/contract/contract.go index bc51c560c..1af5d3d66 100644 --- a/lib/ur/contract/contract.go +++ b/lib/ur/contract/contract.go @@ -111,8 +111,6 @@ type Report struct { ConflictsOther int `json:"conflictsOther,omitempty" metric:"folder_feature{feature=ConflictsOther},summary" since:"3"` DisableSparseFiles int `json:"disableSparseFiles,omitempty" metric:"folder_feature{feature=DisableSparseFiles},summary" since:"3"` DisableTempIndexes int `json:"disableTempIndexes,omitempty" metric:"folder_feature{feature=DisableTempIndexes},summary" since:"3"` - AlwaysWeakHash int `json:"alwaysWeakHash,omitempty" metric:"folder_feature{feature=AlwaysWeakhash},summary" since:"3"` - CustomWeakHashThreshold int `json:"customWeakHashThreshold,omitempty" metric:"folder_feature{feature=CustomWeakhashThreshold},summary" since:"3"` FsWatcherEnabled int `json:"fsWatcherEnabled,omitempty" metric:"folder_feature{feature=FSWatcherEnabled},summary" since:"3"` PullOrder map[string]int `json:"pullOrder,omitempty" metric:"folder_pull_order,summaryVec:order" since:"3"` FilesystemType map[string]int `json:"filesystemType,omitempty" metric:"folder_file_system_type,summaryVec:type" since:"3"` @@ -152,13 +150,12 @@ type Report struct { } `json:"guiStats,omitempty" since:"3"` BlockStats struct { - Total int `json:"total,omitempty" metric:"blocks_processed_total,gauge" since:"3"` - Renamed int `json:"renamed,omitempty" metric:"blocks_processed{source=renamed},gauge" since:"3"` - Reused int `json:"reused,omitempty" metric:"blocks_processed{source=reused},gauge" since:"3"` - Pulled int `json:"pulled,omitempty" metric:"blocks_processed{source=pulled},gauge" since:"3"` - CopyOrigin int `json:"copyOrigin,omitempty" metric:"blocks_processed{source=copy_origin},gauge" since:"3"` - CopyOriginShifted int `json:"copyOriginShifted,omitempty" metric:"blocks_processed{source=copy_origin_shifted},gauge" since:"3"` - CopyElsewhere int `json:"copyElsewhere,omitempty" metric:"blocks_processed{source=copy_elsewhere},gauge" since:"3"` + Total int `json:"total,omitempty" metric:"blocks_processed_total,gauge" since:"3"` + Renamed int `json:"renamed,omitempty" metric:"blocks_processed{source=renamed},gauge" since:"3"` + Reused int `json:"reused,omitempty" metric:"blocks_processed{source=reused},gauge" since:"3"` + Pulled int `json:"pulled,omitempty" metric:"blocks_processed{source=pulled},gauge" since:"3"` + CopyOrigin int `json:"copyOrigin,omitempty" metric:"blocks_processed{source=copy_origin},gauge" since:"3"` + CopyElsewhere int `json:"copyElsewhere,omitempty" metric:"blocks_processed{source=copy_elsewhere},gauge" since:"3"` } `json:"blockStats,omitempty" since:"3"` TransportStats map[string]int `json:"transportStats,omitempty" since:"3"` @@ -175,9 +172,6 @@ type Report struct { Stars int `json:"stars,omitempty" metric:"folder_ignore_lines{kind=stars},summary" since:"3"` } `json:"ignoreStats,omitempty" since:"3"` - // V3 fields added late in the RC - WeakHashEnabled bool `json:"weakHashEnabled,omitempty" metric:"-" since:"3"` // Deprecated and not provided client-side anymore - // Added in post processing Received time.Time `json:"received,omitempty"` Date string `json:"date,omitempty"` diff --git a/lib/ur/usage_report.go b/lib/ur/usage_report.go index 21aea826e..a76e1512e 100644 --- a/lib/ur/usage_report.go +++ b/lib/ur/usage_report.go @@ -116,8 +116,8 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) ( report.TotMiB = int(totBytes / 1024 / 1024) report.FolderMaxMiB = int(maxBytes / 1024 / 1024) report.MemoryUsageMiB = int((mem.Sys - mem.HeapReleased) / 1024 / 1024) - report.SHA256Perf = CpuBench(ctx, 5, 125*time.Millisecond, false) - report.HashPerf = CpuBench(ctx, 5, 125*time.Millisecond, true) + report.SHA256Perf = CpuBench(ctx, 5, 125*time.Millisecond) + report.HashPerf = report.SHA256Perf report.MemorySize = int(memorySize() / 1024 / 1024) report.NumCPU = runtime.NumCPU() @@ -252,11 +252,6 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) ( if cfg.DisableTempIndexes { report.FolderUsesV3.DisableTempIndexes++ } - if cfg.WeakHashThresholdPct < 0 { - report.FolderUsesV3.AlwaysWeakHash++ - } else if cfg.WeakHashThresholdPct != 25 { - report.FolderUsesV3.CustomWeakHashThreshold++ - } if cfg.FSWatcherEnabled { report.FolderUsesV3.FsWatcherEnabled++ } @@ -443,7 +438,7 @@ var ( ) // CpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s -func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWeakHash bool) float64 { +func CpuBench(ctx context.Context, iterations int, duration time.Duration) float64 { blocksResultMut.Lock() defer blocksResultMut.Unlock() @@ -454,7 +449,7 @@ func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWe var perf float64 for i := 0; i < iterations; i++ { - if v := cpuBenchOnce(ctx, duration, useWeakHash, bs); v > perf { + if v := cpuBenchOnce(ctx, duration, bs); v > perf { perf = v } } @@ -467,13 +462,13 @@ func CpuBench(ctx context.Context, iterations int, duration time.Duration, useWe return perf } -func cpuBenchOnce(ctx context.Context, duration time.Duration, useWeakHash bool, bs []byte) float64 { +func cpuBenchOnce(ctx context.Context, duration time.Duration, bs []byte) float64 { t0 := time.Now() b := 0 var err error for time.Since(t0) < duration { r := bytes.NewReader(bs) - blocksResult, err = scanner.Blocks(ctx, r, protocol.MinBlockSize, int64(len(bs)), nil, useWeakHash) + blocksResult, err = scanner.Blocks(ctx, r, protocol.MinBlockSize, int64(len(bs)), nil) if err != nil { return 0 // Context done } diff --git a/lib/weakhash/benchmark_test.go b/lib/weakhash/benchmark_test.go deleted file mode 100644 index bd30e8bbf..000000000 --- a/lib/weakhash/benchmark_test.go +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright (C) 2016 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -package weakhash - -import ( - "bytes" - "context" - "fmt" - "hash" - vadler32 "hash/adler32" - "io" - "math/rand" - "os" - "testing" - - "github.com/chmduquesne/rollinghash/adler32" - "github.com/chmduquesne/rollinghash/bozo32" - "github.com/chmduquesne/rollinghash/buzhash32" - "github.com/chmduquesne/rollinghash/buzhash64" -) - -const ( - testFile = "../model/testdata/tmpfile" - size = 128 << 10 -) - -func BenchmarkFind1MFile(b *testing.B) { - b.ReportAllocs() - b.SetBytes(1 << 20) - for i := 0; i < b.N; i++ { - fd, err := os.Open(testFile) - if err != nil { - b.Fatal(err) - } - _, err = Find(context.Background(), fd, []uint32{0, 1, 2}, size) - if err != nil { - b.Fatal(err) - } - fd.Close() - } -} - -type RollingHash interface { - hash.Hash - Roll(byte) -} - -func BenchmarkBlock(b *testing.B) { - tests := []struct { - name string - hash hash.Hash - }{ - { - "adler32", adler32.New(), - }, - { - "bozo32", bozo32.New(), - }, - { - "buzhash32", buzhash32.New(), - }, - { - "buzhash64", buzhash64.New(), - }, - { - "vanilla-adler32", vadler32.New(), - }, - } - - sizes := []int64{128 << 10, 16 << 20} - - buf := make([]byte, 16<<20) - rand.Read(buf) - - for _, testSize := range sizes { - for _, test := range tests { - b.Run(test.name+"-"+fmt.Sprint(testSize), func(bb *testing.B) { - bb.Run("", func(bbb *testing.B) { - bbb.ResetTimer() - for i := 0; i < bbb.N; i++ { - lr := io.LimitReader(bytes.NewReader(buf), testSize) - n, err := io.Copy(test.hash, lr) - if err != nil { - bbb.Error(err) - } - if n != testSize { - bbb.Errorf("%d != %d", n, testSize) - } - - test.hash.Sum(nil) - test.hash.Reset() - } - - bbb.SetBytes(testSize) - bbb.ReportAllocs() - }) - }) - } - } -} - -func BenchmarkRoll(b *testing.B) { - tests := []struct { - name string - hash RollingHash - }{ - { - "adler32", adler32.New(), - }, - { - "bozo32", bozo32.New(), - }, - { - "buzhash32", buzhash32.New(), - }, - { - "buzhash64", buzhash64.New(), - }, - } - - sizes := []int64{128 << 10, 16 << 20} - - for _, testSize := range sizes { - for _, test := range tests { - b.Run(test.name+"-"+fmt.Sprint(testSize), func(bb *testing.B) { - bb.Run("", func(bbb *testing.B) { - data := make([]byte, testSize) - - if _, err := test.hash.Write(data); err != nil { - bbb.Error(err) - } - - bbb.ResetTimer() - - for i := 0; i < bbb.N; i++ { - for j := int64(0); j <= testSize; j++ { - test.hash.Roll('a') - } - } - - bbb.SetBytes(testSize) - bbb.ReportAllocs() - }) - }) - } - } -} diff --git a/lib/weakhash/weakhash.go b/lib/weakhash/weakhash.go deleted file mode 100644 index 0ecce4c7b..000000000 --- a/lib/weakhash/weakhash.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright (C) 2016 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -package weakhash - -import ( - "bufio" - "context" - "io" - - "github.com/chmduquesne/rollinghash/adler32" -) - -const ( - Size = 4 - - // don't track more hits than this for any given weakhash - maxWeakhashFinderHits = 10 -) - -// Find finds all the blocks of the given size within io.Reader that matches -// the hashes provided, and returns a hash -> slice of offsets within reader -// map, that produces the same weak hash. -func Find(ctx context.Context, ir io.Reader, hashesToFind []uint32, size int) (map[uint32][]int64, error) { - if ir == nil || len(hashesToFind) == 0 { - return nil, nil - } - - r := bufio.NewReader(ir) - hf := adler32.New() - - n, err := io.CopyN(hf, r, int64(size)) - if err == io.EOF { - return nil, nil - } - if err != nil { - return nil, err - } - if n != int64(size) { - return nil, io.ErrShortBuffer - } - - offsets := make(map[uint32][]int64) - for _, hashToFind := range hashesToFind { - offsets[hashToFind] = make([]int64, 0, maxWeakhashFinderHits) - } - - var i int64 - var hash uint32 - for { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - hash = hf.Sum32() - if existing, ok := offsets[hash]; ok && len(existing) < maxWeakhashFinderHits { - offsets[hash] = append(existing, i) - } - i++ - - bt, err := r.ReadByte() - if err == io.EOF { - break - } else if err != nil { - return offsets, err - } - hf.Roll(bt) - } - return offsets, nil -} - -func NewFinder(ctx context.Context, ir io.ReadSeeker, size int, hashesToFind []uint32) (*Finder, error) { - offsets, err := Find(ctx, ir, hashesToFind, size) - if err != nil { - return nil, err - } - - return &Finder{ - reader: ir, - size: size, - offsets: offsets, - }, nil -} - -type Finder struct { - reader io.ReadSeeker - size int - offsets map[uint32][]int64 -} - -// Iterate iterates all available blocks that matches the provided hash, reads -// them into buf, and calls the iterator function. The iterator function should -// return whether it wishes to continue iterating. -func (h *Finder) Iterate(hash uint32, buf []byte, iterFunc func(int64) bool) (bool, error) { - if h == nil || hash == 0 || len(buf) != h.size { - return false, nil - } - - for _, offset := range h.offsets[hash] { - _, err := h.reader.Seek(offset, io.SeekStart) - if err != nil { - return false, err - } - _, err = h.reader.Read(buf) - if err != nil { - return false, err - } - if !iterFunc(offset) { - return true, nil - } - } - return false, nil -} diff --git a/lib/weakhash/weakhash_test.go b/lib/weakhash/weakhash_test.go deleted file mode 100644 index 78c2d3906..000000000 --- a/lib/weakhash/weakhash_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (C) 2016 The Syncthing Authors. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at https://mozilla.org/MPL/2.0/. - -// The existence of this file means we get 0% test coverage rather than no -// test coverage at all. Remove when implementing an actual test. - -package weakhash - -import ( - "bytes" - "context" - "io" - "os" - "reflect" - "testing" -) - -var payload = []byte("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz") - -func TestFinder(t *testing.T) { - f, err := os.CreateTemp("", "") - if err != nil { - t.Error(err) - } - defer os.Remove(f.Name()) - defer f.Close() - - if _, err := f.Write(payload); err != nil { - t.Error(err) - } - if _, err := f.Seek(0, io.SeekStart); err != nil { - t.Error(err) - } - - hashes := []uint32{65143183, 65798547} - finder, err := NewFinder(context.Background(), f, 4, hashes) - if err != nil { - t.Error(err) - } - - expected := map[uint32][]int64{ - 65143183: {1, 27, 53, 79}, - 65798547: {2, 28, 54, 80}, - } - actual := make(map[uint32][]int64) - - b := make([]byte, Size) - - for _, hash := range hashes { - _, err := finder.Iterate(hash, b[:4], func(offset int64) bool { - if !bytes.Equal(b, payload[offset:offset+4]) { - t.Errorf("Not equal at %d: %s != %s", offset, string(b), string(payload[offset:offset+4])) - } - actual[hash] = append(actual[hash], offset) - return true - }) - if err != nil { - t.Error(err) - } - } - - if !reflect.DeepEqual(actual, expected) { - t.Errorf("Not equal: %#v != %#v", actual, expected) - } -} diff --git a/proto/bep/bep.proto b/proto/bep/bep.proto index 4cac50597..0d6d9bd17 100644 --- a/proto/bep/bep.proto +++ b/proto/bep/bep.proto @@ -145,7 +145,7 @@ message BlockInfo { bytes hash = 3; int64 offset = 1; int32 size = 2; - uint32 weak_hash = 4; + reserved 4; } message Vector { @@ -203,8 +203,8 @@ message Request { int32 size = 5; bytes hash = 6; bool from_temporary = 7; - uint32 weak_hash = 8; int32 block_no = 9; + reserved 8; } // Response diff --git a/test/h1/config.xml b/test/h1/config.xml index c55cc5349..c7e965a88 100644 --- a/test/h1/config.xml +++ b/test/h1/config.xml @@ -24,7 +24,6 @@ false false false - 25 .stfolder false 0 @@ -153,7 +152,6 @@ false false false - 25 .stfolder false 0 diff --git a/test/h2/config.xml b/test/h2/config.xml index 8a2e1ab9b..81276c1d1 100644 --- a/test/h2/config.xml +++ b/test/h2/config.xml @@ -24,7 +24,6 @@ false false false - 25 .stfolder false 0 @@ -151,7 +150,6 @@ false false false - 25 .stfolder false 0 diff --git a/test/h3/config.xml b/test/h3/config.xml index ac99d8102..440c15b0d 100644 --- a/test/h3/config.xml +++ b/test/h3/config.xml @@ -20,7 +20,6 @@ false false false - 25 .stfolder false 0 @@ -50,7 +49,6 @@ false false false - 25 .stfolder false 0 diff --git a/test/h4/config.xml b/test/h4/config.xml index d0627cd91..8347f39c6 100644 --- a/test/h4/config.xml +++ b/test/h4/config.xml @@ -17,7 +17,6 @@ false false false - 25 .stfolder false 0