Compare commits

...

39 Commits

Author SHA1 Message Date
Jakob Borg
2df78a9313 Reimplement quick startup scan 2014-03-16 08:14:55 +01:00
Jakob Borg
d2d32f26c7 Don't count deleted files in synced bytes (fixes #88) 2014-03-14 11:12:34 +01:00
Jakob Borg
473df1bd19 Fix integration test 2014-03-14 11:12:34 +01:00
Jakob Borg
880f417ae3 Add CPU usage reporting on Solaris 2014-03-12 10:15:00 +01:00
Jakob Borg
043fa7f489 Add environment variable to inhibit restart 2014-03-12 10:12:35 +01:00
Jakob Borg
9b0768a71b Temporarily remove broken integration tests 2014-03-12 06:33:18 +01:00
Jakob Borg
446b21c568 Package descriptions for godoc 2014-03-12 06:32:26 +01:00
Jakob Borg
b3c2ffc96a Test for previous commit and never ignore entire repo directory 2014-03-09 12:36:26 +01:00
Jakob Borg
b5f652a815 Do not descend into ignored directories (fixes #86) 2014-03-09 12:21:07 +01:00
Jakob Borg
9ec7de643e Refactor profiler startup / logging 2014-03-09 09:18:28 +01:00
Jakob Borg
2553ba0463 Discover & main tracing 2014-03-09 09:15:36 +01:00
Jakob Borg
52ee7d5724 Discovery tracing 2014-03-09 08:58:03 +01:00
Jakob Borg
d4ef6a6285 Document env vars, start profiler based on STPROFILER 2014-03-09 08:48:29 +01:00
Jakob Borg
56b7d3c28d Don't start browser on restart 2014-03-09 08:35:53 +01:00
Jakob Borg
ae94b726a7 Don't expose a -delay paramater 2014-03-09 08:35:38 +01:00
Jakob Borg
a88e4db1ee Option to not start browser (fixes #84) 2014-03-08 23:19:33 +01:00
Jakob Borg
0ebd4a6ba1 Fix relative path open bug 2014-03-08 23:18:50 +01:00
Jakob Borg
1448cfe66a Refactor out file scanner into separate package 2014-03-08 23:02:01 +01:00
Jakob Borg
d6c9afd07f Fix handling of default values in config (fixes #83) 2014-03-04 22:29:48 +01:00
Jakob Borg
799f55e7ae Add basic config tests 2014-03-04 22:17:39 +01:00
Jakob Borg
04a3db132f Merge pull request #81 from filoozom/patch-1
Fix isTempName to work on Windows (fixes #80)
2014-03-04 21:56:23 +01:00
Philippe Schommers
d06204959e Fix isTempName to work on Windows (fixes #80)
```path.Base()``` is for slash-separated paths, whereas Windows uses "\" to separate paths. Just convert the \ to / and it works.
2014-03-04 18:48:03 +01:00
Jakob Borg
2d0600de38 Merge pull request #78 from filoozom/patch-1
Update config.go to handle default slices correctly (fixes #76)
2014-03-04 15:09:47 +01:00
filoozom
6a1c055288 Delete comment in config.go 2014-03-04 15:02:26 +01:00
filoozom
b9ec30ebdb Update config.go 2014-03-04 11:29:51 +01:00
filoozom
428164f395 Update config.go to handle default slices correctly 2014-03-04 11:25:10 +01:00
Jakob Borg
ba59e0d3f0 Use undirected broadcast and WriteTo (fixes #75) 2014-03-03 18:19:32 +01:00
Jakob Borg
5d8f0f835e Merge branch 'filoozom-patch-1'
* filoozom-patch-1:
  Fix divided by zero when the sync folder is empty (tot = 0)
  Delete cfgFile before renaming it on Windows
  Set the right config and home dir for each OS
  Update getHomeDir() to use "os/user"
2014-03-03 14:06:15 +01:00
filoozom
b4a1aadd1b Fix divided by zero when the sync folder is empty (tot = 0) 2014-03-03 08:47:52 +01:00
filoozom
8f41d90ab1 Delete cfgFile before renaming it on Windows 2014-03-03 08:46:20 +01:00
Jakob Borg
9743386166 Re-add inadvertently ignored files 2014-03-02 23:58:24 +01:00
Jakob Borg
0afcb5b7e7 Clean up build.sh 2014-03-02 23:55:08 +01:00
filoozom
043dea760f Set the right config and home dir for each OS
Use %AppData%\syncthing for the config files and %USERPROFILE%\Sync as sync folder for Windows.
2014-03-02 23:49:51 +01:00
Jakob Borg
0618e2b9b4 Don't include timestamp in auto generated files 2014-03-02 23:15:56 +01:00
Jakob Borg
3c171d281c Move cmd files into subdir 2014-03-02 23:13:04 +01:00
Jakob Borg
c217b7cd22 Change default announce server to announce.syncthing.net 2014-03-02 17:29:35 +01:00
filoozom
23593c3d20 Update getHomeDir() to use "os/user"
os.Getenv("HOME") doesn't work properly on Windows (and maybe other systems?) and the package "os/user" gives us a convenient way to find the home directory for every OS.
2014-03-02 16:07:12 +01:00
Jakob Borg
192117dc11 Merge branch 'v0.6'
* v0.6:
  Open GUI on startup
2014-03-02 13:08:05 +01:00
Jakob Borg
51788d6f0e Add some support packages 2014-03-01 11:11:37 +01:00
73 changed files with 13722 additions and 12444 deletions

3
.gitignore vendored
View File

@@ -1,3 +1,4 @@
syncthing
syncthing.exe
*.tar.gz
dist
*.zip

View File

@@ -18,7 +18,7 @@ cd gui
for f in $(find . -type f) ; do
f="${f#./}"
echo "gr, _ = gzip.NewReader(bytes.NewBuffer([]byte{"
gzip -c $f | od -vt x1 | sed 's/^[0-9a-f]*//' | sed 's/\([0-9a-f][0-9a-f]\)/0x\1,/g'
gzip -n -c $f | od -vt x1 | sed 's/^[0-9a-f]*//' | sed 's/\([0-9a-f][0-9a-f]\)/0x\1,/g'
echo "}))"
echo "data, _ = ioutil.ReadAll(gr)"
echo "Assets[\"$f\"] = data"

2
auto/doc.go Normal file
View File

@@ -0,0 +1,2 @@
// Package auto contains auto generated files for web assets.
package auto

View File

File diff suppressed because it is too large Load Diff

View File

@@ -1,3 +1,4 @@
// Package buffers manages a set of reusable byte buffers.
package buffers
const (

118
build.sh
View File

@@ -2,48 +2,90 @@
export COPYFILE_DISABLE=true
distFiles=(README.md LICENSE) # apart from the binary itself
version=$(git describe --always)
buildDir=dist
if [[ $fast != yes ]] ; then
build() {
go build -ldflags "-w -X main.Version $version" ./cmd/syncthing
}
prepare() {
./assets.sh | gofmt > auto/gui.files.go
go get -d
}
test() {
go test ./...
fi
}
if [[ -z $1 ]] ; then
go build -ldflags "-X main.Version $version"
elif [[ $1 == "tar" ]] ; then
go build -ldflags "-X main.Version $version" \
&& mkdir syncthing-dist \
&& cp syncthing README.md LICENSE syncthing-dist \
&& tar zcvf syncthing-dist.tar.gz syncthing-dist \
&& rm -rf syncthing-dist
elif [[ $1 == "all" ]] ; then
rm -rf "$buildDir"
mkdir -p "$buildDir" || exit 1
tarDist() {
name="$1"
mkdir -p "$name"
cp syncthing "${distFiles[@]}" "$name"
tar zcvf "$name.tar.gz" "$name"
rm -rf "$name"
}
export GOARM=7
for os in darwin-amd64 linux-amd64 linux-arm freebsd-amd64 windows-amd64 ; do
echo "$os"
export name="syncthing-$os"
export GOOS=${os%-*}
export GOARCH=${os#*-}
go build -ldflags "-X main.Version $version"
mkdir -p "$name"
cp README.md LICENSE "$name"
case $GOOS in
windows)
cp syncthing.exe "$buildDir/$name.exe"
mv syncthing.exe "$name"
zip -qr "$buildDir/$name.zip" "$name"
;;
*)
cp syncthing "$buildDir/$name"
mv syncthing "$name"
tar zcf "$buildDir/$name.tar.gz" "$name"
;;
esac
rm -r "$name"
done
fi
zipDist() {
name="$1"
mkdir -p "$name"
cp syncthing.exe "${distFiles[@]}" "$name"
zip -r "$name.zip" "$name"
rm -rf "$name"
}
case "$1" in
"")
build
;;
tar)
rm -f *.tar.gz *.zip
prepare
test || exit 1
build
eval $(go env)
name="syncthing-$GOOS-$GOARCH-$version"
tarDist "$name"
;;
all)
rm -f *.tar.gz *.zip
prepare
test || exit 1
export GOARM=7
for os in darwin-amd64 linux-amd64 linux-arm freebsd-amd64 windows-amd64 ; do
export GOOS=${os%-*}
export GOARCH=${os#*-}
build
name="syncthing-$os-$version"
case $GOOS in
windows)
zipDist "$name"
rm -f syncthing.exe
;;
*)
tarDist "$name"
rm -f syncthing
;;
esac
done
;;
upload)
tag=$(git describe)
shopt -s nullglob
for f in *gz *zip ; do
relup calmh/syncthing "$tag" "$f"
done
;;
*)
echo "Unknown build parameter $1"
;;
esac

43
cid/cid.go Normal file
View File

@@ -0,0 +1,43 @@
// Package cid provides a manager for mappings between node ID:s and connection ID:s.
package cid
type Map struct {
toCid map[string]int
toName []string
}
func NewMap() *Map {
return &Map{
toCid: make(map[string]int),
}
}
func (m *Map) Get(name string) int {
cid, ok := m.toCid[name]
if ok {
return cid
}
// Find a free slot to get a new ID
for i, n := range m.toName {
if n == "" {
m.toName[i] = name
m.toCid[name] = i
return i
}
}
// Add it to the end since we didn't find a free slot
m.toName = append(m.toName, name)
cid = len(m.toName) - 1
m.toCid[name] = cid
return cid
}
func (m *Map) Clear(name string) {
cid, ok := m.toCid[name]
if ok {
m.toName[cid] = ""
delete(m.toCid, name)
}
}

1
cmd/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
!syncthing

View File

@@ -36,7 +36,7 @@ type OptionsConfiguration struct {
FollowSymlinks bool `xml:"followSymlinks" default:"true" ini:"follow-symlinks"`
GUIEnabled bool `xml:"guiEnabled" default:"true" ini:"gui-enabled"`
GUIAddress string `xml:"guiAddress" default:"127.0.0.1:8080" ini:"gui-address"`
GlobalAnnServer string `xml:"globalAnnounceServer" default:"syncthing.nym.se:22025" ini:"global-announce-server"`
GlobalAnnServer string `xml:"globalAnnounceServer" default:"announce.syncthing.net:22025" ini:"global-announce-server"`
GlobalAnnEnabled bool `xml:"globalAnnounceEnabled" default:"true" ini:"global-announce-enabled"`
LocalAnnEnabled bool `xml:"localAnnounceEnabled" default:"true" ini:"local-announce-enabled"`
ParallelRequests int `xml:"parallelRequests" default:"16" ini:"parallel-requests"`
@@ -44,6 +44,7 @@ type OptionsConfiguration struct {
RescanIntervalS int `xml:"rescanIntervalS" default:"60" ini:"rescan-interval"`
ReconnectIntervalS int `xml:"reconnectionIntervalS" default:"60" ini:"reconnection-interval"`
MaxChangeKbps int `xml:"maxChangeKbps" default:"1000" ini:"max-change-bw"`
StartBrowser bool `xml:"startBrowser" default:"true"`
}
func setDefaults(data interface{}) error {
@@ -60,11 +61,6 @@ func setDefaults(data interface{}) error {
case string:
f.SetString(v)
case []string:
rv := reflect.MakeSlice(reflect.TypeOf([]string{}), 1, 1)
rv.Index(0).SetString(v)
f.Set(rv)
case int:
i, err := strconv.ParseInt(v, 10, 64)
if err != nil {
@@ -75,6 +71,11 @@ func setDefaults(data interface{}) error {
case bool:
f.SetBool(v == "true")
case []string:
// We don't do anything with string slices here. Any default
// we set will be appended to by the XML decoder, so we fill
// those after decoding.
default:
panic(f.Type())
}
@@ -83,6 +84,30 @@ func setDefaults(data interface{}) error {
return nil
}
// fillNilSlices sets default value on slices that are still nil.
func fillNilSlices(data interface{}) error {
s := reflect.ValueOf(data).Elem()
t := s.Type()
for i := 0; i < s.NumField(); i++ {
f := s.Field(i)
tag := t.Field(i).Tag
v := tag.Get("default")
if len(v) > 0 {
switch f.Interface().(type) {
case []string:
if f.IsNil() {
rv := reflect.MakeSlice(reflect.TypeOf([]string{}), 1, 1)
rv.Index(0).SetString(v)
f.Set(rv)
}
}
}
}
return nil
}
func readConfigINI(m map[string]string, data interface{}) error {
s := reflect.ValueOf(data).Elem()
t := s.Type()
@@ -154,6 +179,8 @@ func readConfigXML(rd io.Reader) (Configuration, error) {
err = xml.NewDecoder(rd).Decode(&cfg)
}
fillNilSlices(&cfg.Options)
cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
return cfg, err
}

View File

@@ -0,0 +1,116 @@
package main
import (
"bytes"
"io"
"reflect"
"testing"
)
func TestDefaultValues(t *testing.T) {
expected := OptionsConfiguration{
ListenAddress: []string{":22000"},
ReadOnly: false,
AllowDelete: true,
FollowSymlinks: true,
GUIEnabled: true,
GUIAddress: "127.0.0.1:8080",
GlobalAnnServer: "announce.syncthing.net:22025",
GlobalAnnEnabled: true,
LocalAnnEnabled: true,
ParallelRequests: 16,
MaxSendKbps: 0,
RescanIntervalS: 60,
ReconnectIntervalS: 60,
MaxChangeKbps: 1000,
StartBrowser: true,
}
cfg, err := readConfigXML(bytes.NewReader(nil))
if err != io.EOF {
t.Error(err)
}
if !reflect.DeepEqual(cfg.Options, expected) {
t.Errorf("Default config differs;\n E: %#v\n A: %#v", expected, cfg.Options)
}
}
func TestNoListenAddress(t *testing.T) {
data := []byte(`<configuration version="1">
<repository directory="~/Sync">
<node id="..." name="...">
<address>dynamic</address>
</node>
</repository>
<options>
<listenAddress></listenAddress>
</options>
</configuration>
`)
cfg, err := readConfigXML(bytes.NewReader(data))
if err != nil {
t.Error(err)
}
expected := []string{""}
if !reflect.DeepEqual(cfg.Options.ListenAddress, expected) {
t.Errorf("Unexpected ListenAddress %#v", cfg.Options.ListenAddress)
}
}
func TestOverriddenValues(t *testing.T) {
data := []byte(`<configuration version="1">
<repository directory="~/Sync">
<node id="..." name="...">
<address>dynamic</address>
</node>
</repository>
<options>
<listenAddress>:23000</listenAddress>
<readOnly>true</readOnly>
<allowDelete>false</allowDelete>
<followSymlinks>false</followSymlinks>
<guiEnabled>false</guiEnabled>
<guiAddress>125.2.2.2:8080</guiAddress>
<globalAnnounceServer>syncthing.nym.se:22025</globalAnnounceServer>
<globalAnnounceEnabled>false</globalAnnounceEnabled>
<localAnnounceEnabled>false</localAnnounceEnabled>
<parallelRequests>32</parallelRequests>
<maxSendKbps>1234</maxSendKbps>
<rescanIntervalS>600</rescanIntervalS>
<reconnectionIntervalS>6000</reconnectionIntervalS>
<maxChangeKbps>2345</maxChangeKbps>
<startBrowser>false</startBrowser>
</options>
</configuration>
`)
expected := OptionsConfiguration{
ListenAddress: []string{":23000"},
ReadOnly: true,
AllowDelete: false,
FollowSymlinks: false,
GUIEnabled: false,
GUIAddress: "125.2.2.2:8080",
GlobalAnnServer: "syncthing.nym.se:22025",
GlobalAnnEnabled: false,
LocalAnnEnabled: false,
ParallelRequests: 32,
MaxSendKbps: 1234,
RescanIntervalS: 600,
ReconnectIntervalS: 6000,
MaxChangeKbps: 2345,
StartBrowser: false,
}
cfg, err := readConfigXML(bytes.NewReader(data))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(cfg.Options, expected) {
t.Errorf("Overridden config differs;\n E: %#v\n A: %#v", expected, cfg.Options)
}
}

15
cmd/syncthing/debug.go Normal file
View File

@@ -0,0 +1,15 @@
package main
import (
"log"
"os"
"strings"
)
var (
dlog = log.New(os.Stderr, "main: ", log.Lmicroseconds|log.Lshortfile)
debugNet = strings.Contains(os.Getenv("STTRACE"), "net")
debugIdx = strings.Contains(os.Getenv("STTRACE"), "idx")
debugNeed = strings.Contains(os.Getenv("STTRACE"), "need")
debugPull = strings.Contains(os.Getenv("STTRACE"), "pull")
)

View File

@@ -4,13 +4,13 @@ import (
"bytes"
"errors"
"fmt"
"log"
"os"
"path"
"sync"
"time"
"github.com/calmh/syncthing/buffers"
"github.com/calmh/syncthing/scanner"
)
type fileMonitor struct {
@@ -18,18 +18,18 @@ type fileMonitor struct {
path string // full path
writeDone sync.WaitGroup
model *Model
global File
localBlocks []Block
global scanner.File
localBlocks []scanner.Block
copyError error
writeError error
}
func (m *fileMonitor) FileBegins(cc <-chan content) error {
if m.model.trace["file"] {
log.Printf("FILE: FileBegins: " + m.name)
if debugPull {
dlog.Println("file begins:", m.name)
}
tmp := tempName(m.path, m.global.Modified)
tmp := defTempNamer.TempName(m.path)
dir := path.Dir(tmp)
_, err := os.Stat(dir)
@@ -109,13 +109,13 @@ func (m *fileMonitor) copyRemoteBlocks(cc <-chan content, outFile *os.File, writ
}
func (m *fileMonitor) FileDone() error {
if m.model.trace["file"] {
log.Printf("FILE: FileDone: " + m.name)
if debugPull {
dlog.Println("file done:", m.name)
}
m.writeDone.Wait()
tmp := tempName(m.path, m.global.Modified)
tmp := defTempNamer.TempName(m.path)
defer os.Remove(tmp)
if m.copyError != nil {
@@ -149,14 +149,14 @@ func (m *fileMonitor) FileDone() error {
return nil
}
func hashCheck(name string, correct []Block) error {
func hashCheck(name string, correct []scanner.Block) error {
rf, err := os.Open(name)
if err != nil {
return err
}
defer rf.Close()
current, err := Blocks(rf, BlockSize)
current, err := scanner.Blocks(rf, BlockSize)
if err != nil {
return err
}

View File

@@ -5,6 +5,8 @@ import (
"sort"
"sync"
"time"
"github.com/calmh/syncthing/scanner"
)
type Monitor interface {
@@ -23,7 +25,7 @@ type FileQueue struct {
type queuedFile struct {
name string
blocks []Block
blocks []scanner.Block
activeBlocks []bool
given int
remaining int
@@ -54,7 +56,7 @@ func (l queuedFileList) Less(a, b int) bool {
type queuedBlock struct {
name string
block Block
block scanner.Block
index int
}
@@ -65,7 +67,7 @@ func NewFileQueue() *FileQueue {
}
}
func (q *FileQueue) Add(name string, blocks []Block, monitor Monitor) {
func (q *FileQueue) Add(name string, blocks []scanner.Block, monitor Monitor) {
q.fmut.Lock()
defer q.fmut.Unlock()

View File

@@ -5,6 +5,8 @@ import (
"sync"
"sync/atomic"
"testing"
"github.com/calmh/syncthing/scanner"
)
func TestFileQueueAdd(t *testing.T) {
@@ -17,8 +19,8 @@ func TestFileQueueAddSorting(t *testing.T) {
q.SetAvailable("zzz", []string{"nodeID"})
q.SetAvailable("aaa", []string{"nodeID"})
q.Add("zzz", []Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
q.Add("aaa", []Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
q.Add("zzz", []scanner.Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
q.Add("aaa", []scanner.Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
b, _ := q.Get("nodeID")
if b.name != "aaa" {
t.Errorf("Incorrectly sorted get: %+v", b)
@@ -28,12 +30,12 @@ func TestFileQueueAddSorting(t *testing.T) {
q.SetAvailable("zzz", []string{"nodeID"})
q.SetAvailable("aaa", []string{"nodeID"})
q.Add("zzz", []Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
q.Add("zzz", []scanner.Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
b, _ = q.Get("nodeID") // Start on zzzz
if b.name != "zzz" {
t.Errorf("Incorrectly sorted get: %+v", b)
}
q.Add("aaa", []Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
q.Add("aaa", []scanner.Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
b, _ = q.Get("nodeID")
if b.name != "zzz" {
// Continue rather than starting a new file
@@ -56,12 +58,12 @@ func TestFileQueueGet(t *testing.T) {
q.SetAvailable("foo", []string{"nodeID"})
q.SetAvailable("bar", []string{"nodeID"})
q.Add("foo", []Block{
q.Add("foo", []scanner.Block{
{Offset: 0, Size: 128, Hash: []byte("some foo hash bytes")},
{Offset: 128, Size: 128, Hash: []byte("some other foo hash bytes")},
{Offset: 256, Size: 128, Hash: []byte("more foo hash bytes")},
}, nil)
q.Add("bar", []Block{
q.Add("bar", []scanner.Block{
{Offset: 0, Size: 128, Hash: []byte("some bar hash bytes")},
{Offset: 128, Size: 128, Hash: []byte("some other bar hash bytes")},
}, nil)
@@ -70,7 +72,7 @@ func TestFileQueueGet(t *testing.T) {
expected := queuedBlock{
name: "bar",
block: Block{
block: scanner.Block{
Offset: 0,
Size: 128,
Hash: []byte("some bar hash bytes"),
@@ -89,7 +91,7 @@ func TestFileQueueGet(t *testing.T) {
expected = queuedBlock{
name: "bar",
block: Block{
block: scanner.Block{
Offset: 128,
Size: 128,
Hash: []byte("some other bar hash bytes"),
@@ -109,7 +111,7 @@ func TestFileQueueGet(t *testing.T) {
expected = queuedBlock{
name: "foo",
block: Block{
block: scanner.Block{
Offset: 0,
Size: 128,
Hash: []byte("some foo hash bytes"),
@@ -150,7 +152,7 @@ func TestFileQueueDone(t *testing.T) {
}()
q := FileQueue{resolver: fakeResolver{}}
q.Add("foo", []Block{
q.Add("foo", []scanner.Block{
{Offset: 0, Length: 128, Hash: []byte("some foo hash bytes")},
{Offset: 128, Length: 128, Hash: []byte("some other foo hash bytes")},
}, ch)
@@ -181,19 +183,19 @@ func TestFileQueueGetNodeIDs(t *testing.T) {
q.SetAvailable("a-foo", []string{"nodeID", "a"})
q.SetAvailable("b-bar", []string{"nodeID", "b"})
q.Add("a-foo", []Block{
q.Add("a-foo", []scanner.Block{
{Offset: 0, Size: 128, Hash: []byte("some foo hash bytes")},
{Offset: 128, Size: 128, Hash: []byte("some other foo hash bytes")},
{Offset: 256, Size: 128, Hash: []byte("more foo hash bytes")},
}, nil)
q.Add("b-bar", []Block{
q.Add("b-bar", []scanner.Block{
{Offset: 0, Size: 128, Hash: []byte("some bar hash bytes")},
{Offset: 128, Size: 128, Hash: []byte("some other bar hash bytes")},
}, nil)
expected := queuedBlock{
name: "b-bar",
block: Block{
block: scanner.Block{
Offset: 0,
Size: 128,
Hash: []byte("some bar hash bytes"),
@@ -209,7 +211,7 @@ func TestFileQueueGetNodeIDs(t *testing.T) {
expected = queuedBlock{
name: "a-foo",
block: Block{
block: scanner.Block{
Offset: 0,
Size: 128,
Hash: []byte("some foo hash bytes"),
@@ -225,7 +227,7 @@ func TestFileQueueGetNodeIDs(t *testing.T) {
expected = queuedBlock{
name: "a-foo",
block: Block{
block: scanner.Block{
Offset: 128,
Size: 128,
Hash: []byte("some other foo hash bytes"),
@@ -246,9 +248,9 @@ func TestFileQueueThreadHandling(t *testing.T) {
const n = 100
var total int
var blocks []Block
var blocks []scanner.Block
for i := 1; i <= n; i++ {
blocks = append(blocks, Block{Offset: int64(i), Size: 1})
blocks = append(blocks, scanner.Block{Offset: int64(i), Size: 1})
total += i
}

View File

@@ -9,6 +9,7 @@ import (
"sync"
"time"
"github.com/calmh/syncthing/scanner"
"github.com/codegangsta/martini"
)
@@ -107,7 +108,7 @@ func restPostRestart(req *http.Request) {
restart()
}
type guiFile File
type guiFile scanner.File
func (f guiFile) MarshalJSON() ([]byte, error) {
type t struct {
@@ -116,7 +117,7 @@ func (f guiFile) MarshalJSON() ([]byte, error) {
}
return json.Marshal(t{
Name: f.Name,
Size: File(f).Size,
Size: scanner.File(f).Size,
})
}

View File

@@ -0,0 +1,92 @@
//+build solaris
package main
import (
"encoding/binary"
"fmt"
"os"
"time"
)
type id_t int32
type ulong_t uint32
type timestruc_t struct {
Tv_sec int64
Tv_nsec int64
}
func (tv timestruc_t) Nano() int64 {
return tv.Tv_sec*1e9 + tv.Tv_nsec
}
type prusage_t struct {
Pr_lwpid id_t /* lwp id. 0: process or defunct */
Pr_count int32 /* number of contributing lwps */
Pr_tstamp timestruc_t /* real time stamp, time of read() */
Pr_create timestruc_t /* process/lwp creation time stamp */
Pr_term timestruc_t /* process/lwp termination time stamp */
Pr_rtime timestruc_t /* total lwp real (elapsed) time */
Pr_utime timestruc_t /* user level CPU time */
Pr_stime timestruc_t /* system call CPU time */
Pr_ttime timestruc_t /* other system trap CPU time */
Pr_tftime timestruc_t /* text page fault sleep time */
Pr_dftime timestruc_t /* data page fault sleep time */
Pr_kftime timestruc_t /* kernel page fault sleep time */
Pr_ltime timestruc_t /* user lock wait sleep time */
Pr_slptime timestruc_t /* all other sleep time */
Pr_wtime timestruc_t /* wait-cpu (latency) time */
Pr_stoptime timestruc_t /* stopped time */
Pr_minf ulong_t /* minor page faults */
Pr_majf ulong_t /* major page faults */
Pr_nswap ulong_t /* swaps */
Pr_inblk ulong_t /* input blocks */
Pr_oublk ulong_t /* output blocks */
Pr_msnd ulong_t /* messages sent */
Pr_mrcv ulong_t /* messages received */
Pr_sigs ulong_t /* signals received */
Pr_vctx ulong_t /* voluntary context switches */
Pr_ictx ulong_t /* involuntary context switches */
Pr_sysc ulong_t /* system calls */
Pr_ioch ulong_t /* chars read and written */
}
func solarisPrusage(pid int, rusage *prusage_t) error {
fd, err := os.Open(fmt.Sprintf("/proc/%d/usage", pid))
if err != nil {
return err
}
err = binary.Read(fd, binary.LittleEndian, rusage)
fd.Close()
return err
}
func init() {
go trackCPUUsage()
}
func trackCPUUsage() {
var prevUsage int64
var prevTime = time.Now().UnixNano()
var rusage prusage_t
var pid = os.Getpid()
for {
time.Sleep(10 * time.Second)
err := solarisPrusage(pid, &rusage)
if err != nil {
warnln(err)
continue
}
curTime := time.Now().UnixNano()
timeDiff := curTime - prevTime
curUsage := rusage.Pr_utime.Nano() + rusage.Pr_stime.Nano()
usageDiff := curUsage - prevUsage
cpuUsageLock.Lock()
cpuUsagePercent = 100 * float64(usageDiff) / float64(timeDiff)
cpuUsageLock.Unlock()
prevTime = curTime
prevUsage = curUsage
}
}

View File

View File

View File

@@ -13,16 +13,6 @@ func init() {
logger = log.New(os.Stderr, "", log.Flags())
}
func debugln(vals ...interface{}) {
s := fmt.Sprintln(vals...)
logger.Output(2, "DEBUG: "+s)
}
func debugf(format string, vals ...interface{}) {
s := fmt.Sprintf(format, vals...)
logger.Output(2, "DEBUG: "+s)
}
func infoln(vals ...interface{}) {
s := fmt.Sprintln(vals...)
logger.Output(2, "INFO: "+s)

View File

@@ -21,8 +21,11 @@ import (
"github.com/calmh/ini"
"github.com/calmh/syncthing/discover"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
)
const BlockSize = 128 * 1024
var cfg Configuration
var Version = "unknown-dev"
@@ -31,26 +34,42 @@ var (
)
var (
showVersion bool
confDir string
trace string
profiler string
verbose bool
startupDelay int
showVersion bool
confDir string
verbose bool
)
const (
usage = "syncthing [options]"
extraUsage = `The following enviroment variables are interpreted by syncthing:
STNORESTART Do not attempt to restart when requested to, instead just exit.
Set this variable when running under a service manager such as
runit, launchd, etc.
STPROFILER Set to a listen address such as "127.0.0.1:9090" to start the
profiler with HTTP access.
STTRACE A comma separated string of facilities to trace. The valid
facility strings:
- "scanner" (the file change scanner)
- "discover" (the node discovery package)
- "net" (connecting and disconnecting, network messages)
- "idx" (index sending and receiving)
- "need" (file need calculations)
- "pull" (file pull activity)`
)
func main() {
flag.StringVar(&confDir, "home", "~/.syncthing", "Set configuration directory")
flag.StringVar(&trace, "debug.trace", "", "(connect,net,idx,file,pull)")
flag.StringVar(&profiler, "debug.profiler", "", "(addr)")
flag.StringVar(&confDir, "home", getDefaultConfDir(), "Set configuration directory")
flag.BoolVar(&showVersion, "version", false, "Show version")
flag.BoolVar(&verbose, "v", false, "Be more verbose")
flag.IntVar(&startupDelay, "delay", 0, "Startup delay (s)")
flag.Usage = usageFor(flag.CommandLine, "syncthing [options]")
flag.Usage = usageFor(flag.CommandLine, usage, extraUsage)
flag.Parse()
if startupDelay > 0 {
time.Sleep(time.Duration(startupDelay) * time.Second)
if len(os.Getenv("STRESTART")) > 0 {
// Give the parent process time to exit and release sockets etc.
time.Sleep(1 * time.Second)
}
if showVersion {
@@ -66,10 +85,6 @@ func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
}
if len(trace) > 0 {
log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime | log.Lmicroseconds)
logger.SetFlags(log.Lshortfile | log.Ldate | log.Ltime | log.Lmicroseconds)
}
confDir = expandTilde(confDir)
// Ensure that our home directory exists and that we have a certificate and key.
@@ -138,7 +153,7 @@ func main() {
cfg, err = readConfigXML(nil)
cfg.Repositories = []RepositoryConfiguration{
{
Directory: "~/Sync",
Directory: path.Join(getHomeDir(), "Sync"),
Nodes: []NodeConfiguration{
{NodeID: myID, Addresses: []string{"dynamic"}},
},
@@ -154,11 +169,12 @@ func main() {
var dir = expandTilde(cfg.Repositories[0].Directory)
if len(profiler) > 0 {
if profiler := os.Getenv("STPROFILER"); len(profiler) > 0 {
go func() {
dlog.Println("Starting profiler on", profiler)
err := http.ListenAndServe(profiler, nil)
if err != nil {
warnln(err)
dlog.Fatal(err)
}
}()
}
@@ -178,9 +194,6 @@ func main() {
ensureDir(dir, -1)
m := NewModel(dir, cfg.Options.MaxChangeKbps*1000)
for _, t := range strings.Split(trace, ",") {
m.Trace(t)
}
if cfg.Options.MaxSendKbps > 0 {
m.LimitRate(cfg.Options.MaxSendKbps)
}
@@ -206,7 +219,9 @@ func main() {
infof("Starting web GUI on http://%s:%d/", hostShow, addr.Port)
startGUI(cfg.Options.GUIAddress, m)
openURL(fmt.Sprintf("http://%s:%d", hostOpen, addr.Port))
if cfg.Options.StartBrowser && len(os.Getenv("STRESTART")) == 0 {
openURL(fmt.Sprintf("http://%s:%d", hostOpen, addr.Port))
}
}
}
@@ -217,7 +232,18 @@ func main() {
infoln("Populating repository index")
}
loadIndex(m)
updateLocalModel(m)
sup := &suppressor{threshold: int64(cfg.Options.MaxChangeKbps)}
w := &scanner.Walker{
Dir: m.dir,
IgnoreFile: ".stignore",
FollowSymlinks: cfg.Options.FollowSymlinks,
BlockSize: BlockSize,
TempNamer: defTempNamer,
Suppressor: sup,
CurrentFiler: m,
}
updateLocalModel(m, w)
connOpts := map[string]string{
"clientId": "syncthing",
@@ -263,7 +289,7 @@ func main() {
for {
time.Sleep(td)
if m.LocalAge() > (td / 2).Seconds() {
updateLocalModel(m)
updateLocalModel(m, w)
}
}
}()
@@ -278,24 +304,23 @@ func main() {
func restart() {
infoln("Restarting")
args := os.Args
doAppend := true
for _, arg := range args {
if arg == "-delay" {
doAppend = false
break
}
if os.Getenv("SMF_FMRI") != "" || os.Getenv("STNORESTART") != "" {
// Solaris SMF
infoln("Service manager detected; exit instead of restart")
os.Exit(0)
}
if doAppend {
args = append(args, "-delay", "2")
env := os.Environ()
if len(os.Getenv("STRESTART")) == 0 {
env = append(env, "STRESTART=1")
}
pgm, err := exec.LookPath(os.Args[0])
if err != nil {
warnln(err)
return
}
proc, err := os.StartProcess(pgm, args, &os.ProcAttr{
Env: os.Environ(),
proc, err := os.StartProcess(pgm, os.Args, &os.ProcAttr{
Env: env,
Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
})
if err != nil {
@@ -328,6 +353,13 @@ func saveConfigLoop(cfgFile string) {
continue
}
if runtime.GOOS == "windows" {
err := os.Remove(cfgFile)
if err != nil && !os.IsNotExist(err) {
warnln(err)
}
}
err = os.Rename(cfgFile+".tmp", cfgFile)
if err != nil {
warnln(err)
@@ -371,8 +403,8 @@ func printStatsLoop(m *Model) {
}
func listen(myID string, addr string, m *Model, tlsCfg *tls.Config, connOpts map[string]string) {
if strings.Contains(trace, "connect") {
debugln("NET: Listening on", addr)
if debugNet {
dlog.Println("listening on", addr)
}
l, err := tls.Listen("tcp", addr, tlsCfg)
fatalErr(err)
@@ -385,8 +417,8 @@ listen:
continue
}
if strings.Contains(trace, "connect") {
debugln("NET: Connect from", conn.RemoteAddr())
if debugNet {
dlog.Println("connect from", conn.RemoteAddr())
}
tc := conn.(*tls.Conn)
@@ -467,13 +499,13 @@ func connect(myID string, disc *discover.Discoverer, m *Model, tlsCfg *tls.Confi
}
}
if strings.Contains(trace, "connect") {
debugln("NET: Dial", nodeCfg.NodeID, addr)
if debugNet {
dlog.Println("dial", nodeCfg.NodeID, addr)
}
conn, err := tls.Dial("tcp", addr, tlsCfg)
if err != nil {
if strings.Contains(trace, "connect") {
debugln("NET:", err)
if debugNet {
dlog.Println(err)
}
continue
}
@@ -495,8 +527,8 @@ func connect(myID string, disc *discover.Discoverer, m *Model, tlsCfg *tls.Confi
}
}
func updateLocalModel(m *Model) {
files, _ := m.Walk(cfg.Options.FollowSymlinks)
func updateLocalModel(m *Model, w *scanner.Walker) {
files, _ := w.Walk()
m.ReplaceLocal(files)
saveIndex(m)
}
@@ -554,16 +586,38 @@ func ensureDir(dir string, mode int) {
}
func expandTilde(p string) string {
if runtime.GOOS == "windows" {
return p
}
if strings.HasPrefix(p, "~/") {
return strings.Replace(p, "~", getHomeDir(), 1)
return strings.Replace(p, "~", getUnixHomeDir(), 1)
}
return p
}
func getHomeDir() string {
func getUnixHomeDir() string {
home := os.Getenv("HOME")
if home == "" {
fatalln("No home directory?")
}
return home
}
func getHomeDir() string {
if runtime.GOOS == "windows" {
home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
if home == "" {
home = os.Getenv("USERPROFILE")
}
return home
}
return getUnixHomeDir()
}
func getDefaultConfDir() string {
if runtime.GOOS == "windows" {
return path.Join(os.Getenv("AppData"), "syncthing")
}
return expandTilde("~/.syncthing")
}

View File

@@ -13,16 +13,17 @@ import (
"github.com/calmh/syncthing/buffers"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
)
type Model struct {
dir string
global map[string]File // the latest version of each file as it exists in the cluster
gmut sync.RWMutex // protects global
local map[string]File // the files we currently have locally on disk
lmut sync.RWMutex // protects local
remote map[string]map[string]File
global map[string]scanner.File // the latest version of each file as it exists in the cluster
gmut sync.RWMutex // protects global
local map[string]scanner.File // the files we currently have locally on disk
lmut sync.RWMutex // protects local
remote map[string]map[string]scanner.File
rmut sync.RWMutex // protects remote
protoConn map[string]Connection
rawConn map[string]io.Closer
@@ -31,7 +32,7 @@ type Model struct {
// Queue for files to fetch. fq can call back into the model, so we must ensure
// to hold no locks when calling methods on fq.
fq *FileQueue
dq chan File // queue for files to delete
dq chan scanner.File // queue for files to delete
updatedLocal int64 // timestamp of last update to local
updateGlobal int64 // timestamp of last update to remote
@@ -43,8 +44,6 @@ type Model struct {
delete bool
initmut sync.Mutex // protects rwRunning and delete
trace map[string]bool
sup suppressor
parallelRequests int
@@ -77,16 +76,15 @@ var (
func NewModel(dir string, maxChangeBw int) *Model {
m := &Model{
dir: dir,
global: make(map[string]File),
local: make(map[string]File),
remote: make(map[string]map[string]File),
global: make(map[string]scanner.File),
local: make(map[string]scanner.File),
remote: make(map[string]map[string]scanner.File),
protoConn: make(map[string]Connection),
rawConn: make(map[string]io.Closer),
lastIdxBcast: time.Now(),
trace: make(map[string]bool),
sup: suppressor{threshold: int64(maxChangeBw)},
fq: NewFileQueue(),
dq: make(chan File),
dq: make(chan scanner.File),
}
go m.broadcastIndexLoop()
@@ -108,11 +106,6 @@ func (m *Model) LimitRate(kbps int) {
}()
}
// Trace enables trace logging of the given facility. This is a debugging function; grep for m.trace.
func (m *Model) Trace(t string) {
m.trace[t] = true
}
// StartRW starts read/write processing on the current model. When in
// read/write mode the model will attempt to keep in sync with the cluster by
// pulling needed files from peer nodes.
@@ -128,7 +121,6 @@ func (m *Model) StartRW(del bool, threads int) {
m.delete = del
m.parallelRequests = threads
go m.cleanTempFiles()
if del {
go m.deleteLoop()
}
@@ -170,7 +162,9 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
var tot int64
for _, f := range m.global {
tot += f.Size
if f.Flags&protocol.FlagDeleted == 0 {
tot += f.Size
}
}
var res = make(map[string]ConnectionInfo)
@@ -186,12 +180,15 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
var have int64
for _, f := range m.remote[node] {
if f.Equals(m.global[f.Name]) {
if f.Equals(m.global[f.Name]) && f.Flags&protocol.FlagDeleted == 0 {
have += f.Size
}
}
ci.Completion = int(100 * have / tot)
ci.Completion = 100
if tot != 0 {
ci.Completion = int(100 * have / tot)
}
res[node] = ci
}
@@ -246,8 +243,10 @@ func (m *Model) InSyncSize() (files, bytes int64) {
for n, f := range m.local {
if gf, ok := m.global[n]; ok && f.Equals(gf) {
files++
bytes += f.Size
if f.Flags&protocol.FlagDeleted == 0 {
files++
bytes += f.Size
}
}
}
@@ -257,7 +256,7 @@ func (m *Model) InSyncSize() (files, bytes int64) {
}
// NeedFiles returns the list of currently needed files and the total size.
func (m *Model) NeedFiles() (files []File, bytes int64) {
func (m *Model) NeedFiles() (files []scanner.File, bytes int64) {
qf := m.fq.QueuedFiles()
m.gmut.RLock()
@@ -275,7 +274,7 @@ func (m *Model) NeedFiles() (files []File, bytes int64) {
// Index is called when a new node is connected and we receive their full index.
// Implements the protocol.Model interface.
func (m *Model) Index(nodeID string, fs []protocol.FileInfo) {
var files = make([]File, len(fs))
var files = make([]scanner.File, len(fs))
for i := range fs {
files[i] = fileFromFileInfo(fs[i])
}
@@ -283,11 +282,11 @@ func (m *Model) Index(nodeID string, fs []protocol.FileInfo) {
m.imut.Lock()
defer m.imut.Unlock()
if m.trace["net"] {
debugf("NET IDX(in): %s: %d files", nodeID, len(fs))
if debugNet {
dlog.Printf("IDX(in): %s: %d files", nodeID, len(fs))
}
repo := make(map[string]File)
repo := make(map[string]scanner.File)
for _, f := range files {
m.indexUpdate(repo, f)
}
@@ -303,7 +302,7 @@ func (m *Model) Index(nodeID string, fs []protocol.FileInfo) {
// IndexUpdate is called for incremental updates to connected nodes' indexes.
// Implements the protocol.Model interface.
func (m *Model) IndexUpdate(nodeID string, fs []protocol.FileInfo) {
var files = make([]File, len(fs))
var files = make([]scanner.File, len(fs))
for i := range fs {
files[i] = fileFromFileInfo(fs[i])
}
@@ -311,8 +310,8 @@ func (m *Model) IndexUpdate(nodeID string, fs []protocol.FileInfo) {
m.imut.Lock()
defer m.imut.Unlock()
if m.trace["net"] {
debugf("NET IDXUP(in): %s: %d files", nodeID, len(files))
if debugNet {
dlog.Printf("IDXUP(in): %s: %d files", nodeID, len(files))
}
m.rmut.Lock()
@@ -332,13 +331,13 @@ func (m *Model) IndexUpdate(nodeID string, fs []protocol.FileInfo) {
m.recomputeNeedForFiles(files)
}
func (m *Model) indexUpdate(repo map[string]File, f File) {
if m.trace["idx"] {
func (m *Model) indexUpdate(repo map[string]scanner.File, f scanner.File) {
if debugIdx {
var flagComment string
if f.Flags&protocol.FlagDeleted != 0 {
flagComment = " (deleted)"
}
debugf("IDX(in): %q m=%d f=%o%s v=%d (%d blocks)", f.Name, f.Modified, f.Flags, flagComment, f.Version, len(f.Blocks))
dlog.Printf("IDX(in): %q m=%d f=%o%s v=%d (%d blocks)", f.Name, f.Modified, f.Flags, flagComment, f.Version, len(f.Blocks))
}
if extraFlags := f.Flags &^ (protocol.FlagInvalid | protocol.FlagDeleted | 0xfff); extraFlags != 0 {
@@ -352,8 +351,8 @@ func (m *Model) indexUpdate(repo map[string]File, f File) {
// Close removes the peer from the model and closes the underlying connection if possible.
// Implements the protocol.Model interface.
func (m *Model) Close(node string, err error) {
if m.trace["net"] {
debugf("NET: %s: %v", node, err)
if debugNet {
dlog.Printf("%s: %v", node, err)
}
if err == protocol.ErrClusterHash {
warnf("Connection to %s closed due to mismatched cluster hash. Ensure that the configured cluster members are identical on both nodes.", node)
@@ -398,12 +397,12 @@ func (m *Model) Request(nodeID, repo, name string, offset int64, size int) ([]by
warnf("SECURITY (nonexistent file) REQ(in): %s: %q o=%d s=%d", nodeID, name, offset, size)
return nil, ErrNoSuchFile
}
if lf.Flags&protocol.FlagInvalid != 0 {
if lf.Suppressed {
return nil, ErrInvalid
}
if m.trace["net"] && nodeID != "<local>" {
debugf("NET REQ(in): %s: %q o=%d s=%d", nodeID, name, offset, size)
if debugNet && nodeID != "<local>" {
dlog.Printf("REQ(in): %s: %q o=%d s=%d", nodeID, name, offset, size)
}
fn := path.Join(m.dir, name)
fd, err := os.Open(fn) // XXX: Inefficient, should cache fd?
@@ -428,9 +427,9 @@ func (m *Model) Request(nodeID, repo, name string, offset int64, size int) ([]by
}
// ReplaceLocal replaces the local repository index with the given list of files.
func (m *Model) ReplaceLocal(fs []File) {
func (m *Model) ReplaceLocal(fs []scanner.File) {
var updated bool
var newLocal = make(map[string]File)
var newLocal = make(map[string]scanner.File)
m.lmut.RLock()
for _, f := range fs {
@@ -471,7 +470,7 @@ func (m *Model) ReplaceLocal(fs []File) {
// the local index from a cache file at startup.
func (m *Model) SeedLocal(fs []protocol.FileInfo) {
m.lmut.Lock()
m.local = make(map[string]File)
m.local = make(map[string]scanner.File)
for _, f := range fs {
m.local[f.Name] = fileFromFileInfo(f)
}
@@ -481,6 +480,14 @@ func (m *Model) SeedLocal(fs []protocol.FileInfo) {
m.recomputeNeedForGlobal()
}
// Implements scanner.CurrentFiler
func (m *Model) CurrentFile(file string) scanner.File {
m.lmut.RLock()
f := m.local[file]
m.lmut.RUnlock()
return f
}
// ConnectedTo returns true if we are connected to the named node.
func (m *Model) ConnectedTo(nodeID string) bool {
m.pmut.RLock()
@@ -506,6 +513,9 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn Connection) {
go func() {
idx := m.ProtocolIndex()
if debugNet {
dlog.Printf("IDX(out/initial): %s: %d files", nodeID, len(idx))
}
protoConn.Index("default", idx)
}()
@@ -519,14 +529,14 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn Connection) {
for i := 0; i < m.parallelRequests; i++ {
i := i
go func() {
if m.trace["pull"] {
debugln("PULL: Starting", nodeID, i)
if debugPull {
dlog.Println("starting puller:", nodeID, i)
}
for {
m.pmut.RLock()
if _, ok := m.protoConn[nodeID]; !ok {
if m.trace["pull"] {
debugln("PULL: Exiting", nodeID, i)
if debugPull {
dlog.Println("stopping puller:", nodeID, i)
}
m.pmut.RUnlock()
return
@@ -535,8 +545,8 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn Connection) {
qb, ok := m.fq.Get(nodeID)
if ok {
if m.trace["pull"] {
debugln("PULL: Request", nodeID, i, qb.name, qb.block.Offset)
if debugPull {
dlog.Println("request: out", nodeID, i, qb.name, qb.block.Offset)
}
data, _ := protoConn.Request("default", qb.name, qb.block.Offset, int(qb.block.Size))
m.fq.Done(qb.name, qb.block.Offset, data)
@@ -557,12 +567,12 @@ func (m *Model) ProtocolIndex() []protocol.FileInfo {
for _, f := range m.local {
mf := fileInfoFromFile(f)
if m.trace["idx"] {
if debugIdx {
var flagComment string
if mf.Flags&protocol.FlagDeleted != 0 {
flagComment = " (deleted)"
}
debugf("IDX(out): %q m=%d f=%o%s v=%d (%d blocks)", mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks))
dlog.Printf("IDX(out): %q m=%d f=%o%s v=%d (%d blocks)", mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks))
}
index = append(index, mf)
}
@@ -580,8 +590,8 @@ func (m *Model) requestGlobal(nodeID, name string, offset int64, size int, hash
return nil, fmt.Errorf("requestGlobal: no such node: %s", nodeID)
}
if m.trace["net"] {
debugf("NET REQ(out): %s: %q o=%d s=%d h=%x", nodeID, name, offset, size, hash)
if debugNet {
dlog.Printf("REQ(out): %s: %q o=%d s=%d h=%x", nodeID, name, offset, size, hash)
}
return nc.Request("default", name, offset, size)
@@ -608,8 +618,8 @@ func (m *Model) broadcastIndexLoop() {
m.pmut.RLock()
for _, node := range m.protoConn {
node := node
if m.trace["net"] {
debugf("NET IDX(out/loop): %s: %d files", node.ID(), len(idx))
if debugNet {
dlog.Printf("IDX(out/loop): %s: %d files", node.ID(), len(idx))
}
go func() {
node.Index("default", idx)
@@ -625,7 +635,7 @@ func (m *Model) broadcastIndexLoop() {
}
// markDeletedLocals sets the deleted flag on files that have gone missing locally.
func (m *Model) markDeletedLocals(newLocal map[string]File) bool {
func (m *Model) markDeletedLocals(newLocal map[string]scanner.File) bool {
// For every file in the existing local table, check if they are also
// present in the new local table. If they are not, check that we already
// had the newest version available according to the global table and if so
@@ -655,7 +665,7 @@ func (m *Model) markDeletedLocals(newLocal map[string]File) bool {
return updated
}
func (m *Model) updateLocal(f File) {
func (m *Model) updateLocal(f scanner.File) {
var updated bool
m.lmut.Lock()
@@ -682,7 +692,7 @@ func (m *Model) updateLocal(f File) {
/*
XXX: Not done, needs elegant handling of availability
func (m *Model) recomputeGlobalFor(files []File) bool {
func (m *Model) recomputeGlobalFor(files []scanner.File) bool {
m.gmut.Lock()
defer m.gmut.Unlock()
@@ -699,7 +709,7 @@ func (m *Model) recomputeGlobalFor(files []File) bool {
*/
func (m *Model) recomputeGlobal() {
var newGlobal = make(map[string]File)
var newGlobal = make(map[string]scanner.File)
m.lmut.RLock()
for n, f := range m.local {
@@ -758,12 +768,12 @@ func (m *Model) recomputeGlobal() {
type addOrder struct {
n string
remote []Block
remote []scanner.Block
fm *fileMonitor
}
func (m *Model) recomputeNeedForGlobal() {
var toDelete []File
var toDelete []scanner.File
var toAdd []addOrder
m.gmut.RLock()
@@ -782,8 +792,8 @@ func (m *Model) recomputeNeedForGlobal() {
}
}
func (m *Model) recomputeNeedForFiles(files []File) {
var toDelete []File
func (m *Model) recomputeNeedForFiles(files []scanner.File) {
var toDelete []scanner.File
var toAdd []addOrder
m.gmut.RLock()
@@ -802,13 +812,13 @@ func (m *Model) recomputeNeedForFiles(files []File) {
}
}
func (m *Model) recomputeNeedForFile(gf File, toAdd []addOrder, toDelete []File) ([]addOrder, []File) {
func (m *Model) recomputeNeedForFile(gf scanner.File, toAdd []addOrder, toDelete []scanner.File) ([]addOrder, []scanner.File) {
m.lmut.RLock()
lf, ok := m.local[gf.Name]
m.lmut.RUnlock()
if !ok || gf.NewerThan(lf) {
if gf.Flags&protocol.FlagInvalid != 0 {
if gf.Suppressed {
// Never attempt to sync invalid files
return toAdd, toDelete
}
@@ -820,14 +830,14 @@ func (m *Model) recomputeNeedForFile(gf File, toAdd []addOrder, toDelete []File)
// Don't have the file, so don't need to delete it
return toAdd, toDelete
}
if m.trace["need"] {
debugf("NEED: lf:%v gf:%v", lf, gf)
if debugNeed {
dlog.Printf("need: lf:%v gf:%v", lf, gf)
}
if gf.Flags&protocol.FlagDeleted != 0 {
toDelete = append(toDelete, gf)
} else {
local, remote := BlockDiff(lf.Blocks, gf.Blocks)
local, remote := scanner.BlockDiff(lf.Blocks, gf.Blocks)
fm := fileMonitor{
name: gf.Name,
path: path.Clean(path.Join(m.dir, gf.Name)),
@@ -862,8 +872,8 @@ func (m *Model) WhoHas(name string) []string {
func (m *Model) deleteLoop() {
for file := range m.dq {
if m.trace["file"] {
debugln("FILE: Delete", file.Name)
if debugPull {
dlog.Println("delete", file.Name)
}
path := path.Clean(path.Join(m.dir, file.Name))
err := os.Remove(path)
@@ -875,28 +885,29 @@ func (m *Model) deleteLoop() {
}
}
func fileFromFileInfo(f protocol.FileInfo) File {
var blocks = make([]Block, len(f.Blocks))
func fileFromFileInfo(f protocol.FileInfo) scanner.File {
var blocks = make([]scanner.Block, len(f.Blocks))
var offset int64
for i, b := range f.Blocks {
blocks[i] = Block{
blocks[i] = scanner.Block{
Offset: offset,
Size: b.Size,
Hash: b.Hash,
}
offset += int64(b.Size)
}
return File{
Name: f.Name,
Size: offset,
Flags: f.Flags,
Modified: f.Modified,
Version: f.Version,
Blocks: blocks,
return scanner.File{
Name: f.Name,
Size: offset,
Flags: f.Flags &^ protocol.FlagInvalid,
Modified: f.Modified,
Version: f.Version,
Blocks: blocks,
Suppressed: f.Flags&protocol.FlagInvalid != 0,
}
}
func fileInfoFromFile(f File) protocol.FileInfo {
func fileInfoFromFile(f scanner.File) protocol.FileInfo {
var blocks = make([]protocol.BlockInfo, len(f.Blocks))
for i, b := range f.Blocks {
blocks[i] = protocol.BlockInfo{
@@ -904,11 +915,15 @@ func fileInfoFromFile(f File) protocol.FileInfo {
Hash: b.Hash,
}
}
return protocol.FileInfo{
pf := protocol.FileInfo{
Name: f.Name,
Flags: f.Flags,
Modified: f.Modified,
Version: f.Version,
Blocks: blocks,
}
if f.Suppressed {
pf.Flags |= protocol.FlagInvalid
}
return pf
}

View File

@@ -9,6 +9,7 @@ import (
"time"
"github.com/calmh/syncthing/protocol"
"github.com/calmh/syncthing/scanner"
)
func TestNewModel(t *testing.T) {
@@ -27,27 +28,27 @@ func TestNewModel(t *testing.T) {
}
}
var testDataExpected = map[string]File{
"foo": File{
var testDataExpected = map[string]scanner.File{
"foo": scanner.File{
Name: "foo",
Flags: 0,
Modified: 0,
Size: 7,
Blocks: []Block{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
Blocks: []scanner.Block{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
},
"empty": File{
"empty": scanner.File{
Name: "empty",
Flags: 0,
Modified: 0,
Size: 0,
Blocks: []Block{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
Blocks: []scanner.Block{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
},
"bar": File{
"bar": scanner.File{
Name: "bar",
Flags: 0,
Modified: 0,
Size: 10,
Blocks: []Block{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
Blocks: []scanner.Block{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
},
}
@@ -63,7 +64,8 @@ func init() {
func TestUpdateLocal(t *testing.T) {
m := NewModel("testdata", 1e6)
fs, _ := m.Walk(false)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
if fs, _ := m.NeedFiles(); len(fs) > 0 {
@@ -105,7 +107,8 @@ func TestUpdateLocal(t *testing.T) {
func TestRemoteUpdateExisting(t *testing.T) {
m := NewModel("testdata", 1e6)
fs, _ := m.Walk(false)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
newFile := protocol.FileInfo{
@@ -122,7 +125,8 @@ func TestRemoteUpdateExisting(t *testing.T) {
func TestRemoteAddNew(t *testing.T) {
m := NewModel("testdata", 1e6)
fs, _ := m.Walk(false)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
newFile := protocol.FileInfo{
@@ -139,7 +143,8 @@ func TestRemoteAddNew(t *testing.T) {
func TestRemoteUpdateOld(t *testing.T) {
m := NewModel("testdata", 1e6)
fs, _ := m.Walk(false)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
oldTimeStamp := int64(1234)
@@ -157,7 +162,8 @@ func TestRemoteUpdateOld(t *testing.T) {
func TestRemoteIndexUpdate(t *testing.T) {
m := NewModel("testdata", 1e6)
fs, _ := m.Walk(false)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
foo := protocol.FileInfo{
@@ -190,7 +196,8 @@ func TestRemoteIndexUpdate(t *testing.T) {
func TestDelete(t *testing.T) {
m := NewModel("testdata", 1e6)
fs, _ := m.Walk(false)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
if l1, l2 := len(m.local), len(fs); l1 != l2 {
@@ -201,10 +208,10 @@ func TestDelete(t *testing.T) {
}
ot := time.Now().Unix()
newFile := File{
newFile := scanner.File{
Name: "a new file",
Modified: ot,
Blocks: []Block{{0, 100, []byte("some hash bytes")}},
Blocks: []scanner.Block{{0, 100, []byte("some hash bytes")}},
}
m.updateLocal(newFile)
@@ -292,7 +299,8 @@ func TestDelete(t *testing.T) {
func TestForgetNode(t *testing.T) {
m := NewModel("testdata", 1e6)
fs, _ := m.Walk(false)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
if l1, l2 := len(m.local), len(fs); l1 != l2 {
@@ -345,7 +353,8 @@ func TestForgetNode(t *testing.T) {
func TestRequest(t *testing.T) {
m := NewModel("testdata", 1e6)
fs, _ := m.Walk(false)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
bs, err := m.Request("some node", "default", "foo", 0, 6)
@@ -367,7 +376,8 @@ func TestRequest(t *testing.T) {
func TestIgnoreWithUnknownFlags(t *testing.T) {
m := NewModel("testdata", 1e6)
fs, _ := m.Walk(false)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
valid := protocol.FileInfo{
@@ -410,7 +420,8 @@ func genFiles(n int) []protocol.FileInfo {
func BenchmarkIndex10000(b *testing.B) {
m := NewModel("testdata", 1e6)
fs, _ := m.Walk(false)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
files := genFiles(10000)
@@ -422,7 +433,8 @@ func BenchmarkIndex10000(b *testing.B) {
func BenchmarkIndex00100(b *testing.B) {
m := NewModel("testdata", 1e6)
fs, _ := m.Walk(false)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
files := genFiles(100)
@@ -434,7 +446,8 @@ func BenchmarkIndex00100(b *testing.B) {
func BenchmarkIndexUpdate10000f10000(b *testing.B) {
m := NewModel("testdata", 1e6)
fs, _ := m.Walk(false)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
files := genFiles(10000)
m.Index("42", files)
@@ -447,7 +460,8 @@ func BenchmarkIndexUpdate10000f10000(b *testing.B) {
func BenchmarkIndexUpdate10000f00100(b *testing.B) {
m := NewModel("testdata", 1e6)
fs, _ := m.Walk(false)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
files := genFiles(10000)
m.Index("42", files)
@@ -461,7 +475,8 @@ func BenchmarkIndexUpdate10000f00100(b *testing.B) {
func BenchmarkIndexUpdate10000f00001(b *testing.B) {
m := NewModel("testdata", 1e6)
fs, _ := m.Walk(false)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
files := genFiles(10000)
m.Index("42", files)
@@ -506,7 +521,8 @@ func (FakeConnection) Statistics() protocol.Statistics {
func BenchmarkRequest(b *testing.B) {
m := NewModel("testdata", 1e6)
fs, _ := m.Walk(false)
w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
fs, _ := w.Walk()
m.ReplaceLocal(fs)
const n = 1000

View File

View File

@@ -1,6 +1,7 @@
package main
import (
"os"
"sync"
"time"
)
@@ -51,6 +52,11 @@ func (h *changeHistory) append(size int64, t time.Time) {
h.changes = append(h.changes, c)
}
func (s *suppressor) Suppress(name string, fi os.FileInfo) bool {
sup, _ := s.suppress(name, fi.Size(), time.Now())
return sup
}
func (s *suppressor) suppress(name string, size int64, t time.Time) (bool, bool) {
s.Lock()

28
cmd/syncthing/tempname.go Normal file
View File

@@ -0,0 +1,28 @@
package main
import (
"fmt"
"path"
"path/filepath"
"runtime"
"strings"
)
type tempNamer struct {
prefix string
}
var defTempNamer = tempNamer{".syncthing"}
func (t tempNamer) IsTemporary(name string) bool {
if runtime.GOOS == "windows" {
name = filepath.ToSlash(name)
}
return strings.HasPrefix(path.Base(name), t.prefix)
}
func (t tempNamer) TempName(name string) string {
tdir := path.Dir(name)
tname := fmt.Sprintf("%s.%s", t.prefix, path.Base(name))
return path.Join(tdir, tname)
}

View File

View File

View File

View File

View File

View File

View File

@@ -22,18 +22,14 @@ func optionTable(w io.Writer, rows [][]string) {
tw.Flush()
}
func usageFor(fs *flag.FlagSet, usage string) func() {
func usageFor(fs *flag.FlagSet, usage string, extra string) func() {
return func() {
var b bytes.Buffer
b.WriteString("Usage:\n " + usage + "\n")
var options [][]string
fs.VisitAll(func(f *flag.Flag) {
var dash = "-"
if len(f.Name) > 1 {
dash = "--"
}
var opt = " " + dash + f.Name
var opt = " -" + f.Name
if f.DefValue != "false" {
opt += "=" + f.DefValue
@@ -48,5 +44,9 @@ func usageFor(fs *flag.FlagSet, usage string) func() {
}
fmt.Println(b.String())
if len(extra) > 0 {
fmt.Println(extra)
}
}
}

View File

12
discover/debug.go Normal file
View File

@@ -0,0 +1,12 @@
package discover
import (
"log"
"os"
"strings"
)
var (
dlog = log.New(os.Stderr, "discover: ", log.Lmicroseconds|log.Lshortfile)
debug = strings.Contains(os.Getenv("STTRACE"), "discover")
)

View File

@@ -15,7 +15,6 @@ import (
const (
AnnouncementPort = 21025
Debug = false
)
type Discoverer struct {
@@ -80,24 +79,75 @@ func (d *Discoverer) sendAnnouncements() {
var errCounter = 0
var err error
remote := &net.UDPAddr{
IP: net.IP{255, 255, 255, 255},
Port: AnnouncementPort,
}
for errCounter < maxErrors {
for _, ipStr := range allBroadcasts() {
var addrStr = ipStr + ":21025"
intfs, err := net.Interfaces()
if err != nil {
log.Printf("discover/listInterfaces: %v; no local announcements", err)
return
}
remote, err := net.ResolveUDPAddr("udp4", addrStr)
if err != nil {
log.Printf("discover/external: %v; no external announcements", err)
return
}
for _, intf := range intfs {
if intf.Flags&(net.FlagBroadcast|net.FlagLoopback) == net.FlagBroadcast {
addrs, err := intf.Addrs()
if err != nil {
log.Println("discover/listAddrs: warning:", err)
errCounter++
continue
}
if Debug {
log.Println("send announcement -> ", remote)
}
_, _, err = d.conn.WriteMsgUDP(buf, nil, remote)
if err != nil {
log.Println("discover/write: warning:", err)
errCounter++
} else {
var srcAddr string
for _, addr := range addrs {
if strings.Contains(addr.String(), ".") {
// Found an IPv4 adress
parts := strings.Split(addr.String(), "/")
srcAddr = parts[0]
break
}
}
if len(srcAddr) == 0 {
if debug {
dlog.Println("no source address found on interface", intf.Name)
}
continue
}
iaddr, err := net.ResolveUDPAddr("udp4", srcAddr+":0")
if err != nil {
log.Println("discover/resolve: warning:", err)
errCounter++
continue
}
conn, err := net.ListenUDP("udp4", iaddr)
if err != nil {
log.Println("discover/listen: warning:", err)
errCounter++
continue
}
if debug {
dlog.Println("send announcement from", conn.LocalAddr(), "to", remote, "on", intf.Name)
}
_, err = conn.WriteTo(buf, remote)
if err != nil {
// Some interfaces don't seem to support broadcast even though the flags claims they do, i.e. vmnet
conn.Close()
if debug {
log.Println(err)
}
errCounter++
continue
}
conn.Close()
errCounter = 0
}
}
@@ -122,10 +172,10 @@ func (d *Discoverer) sendExtAnnouncements() {
var errCounter = 0
for errCounter < maxErrors {
if Debug {
log.Println("send announcement -> ", remote)
if debug {
dlog.Println("send announcement -> ", remote)
}
_, _, err = d.conn.WriteMsgUDP(buf, nil, remote)
_, err = d.conn.WriteTo(buf, remote)
if err != nil {
log.Println("discover/write: warning:", err)
errCounter++
@@ -149,8 +199,8 @@ func (d *Discoverer) recvAnnouncements() {
continue
}
if Debug {
log.Printf("read announcement:\n%s", hex.Dump(buf[:n]))
if debug {
dlog.Printf("read announcement:\n%s", hex.Dump(buf[:n]))
}
var pkt AnnounceV2
@@ -161,8 +211,8 @@ func (d *Discoverer) recvAnnouncements() {
continue
}
if Debug {
log.Printf("read announcement: %#v", pkt)
if debug {
dlog.Printf("parsed announcement: %#v", pkt)
}
errCounter = 0
@@ -178,8 +228,8 @@ func (d *Discoverer) recvAnnouncements() {
}
addrs = append(addrs, nodeAddr)
}
if Debug {
log.Printf("register: %#v", addrs)
if debug {
dlog.Printf("register: %#v", addrs)
}
d.registryLock.Lock()
_, seen := d.registry[pkt.NodeID]
@@ -236,8 +286,8 @@ func (d *Discoverer) externalLookup(node string) []string {
return nil
}
if Debug {
log.Printf("read external:\n%s", hex.Dump(buf[:n]))
if debug {
dlog.Printf("read external:\n%s", hex.Dump(buf[:n]))
}
var pkt AnnounceV2
@@ -247,8 +297,8 @@ func (d *Discoverer) externalLookup(node string) []string {
return nil
}
if Debug {
log.Printf("read external: %#v", pkt)
if debug {
dlog.Printf("parsed external: %#v", pkt)
}
var addrs []string
@@ -289,38 +339,3 @@ func ipStr(ip []byte) string {
}
return strings.Join(ss, s)
}
func allBroadcasts() []string {
var bcasts = make(map[string]bool)
addrs, err := net.InterfaceAddrs()
if err != nil {
return nil
}
for _, addr := range addrs {
switch {
case strings.HasPrefix(addr.String(), "127."):
// Ignore v4 localhost
case strings.Contains(addr.String(), ":"):
// Ignore all v6, because we need link local multicast there which I haven't implemented
default:
if in, ok := addr.(*net.IPNet); ok {
il := len(in.IP) - 1
ml := len(in.Mask) - 1
for i := range in.Mask {
in.IP[il-i] = in.IP[il-i] | ^in.Mask[ml-i]
}
parts := strings.Split(in.String(), "/")
bcasts[parts[0]] = true
}
}
}
var l []string
for ip := range bcasts {
l = append(l, ip)
}
return l
}

2
discover/doc.go Normal file
View File

@@ -0,0 +1,2 @@
// Package discover implements the node discovery protocol.
package discover

174
files/set.go Normal file
View File

@@ -0,0 +1,174 @@
// Package fileset provides a set type to track local/remote files with newness checks.
package fileset
import "sync"
type File struct {
Key Key
Modified int64
Flags uint32
Data interface{}
}
type Key struct {
Name string
Version uint32
}
type fileRecord struct {
Usage int
File File
}
type bitset uint64
func (a Key) newerThan(b Key) bool {
return a.Version > b.Version
}
type Set struct {
mutex sync.RWMutex
files map[Key]fileRecord
remoteKey [64]map[string]Key
globalAvailability map[string]bitset
globalKey map[string]Key
}
func NewSet() *Set {
var m = Set{
files: make(map[Key]fileRecord),
globalAvailability: make(map[string]bitset),
globalKey: make(map[string]Key),
}
return &m
}
func (m *Set) AddLocal(fs []File) {
m.mutex.Lock()
m.unlockedAddRemote(0, fs)
m.mutex.Unlock()
}
func (m *Set) SetLocal(fs []File) {
m.mutex.Lock()
m.unlockedSetRemote(0, fs)
m.mutex.Unlock()
}
func (m *Set) AddRemote(cid uint, fs []File) {
if cid < 1 || cid > 63 {
panic("Connection ID must be in the range 1 - 63 inclusive")
}
m.mutex.Lock()
m.unlockedAddRemote(cid, fs)
m.mutex.Unlock()
}
func (m *Set) SetRemote(cid uint, fs []File) {
if cid < 1 || cid > 63 {
panic("Connection ID must be in the range 1 - 63 inclusive")
}
m.mutex.Lock()
m.unlockedSetRemote(cid, fs)
m.mutex.Unlock()
}
func (m *Set) unlockedAddRemote(cid uint, fs []File) {
remFiles := m.remoteKey[cid]
for _, f := range fs {
n := f.Key.Name
if ck, ok := remFiles[n]; ok && ck == f.Key {
// The remote already has exactly this file, skip it
continue
}
remFiles[n] = f.Key
// Keep the block list or increment the usage
if br, ok := m.files[f.Key]; !ok {
m.files[f.Key] = fileRecord{
Usage: 1,
File: f,
}
} else {
br.Usage++
m.files[f.Key] = br
}
// Update global view
gk, ok := m.globalKey[n]
switch {
case ok && f.Key == gk:
av := m.globalAvailability[n]
av |= 1 << cid
m.globalAvailability[n] = av
case f.Key.newerThan(gk):
m.globalKey[n] = f.Key
m.globalAvailability[n] = 1 << cid
}
}
}
func (m *Set) unlockedSetRemote(cid uint, fs []File) {
// Decrement usage for all files belonging to this remote, and remove
// those that are no longer needed.
for _, fk := range m.remoteKey[cid] {
br, ok := m.files[fk]
switch {
case ok && br.Usage == 1:
delete(m.files, fk)
case ok && br.Usage > 1:
br.Usage--
m.files[fk] = br
}
}
// Clear existing remote remoteKey
m.remoteKey[cid] = make(map[string]Key)
// Recalculate global based on all remaining remoteKey
for n := range m.globalKey {
var nk Key // newest key
var na bitset // newest availability
for i, rem := range m.remoteKey {
if rk, ok := rem[n]; ok {
switch {
case rk == nk:
na |= 1 << uint(i)
case rk.newerThan(nk):
nk = rk
na = 1 << uint(i)
}
}
}
if na != 0 {
// Someone had the file
m.globalKey[n] = nk
m.globalAvailability[n] = na
} else {
// Noone had the file
delete(m.globalKey, n)
delete(m.globalAvailability, n)
}
}
// Add new remote remoteKey to the mix
m.unlockedAddRemote(cid, fs)
}
func (m *Set) Need(cid uint) []File {
var fs []File
m.mutex.Lock()
for name, gk := range m.globalKey {
if gk.newerThan(m.remoteKey[cid][name]) {
fs = append(fs, m.files[gk].File)
}
}
m.mutex.Unlock()
return fs
}

207
files/set_test.go Normal file
View File

@@ -0,0 +1,207 @@
package fileset
import (
"fmt"
"reflect"
"testing"
)
func TestGlobalSet(t *testing.T) {
m := NewSet()
local := []File{
File{Key{"a", 1000}, 0, 0, nil},
File{Key{"b", 1000}, 0, 0, nil},
File{Key{"c", 1000}, 0, 0, nil},
File{Key{"d", 1000}, 0, 0, nil},
}
remote := []File{
File{Key{"a", 1000}, 0, 0, nil},
File{Key{"b", 1001}, 0, 0, nil},
File{Key{"c", 1002}, 0, 0, nil},
File{Key{"e", 1000}, 0, 0, nil},
}
expectedGlobal := map[string]Key{
"a": local[0].Key,
"b": remote[1].Key,
"c": remote[2].Key,
"d": local[3].Key,
"e": remote[3].Key,
}
m.SetLocal(local)
m.SetRemote(1, remote)
if !reflect.DeepEqual(m.globalKey, expectedGlobal) {
t.Errorf("Global incorrect;\n%v !=\n%v", m.globalKey, expectedGlobal)
}
if lb := len(m.files); lb != 7 {
t.Errorf("Num files incorrect %d != 7\n%v", lb, m.files)
}
}
func BenchmarkSetLocal10k(b *testing.B) {
m := NewSet()
var local []File
for i := 0; i < 10000; i++ {
local = append(local, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
}
var remote []File
for i := 0; i < 10000; i++ {
remote = append(remote, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
}
m.SetRemote(1, remote)
b.ResetTimer()
for i := 0; i < b.N; i++ {
m.SetLocal(local)
}
}
func BenchmarkSetLocal10(b *testing.B) {
m := NewSet()
var local []File
for i := 0; i < 10; i++ {
local = append(local, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
}
var remote []File
for i := 0; i < 10000; i++ {
remote = append(remote, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
}
m.SetRemote(1, remote)
b.ResetTimer()
for i := 0; i < b.N; i++ {
m.SetLocal(local)
}
}
func BenchmarkAddLocal10k(b *testing.B) {
m := NewSet()
var local []File
for i := 0; i < 10000; i++ {
local = append(local, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
}
var remote []File
for i := 0; i < 10000; i++ {
remote = append(remote, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
}
m.SetRemote(1, remote)
m.SetLocal(local)
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
for j := range local {
local[j].Key.Version++
}
b.StartTimer()
m.AddLocal(local)
}
}
func BenchmarkAddLocal10(b *testing.B) {
m := NewSet()
var local []File
for i := 0; i < 10; i++ {
local = append(local, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
}
var remote []File
for i := 0; i < 10000; i++ {
remote = append(remote, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
}
m.SetRemote(1, remote)
m.SetLocal(local)
b.ResetTimer()
for i := 0; i < b.N; i++ {
for j := range local {
local[j].Key.Version++
}
m.AddLocal(local)
}
}
func TestGlobalReset(t *testing.T) {
m := NewSet()
local := []File{
File{Key{"a", 1000}, 0, 0, nil},
File{Key{"b", 1000}, 0, 0, nil},
File{Key{"c", 1000}, 0, 0, nil},
File{Key{"d", 1000}, 0, 0, nil},
}
remote := []File{
File{Key{"a", 1000}, 0, 0, nil},
File{Key{"b", 1001}, 0, 0, nil},
File{Key{"c", 1002}, 0, 0, nil},
File{Key{"e", 1000}, 0, 0, nil},
}
expectedGlobalKey := map[string]Key{
"a": local[0].Key,
"b": local[1].Key,
"c": local[2].Key,
"d": local[3].Key,
}
m.SetLocal(local)
m.SetRemote(1, remote)
m.SetRemote(1, nil)
if !reflect.DeepEqual(m.globalKey, expectedGlobalKey) {
t.Errorf("Global incorrect;\n%v !=\n%v", m.globalKey, expectedGlobalKey)
}
if lb := len(m.files); lb != 4 {
t.Errorf("Num files incorrect %d != 4\n%v", lb, m.files)
}
}
func TestNeed(t *testing.T) {
m := NewSet()
local := []File{
File{Key{"a", 1000}, 0, 0, nil},
File{Key{"b", 1000}, 0, 0, nil},
File{Key{"c", 1000}, 0, 0, nil},
File{Key{"d", 1000}, 0, 0, nil},
}
remote := []File{
File{Key{"a", 1000}, 0, 0, nil},
File{Key{"b", 1001}, 0, 0, nil},
File{Key{"c", 1002}, 0, 0, nil},
File{Key{"e", 1000}, 0, 0, nil},
}
shouldNeed := []File{
File{Key{"b", 1001}, 0, 0, nil},
File{Key{"c", 1002}, 0, 0, nil},
File{Key{"e", 1000}, 0, 0, nil},
}
m.SetLocal(local)
m.SetRemote(1, remote)
need := m.Need(0)
if !reflect.DeepEqual(need, shouldNeed) {
t.Errorf("Need incorrect;\n%v !=\n%v", need, shouldNeed)
}
}

View File

@@ -32,6 +32,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
{id: 'FollowSymlinks', descr: 'Follow Symlinks', type: 'bool', restart: true},
{id: 'GlobalAnnEnabled', descr: 'Global Announce', type: 'bool', restart: true},
{id: 'LocalAnnEnabled', descr: 'Local Announce', type: 'bool', restart: true},
{id: 'StartBrowser', descr: 'Start Browser', type: 'bool'},
];
function modelGetSucceeded() {
@@ -317,7 +318,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
}
return errors;
};
$scope.clearErrors = function () {
$scope.seenError = $scope.errors[$scope.errors.length - 1].Time;
};

View File

@@ -1,5 +1,8 @@
files-*
conf-*
s1
s2
s3
md5-*
genfiles
md5r
json
*.idx.gz

View File

@@ -8,6 +8,7 @@ import (
mr "math/rand"
"os"
"path"
"time"
)
func name() string {
@@ -38,5 +39,10 @@ func main() {
rand.Reader.Read(b)
p1 := path.Join(p0, n)
ioutil.WriteFile(p1, b, 0644)
os.Chmod(p1, os.FileMode(mr.Intn(0777)|0400))
t := time.Now().Add(-time.Duration(mr.Intn(30*86400)) * time.Second)
os.Chtimes(p1, t, t)
}
}

23
integration/h1/cert.pem Normal file
View File

@@ -0,0 +1,23 @@
-----BEGIN CERTIFICATE-----
MIID3jCCAkigAwIBAgIBADALBgkqhkiG9w0BAQUwFDESMBAGA1UEAxMJc3luY3Ro
aW5nMB4XDTE0MDMxNDA3MDA1M1oXDTQ5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJ
c3luY3RoaW5nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEArDOcd5ft
R7SnalxF1ckU3lDQpgfMIPhFDU//4dvdSSFevrMuVDTbUYhyCfGtg/g+F5TmKhZg
E2peYhllITupz5MP7OHGaO2GHf2XnUDD4QUO3E+KVAUw7dyFSwy09esqApVLzH3+
ov+QXyyzmRWPsJe9u18BHU1Hob/RmBhS9m2CAJgzN6EJ8KGjApiW3iR8lD/hjVyi
IVde8IRD6qYHEJYiPJuziTVcQpCblVYxTz3ScmmT190/O9UvViIpcOPQdwgOdewP
NNMK35c9Edt0AH5flYp6jgrja9NkLQJ3+KOiro6yl9IUS5w87GMxI8qzI8SgCAZZ
pYSoLbu1FJPvxV4p5eHwuprBCwmFYZWw6Y7rqH0sN52C+3TeObJCMNP9ilPadqRI
+G0Q99TCaloeR022x33r/8D8SIn3FP35zrlFM+DvqlxoS6glbNb/Bj3p9vN0XONO
RCuynOGe9F/4h/DaNnrbrRWqJOxBsZTsbbcJaKATfWU/Z9GcC+pUpPRhAgMBAAGj
PzA9MA4GA1UdDwEB/wQEAwIAoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH
AwIwDAYDVR0TAQH/BAIwADALBgkqhkiG9w0BAQUDggGBAFF8dklGoC43fMrUZfb4
6areRWG8quO6cSX6ATzRQVJ8WJ5VcC7OJk8/FeiYA+wcvUJ/1Zm/VHMYugtOz5M8
CrWAF1r9D3Xfe5D8qfrEOYG2XjxD2nFHCnkbY4fP+SMSuXaDs7ixQnzw0UFh1wsV
9Jy/QrgXFAIFZtu1Nz+rrvoAgw24gkDhY3557MbmYfmfPsJ8cw+WJ845sxGMPFF2
c+5EN0jiSm0AwZK11BMJda36ke829UZctDkopbGEg1peydDR5LiyhiTAPtWn7uT/
PkzHYLuaECAkVbWC3bZLocMGOP6F1pG+BMr00NJgVy05ASQzi4FPjcZQNNY8s69R
ZgoCIBaJZq3ti1EsZQ1H0Ynm2c2NMVKdj4czoy8a9ZC+DCuhG7EV5Foh20VhCWgA
RfPhlHVJthuimsWBx39X85gjSBR017uk0AxOJa6pzh/b/RPCRtUfX8EArInS3XCf
RvRtdrnBZNI3tiREopZGt0SzgDZUs4uDVBUX8HnHzyFJrg==
-----END CERTIFICATE-----

30
integration/h1/config.xml Normal file
View File

@@ -0,0 +1,30 @@
<configuration version="1">
<repository directory="s1">
<node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1">
<address>127.0.0.1:22001</address>
</node>
<node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ" name="s2">
<address>127.0.0.1:22002</address>
</node>
<node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3">
<address>127.0.0.1:22003</address>
</node>
</repository>
<options>
<listenAddress>127.0.0.1:22001</listenAddress>
<readOnly>false</readOnly>
<allowDelete>true</allowDelete>
<followSymlinks>true</followSymlinks>
<guiEnabled>true</guiEnabled>
<guiAddress>127.0.0.1:8081</guiAddress>
<globalAnnounceServer>announce.syncthing.net:22025</globalAnnounceServer>
<globalAnnounceEnabled>false</globalAnnounceEnabled>
<localAnnounceEnabled>false</localAnnounceEnabled>
<parallelRequests>16</parallelRequests>
<maxSendKbps>0</maxSendKbps>
<rescanIntervalS>60</rescanIntervalS>
<reconnectionIntervalS>5</reconnectionIntervalS>
<maxChangeKbps>1000</maxChangeKbps>
<startBrowser>false</startBrowser>
</options>
</configuration>

39
integration/h1/key.pem Normal file
View File

@@ -0,0 +1,39 @@
-----BEGIN RSA PRIVATE KEY-----
MIIG5AIBAAKCAYEArDOcd5ftR7SnalxF1ckU3lDQpgfMIPhFDU//4dvdSSFevrMu
VDTbUYhyCfGtg/g+F5TmKhZgE2peYhllITupz5MP7OHGaO2GHf2XnUDD4QUO3E+K
VAUw7dyFSwy09esqApVLzH3+ov+QXyyzmRWPsJe9u18BHU1Hob/RmBhS9m2CAJgz
N6EJ8KGjApiW3iR8lD/hjVyiIVde8IRD6qYHEJYiPJuziTVcQpCblVYxTz3ScmmT
190/O9UvViIpcOPQdwgOdewPNNMK35c9Edt0AH5flYp6jgrja9NkLQJ3+KOiro6y
l9IUS5w87GMxI8qzI8SgCAZZpYSoLbu1FJPvxV4p5eHwuprBCwmFYZWw6Y7rqH0s
N52C+3TeObJCMNP9ilPadqRI+G0Q99TCaloeR022x33r/8D8SIn3FP35zrlFM+Dv
qlxoS6glbNb/Bj3p9vN0XONORCuynOGe9F/4h/DaNnrbrRWqJOxBsZTsbbcJaKAT
fWU/Z9GcC+pUpPRhAgMBAAECggGAL8+Unc/c3Y/W+7zq1tShqqgdhjub/XtxEKUp
kngNFITjXWc6cb7LNfQAVap4Vq/R7ZI15XGY80sRMYODhJqgJzXZshdtkyx/lEwY
kFyvBgb1fU3IRlO6phAYIiJBDBZi75ysEvbYgEEcwJAUvWgzIQDAeQmDsbMHNG2h
r+zw++Kjua6IaeWYcOsv60Safsr6m96wrSMPENrFTVor0TaPt5c3okRIsMvT9ddY
mzn3Lt0nVQTjO4f+SoqCPhP2FZXqksfKlZlKlr6BLxXGt6b49OrLSXM5eQXIcIZn
ZDRsO24X5z8156qPgM9cA8oNEjuSdnArUTreBOsTwNoSpf24Qadsv/uTZlaHM19V
q6zQvkjH3ERcOpixmg48TKdIj8cPYxezvcbNqSbZmdyQuaVlgDbUxwYI8A4IhhWl
6xhwpX3qPDgw/QHIEngFIWfiIfCk11EPY0SN4cGO6f1rLYug8kqxMPuIQ5Jz9Hhx
eFSRnr/fWoJcVYG6bMDKn9YWObQBAoHBAM8NahsLbjl8mdT43LH1Od1tDmDch+0Y
JM7TgiIN/GM3piZSpGMOFqToLAqvY+Gf3l4sPgNs10cqdPAEpMk8MJ/IXGmbKq38
iVmMaqHTQorCxyUbc54q9AbFU4HKv//F6ZN6K1wSaJt2RBeZpYI+MyBXr5baFiBZ
ddXtXlqoEcCFyNR0DhlXrlZPs+cnyM2ZDp++lpn9Wfy+zkv36+NWpAkXVnARjxdF
l6M+L7OlurYAWiyJE4uHUjawAM82i5+w8QKBwQDU6RCN6/AMmVrYqPy+7QcnAq67
tPDv25gzVExeMKLBAMoz1TkMS+jIF1NMp3cYg5GbLqvx8Qd27fjFbWe/GPeZvlgL
qdQI/T8J60dHAySMeOFOB2QWXhI1kwh0b2X0SDkTgfdJBKGdrKVcLTuLyVE24exu
yRc8cXpYwBtVkXNBYFd7XEM+tC4b1khO23OJXHJUen9+hgsmn8/zUjASAoq3+Zly
J+OHwwXcDcTFLeok3kX3A9NuqIV/Fa9DOGYlenECgcEAvO1onDTZ5uqjE4nhFyDE
JB+WtxuDi/wz2eV1IM3SNlZY7S8LgLciQmb3iOhxIzdVGGkWTNnLtcwv17LlCho5
5BJXAKXtU8TTLzrJMdArL6J7RIi//tsCwAreH9h5SVG1yDP5zJGfkftgNoikVSuc
Sy63sdZdyjbXJtTo+5/QUvPARNuA4e73zRn89jd/Kts2VNz7XpemvND+PKOEQnSU
SRdab/gVsQ53RyU/MZVPwTKhFXIeu3pGsk/27RzAWn6BAoHBAMIRYwaKDffd/SHJ
/v+lHEThvBXa21c26ae36hhc6q1UI/tVGrfrpVZldIdFilgs7RbvVsmksvIj/gMv
M0bL4j0gdC7FcUF0XPaUoBbJdZIZSP0P3ZpJyv1MdYN0WxFsl6IBcD79WrdXPC8m
B8XmDgIhsppU77onkaa+DOxVNSJdR8BpG95W7ERxcN14SPrm6ku4kOfqFNXzC+C1
hJ2V9Y22lLiqRUplaLzpS/eTX36VoF6E/T87mtt5D5UNHoaA8QKBwH5sRqZXoatU
X+vw1MHU5eptMwG7LXR0gw2xmvG3cCN4hbnnBp5YaXlWPiIMmaWhpvschgBIo1TP
qGWUpMEETGES18NenLBym+tWIXlfuyZH3B4NUi4kItiZaKb09LzmTjFvzdfQzun4
HzIeigTNBDHdS0rdicNIn83QLZ4pJaOZJHq79+mFYkp+9It7UUoWsws6DGl/qX8o
0cj4NmJB6QiJa1QCzrGkaajbtThbFoQal9Twk2h3jHgJzX3FbwCpLw==
-----END RSA PRIVATE KEY-----

23
integration/h2/cert.pem Normal file
View File

@@ -0,0 +1,23 @@
-----BEGIN CERTIFICATE-----
MIID3jCCAkigAwIBAgIBADALBgkqhkiG9w0BAQUwFDESMBAGA1UEAxMJc3luY3Ro
aW5nMB4XDTE0MDMxNDA3MDEwNFoXDTQ5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJ
c3luY3RoaW5nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAsIV0syyR
O56BvIOro4bIqB6iFJsNc4zX8MiM4QPTWgqGlYwsKSVmNppTdlACZCJIqyzoscrF
qJPto8/e2Fc3oaTdEREGIs7cmc7LSXfot/mAgPpy71SVWtb7xNmXro2JJPZjRBCS
pl1ulPug+/8w7fSKQdLMjh4Hp2YlwVBfVu0bYEEW+7Vl9PZVTv+NbTqXYvYVc9R6
QFIbN/njWAuo2wpjJlY7vqNnSYZyskAaaAC17fFJkVQKKblTeTk1C9PxTmVTB1j9
yOoD3+V/6IrTYKXdTHGJ1MqdieTHj1jHXe5TOeSB+Hjgq4tr25mPfQ4ixXqDqIcx
5390DAjInuSKNUJ5pqiFrVe9eIDmySZCg5/JIL3c8phy6g1bxiJN14+Dn0om/0+9
UrHK8LVzWMmtFRVycWVUYmARWFY3EE10k0RXU2HtzmjfnBkRrl13b0ExizlA1qJ3
3ngxF5rNEDSMpwf4og5uYOjRUPYuvCL9XtQKr254NFO/sg/qqPV4hFWTAgMBAAGj
PzA9MA4GA1UdDwEB/wQEAwIAoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH
AwIwDAYDVR0TAQH/BAIwADALBgkqhkiG9w0BAQUDggGBAAZSU08zAzyuGqKqqU/c
Pr+xML8oKiJqko5pb3ETDQC+uVw+qUHwiGYsvHI1cih4ix+tKvf+Yaiizp/35VkP
qwls3a4ljq1Ww0Sf7J87QX0DumYpBGOfoCpmV4MacyjLhpLRKRGZHwIbOeFsmEu9
oO38co+GvDy4CiAt3tuOdjBNs0gNOAdTTxqgm97raB9oXeg2i4Fb4MCT4UBUdXLM
ZNLCifza+PWkBxmfBORvlKGeJBruLpXHBWnWEigZSLXIFjn3JJUy4fKd+/JMp063
8Pjo6zUOckBCH8Lv90vzfrmdlQK555jWpcebN0l9neESEXw19l0OlqkJGVTr6JKq
w5kjiL4eP7kpKKwCezhDSX3jf4P36wdF8MpOUBxVqfM+Oh5tHIcZctnurhYV7rXs
jR70FMqWjHBmwemsXGrObNVt8c75yB+19U6DAulr2RhRw5GD74U1znP00eGZ8TJf
RN1FYilUPCawMYeQoB8WIn9So7zIm0MfOl4KXNWDX02+Kw==
-----END CERTIFICATE-----

30
integration/h2/config.xml Normal file
View File

@@ -0,0 +1,30 @@
<configuration version="1">
<repository directory="s2">
<node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1">
<address>127.0.0.1:22001</address>
</node>
<node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ" name="s2">
<address>127.0.0.1:22002</address>
</node>
<node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3">
<address>127.0.0.1:22003</address>
</node>
</repository>
<options>
<listenAddress>127.0.0.2:22002</listenAddress>
<readOnly>false</readOnly>
<allowDelete>true</allowDelete>
<followSymlinks>true</followSymlinks>
<guiEnabled>true</guiEnabled>
<guiAddress>127.0.0.1:8082</guiAddress>
<globalAnnounceServer>announce.syncthing.net:22025</globalAnnounceServer>
<globalAnnounceEnabled>false</globalAnnounceEnabled>
<localAnnounceEnabled>false</localAnnounceEnabled>
<parallelRequests>16</parallelRequests>
<maxSendKbps>0</maxSendKbps>
<rescanIntervalS>60</rescanIntervalS>
<reconnectionIntervalS>5</reconnectionIntervalS>
<maxChangeKbps>1000</maxChangeKbps>
<startBrowser>false</startBrowser>
</options>
</configuration>

39
integration/h2/key.pem Normal file
View File

@@ -0,0 +1,39 @@
-----BEGIN RSA PRIVATE KEY-----
MIIG5AIBAAKCAYEAsIV0syyRO56BvIOro4bIqB6iFJsNc4zX8MiM4QPTWgqGlYws
KSVmNppTdlACZCJIqyzoscrFqJPto8/e2Fc3oaTdEREGIs7cmc7LSXfot/mAgPpy
71SVWtb7xNmXro2JJPZjRBCSpl1ulPug+/8w7fSKQdLMjh4Hp2YlwVBfVu0bYEEW
+7Vl9PZVTv+NbTqXYvYVc9R6QFIbN/njWAuo2wpjJlY7vqNnSYZyskAaaAC17fFJ
kVQKKblTeTk1C9PxTmVTB1j9yOoD3+V/6IrTYKXdTHGJ1MqdieTHj1jHXe5TOeSB
+Hjgq4tr25mPfQ4ixXqDqIcx5390DAjInuSKNUJ5pqiFrVe9eIDmySZCg5/JIL3c
8phy6g1bxiJN14+Dn0om/0+9UrHK8LVzWMmtFRVycWVUYmARWFY3EE10k0RXU2Ht
zmjfnBkRrl13b0ExizlA1qJ33ngxF5rNEDSMpwf4og5uYOjRUPYuvCL9XtQKr254
NFO/sg/qqPV4hFWTAgMBAAECggGAH6SMuuGuVyWe1BA2YGX06k4zd8Yjryb8Pql0
t5Fb/bQNVBmAgQ+3NuqLM5Y8F38dz7GJNPXIYOPDoa3NoLJhwpQvHLQUiYDTgq7T
OiRIj1ImevhqSgS7kUEgeLUYv62XfAy+1qCx6Siuff5taT7hooZHkm0bRg6UCKoC
8phZvtdaJPMGD7EAydyuhi7BR2dNY+wBBHZ+Q7F0N6CP5GSSrFE8XM7wfsgD5+Y2
AUYEdchK1JCAQ5DxEXGrSPu8SpZ/SuhMjLc3/JDwB8SZPT0C1jX7YMeUiPONy9VK
J6Fdnl0FMhS9VJHocL4o5IU9OLoahAcpq/Z25arm9z7yyxUoO4nVUAl3H9N7+N7A
cwpbSgMld15bQ9iPV8MCB/eVKzfgLbWuhpZr6h6oJF9pgIq9DDCK/mc9KYzSGd1J
dOVuizi0dMYS+iOJRFR3kIrNW7dGCecniigZfrrprqqkycl7823VTi0zIU4CHbDm
ypu/b8sbs+h6mHN71muWAlmChz3hAoHBAN5Cm3ZZeQJj/p7Kb3sn6WAXlRqnnDz9
fJDaa3788o4VQ3ie4odDNzALF7bHhYnfovXWrh/4XGkjiW98GPczpEFEdYF9gCGO
mAaHV/unvtjbGF7Wk3xjgaXwPeKXGU8vZrQ4y41u5eZWpA1fwSK3T+AQ79t4R2jr
kRgFz7iIJ8iQGleI+F9X4PRjhoOSsdaUkJRB6pxvxcsiYIKxDi7VScTx8iD0pgwn
tgcQ0do1A0ZQsnJMtBnfIj1/J0sSMHEE9QKBwQDLUVyLmVjv0apqDxnNCw0laIFm
ofp7S/q4pXfDDg3SqrM05Wgm4CHijzKzoqvFLILQvI004LShRcNXTMAsAbbIRzcY
YbEOYytHB+k9WfEjAFJkNM4qB4w8erELKwnvjflodLgBw9k97cybhYIZCnwWIIHp
SwXPT9AI5Ck8E6wifo1nWjpgMZtg5PaH6yfGa+o+ahmetKoU+4ENzVeU95XuKbVa
x/6UW+wNbPqo3oEfV/K25U6WGHoGfX2X8wn/m2cCgcEAnlABXi5i9GH3ZnG5MJcA
M3L4wNCsiADirmb19LEFsFDTC2LY5hHpiG4OSSIbK1bBQ6zTwG/umvE2HtPdEI+X
KuoxbLfRAZYJEXVsJROZ6+s7k6nxycMzANh7rB+GZpHT7QEbdDWOyh/ioKgY8Lpz
yZ0mzEQDUWehpOPWzpElDUYfjURB7d+xm0Ic+TEPPVH7Ha9KBn3S/FsTNWQaPx+r
eP4BQpoggD30+VlwsKXcHES0ppeeHWODhxxAB8f/+zDVAoHBALJY3GVYTruPn30J
YgiK+S0nTttImwAs1fHCtBtV6KozMp/j3Ei9svuZwU/yEdsUAGw5+WO4+Lm/CGs7
2BbCKiPk1F9+0mFcfEoCloZKr0uUrLFZ4L7dgBZNSaASUNTiJTWLrR1fPuEkB6ck
pcpxeAew3ERYmvAPgt1JxyH737Mib8eJTkuzOCj2r4rqrClR4Fh/mZmtwMRHGh2R
UpJJ3CreS0cmyBo7yAS+4+HdzEZCT5Y/73+aWO/4hIMVnl+pYQKBwCpUb85zm5zg
UnZ8nBS22FLGTcvBs8hbyXUtioSNadNteuqk6jsN2F+Pwsh6eHbVHW4Lu9j6Gn+J
S1ss/ztgGkErvQF/9DpxMeYt01FpvZaUJthThQVQ9xvr9i7utgthtdspNvQ0fux1
9Xg2fhLnDz707PUt7OhmVW7d+XOfoc19mYZlN0IOHsqMUMphIW97Lp5QWlZXxr23
Zrv2j5mTvv3Fq2TRDNfz5dwijFMvv7kpGfHA1950ZIbobQvYYsoC7A==
-----END RSA PRIVATE KEY-----

23
integration/h3/cert.pem Normal file
View File

@@ -0,0 +1,23 @@
-----BEGIN CERTIFICATE-----
MIID3jCCAkigAwIBAgIBADALBgkqhkiG9w0BAQUwFDESMBAGA1UEAxMJc3luY3Ro
aW5nMB4XDTE0MDMxNDA3MDExMVoXDTQ5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJ
c3luY3RoaW5nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAypziXAfA
GG5mEwTIo+dUt3a4SxKnCVWfTXq7RF5ukCdz3jRRRoqy2ujbW5oe7qmqELTUzkJU
eJkuCyvLIhVsKgt2r1peThy5+3/kUst9Faitnke1/jYtWwSf7WmK9TQBN/tFxdgR
SM0LpdvsIJQqB5eodR51KrsjkJcKzEHeE2IQA8EKD1LcfXz8p22zdaZ4CNhz7mgF
ghkRT6NINDZiMtm6R4qXqYyW1MNlYLw+DqXvE2eCQMQOcqI7SSZTeS3eSTP5NQum
AmaLKn3JZQ9G6Ldn0VSWSPlXqVSaFTI9LnYICTDa4Nj4+L6idwzjmBkxPr8vBjlj
15eo6xcU4fKqh2/xHIcZYpas1bwo5ljGdcfRm4L1zpxDY+nJFIgRv0Ndpltve+T8
o3qiqFvIMiphFUtdb2pzgaTi8FW9SHMRFiuj6sNLI0Yb9u8BoN/QsWZnFh6fQoHW
PKtkJrPZGw4GMMTHeEQe/eLW9VAH7ywPyc2hSzSri44aR44s4ErSV+XfAgMBAAGj
PzA9MA4GA1UdDwEB/wQEAwIAoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH
AwIwDAYDVR0TAQH/BAIwADALBgkqhkiG9w0BAQUDggGBAJshUcNfAveVE115S/NQ
h+HfqahB3Gw69KmSWgFNXAEyXwOftmz9p2gL6DwfbU1SkbakU/YLJ3/mP4Q69EmX
axzr108hUJ2qxH091IfKPCuTDtLsFY7g0PI+SBzADFSkoFe79aF/PSjaZk6fjvEl
44PXzhxqqnSS+9lXQk+DIS6/jSVuX1XYvkkdHRXd9d7nw42sONDdSX8oQAUvGOeR
CBleWB4AqS6HH2zFVIzRvjDsSg2wUXQjcqMqef7zBMpuWKTIQf+uaXWsEuPK2a3V
IPwreWGwkxqGTToLagqOdAhgN+owvTJS9PaqxFqlxp8ypIYBnZ5VL5G5p115C17/
xVENRxX0/oB99L5HGRhAuIGE4LhXq858DzmTZFcwm3fHragPKS/MMI71Z6U5RH2p
VouosWzxAEx3Piq+ccenPLt6VPO5iFn+ppoBfwAFtNKyTrXcTQm33oJWFz+NSMAl
vCq5H4c9x5CoWDpX2bdjpnZ3xcSj84fPYpkKBI52eZAvcw==
-----END CERTIFICATE-----

30
integration/h3/config.xml Normal file
View File

@@ -0,0 +1,30 @@
<configuration version="1">
<repository directory="s3">
<node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1">
<address>127.0.0.1:22001</address>
</node>
<node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ" name="s2">
<address>127.0.0.1:22002</address>
</node>
<node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3">
<address>127.0.0.1:22003</address>
</node>
</repository>
<options>
<listenAddress>127.0.0.1:22003</listenAddress>
<readOnly>false</readOnly>
<allowDelete>true</allowDelete>
<followSymlinks>true</followSymlinks>
<guiEnabled>true</guiEnabled>
<guiAddress>127.0.0.1:8083</guiAddress>
<globalAnnounceServer>announce.syncthing.net:22025</globalAnnounceServer>
<globalAnnounceEnabled>false</globalAnnounceEnabled>
<localAnnounceEnabled>false</localAnnounceEnabled>
<parallelRequests>16</parallelRequests>
<maxSendKbps>0</maxSendKbps>
<rescanIntervalS>60</rescanIntervalS>
<reconnectionIntervalS>5</reconnectionIntervalS>
<maxChangeKbps>1000</maxChangeKbps>
<startBrowser>false</startBrowser>
</options>
</configuration>

39
integration/h3/key.pem Normal file
View File

@@ -0,0 +1,39 @@
-----BEGIN RSA PRIVATE KEY-----
MIIG5AIBAAKCAYEAypziXAfAGG5mEwTIo+dUt3a4SxKnCVWfTXq7RF5ukCdz3jRR
Roqy2ujbW5oe7qmqELTUzkJUeJkuCyvLIhVsKgt2r1peThy5+3/kUst9Faitnke1
/jYtWwSf7WmK9TQBN/tFxdgRSM0LpdvsIJQqB5eodR51KrsjkJcKzEHeE2IQA8EK
D1LcfXz8p22zdaZ4CNhz7mgFghkRT6NINDZiMtm6R4qXqYyW1MNlYLw+DqXvE2eC
QMQOcqI7SSZTeS3eSTP5NQumAmaLKn3JZQ9G6Ldn0VSWSPlXqVSaFTI9LnYICTDa
4Nj4+L6idwzjmBkxPr8vBjlj15eo6xcU4fKqh2/xHIcZYpas1bwo5ljGdcfRm4L1
zpxDY+nJFIgRv0Ndpltve+T8o3qiqFvIMiphFUtdb2pzgaTi8FW9SHMRFiuj6sNL
I0Yb9u8BoN/QsWZnFh6fQoHWPKtkJrPZGw4GMMTHeEQe/eLW9VAH7ywPyc2hSzSr
i44aR44s4ErSV+XfAgMBAAECggGBALolpuXsjPUlQIyKoZfMag3gefMnIOW3j5NM
hg6LP8MbLB3jLSTFOwtaUmZ3U6HrqP6OVNFnKVpfSWkkBA29ZtG+FH2IZgoX5FsH
JgtXPwWOImy/75mtxr/PoOsrQ2qCK/h116WsHD0pfWEVi3xnA7JUCIYJxJXMtyEZ
U+dTQKfIOXRpf0eS1lZIZYuhgvY9Shy+WMyZLy5Wv4vONQEbUd7sIHOoBizUqKKz
HkngyJcGpn7KY2YDek6hdByBpME5fFRo1DmXPeE+pTycDBP3C55JTB6v6kGkqxAl
hTFK+x8tU9N+d+Wmuvfdm4MdQ1CnLjv+TL01OTYJx4nTDRk8PkVxVfrXdSY/LFib
4mhghDPglvPHyFljbwkdiah60dAd2dIUhZwax79oxliyV6Ivf3teB5bvvrWIsMCX
harESDSC6sgkHRGb05u1uYOPpsnAVOmi6CVgsKQtTmRZ6tqBA/O97RBWj92ROJi+
0PqFs+cLXvpd4u36sI0MeWYO7Eko2QKBwQDo1Mc/fUsquniFrkjjXnGC8Q+tUpRC
/TS2vjP5zFlgIOD8fSOoNF4VwFapCoG8l5NsOZrFOYvzHKZRtuBHLL0xdAOuxzge
72mJYDsfO+vwOtg8xMrdtlJqDOnQWU2NVgOB+xSQKILEOm422BRyXbOM3hylsL7Y
GPSTdqst9jcLODd6VFY4xq9VDebO63z5Ku62K+BerUJsUfrs64ES6U//Pivq4c06
57BQq1AbVXqd8q9aInOsSQ1nJwg28PuBREUCgcEA3sZQRJSpktsdsnlvk4Nvf+KK
OdYEAa2hPcflWDn/ZiEzaP0NR5s6NFSYhzGWVAX39lPUuHJ9MZ17CO4GqMojbXJc
hneWi7+SWNqZn1rxdCl8FU7wJccX479G45lEL0aIN91i+9K0ZFGZg198EQcpX4kt
s8J3O3n81rLnHg4bgMhJ6BGRIAxtYwmdOSfTmdyewVJfNUookswjC0sD5JDL/jd1
Yi4CeaFOqRFVY+4ge+whYz9OcfBBj4kBl1OYoq3TAoHAZe35CJd+l8coykVhjYCk
KxIDrfpQA/+72yDrujk3C3l5ZbAXMyUx503b4odCAuFM3f1d/2fRF+579ZwdFavF
a+gBULvQmuJvDoA9gdAG25W3Yus4cNXHwLvEhL0D0ZNNV9MmznGdxfBandH9KZBR
8aKvYe49rndGY2R5TlbTBCtpRjmzwYlh6td6Ky42+RJNjR0qTeiGAsvlEWGMkU3p
ArIiIeMWqOoTa02EdWL3mjxLfidFArC8mGMjGoJnaNENAoHBAIQ9ZnJ/aPXS+hry
uQfw0qQwXuscHr68SeW5nmuz6ea/OJxO8q4Z+AAOY4iFJ/5ymJHxi2l+FND58YoI
eY2CiGs0orXzkTsdmgsCoISW4JOa+JxRgn56Y7T+217JoU8K6Ft5IIPpvMl8Ist4
R9Z30Nh0PtvhSRPWQv9TrZwKtcrJmg2XN/W1Ss2qbFj8SkCgVODfO8MPZWxWn0rZ
2dK5HU6nrxd7xl5bIa18q5qpRUEql1spvjAmdVR1+KrSpd2TnwKBwDGptuY2og9y
PeEipbGrxww44DMFp6026xn0r8NXSlylD8Rihm9z2wQ0vFPjG1VHjupcQYI5bQLB
piYt1PyPzCs27WrnXU6eEFjhVP4CzZHFKHkPeW6uBt7XbHpR14hxJ54FLN33Rewr
Vt6xys+K3OHFzgDfBnhtvROYHtNQENiyJ3rMwRZcxXROmR/zj0cVl1R8wdE3XCwA
rFPm/n3hzKv2OG3KX0KHh1bH0eXPVhIEmPcgz8RgL3uPGkqh8H+sgQ==
-----END RSA PRIVATE KEY-----

39
integration/json.go Normal file
View File

@@ -0,0 +1,39 @@
package main
import (
"encoding/json"
"flag"
"fmt"
"log"
"os"
"strconv"
"strings"
)
func main() {
log.SetFlags(0)
flag.Parse()
path := strings.Split(flag.Arg(0), "/")
var obj map[string]interface{}
dec := json.NewDecoder(os.Stdin)
dec.UseNumber()
dec.Decode(&obj)
var v interface{} = obj
for _, p := range path {
switch tv := v.(type) {
case map[string]interface{}:
v = tv[p]
case []interface{}:
i, err := strconv.Atoi(p)
if err != nil {
log.Fatal(err)
}
v = tv[i]
default:
return // Silence is golden
}
}
fmt.Println(v)
}

View File

@@ -9,7 +9,10 @@ import (
"path/filepath"
)
var long bool
func main() {
flag.BoolVar(&long, "l", false, "Long output")
flag.Parse()
args := flag.Args()
@@ -37,7 +40,12 @@ func walker(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
fmt.Printf("%s %s\n", sum, path)
if long {
fmt.Printf("%s %s 0%03o %d\n", sum, path, info.Mode(), info.ModTime().Unix())
} else {
fmt.Printf("%s %s\n", sum, path)
}
}
return nil

View File

@@ -1,74 +1,94 @@
#!/bin/bash
rm -rf files-* conf-* md5-*
export STNORESTART=1
extraopts=""
p=$(pwd)
id1=I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA
id2=JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ
id3=373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA
go build genfiles.go
go build md5r.go
go build json.go
echo "Setting up (keys)..."
i1=$(syncthing --home conf-1 2>&1 | awk '/My ID/ {print $7}')
echo $i1
i2=$(syncthing --home conf-2 2>&1 | awk '/My ID/ {print $7}')
echo $i2
i3=$(syncthing --home conf-3 2>&1 | awk '/My ID/ {print $7}')
echo $i3
echo "Setting up (files)..."
for i in 1 2 3 ; do
cat >conf-$i/syncthing.ini <<EOT
[repository]
dir = $p/files-$i
[nodes]
$i1 = 127.0.0.1:22001
$i2 = 127.0.0.1:22002
$i3 = 127.0.0.1:22003
[settings]
gui-enabled = false
listen-address = :2200$i
EOT
mkdir files-$i
pushd files-$i >/dev/null
../genfiles -maxexp 21 -files 400
touch empty-$i
../md5r > ../md5-$i
popd >/dev/null
done
echo "Starting..."
for i in 1 2 3 ; do
sleep 1
syncthing --home conf-$i $extraopts &
done
cat md5-* | sort > md5-tot
while true ; do
read
echo Verifying...
conv=0
testConvergence() {
echo "Starting..."
for i in 1 2 3 ; do
pushd files-$i >/dev/null
../md5r | sort > ../md5-$i
popd >/dev/null
if ! cmp md5-$i md5-tot >/dev/null ; then
echo $i unconverged
else
conv=$((conv + 1))
echo $i converged
sleep 1
syncthing -home "h$i" &
done
while true ; do
sleep 5
s1comp=$(curl -s "http://localhost:8082/rest/connections" | ./json "$id1/Completion")
s2comp=$(curl -s "http://localhost:8083/rest/connections" | ./json "$id2/Completion")
s3comp=$(curl -s "http://localhost:8081/rest/connections" | ./json "$id3/Completion")
s1comp=${s1comp:-0}
s2comp=${s2comp:-0}
s3comp=${s3comp:-0}
tot=$(($s1comp + $s2comp + $s3comp))
echo $tot / 300
if [[ $tot == 300 ]] ; then
echo "Stopping..."
pkill syncthing
break
fi
done
if [[ $conv == 3 ]] ; then
kill %1
kill %2
kill %3
exit
fi
echo "Verifying..."
cat md5-* | sort | uniq > md5-tot
for i in 1 2 3 ; do
pushd "s$i" >/dev/null
../md5r -l | sort > ../md5-$i
popd >/dev/null
if ! cmp "md5-$i" md5-tot >/dev/null ; then
echo "Fail: instance $i unconverged"
diff -u md5-tot "md5-$i"
exit
else
echo "OK: instance $i converged"
fi
done
}
echo "Setting up files..."
for i in 1 2 3 ; do
rm -f h$i/*.idx.gz
rm -rf "s$i"
mkdir "s$i"
pushd "s$i" >/dev/null
echo " $i: random nonoverlapping"
../genfiles -maxexp 22 -files 600
echo " $i: empty file"
touch "empty-$i"
echo " $i: common file"
dd if=/dev/urandom of=common bs=1000 count=1000 2>/dev/null
popd >/dev/null
done
# instance 1 common file should be the newest, the other should disappear
sleep 2
touch "s1/common"
echo "MD5-summing..."
for i in 1 2 3 ; do
pushd "s$i" >/dev/null
../md5r -l > ../md5-$i
popd >/dev/null
done
grep -v common md5-2 > t ; mv t md5-2
grep -v common md5-3 > t ; mv t md5-3
testConvergence
echo "Add and remove random files..."
for i in 1 2 3 ; do
pushd "s$i" >/dev/null
rm -rf */?[02468ace]
../genfiles -maxexp 22 -files 600
../md5r -l > ../md5-$i
popd >/dev/null
done
testConvergence

2
protocol/doc.go Normal file
View File

@@ -0,0 +1,2 @@
// Package protocol implements the Block Exchange Protocol.
package protocol

View File

@@ -26,8 +26,8 @@ const (
)
const (
FlagDeleted = 1 << 12
FlagInvalid = 1 << 13
FlagDeleted uint32 = 1 << 12
FlagInvalid = 1 << 13
)
var (

View File

@@ -1,4 +1,4 @@
package main
package scanner
import (
"bytes"

View File

@@ -1,4 +1,4 @@
package main
package scanner
import (
"bytes"

12
scanner/debug.go Normal file
View File

@@ -0,0 +1,12 @@
package scanner
import (
"log"
"os"
"strings"
)
var (
dlog = log.New(os.Stderr, "scanner: ", log.Lmicroseconds|log.Lshortfile)
debug = strings.Contains(os.Getenv("STTRACE"), "scanner")
)

2
scanner/doc.go Normal file
View File

@@ -0,0 +1,2 @@
// Package scanner implements a file system scanner and hasher.
package scanner

26
scanner/file.go Normal file
View File

@@ -0,0 +1,26 @@
package scanner
import "fmt"
type File struct {
Name string
Flags uint32
Modified int64
Version uint32
Size int64
Blocks []Block
Suppressed bool
}
func (f File) String() string {
return fmt.Sprintf("File{Name:%q, Flags:0x%x, Modified:%d, Version:%d, Size:%d, NumBlocks:%d}",
f.Name, f.Flags, f.Modified, f.Version, f.Size, len(f.Blocks))
}
func (f File) Equals(o File) bool {
return f.Modified == o.Modified && f.Version == o.Version
}
func (f File) NewerThan(o File) bool {
return f.Modified > o.Modified || (f.Modified == o.Modified && f.Version > o.Version)
}

0
scanner/testdata/.foo/bar vendored Normal file
View File

2
scanner/testdata/.stignore vendored Normal file
View File

@@ -0,0 +1,2 @@
.*
quux

1
scanner/testdata/bar vendored Normal file
View File

@@ -0,0 +1 @@
foobarbaz

1
scanner/testdata/baz/quux vendored Normal file
View File

@@ -0,0 +1 @@
baazquux

0
scanner/testdata/empty vendored Normal file
View File

1
scanner/testdata/foo vendored Normal file
View File

@@ -0,0 +1 @@
foobar

262
scanner/walk.go Normal file
View File

@@ -0,0 +1,262 @@
package scanner
import (
"bytes"
"io/ioutil"
"log"
"os"
"path"
"path/filepath"
"strings"
"time"
)
type Walker struct {
// Dir is the base directory for the walk
Dir string
// If FollowSymlinks is true, symbolic links directly under Dir will be followed.
// Symbolic links at deeper levels are never followed regardless of this flag.
FollowSymlinks bool
// BlockSize controls the size of the block used when hashing.
BlockSize int
// If IgnoreFile is not empty, it is the name used for the file that holds ignore patterns.
IgnoreFile string
// If TempNamer is not nil, it is used to ignore tempory files when walking.
TempNamer TempNamer
// If CurrentFiler is not nil, it is queried for the current file before rescanning.
CurrentFiler CurrentFiler
// If Suppressor is not nil, it is queried for supression of modified files.
// Suppressed files will be returned with empty metadata and the Suppressed flag set.
// Requires CurrentFiler to be set.
Suppressor Suppressor
suppressed map[string]bool // file name -> suppression status
}
type TempNamer interface {
// Temporary returns a temporary name for the filed referred to by path.
TempName(path string) string
// IsTemporary returns true if path refers to the name of temporary file.
IsTemporary(path string) bool
}
type Suppressor interface {
// Supress returns true if the update to the named file should be ignored.
Suppress(name string, fi os.FileInfo) bool
}
type CurrentFiler interface {
// CurrentFile returns the file as seen at last scan.
CurrentFile(name string) File
}
// Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed.
func (w *Walker) Walk() (files []File, ignore map[string][]string) {
w.lazyInit()
if debug {
dlog.Println("Walk", w.Dir, w.FollowSymlinks, w.BlockSize, w.IgnoreFile)
}
t0 := time.Now()
ignore = make(map[string][]string)
hashFiles := w.walkAndHashFiles(&files, ignore)
filepath.Walk(w.Dir, w.loadIgnoreFiles(w.Dir, ignore))
filepath.Walk(w.Dir, hashFiles)
if w.FollowSymlinks {
d, err := os.Open(w.Dir)
if err != nil {
return
}
defer d.Close()
fis, err := d.Readdir(-1)
if err != nil {
return
}
for _, info := range fis {
if info.Mode()&os.ModeSymlink != 0 {
dir := path.Join(w.Dir, info.Name()) + "/"
filepath.Walk(dir, w.loadIgnoreFiles(dir, ignore))
filepath.Walk(dir, hashFiles)
}
}
}
if debug {
t1 := time.Now()
d := t1.Sub(t0).Seconds()
dlog.Printf("Walk in %.02f ms, %.0f files/s", d*1000, float64(len(files))/d)
}
return
}
// CleanTempFiles removes all files that match the temporary filename pattern.
func (w *Walker) CleanTempFiles() {
filepath.Walk(w.Dir, w.cleanTempFile)
}
func (w *Walker) lazyInit() {
if w.suppressed == nil {
w.suppressed = make(map[string]bool)
}
}
func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.WalkFunc {
return func(p string, info os.FileInfo, err error) error {
if err != nil {
return nil
}
rn, err := filepath.Rel(dir, p)
if err != nil {
return nil
}
if pn, sn := path.Split(rn); sn == w.IgnoreFile {
pn := strings.Trim(pn, "/")
bs, _ := ioutil.ReadFile(p)
lines := bytes.Split(bs, []byte("\n"))
var patterns []string
for _, line := range lines {
if len(line) > 0 {
patterns = append(patterns, string(line))
}
}
ign[pn] = patterns
}
return nil
}
}
func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
return func(p string, info os.FileInfo, err error) error {
if err != nil {
if debug {
dlog.Println("error:", p, info, err)
}
return nil
}
rn, err := filepath.Rel(w.Dir, p)
if err != nil {
if debug {
dlog.Println("rel error:", p, err)
}
return nil
}
if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
if debug {
dlog.Println("temporary:", rn)
}
return nil
}
if _, sn := path.Split(rn); sn == w.IgnoreFile {
if debug {
dlog.Println("ignorefile:", rn)
}
return nil
}
if rn != "." && w.ignoreFile(ign, rn) {
if debug {
dlog.Println("ignored:", rn)
}
if info.IsDir() {
return filepath.SkipDir
}
return nil
}
if info.Mode()&os.ModeType == 0 {
if w.CurrentFiler != nil {
cf := w.CurrentFiler.CurrentFile(rn)
if cf.Modified == info.ModTime().Unix() {
if debug {
dlog.Println("unchanged:", rn)
}
*res = append(*res, cf)
return nil
}
if w.Suppressor != nil && w.Suppressor.Suppress(rn, info) {
if debug {
dlog.Println("suppressed:", rn)
}
if !w.suppressed[rn] {
w.suppressed[rn] = true
log.Printf("INFO: Changes to %q are being temporarily suppressed because it changes too frequently.", p)
}
cf.Suppressed = true
*res = append(*res, cf)
} else if w.suppressed[rn] {
log.Printf("INFO: Changes to %q are no longer suppressed.", p)
delete(w.suppressed, rn)
}
}
fd, err := os.Open(p)
if err != nil {
if debug {
dlog.Println("open:", p, err)
}
return nil
}
defer fd.Close()
t0 := time.Now()
blocks, err := Blocks(fd, w.BlockSize)
if err != nil {
if debug {
dlog.Println("hash error:", rn, err)
}
return nil
}
if debug {
t1 := time.Now()
dlog.Println("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s")
}
f := File{
Name: rn,
Size: info.Size(),
Flags: uint32(info.Mode()),
Modified: info.ModTime().Unix(),
Blocks: blocks,
}
*res = append(*res, f)
}
return nil
}
}
func (w *Walker) cleanTempFile(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Mode()&os.ModeType == 0 && w.TempNamer.IsTemporary(path) {
os.Remove(path)
}
return nil
}
func (w *Walker) ignoreFile(patterns map[string][]string, file string) bool {
first, last := path.Split(file)
for prefix, pats := range patterns {
if len(prefix) == 0 || prefix == first || strings.HasPrefix(first, prefix+"/") {
for _, pattern := range pats {
if match, _ := path.Match(pattern, last); match {
return true
}
}
}
}
return false
}

View File

@@ -1,4 +1,4 @@
package main
package scanner
import (
"fmt"
@@ -22,8 +22,12 @@ var correctIgnores = map[string][]string{
}
func TestWalk(t *testing.T) {
m := NewModel("testdata", 1e6)
files, ignores := m.Walk(false)
w := Walker{
Dir: "testdata",
BlockSize: 128 * 1024,
IgnoreFile: ".stignore",
}
files, ignores := w.Walk()
if l1, l2 := len(files), len(testdata); l1 != l2 {
t.Fatalf("Incorrect number of walked files %d != %d", l1, l2)
@@ -75,8 +79,9 @@ func TestIgnore(t *testing.T) {
{"foo/bazz/quux", false},
}
w := Walker{}
for i, tc := range tests {
if r := ignoreFile(patterns, tc.f); r != tc.r {
if r := w.ignoreFile(patterns, tc.f); r != tc.r {
t.Errorf("Incorrect ignoreFile() #%d; E: %v, A: %v", i, tc.r, r)
}
}

238
walk.go
View File

@@ -1,238 +0,0 @@
package main
import (
"bytes"
"fmt"
"io/ioutil"
"log"
"os"
"path"
"path/filepath"
"strings"
"time"
"github.com/calmh/syncthing/protocol"
)
const BlockSize = 128 * 1024
type File struct {
Name string
Flags uint32
Modified int64
Version uint32
Size int64
Blocks []Block
}
func (f File) String() string {
return fmt.Sprintf("File{Name:%q, Flags:0x%x, Modified:%d, Version:%d, Size:%d, NumBlocks:%d}",
f.Name, f.Flags, f.Modified, f.Version, f.Size, len(f.Blocks))
}
func (f File) Equals(o File) bool {
return f.Modified == o.Modified && f.Version == o.Version
}
func (f File) NewerThan(o File) bool {
return f.Modified > o.Modified || (f.Modified == o.Modified && f.Version > o.Version)
}
func isTempName(name string) bool {
return strings.HasPrefix(path.Base(name), ".syncthing.")
}
func tempName(name string, modified int64) string {
tdir := path.Dir(name)
tname := fmt.Sprintf(".syncthing.%s.%d", path.Base(name), modified)
return path.Join(tdir, tname)
}
func (m *Model) loadIgnoreFiles(ign map[string][]string) filepath.WalkFunc {
return func(p string, info os.FileInfo, err error) error {
if err != nil {
return nil
}
rn, err := filepath.Rel(m.dir, p)
if err != nil {
return nil
}
if pn, sn := path.Split(rn); sn == ".stignore" {
pn := strings.Trim(pn, "/")
bs, _ := ioutil.ReadFile(p)
lines := bytes.Split(bs, []byte("\n"))
var patterns []string
for _, line := range lines {
if len(line) > 0 {
patterns = append(patterns, string(line))
}
}
ign[pn] = patterns
}
return nil
}
}
func (m *Model) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
return func(p string, info os.FileInfo, err error) error {
if err != nil {
if m.trace["file"] {
log.Printf("FILE: %q: %v", p, err)
}
return nil
}
if isTempName(p) {
return nil
}
rn, err := filepath.Rel(m.dir, p)
if err != nil {
return nil
}
if _, sn := path.Split(rn); sn == ".stignore" {
// We never sync the .stignore files
return nil
}
if ignoreFile(ign, rn) {
if m.trace["file"] {
log.Println("FILE: IGNORE:", rn)
}
return nil
}
if info.Mode()&os.ModeType == 0 {
modified := info.ModTime().Unix()
m.lmut.RLock()
lf, ok := m.local[rn]
m.lmut.RUnlock()
if ok && lf.Modified == modified {
if nf := uint32(info.Mode()); nf != lf.Flags {
lf.Flags = nf
lf.Version++
}
*res = append(*res, lf)
} else {
if cur, prev := m.sup.suppress(rn, info.Size(), time.Now()); cur {
if m.trace["file"] {
log.Printf("FILE: SUPPRESS: %q change bw over threshold", rn)
}
if !prev {
log.Printf("INFO: Changes to %q are being temporarily suppressed because it changes too frequently.", rn)
}
if ok {
lf.Flags = protocol.FlagInvalid
lf.Version++
*res = append(*res, lf)
}
return nil
} else if prev && !cur {
log.Printf("INFO: Changes to %q are no longer suppressed.", rn)
}
if m.trace["file"] {
log.Printf("FILE: Hash %q", p)
}
fd, err := os.Open(p)
if err != nil {
if m.trace["file"] {
log.Printf("FILE: %q: %v", p, err)
}
return nil
}
defer fd.Close()
blocks, err := Blocks(fd, BlockSize)
if err != nil {
if m.trace["file"] {
log.Printf("FILE: %q: %v", p, err)
}
return nil
}
f := File{
Name: rn,
Size: info.Size(),
Flags: uint32(info.Mode()),
Modified: modified,
Blocks: blocks,
}
*res = append(*res, f)
}
}
return nil
}
}
// Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed.
func (m *Model) Walk(followSymlinks bool) (files []File, ignore map[string][]string) {
ignore = make(map[string][]string)
hashFiles := m.walkAndHashFiles(&files, ignore)
filepath.Walk(m.dir, m.loadIgnoreFiles(ignore))
filepath.Walk(m.dir, hashFiles)
if followSymlinks {
d, err := os.Open(m.dir)
if err != nil {
return
}
defer d.Close()
fis, err := d.Readdir(-1)
if err != nil {
return
}
for _, info := range fis {
if info.Mode()&os.ModeSymlink != 0 {
dir := path.Join(m.dir, info.Name()) + "/"
filepath.Walk(dir, m.loadIgnoreFiles(ignore))
filepath.Walk(dir, hashFiles)
}
}
}
return
}
func (m *Model) cleanTempFile(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.Mode()&os.ModeType == 0 && isTempName(path) {
if m.trace["file"] {
log.Printf("FILE: Remove %q", path)
}
os.Remove(path)
}
return nil
}
func (m *Model) cleanTempFiles() {
filepath.Walk(m.dir, m.cleanTempFile)
}
func ignoreFile(patterns map[string][]string, file string) bool {
first, last := path.Split(file)
for prefix, pats := range patterns {
if len(prefix) == 0 || prefix == first || strings.HasPrefix(first, prefix+"/") {
for _, pattern := range pats {
if match, _ := path.Match(pattern, last); match {
return true
}
}
}
}
return false
}

2
xdr/doc.go Normal file
View File

@@ -0,0 +1,2 @@
// Package xdr implements an XDR (RFC 4506) encoder/decoder.
package xdr