Reimplement quick startup scan

Fix isTempName to work on Windows (fixes #80)

.gitignore

@@ -1,3 +1,4 @@
 syncthing
+syncthing.exe
 *.tar.gz
-dist
+*.zip

README.md

@@ -1,4 +1,4 @@
-syncthing
+syncthing [![Build Status](https://drone.io/github.com/calmh/syncthing/status.png)](https://drone.io/github.com/calmh/syncthing/latest)
 =========
 
 This is the `syncthing` project. The following are the project goals:

assets.sh

@@ -18,7 +18,7 @@ cd gui
 for f in $(find . -type f) ; do 
 	f="${f#./}"
 	echo "gr, _ = gzip.NewReader(bytes.NewBuffer([]byte{"
-	gzip -c $f | od -vt x1 | sed 's/^[0-9a-f]*//' | sed 's/\([0-9a-f][0-9a-f]\)/0x\1,/g'
+	gzip -n -c $f | od -vt x1 | sed 's/^[0-9a-f]*//' | sed 's/\([0-9a-f][0-9a-f]\)/0x\1,/g'
 	echo "}))"
 	echo "data, _ = ioutil.ReadAll(gr)"
 	echo "Assets[\"$f\"] = data"

auto/doc.go

@@ -0,0 +1,2 @@
+// Package auto contains auto generated files for web assets.
+package auto

buffers/buffers.go

@@ -1,3 +1,4 @@
+// Package buffers manages a set of reusable byte buffers.
 package buffers
 
 const (

build.sh

@@ -2,48 +2,90 @@
 
 export COPYFILE_DISABLE=true
 
+distFiles=(README.md LICENSE) # apart from the binary itself
 version=$(git describe --always)
-buildDir=dist
 
-if [[ $fast != yes ]] ; then
+build() {
+	go build -ldflags "-w -X main.Version $version" ./cmd/syncthing	
+}
+
+prepare() {
 	./assets.sh | gofmt > auto/gui.files.go
 	go get -d
+}
+
+test() {
 	go test ./...
-fi
+}	
 
-if [[ -z $1 ]] ; then
-	go build -ldflags "-X main.Version $version"
-elif [[ $1 == "tar" ]] ; then
-	go build -ldflags "-X main.Version $version" \
-	&& mkdir syncthing-dist \
-	&& cp syncthing README.md LICENSE syncthing-dist \
-	&& tar zcvf syncthing-dist.tar.gz syncthing-dist \
-	&& rm -rf syncthing-dist
-elif [[ $1 == "all" ]] ; then
-	rm -rf "$buildDir"
-	mkdir -p "$buildDir" || exit 1
+tarDist() {
+	name="$1"
+	mkdir -p "$name"
+	cp syncthing "${distFiles[@]}" "$name"
+	tar zcvf "$name.tar.gz" "$name"
+	rm -rf "$name"
+}
 
-	export GOARM=7
-	for os in darwin-amd64 linux-amd64 linux-arm freebsd-amd64 windows-amd64 ; do
-		echo "$os"
-		export name="syncthing-$os"
-		export GOOS=${os%-*}
-		export GOARCH=${os#*-}
-		go build -ldflags "-X main.Version $version"
-		mkdir -p "$name"
-		cp README.md LICENSE "$name"
-		case $GOOS in
-			windows)
-				cp syncthing.exe "$buildDir/$name.exe"
-				mv syncthing.exe "$name"
-				zip -qr "$buildDir/$name.zip" "$name"
-				;;
-			*)
-				cp syncthing "$buildDir/$name"
-				mv syncthing "$name"
-				tar zcf "$buildDir/$name.tar.gz" "$name"
-				;;
-		esac
-		rm -r  "$name"
-	done
-fi
+zipDist() {
+	name="$1"
+	mkdir -p "$name"
+	cp syncthing.exe "${distFiles[@]}" "$name"
+	zip -r "$name.zip" "$name"
+	rm -rf "$name"
+}
+
+case "$1" in
+	"")
+		build
+		;;
+
+	tar)
+		rm -f *.tar.gz *.zip
+		prepare
+		test || exit 1
+		build
+
+		eval $(go env)
+		name="syncthing-$GOOS-$GOARCH-$version"
+
+		tarDist "$name"
+		;;
+
+	all)
+		rm -f *.tar.gz *.zip
+		prepare
+		test || exit 1
+
+		export GOARM=7
+		for os in darwin-amd64 linux-amd64 linux-arm freebsd-amd64 windows-amd64 ; do
+			export GOOS=${os%-*}
+			export GOARCH=${os#*-}
+
+			build
+			
+			name="syncthing-$os-$version"
+			case $GOOS in
+				windows)
+					zipDist "$name"
+					rm -f syncthing.exe
+					;;
+				*)
+					tarDist "$name"
+					rm -f syncthing
+					;;
+			esac
+		done
+		;;
+
+	upload)
+		tag=$(git describe)
+		shopt -s nullglob
+		for f in *gz *zip ; do
+			relup calmh/syncthing "$tag" "$f"
+		done
+		;;
+
+	*)
+		echo "Unknown build parameter $1"
+		;;
+esac

cid/cid.go

@@ -0,0 +1,43 @@
+// Package cid provides a manager for mappings between node ID:s and connection ID:s.
+package cid
+
+type Map struct {
+	toCid  map[string]int
+	toName []string
+}
+
+func NewMap() *Map {
+	return &Map{
+		toCid: make(map[string]int),
+	}
+}
+
+func (m *Map) Get(name string) int {
+	cid, ok := m.toCid[name]
+	if ok {
+		return cid
+	}
+
+	// Find a free slot to get a new ID
+	for i, n := range m.toName {
+		if n == "" {
+			m.toName[i] = name
+			m.toCid[name] = i
+			return i
+		}
+	}
+
+	// Add it to the end since we didn't find a free slot
+	m.toName = append(m.toName, name)
+	cid = len(m.toName) - 1
+	m.toCid[name] = cid
+	return cid
+}
+
+func (m *Map) Clear(name string) {
+	cid, ok := m.toCid[name]
+	if ok {
+		m.toName[cid] = ""
+		delete(m.toCid, name)
+	}
+}

cmd/.gitignore

@@ -0,0 +1 @@
+!syncthing

cmd/syncthing/config.go

@@ -36,7 +36,7 @@ type OptionsConfiguration struct {
 	FollowSymlinks     bool     `xml:"followSymlinks" default:"true" ini:"follow-symlinks"`
 	GUIEnabled         bool     `xml:"guiEnabled" default:"true" ini:"gui-enabled"`
 	GUIAddress         string   `xml:"guiAddress" default:"127.0.0.1:8080" ini:"gui-address"`
-	GlobalAnnServer    string   `xml:"globalAnnounceServer" default:"syncthing.nym.se:22025" ini:"global-announce-server"`
+	GlobalAnnServer    string   `xml:"globalAnnounceServer" default:"announce.syncthing.net:22025" ini:"global-announce-server"`
 	GlobalAnnEnabled   bool     `xml:"globalAnnounceEnabled" default:"true" ini:"global-announce-enabled"`
 	LocalAnnEnabled    bool     `xml:"localAnnounceEnabled" default:"true" ini:"local-announce-enabled"`
 	ParallelRequests   int      `xml:"parallelRequests" default:"16" ini:"parallel-requests"`
@@ -44,6 +44,7 @@ type OptionsConfiguration struct {
 	RescanIntervalS    int      `xml:"rescanIntervalS" default:"60" ini:"rescan-interval"`
 	ReconnectIntervalS int      `xml:"reconnectionIntervalS" default:"60" ini:"reconnection-interval"`
 	MaxChangeKbps      int      `xml:"maxChangeKbps" default:"1000" ini:"max-change-bw"`
+	StartBrowser       bool     `xml:"startBrowser" default:"true"`
 }
 
 func setDefaults(data interface{}) error {
@@ -60,11 +61,6 @@ func setDefaults(data interface{}) error {
 			case string:
 				f.SetString(v)
 
-			case []string:
-				rv := reflect.MakeSlice(reflect.TypeOf([]string{}), 1, 1)
-				rv.Index(0).SetString(v)
-				f.Set(rv)
-
 			case int:
 				i, err := strconv.ParseInt(v, 10, 64)
 				if err != nil {
@@ -75,6 +71,11 @@ func setDefaults(data interface{}) error {
 			case bool:
 				f.SetBool(v == "true")
 
+			case []string:
+				// We don't do anything with string slices here. Any default
+				// we set will be appended to by the XML decoder, so we fill
+				// those after decoding.
+
 			default:
 				panic(f.Type())
 			}
@@ -83,6 +84,30 @@ func setDefaults(data interface{}) error {
 	return nil
 }
 
+// fillNilSlices sets default value on slices that are still nil.
+func fillNilSlices(data interface{}) error {
+	s := reflect.ValueOf(data).Elem()
+	t := s.Type()
+
+	for i := 0; i < s.NumField(); i++ {
+		f := s.Field(i)
+		tag := t.Field(i).Tag
+
+		v := tag.Get("default")
+		if len(v) > 0 {
+			switch f.Interface().(type) {
+			case []string:
+				if f.IsNil() {
+					rv := reflect.MakeSlice(reflect.TypeOf([]string{}), 1, 1)
+					rv.Index(0).SetString(v)
+					f.Set(rv)
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func readConfigINI(m map[string]string, data interface{}) error {
 	s := reflect.ValueOf(data).Elem()
 	t := s.Type()
@@ -154,6 +179,8 @@ func readConfigXML(rd io.Reader) (Configuration, error) {
 		err = xml.NewDecoder(rd).Decode(&cfg)
 	}
 
+	fillNilSlices(&cfg.Options)
+
 	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
 	return cfg, err
 }

cmd/syncthing/config_test.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"bytes"
+	"io"
+	"reflect"
+	"testing"
+)
+
+func TestDefaultValues(t *testing.T) {
+	expected := OptionsConfiguration{
+		ListenAddress:      []string{":22000"},
+		ReadOnly:           false,
+		AllowDelete:        true,
+		FollowSymlinks:     true,
+		GUIEnabled:         true,
+		GUIAddress:         "127.0.0.1:8080",
+		GlobalAnnServer:    "announce.syncthing.net:22025",
+		GlobalAnnEnabled:   true,
+		LocalAnnEnabled:    true,
+		ParallelRequests:   16,
+		MaxSendKbps:        0,
+		RescanIntervalS:    60,
+		ReconnectIntervalS: 60,
+		MaxChangeKbps:      1000,
+		StartBrowser:       true,
+	}
+
+	cfg, err := readConfigXML(bytes.NewReader(nil))
+	if err != io.EOF {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(cfg.Options, expected) {
+		t.Errorf("Default config differs;\n  E: %#v\n  A: %#v", expected, cfg.Options)
+	}
+}
+
+func TestNoListenAddress(t *testing.T) {
+	data := []byte(`<configuration version="1">
+    <repository directory="~/Sync">
+        <node id="..." name="...">
+            <address>dynamic</address>
+        </node>
+    </repository>
+    <options>
+        <listenAddress></listenAddress>
+    </options>
+</configuration>
+`)
+
+	cfg, err := readConfigXML(bytes.NewReader(data))
+	if err != nil {
+		t.Error(err)
+	}
+
+	expected := []string{""}
+	if !reflect.DeepEqual(cfg.Options.ListenAddress, expected) {
+		t.Errorf("Unexpected ListenAddress %#v", cfg.Options.ListenAddress)
+	}
+}
+
+func TestOverriddenValues(t *testing.T) {
+	data := []byte(`<configuration version="1">
+    <repository directory="~/Sync">
+        <node id="..." name="...">
+            <address>dynamic</address>
+        </node>
+    </repository>
+    <options>
+       <listenAddress>:23000</listenAddress>
+        <readOnly>true</readOnly>
+        <allowDelete>false</allowDelete>
+        <followSymlinks>false</followSymlinks>
+        <guiEnabled>false</guiEnabled>
+        <guiAddress>125.2.2.2:8080</guiAddress>
+        <globalAnnounceServer>syncthing.nym.se:22025</globalAnnounceServer>
+        <globalAnnounceEnabled>false</globalAnnounceEnabled>
+        <localAnnounceEnabled>false</localAnnounceEnabled>
+        <parallelRequests>32</parallelRequests>
+        <maxSendKbps>1234</maxSendKbps>
+        <rescanIntervalS>600</rescanIntervalS>
+        <reconnectionIntervalS>6000</reconnectionIntervalS>
+        <maxChangeKbps>2345</maxChangeKbps>
+        <startBrowser>false</startBrowser>
+    </options>
+</configuration>
+`)
+
+	expected := OptionsConfiguration{
+		ListenAddress:      []string{":23000"},
+		ReadOnly:           true,
+		AllowDelete:        false,
+		FollowSymlinks:     false,
+		GUIEnabled:         false,
+		GUIAddress:         "125.2.2.2:8080",
+		GlobalAnnServer:    "syncthing.nym.se:22025",
+		GlobalAnnEnabled:   false,
+		LocalAnnEnabled:    false,
+		ParallelRequests:   32,
+		MaxSendKbps:        1234,
+		RescanIntervalS:    600,
+		ReconnectIntervalS: 6000,
+		MaxChangeKbps:      2345,
+		StartBrowser:       false,
+	}
+
+	cfg, err := readConfigXML(bytes.NewReader(data))
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(cfg.Options, expected) {
+		t.Errorf("Overridden config differs;\n  E: %#v\n  A: %#v", expected, cfg.Options)
+	}
+}

cmd/syncthing/debug.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"log"
+	"os"
+	"strings"
+)
+
+var (
+	dlog      = log.New(os.Stderr, "main: ", log.Lmicroseconds|log.Lshortfile)
+	debugNet  = strings.Contains(os.Getenv("STTRACE"), "net")
+	debugIdx  = strings.Contains(os.Getenv("STTRACE"), "idx")
+	debugNeed = strings.Contains(os.Getenv("STTRACE"), "need")
+	debugPull = strings.Contains(os.Getenv("STTRACE"), "pull")
+)

cmd/syncthing/filemonitor.go

@@ -4,13 +4,13 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"log"
 	"os"
 	"path"
 	"sync"
 	"time"
 
 	"github.com/calmh/syncthing/buffers"
+	"github.com/calmh/syncthing/scanner"
 )
 
 type fileMonitor struct {
@@ -18,18 +18,18 @@ type fileMonitor struct {
 	path        string // full path
 	writeDone   sync.WaitGroup
 	model       *Model
-	global      File
-	localBlocks []Block
+	global      scanner.File
+	localBlocks []scanner.Block
 	copyError   error
 	writeError  error
 }
 
 func (m *fileMonitor) FileBegins(cc <-chan content) error {
-	if m.model.trace["file"] {
-		log.Printf("FILE: FileBegins: " + m.name)
+	if debugPull {
+		dlog.Println("file begins:", m.name)
 	}
 
-	tmp := tempName(m.path, m.global.Modified)
+	tmp := defTempNamer.TempName(m.path)
 
 	dir := path.Dir(tmp)
 	_, err := os.Stat(dir)
@@ -109,13 +109,13 @@ func (m *fileMonitor) copyRemoteBlocks(cc <-chan content, outFile *os.File, writ
 }
 
 func (m *fileMonitor) FileDone() error {
-	if m.model.trace["file"] {
-		log.Printf("FILE: FileDone: " + m.name)
+	if debugPull {
+		dlog.Println("file done:", m.name)
 	}
 
 	m.writeDone.Wait()
 
-	tmp := tempName(m.path, m.global.Modified)
+	tmp := defTempNamer.TempName(m.path)
 	defer os.Remove(tmp)
 
 	if m.copyError != nil {
@@ -149,14 +149,14 @@ func (m *fileMonitor) FileDone() error {
 	return nil
 }
 
-func hashCheck(name string, correct []Block) error {
+func hashCheck(name string, correct []scanner.Block) error {
 	rf, err := os.Open(name)
 	if err != nil {
 		return err
 	}
 	defer rf.Close()
 
-	current, err := Blocks(rf, BlockSize)
+	current, err := scanner.Blocks(rf, BlockSize)
 	if err != nil {
 		return err
 	}

cmd/syncthing/filequeue.go

@@ -5,6 +5,8 @@ import (
 	"sort"
 	"sync"
 	"time"
+
+	"github.com/calmh/syncthing/scanner"
 )
 
 type Monitor interface {
@@ -23,7 +25,7 @@ type FileQueue struct {
 
 type queuedFile struct {
 	name         string
-	blocks       []Block
+	blocks       []scanner.Block
 	activeBlocks []bool
 	given        int
 	remaining    int
@@ -54,7 +56,7 @@ func (l queuedFileList) Less(a, b int) bool {
 
 type queuedBlock struct {
 	name  string
-	block Block
+	block scanner.Block
 	index int
 }
 
@@ -65,7 +67,7 @@ func NewFileQueue() *FileQueue {
 	}
 }
 
-func (q *FileQueue) Add(name string, blocks []Block, monitor Monitor) {
+func (q *FileQueue) Add(name string, blocks []scanner.Block, monitor Monitor) {
 	q.fmut.Lock()
 	defer q.fmut.Unlock()
 

cmd/syncthing/filequeue_test.go

@@ -5,6 +5,8 @@ import (
 	"sync"
 	"sync/atomic"
 	"testing"
+
+	"github.com/calmh/syncthing/scanner"
 )
 
 func TestFileQueueAdd(t *testing.T) {
@@ -17,8 +19,8 @@ func TestFileQueueAddSorting(t *testing.T) {
 	q.SetAvailable("zzz", []string{"nodeID"})
 	q.SetAvailable("aaa", []string{"nodeID"})
 
-	q.Add("zzz", []Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
-	q.Add("aaa", []Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
+	q.Add("zzz", []scanner.Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
+	q.Add("aaa", []scanner.Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
 	b, _ := q.Get("nodeID")
 	if b.name != "aaa" {
 		t.Errorf("Incorrectly sorted get: %+v", b)
@@ -28,12 +30,12 @@ func TestFileQueueAddSorting(t *testing.T) {
 	q.SetAvailable("zzz", []string{"nodeID"})
 	q.SetAvailable("aaa", []string{"nodeID"})
 
-	q.Add("zzz", []Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
+	q.Add("zzz", []scanner.Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
 	b, _ = q.Get("nodeID") // Start on zzzz
 	if b.name != "zzz" {
 		t.Errorf("Incorrectly sorted get: %+v", b)
 	}
-	q.Add("aaa", []Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
+	q.Add("aaa", []scanner.Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
 	b, _ = q.Get("nodeID")
 	if b.name != "zzz" {
 		// Continue rather than starting a new file
@@ -56,12 +58,12 @@ func TestFileQueueGet(t *testing.T) {
 	q.SetAvailable("foo", []string{"nodeID"})
 	q.SetAvailable("bar", []string{"nodeID"})
 
-	q.Add("foo", []Block{
+	q.Add("foo", []scanner.Block{
 		{Offset: 0, Size: 128, Hash: []byte("some foo hash bytes")},
 		{Offset: 128, Size: 128, Hash: []byte("some other foo hash bytes")},
 		{Offset: 256, Size: 128, Hash: []byte("more foo hash bytes")},
 	}, nil)
-	q.Add("bar", []Block{
+	q.Add("bar", []scanner.Block{
 		{Offset: 0, Size: 128, Hash: []byte("some bar hash bytes")},
 		{Offset: 128, Size: 128, Hash: []byte("some other bar hash bytes")},
 	}, nil)
@@ -70,7 +72,7 @@ func TestFileQueueGet(t *testing.T) {
 
 	expected := queuedBlock{
 		name: "bar",
-		block: Block{
+		block: scanner.Block{
 			Offset: 0,
 			Size:   128,
 			Hash:   []byte("some bar hash bytes"),
@@ -89,7 +91,7 @@ func TestFileQueueGet(t *testing.T) {
 
 	expected = queuedBlock{
 		name: "bar",
-		block: Block{
+		block: scanner.Block{
 			Offset: 128,
 			Size:   128,
 			Hash:   []byte("some other bar hash bytes"),
@@ -109,7 +111,7 @@ func TestFileQueueGet(t *testing.T) {
 
 	expected = queuedBlock{
 		name: "foo",
-		block: Block{
+		block: scanner.Block{
 			Offset: 0,
 			Size:   128,
 			Hash:   []byte("some foo hash bytes"),
@@ -150,7 +152,7 @@ func TestFileQueueDone(t *testing.T) {
 	}()
 
 	q := FileQueue{resolver: fakeResolver{}}
-	q.Add("foo", []Block{
+	q.Add("foo", []scanner.Block{
 		{Offset: 0, Length: 128, Hash: []byte("some foo hash bytes")},
 		{Offset: 128, Length: 128, Hash: []byte("some other foo hash bytes")},
 	}, ch)
@@ -181,19 +183,19 @@ func TestFileQueueGetNodeIDs(t *testing.T) {
 	q.SetAvailable("a-foo", []string{"nodeID", "a"})
 	q.SetAvailable("b-bar", []string{"nodeID", "b"})
 
-	q.Add("a-foo", []Block{
+	q.Add("a-foo", []scanner.Block{
 		{Offset: 0, Size: 128, Hash: []byte("some foo hash bytes")},
 		{Offset: 128, Size: 128, Hash: []byte("some other foo hash bytes")},
 		{Offset: 256, Size: 128, Hash: []byte("more foo hash bytes")},
 	}, nil)
-	q.Add("b-bar", []Block{
+	q.Add("b-bar", []scanner.Block{
 		{Offset: 0, Size: 128, Hash: []byte("some bar hash bytes")},
 		{Offset: 128, Size: 128, Hash: []byte("some other bar hash bytes")},
 	}, nil)
 
 	expected := queuedBlock{
 		name: "b-bar",
-		block: Block{
+		block: scanner.Block{
 			Offset: 0,
 			Size:   128,
 			Hash:   []byte("some bar hash bytes"),
@@ -209,7 +211,7 @@ func TestFileQueueGetNodeIDs(t *testing.T) {
 
 	expected = queuedBlock{
 		name: "a-foo",
-		block: Block{
+		block: scanner.Block{
 			Offset: 0,
 			Size:   128,
 			Hash:   []byte("some foo hash bytes"),
@@ -225,7 +227,7 @@ func TestFileQueueGetNodeIDs(t *testing.T) {
 
 	expected = queuedBlock{
 		name: "a-foo",
-		block: Block{
+		block: scanner.Block{
 			Offset: 128,
 			Size:   128,
 			Hash:   []byte("some other foo hash bytes"),
@@ -246,9 +248,9 @@ func TestFileQueueThreadHandling(t *testing.T) {
 
 	const n = 100
 	var total int
-	var blocks []Block
+	var blocks []scanner.Block
 	for i := 1; i <= n; i++ {
-		blocks = append(blocks, Block{Offset: int64(i), Size: 1})
+		blocks = append(blocks, scanner.Block{Offset: int64(i), Size: 1})
 		total += i
 	}
 
@@ -272,7 +274,7 @@ func TestFileQueueThreadHandling(t *testing.T) {
 	close(start)
 	wg.Wait()
 	if int(gotTot) != total {
-		t.Error("Total mismatch; %d != %d", gotTot, total)
+		t.Errorf("Total mismatch; %d != %d", gotTot, total)
 	}
 }
 
@@ -283,13 +285,13 @@ func TestDeleteAt(t *testing.T) {
 		q.files = queuedFileList{{name: "a"}, {name: "b"}, {name: "c"}, {name: "d"}}
 		q.deleteAt(i)
 		if l := len(q.files); l != 3 {
-			t.Fatal("deleteAt(%d) failed; %d != 3", i, l)
+			t.Fatalf("deleteAt(%d) failed; %d != 3", i, l)
 		}
 	}
 
 	q.files = queuedFileList{{name: "a"}}
 	q.deleteAt(0)
 	if l := len(q.files); l != 0 {
-		t.Fatal("deleteAt(only) failed; %d != 0", l)
+		t.Fatalf("deleteAt(only) failed; %d != 0", l)
 	}
 }

cmd/syncthing/gui.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/calmh/syncthing/scanner"
 	"github.com/codegangsta/martini"
 )
 
@@ -107,7 +108,7 @@ func restPostRestart(req *http.Request) {
 	restart()
 }
 
-type guiFile File
+type guiFile scanner.File
 
 func (f guiFile) MarshalJSON() ([]byte, error) {
 	type t struct {
@@ -116,7 +117,7 @@ func (f guiFile) MarshalJSON() ([]byte, error) {
 	}
 	return json.Marshal(t{
 		Name: f.Name,
-		Size: File(f).Size,
+		Size: scanner.File(f).Size,
 	})
 }
 

cmd/syncthing/gui_solaris.go

@@ -0,0 +1,92 @@
+//+build solaris
+
+package main
+
+import (
+	"encoding/binary"
+	"fmt"
+	"os"
+	"time"
+)
+
+type id_t int32
+type ulong_t uint32
+
+type timestruc_t struct {
+	Tv_sec  int64
+	Tv_nsec int64
+}
+
+func (tv timestruc_t) Nano() int64 {
+	return tv.Tv_sec*1e9 + tv.Tv_nsec
+}
+
+type prusage_t struct {
+	Pr_lwpid    id_t        /* lwp id.  0: process or defunct */
+	Pr_count    int32       /* number of contributing lwps */
+	Pr_tstamp   timestruc_t /* real time stamp, time of read() */
+	Pr_create   timestruc_t /* process/lwp creation time stamp */
+	Pr_term     timestruc_t /* process/lwp termination time stamp */
+	Pr_rtime    timestruc_t /* total lwp real (elapsed) time */
+	Pr_utime    timestruc_t /* user level CPU time */
+	Pr_stime    timestruc_t /* system call CPU time */
+	Pr_ttime    timestruc_t /* other system trap CPU time */
+	Pr_tftime   timestruc_t /* text page fault sleep time */
+	Pr_dftime   timestruc_t /* data page fault sleep time */
+	Pr_kftime   timestruc_t /* kernel page fault sleep time */
+	Pr_ltime    timestruc_t /* user lock wait sleep time */
+	Pr_slptime  timestruc_t /* all other sleep time */
+	Pr_wtime    timestruc_t /* wait-cpu (latency) time */
+	Pr_stoptime timestruc_t /* stopped time */
+	Pr_minf     ulong_t     /* minor page faults */
+	Pr_majf     ulong_t     /* major page faults */
+	Pr_nswap    ulong_t     /* swaps */
+	Pr_inblk    ulong_t     /* input blocks */
+	Pr_oublk    ulong_t     /* output blocks */
+	Pr_msnd     ulong_t     /* messages sent */
+	Pr_mrcv     ulong_t     /* messages received */
+	Pr_sigs     ulong_t     /* signals received */
+	Pr_vctx     ulong_t     /* voluntary context switches */
+	Pr_ictx     ulong_t     /* involuntary context switches */
+	Pr_sysc     ulong_t     /* system calls */
+	Pr_ioch     ulong_t     /* chars read and written */
+
+}
+
+func solarisPrusage(pid int, rusage *prusage_t) error {
+	fd, err := os.Open(fmt.Sprintf("/proc/%d/usage", pid))
+	if err != nil {
+		return err
+	}
+	err = binary.Read(fd, binary.LittleEndian, rusage)
+	fd.Close()
+	return err
+}
+
+func init() {
+	go trackCPUUsage()
+}
+
+func trackCPUUsage() {
+	var prevUsage int64
+	var prevTime = time.Now().UnixNano()
+	var rusage prusage_t
+	var pid = os.Getpid()
+	for {
+		time.Sleep(10 * time.Second)
+		err := solarisPrusage(pid, &rusage)
+		if err != nil {
+			warnln(err)
+			continue
+		}
+		curTime := time.Now().UnixNano()
+		timeDiff := curTime - prevTime
+		curUsage := rusage.Pr_utime.Nano() + rusage.Pr_stime.Nano()
+		usageDiff := curUsage - prevUsage
+		cpuUsageLock.Lock()
+		cpuUsagePercent = 100 * float64(usageDiff) / float64(timeDiff)
+		cpuUsageLock.Unlock()
+		prevTime = curTime
+		prevUsage = curUsage
+	}
+}

cmd/syncthing/logger.go

@@ -13,16 +13,6 @@ func init() {
 	logger = log.New(os.Stderr, "", log.Flags())
 }
 
-func debugln(vals ...interface{}) {
-	s := fmt.Sprintln(vals...)
-	logger.Output(2, "DEBUG: "+s)
-}
-
-func debugf(format string, vals ...interface{}) {
-	s := fmt.Sprintf(format, vals...)
-	logger.Output(2, "DEBUG: "+s)
-}
-
 func infoln(vals ...interface{}) {
 	s := fmt.Sprintln(vals...)
 	logger.Output(2, "INFO: "+s)

cmd/syncthing/main.go

@@ -21,37 +21,55 @@ import (
 	"github.com/calmh/ini"
 	"github.com/calmh/syncthing/discover"
 	"github.com/calmh/syncthing/protocol"
+	"github.com/calmh/syncthing/scanner"
 )
 
+const BlockSize = 128 * 1024
+
 var cfg Configuration
-var Version string = "unknown-dev"
+var Version = "unknown-dev"
 
 var (
-	myID   string
-	config ini.Config
+	myID string
 )
 
 var (
-	showVersion  bool
-	confDir      string
-	trace        string
-	profiler     string
-	verbose      bool
-	startupDelay int
+	showVersion bool
+	confDir     string
+	verbose     bool
+)
+
+const (
+	usage      = "syncthing [options]"
+	extraUsage = `The following enviroment variables are interpreted by syncthing:
+
+ STNORESTART  Do not attempt to restart when requested to, instead just exit.
+              Set this variable when running under a service manager such as
+              runit, launchd, etc.
+
+ STPROFILER   Set to a listen address such as "127.0.0.1:9090" to start the
+              profiler with HTTP access.
+
+ STTRACE      A comma separated string of facilities to trace. The valid
+              facility strings:
+              - "scanner"  (the file change scanner)
+              - "discover" (the node discovery package)
+              - "net"      (connecting and disconnecting, network messages)
+              - "idx"      (index sending and receiving)
+              - "need"     (file need calculations)
+              - "pull"     (file pull activity)`
 )
 
 func main() {
-	flag.StringVar(&confDir, "home", "~/.syncthing", "Set configuration directory")
-	flag.StringVar(&trace, "debug.trace", "", "(connect,net,idx,file,pull)")
-	flag.StringVar(&profiler, "debug.profiler", "", "(addr)")
+	flag.StringVar(&confDir, "home", getDefaultConfDir(), "Set configuration directory")
 	flag.BoolVar(&showVersion, "version", false, "Show version")
 	flag.BoolVar(&verbose, "v", false, "Be more verbose")
-	flag.IntVar(&startupDelay, "delay", 0, "Startup delay (s)")
-	flag.Usage = usageFor(flag.CommandLine, "syncthing [options]")
+	flag.Usage = usageFor(flag.CommandLine, usage, extraUsage)
 	flag.Parse()
 
-	if startupDelay > 0 {
-		time.Sleep(time.Duration(startupDelay) * time.Second)
+	if len(os.Getenv("STRESTART")) > 0 {
+		// Give the parent process time to exit and release sockets etc.
+		time.Sleep(1 * time.Second)
 	}
 
 	if showVersion {
@@ -67,10 +85,6 @@ func main() {
 		runtime.GOMAXPROCS(runtime.NumCPU())
 	}
 
-	if len(trace) > 0 {
-		log.SetFlags(log.Lshortfile | log.Ldate | log.Ltime | log.Lmicroseconds)
-		logger.SetFlags(log.Lshortfile | log.Ldate | log.Ltime | log.Lmicroseconds)
-	}
 	confDir = expandTilde(confDir)
 
 	// Ensure that our home directory exists and that we have a certificate and key.
@@ -83,7 +97,7 @@ func main() {
 		fatalErr(err)
 	}
 
-	myID = string(certId(cert.Certificate[0]))
+	myID = string(certID(cert.Certificate[0]))
 	log.SetPrefix("[" + myID[0:5] + "] ")
 	logger.SetPrefix("[" + myID[0:5] + "] ")
 
@@ -139,7 +153,7 @@ func main() {
 		cfg, err = readConfigXML(nil)
 		cfg.Repositories = []RepositoryConfiguration{
 			{
-				Directory: "~/Sync",
+				Directory: path.Join(getHomeDir(), "Sync"),
 				Nodes: []NodeConfiguration{
 					{NodeID: myID, Addresses: []string{"dynamic"}},
 				},
@@ -155,11 +169,12 @@ func main() {
 
 	var dir = expandTilde(cfg.Repositories[0].Directory)
 
-	if len(profiler) > 0 {
+	if profiler := os.Getenv("STPROFILER"); len(profiler) > 0 {
 		go func() {
+			dlog.Println("Starting profiler on", profiler)
 			err := http.ListenAndServe(profiler, nil)
 			if err != nil {
-				warnln(err)
+				dlog.Fatal(err)
 			}
 		}()
 	}
@@ -179,25 +194,34 @@ func main() {
 
 	ensureDir(dir, -1)
 	m := NewModel(dir, cfg.Options.MaxChangeKbps*1000)
-	for _, t := range strings.Split(trace, ",") {
-		m.Trace(t)
-	}
 	if cfg.Options.MaxSendKbps > 0 {
 		m.LimitRate(cfg.Options.MaxSendKbps)
 	}
 
 	// GUI
 	if cfg.Options.GUIEnabled && cfg.Options.GUIAddress != "" {
-		host, port, err := net.SplitHostPort(cfg.Options.GUIAddress)
+		addr, err := net.ResolveTCPAddr("tcp", cfg.Options.GUIAddress)
 		if err != nil {
 			warnf("Cannot start GUI on %q: %v", cfg.Options.GUIAddress, err)
 		} else {
-			if len(host) > 0 {
-				infof("Starting web GUI on http://%s", cfg.Options.GUIAddress)
-			} else {
-				infof("Starting web GUI on port %s", port)
+			var hostOpen, hostShow string
+			switch {
+			case addr.IP == nil:
+				hostOpen = "localhost"
+				hostShow = "0.0.0.0"
+			case addr.IP.IsUnspecified():
+				hostOpen = "localhost"
+				hostShow = addr.IP.String()
+			default:
+				hostOpen = addr.IP.String()
+				hostShow = hostOpen
 			}
+
+			infof("Starting web GUI on http://%s:%d/", hostShow, addr.Port)
 			startGUI(cfg.Options.GUIAddress, m)
+			if cfg.Options.StartBrowser && len(os.Getenv("STRESTART")) == 0 {
+				openURL(fmt.Sprintf("http://%s:%d", hostOpen, addr.Port))
+			}
 		}
 	}
 
@@ -208,7 +232,18 @@ func main() {
 		infoln("Populating repository index")
 	}
 	loadIndex(m)
-	updateLocalModel(m)
+
+	sup := &suppressor{threshold: int64(cfg.Options.MaxChangeKbps)}
+	w := &scanner.Walker{
+		Dir:            m.dir,
+		IgnoreFile:     ".stignore",
+		FollowSymlinks: cfg.Options.FollowSymlinks,
+		BlockSize:      BlockSize,
+		TempNamer:      defTempNamer,
+		Suppressor:     sup,
+		CurrentFiler:   m,
+	}
+	updateLocalModel(m, w)
 
 	connOpts := map[string]string{
 		"clientId":      "syncthing",
@@ -254,7 +289,7 @@ func main() {
 		for {
 			time.Sleep(td)
 			if m.LocalAge() > (td / 2).Seconds() {
-				updateLocalModel(m)
+				updateLocalModel(m, w)
 			}
 		}
 	}()
@@ -269,24 +304,23 @@ func main() {
 
 func restart() {
 	infoln("Restarting")
-	args := os.Args
-	doAppend := true
-	for _, arg := range args {
-		if arg == "-delay" {
-			doAppend = false
-			break
-		}
+	if os.Getenv("SMF_FMRI") != "" || os.Getenv("STNORESTART") != "" {
+		// Solaris SMF
+		infoln("Service manager detected; exit instead of restart")
+		os.Exit(0)
 	}
-	if doAppend {
-		args = append(args, "-delay", "2")
+
+	env := os.Environ()
+	if len(os.Getenv("STRESTART")) == 0 {
+		env = append(env, "STRESTART=1")
 	}
 	pgm, err := exec.LookPath(os.Args[0])
 	if err != nil {
 		warnln(err)
 		return
 	}
-	proc, err := os.StartProcess(pgm, args, &os.ProcAttr{
-		Env:   os.Environ(),
+	proc, err := os.StartProcess(pgm, os.Args, &os.ProcAttr{
+		Env:   env,
 		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
 	})
 	if err != nil {
@@ -319,6 +353,13 @@ func saveConfigLoop(cfgFile string) {
 			continue
 		}
 
+		if runtime.GOOS == "windows" {
+			err := os.Remove(cfgFile)
+			if err != nil && !os.IsNotExist(err) {
+				warnln(err)
+			}
+		}
+
 		err = os.Rename(cfgFile+".tmp", cfgFile)
 		if err != nil {
 			warnln(err)
@@ -362,8 +403,8 @@ func printStatsLoop(m *Model) {
 }
 
 func listen(myID string, addr string, m *Model, tlsCfg *tls.Config, connOpts map[string]string) {
-	if strings.Contains(trace, "connect") {
-		debugln("NET: Listening on", addr)
+	if debugNet {
+		dlog.Println("listening on", addr)
 	}
 	l, err := tls.Listen("tcp", addr, tlsCfg)
 	fatalErr(err)
@@ -376,8 +417,8 @@ listen:
 			continue
 		}
 
-		if strings.Contains(trace, "connect") {
-			debugln("NET: Connect from", conn.RemoteAddr())
+		if debugNet {
+			dlog.Println("connect from", conn.RemoteAddr())
 		}
 
 		tc := conn.(*tls.Conn)
@@ -388,7 +429,7 @@ listen:
 			continue
 		}
 
-		remoteID := certId(tc.ConnectionState().PeerCertificates[0].Raw)
+		remoteID := certID(tc.ConnectionState().PeerCertificates[0].Raw)
 
 		if remoteID == myID {
 			warnf("Connect from myself (%s) - should not happen", remoteID)
@@ -458,18 +499,18 @@ func connect(myID string, disc *discover.Discoverer, m *Model, tlsCfg *tls.Confi
 					}
 				}
 
-				if strings.Contains(trace, "connect") {
-					debugln("NET: Dial", nodeCfg.NodeID, addr)
+				if debugNet {
+					dlog.Println("dial", nodeCfg.NodeID, addr)
 				}
 				conn, err := tls.Dial("tcp", addr, tlsCfg)
 				if err != nil {
-					if strings.Contains(trace, "connect") {
-						debugln("NET:", err)
+					if debugNet {
+						dlog.Println(err)
 					}
 					continue
 				}
 
-				remoteID := certId(conn.ConnectionState().PeerCertificates[0].Raw)
+				remoteID := certID(conn.ConnectionState().PeerCertificates[0].Raw)
 				if remoteID != nodeCfg.NodeID {
 					warnln("Unexpected nodeID", remoteID, "!=", nodeCfg.NodeID)
 					conn.Close()
@@ -486,8 +527,8 @@ func connect(myID string, disc *discover.Discoverer, m *Model, tlsCfg *tls.Confi
 	}
 }
 
-func updateLocalModel(m *Model) {
-	files, _ := m.Walk(cfg.Options.FollowSymlinks)
+func updateLocalModel(m *Model, w *scanner.Walker) {
+	files, _ := w.Walk()
 	m.ReplaceLocal(files)
 	saveIndex(m)
 }
@@ -502,7 +543,10 @@ func saveIndex(m *Model) {
 
 	gzw := gzip.NewWriter(idxf)
 
-	protocol.IndexMessage{"local", m.ProtocolIndex()}.EncodeXDR(gzw)
+	protocol.IndexMessage{
+		Repository: "local",
+		Files:      m.ProtocolIndex(),
+	}.EncodeXDR(gzw)
 	gzw.Close()
 	idxf.Close()
 	os.Rename(fullName+".tmp", fullName)
@@ -542,16 +586,38 @@ func ensureDir(dir string, mode int) {
 }
 
 func expandTilde(p string) string {
+	if runtime.GOOS == "windows" {
+		return p
+	}
+
 	if strings.HasPrefix(p, "~/") {
-		return strings.Replace(p, "~", getHomeDir(), 1)
+		return strings.Replace(p, "~", getUnixHomeDir(), 1)
 	}
 	return p
 }
 
-func getHomeDir() string {
+func getUnixHomeDir() string {
 	home := os.Getenv("HOME")
 	if home == "" {
 		fatalln("No home directory?")
 	}
 	return home
 }
+
+func getHomeDir() string {
+	if runtime.GOOS == "windows" {
+		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
+		if home == "" {
+			home = os.Getenv("USERPROFILE")
+		}
+		return home
+	}
+	return getUnixHomeDir()
+}
+
+func getDefaultConfDir() string {
+	if runtime.GOOS == "windows" {
+		return path.Join(os.Getenv("AppData"), "syncthing")
+	}
+	return expandTilde("~/.syncthing")
+}

cmd/syncthing/model.go

@@ -13,16 +13,17 @@ import (
 
 	"github.com/calmh/syncthing/buffers"
 	"github.com/calmh/syncthing/protocol"
+	"github.com/calmh/syncthing/scanner"
 )
 
 type Model struct {
 	dir string
 
-	global    map[string]File // the latest version of each file as it exists in the cluster
-	gmut      sync.RWMutex    // protects global
-	local     map[string]File // the files we currently have locally on disk
-	lmut      sync.RWMutex    // protects local
-	remote    map[string]map[string]File
+	global    map[string]scanner.File // the latest version of each file as it exists in the cluster
+	gmut      sync.RWMutex            // protects global
+	local     map[string]scanner.File // the files we currently have locally on disk
+	lmut      sync.RWMutex            // protects local
+	remote    map[string]map[string]scanner.File
 	rmut      sync.RWMutex // protects remote
 	protoConn map[string]Connection
 	rawConn   map[string]io.Closer
@@ -31,7 +32,7 @@ type Model struct {
 	// Queue for files to fetch. fq can call back into the model, so we must ensure
 	// to hold no locks when calling methods on fq.
 	fq *FileQueue
-	dq chan File // queue for files to delete
+	dq chan scanner.File // queue for files to delete
 
 	updatedLocal        int64 // timestamp of last update to local
 	updateGlobal        int64 // timestamp of last update to remote
@@ -43,8 +44,6 @@ type Model struct {
 	delete    bool
 	initmut   sync.Mutex // protects rwRunning and delete
 
-	trace map[string]bool
-
 	sup suppressor
 
 	parallelRequests int
@@ -64,9 +63,6 @@ type Connection interface {
 const (
 	idxBcastHoldtime = 15 * time.Second  // Wait at least this long after the last index modification
 	idxBcastMaxDelay = 120 * time.Second // Unless we've already waited this long
-
-	minFileHoldTimeS = 60  // Never allow file changes more often than this
-	maxFileHoldTimeS = 600 // Always allow file changes at least this often
 )
 
 var (
@@ -80,16 +76,15 @@ var (
 func NewModel(dir string, maxChangeBw int) *Model {
 	m := &Model{
 		dir:          dir,
-		global:       make(map[string]File),
-		local:        make(map[string]File),
-		remote:       make(map[string]map[string]File),
+		global:       make(map[string]scanner.File),
+		local:        make(map[string]scanner.File),
+		remote:       make(map[string]map[string]scanner.File),
 		protoConn:    make(map[string]Connection),
 		rawConn:      make(map[string]io.Closer),
 		lastIdxBcast: time.Now(),
-		trace:        make(map[string]bool),
 		sup:          suppressor{threshold: int64(maxChangeBw)},
 		fq:           NewFileQueue(),
-		dq:           make(chan File),
+		dq:           make(chan scanner.File),
 	}
 
 	go m.broadcastIndexLoop()
@@ -111,11 +106,6 @@ func (m *Model) LimitRate(kbps int) {
 	}()
 }
 
-// Trace enables trace logging of the given facility. This is a debugging function; grep for m.trace.
-func (m *Model) Trace(t string) {
-	m.trace[t] = true
-}
-
 // StartRW starts read/write processing on the current model. When in
 // read/write mode the model will attempt to keep in sync with the cluster by
 // pulling needed files from peer nodes.
@@ -131,7 +121,6 @@ func (m *Model) StartRW(del bool, threads int) {
 	m.delete = del
 	m.parallelRequests = threads
 
-	go m.cleanTempFiles()
 	if del {
 		go m.deleteLoop()
 	}
@@ -173,7 +162,9 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
 
 	var tot int64
 	for _, f := range m.global {
-		tot += f.Size
+		if f.Flags&protocol.FlagDeleted == 0 {
+			tot += f.Size
+		}
 	}
 
 	var res = make(map[string]ConnectionInfo)
@@ -189,12 +180,15 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
 
 		var have int64
 		for _, f := range m.remote[node] {
-			if f.Equals(m.global[f.Name]) {
+			if f.Equals(m.global[f.Name]) && f.Flags&protocol.FlagDeleted == 0 {
 				have += f.Size
 			}
 		}
 
-		ci.Completion = int(100 * have / tot)
+		ci.Completion = 100
+		if tot != 0 {
+			ci.Completion = int(100 * have / tot)
+		}
 
 		res[node] = ci
 	}
@@ -249,8 +243,10 @@ func (m *Model) InSyncSize() (files, bytes int64) {
 
 	for n, f := range m.local {
 		if gf, ok := m.global[n]; ok && f.Equals(gf) {
-			files++
-			bytes += f.Size
+			if f.Flags&protocol.FlagDeleted == 0 {
+				files++
+				bytes += f.Size
+			}
 		}
 	}
 
@@ -260,7 +256,7 @@ func (m *Model) InSyncSize() (files, bytes int64) {
 }
 
 // NeedFiles returns the list of currently needed files and the total size.
-func (m *Model) NeedFiles() (files []File, bytes int64) {
+func (m *Model) NeedFiles() (files []scanner.File, bytes int64) {
 	qf := m.fq.QueuedFiles()
 
 	m.gmut.RLock()
@@ -278,7 +274,7 @@ func (m *Model) NeedFiles() (files []File, bytes int64) {
 // Index is called when a new node is connected and we receive their full index.
 // Implements the protocol.Model interface.
 func (m *Model) Index(nodeID string, fs []protocol.FileInfo) {
-	var files = make([]File, len(fs))
+	var files = make([]scanner.File, len(fs))
 	for i := range fs {
 		files[i] = fileFromFileInfo(fs[i])
 	}
@@ -286,11 +282,11 @@ func (m *Model) Index(nodeID string, fs []protocol.FileInfo) {
 	m.imut.Lock()
 	defer m.imut.Unlock()
 
-	if m.trace["net"] {
-		debugf("NET IDX(in): %s: %d files", nodeID, len(fs))
+	if debugNet {
+		dlog.Printf("IDX(in): %s: %d files", nodeID, len(fs))
 	}
 
-	repo := make(map[string]File)
+	repo := make(map[string]scanner.File)
 	for _, f := range files {
 		m.indexUpdate(repo, f)
 	}
@@ -306,7 +302,7 @@ func (m *Model) Index(nodeID string, fs []protocol.FileInfo) {
 // IndexUpdate is called for incremental updates to connected nodes' indexes.
 // Implements the protocol.Model interface.
 func (m *Model) IndexUpdate(nodeID string, fs []protocol.FileInfo) {
-	var files = make([]File, len(fs))
+	var files = make([]scanner.File, len(fs))
 	for i := range fs {
 		files[i] = fileFromFileInfo(fs[i])
 	}
@@ -314,8 +310,8 @@ func (m *Model) IndexUpdate(nodeID string, fs []protocol.FileInfo) {
 	m.imut.Lock()
 	defer m.imut.Unlock()
 
-	if m.trace["net"] {
-		debugf("NET IDXUP(in): %s: %d files", nodeID, len(files))
+	if debugNet {
+		dlog.Printf("IDXUP(in): %s: %d files", nodeID, len(files))
 	}
 
 	m.rmut.Lock()
@@ -335,13 +331,13 @@ func (m *Model) IndexUpdate(nodeID string, fs []protocol.FileInfo) {
 	m.recomputeNeedForFiles(files)
 }
 
-func (m *Model) indexUpdate(repo map[string]File, f File) {
-	if m.trace["idx"] {
+func (m *Model) indexUpdate(repo map[string]scanner.File, f scanner.File) {
+	if debugIdx {
 		var flagComment string
 		if f.Flags&protocol.FlagDeleted != 0 {
 			flagComment = " (deleted)"
 		}
-		debugf("IDX(in): %q m=%d f=%o%s v=%d (%d blocks)", f.Name, f.Modified, f.Flags, flagComment, f.Version, len(f.Blocks))
+		dlog.Printf("IDX(in): %q m=%d f=%o%s v=%d (%d blocks)", f.Name, f.Modified, f.Flags, flagComment, f.Version, len(f.Blocks))
 	}
 
 	if extraFlags := f.Flags &^ (protocol.FlagInvalid | protocol.FlagDeleted | 0xfff); extraFlags != 0 {
@@ -355,8 +351,8 @@ func (m *Model) indexUpdate(repo map[string]File, f File) {
 // Close removes the peer from the model and closes the underlying connection if possible.
 // Implements the protocol.Model interface.
 func (m *Model) Close(node string, err error) {
-	if m.trace["net"] {
-		debugf("NET: %s: %v", node, err)
+	if debugNet {
+		dlog.Printf("%s: %v", node, err)
 	}
 	if err == protocol.ErrClusterHash {
 		warnf("Connection to %s closed due to mismatched cluster hash. Ensure that the configured cluster members are identical on both nodes.", node)
@@ -401,12 +397,12 @@ func (m *Model) Request(nodeID, repo, name string, offset int64, size int) ([]by
 		warnf("SECURITY (nonexistent file) REQ(in): %s: %q o=%d s=%d", nodeID, name, offset, size)
 		return nil, ErrNoSuchFile
 	}
-	if lf.Flags&protocol.FlagInvalid != 0 {
+	if lf.Suppressed {
 		return nil, ErrInvalid
 	}
 
-	if m.trace["net"] && nodeID != "<local>" {
-		debugf("NET REQ(in): %s: %q o=%d s=%d", nodeID, name, offset, size)
+	if debugNet && nodeID != "<local>" {
+		dlog.Printf("REQ(in): %s: %q o=%d s=%d", nodeID, name, offset, size)
 	}
 	fn := path.Join(m.dir, name)
 	fd, err := os.Open(fn) // XXX: Inefficient, should cache fd?
@@ -431,9 +427,9 @@ func (m *Model) Request(nodeID, repo, name string, offset int64, size int) ([]by
 }
 
 // ReplaceLocal replaces the local repository index with the given list of files.
-func (m *Model) ReplaceLocal(fs []File) {
+func (m *Model) ReplaceLocal(fs []scanner.File) {
 	var updated bool
-	var newLocal = make(map[string]File)
+	var newLocal = make(map[string]scanner.File)
 
 	m.lmut.RLock()
 	for _, f := range fs {
@@ -474,7 +470,7 @@ func (m *Model) ReplaceLocal(fs []File) {
 // the local index from a cache file at startup.
 func (m *Model) SeedLocal(fs []protocol.FileInfo) {
 	m.lmut.Lock()
-	m.local = make(map[string]File)
+	m.local = make(map[string]scanner.File)
 	for _, f := range fs {
 		m.local[f.Name] = fileFromFileInfo(f)
 	}
@@ -484,6 +480,14 @@ func (m *Model) SeedLocal(fs []protocol.FileInfo) {
 	m.recomputeNeedForGlobal()
 }
 
+// Implements scanner.CurrentFiler
+func (m *Model) CurrentFile(file string) scanner.File {
+	m.lmut.RLock()
+	f := m.local[file]
+	m.lmut.RUnlock()
+	return f
+}
+
 // ConnectedTo returns true if we are connected to the named node.
 func (m *Model) ConnectedTo(nodeID string) bool {
 	m.pmut.RLock()
@@ -509,6 +513,9 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn Connection) {
 
 	go func() {
 		idx := m.ProtocolIndex()
+		if debugNet {
+			dlog.Printf("IDX(out/initial): %s: %d files", nodeID, len(idx))
+		}
 		protoConn.Index("default", idx)
 	}()
 
@@ -522,14 +529,14 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn Connection) {
 	for i := 0; i < m.parallelRequests; i++ {
 		i := i
 		go func() {
-			if m.trace["pull"] {
-				debugln("PULL: Starting", nodeID, i)
+			if debugPull {
+				dlog.Println("starting puller:", nodeID, i)
 			}
 			for {
 				m.pmut.RLock()
 				if _, ok := m.protoConn[nodeID]; !ok {
-					if m.trace["pull"] {
-						debugln("PULL: Exiting", nodeID, i)
+					if debugPull {
+						dlog.Println("stopping puller:", nodeID, i)
 					}
 					m.pmut.RUnlock()
 					return
@@ -538,8 +545,8 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn Connection) {
 
 				qb, ok := m.fq.Get(nodeID)
 				if ok {
-					if m.trace["pull"] {
-						debugln("PULL: Request", nodeID, i, qb.name, qb.block.Offset)
+					if debugPull {
+						dlog.Println("request: out", nodeID, i, qb.name, qb.block.Offset)
 					}
 					data, _ := protoConn.Request("default", qb.name, qb.block.Offset, int(qb.block.Size))
 					m.fq.Done(qb.name, qb.block.Offset, data)
@@ -560,12 +567,12 @@ func (m *Model) ProtocolIndex() []protocol.FileInfo {
 
 	for _, f := range m.local {
 		mf := fileInfoFromFile(f)
-		if m.trace["idx"] {
+		if debugIdx {
 			var flagComment string
 			if mf.Flags&protocol.FlagDeleted != 0 {
 				flagComment = " (deleted)"
 			}
-			debugf("IDX(out): %q m=%d f=%o%s v=%d (%d blocks)", mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks))
+			dlog.Printf("IDX(out): %q m=%d f=%o%s v=%d (%d blocks)", mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks))
 		}
 		index = append(index, mf)
 	}
@@ -583,8 +590,8 @@ func (m *Model) requestGlobal(nodeID, name string, offset int64, size int, hash
 		return nil, fmt.Errorf("requestGlobal: no such node: %s", nodeID)
 	}
 
-	if m.trace["net"] {
-		debugf("NET REQ(out): %s: %q o=%d s=%d h=%x", nodeID, name, offset, size, hash)
+	if debugNet {
+		dlog.Printf("REQ(out): %s: %q o=%d s=%d h=%x", nodeID, name, offset, size, hash)
 	}
 
 	return nc.Request("default", name, offset, size)
@@ -611,8 +618,8 @@ func (m *Model) broadcastIndexLoop() {
 			m.pmut.RLock()
 			for _, node := range m.protoConn {
 				node := node
-				if m.trace["net"] {
-					debugf("NET IDX(out/loop): %s: %d files", node.ID(), len(idx))
+				if debugNet {
+					dlog.Printf("IDX(out/loop): %s: %d files", node.ID(), len(idx))
 				}
 				go func() {
 					node.Index("default", idx)
@@ -628,7 +635,7 @@ func (m *Model) broadcastIndexLoop() {
 }
 
 // markDeletedLocals sets the deleted flag on files that have gone missing locally.
-func (m *Model) markDeletedLocals(newLocal map[string]File) bool {
+func (m *Model) markDeletedLocals(newLocal map[string]scanner.File) bool {
 	// For every file in the existing local table, check if they are also
 	// present in the new local table. If they are not, check that we already
 	// had the newest version available according to the global table and if so
@@ -658,7 +665,7 @@ func (m *Model) markDeletedLocals(newLocal map[string]File) bool {
 	return updated
 }
 
-func (m *Model) updateLocal(f File) {
+func (m *Model) updateLocal(f scanner.File) {
 	var updated bool
 
 	m.lmut.Lock()
@@ -685,7 +692,7 @@ func (m *Model) updateLocal(f File) {
 /*
 XXX: Not done, needs elegant handling of availability
 
-func (m *Model) recomputeGlobalFor(files []File) bool {
+func (m *Model) recomputeGlobalFor(files []scanner.File) bool {
 	m.gmut.Lock()
 	defer m.gmut.Unlock()
 
@@ -702,7 +709,7 @@ func (m *Model) recomputeGlobalFor(files []File) bool {
 */
 
 func (m *Model) recomputeGlobal() {
-	var newGlobal = make(map[string]File)
+	var newGlobal = make(map[string]scanner.File)
 
 	m.lmut.RLock()
 	for n, f := range m.local {
@@ -761,12 +768,12 @@ func (m *Model) recomputeGlobal() {
 
 type addOrder struct {
 	n      string
-	remote []Block
+	remote []scanner.Block
 	fm     *fileMonitor
 }
 
 func (m *Model) recomputeNeedForGlobal() {
-	var toDelete []File
+	var toDelete []scanner.File
 	var toAdd []addOrder
 
 	m.gmut.RLock()
@@ -785,8 +792,8 @@ func (m *Model) recomputeNeedForGlobal() {
 	}
 }
 
-func (m *Model) recomputeNeedForFiles(files []File) {
-	var toDelete []File
+func (m *Model) recomputeNeedForFiles(files []scanner.File) {
+	var toDelete []scanner.File
 	var toAdd []addOrder
 
 	m.gmut.RLock()
@@ -805,13 +812,13 @@ func (m *Model) recomputeNeedForFiles(files []File) {
 	}
 }
 
-func (m *Model) recomputeNeedForFile(gf File, toAdd []addOrder, toDelete []File) ([]addOrder, []File) {
+func (m *Model) recomputeNeedForFile(gf scanner.File, toAdd []addOrder, toDelete []scanner.File) ([]addOrder, []scanner.File) {
 	m.lmut.RLock()
 	lf, ok := m.local[gf.Name]
 	m.lmut.RUnlock()
 
 	if !ok || gf.NewerThan(lf) {
-		if gf.Flags&protocol.FlagInvalid != 0 {
+		if gf.Suppressed {
 			// Never attempt to sync invalid files
 			return toAdd, toDelete
 		}
@@ -823,14 +830,14 @@ func (m *Model) recomputeNeedForFile(gf File, toAdd []addOrder, toDelete []File)
 			// Don't have the file, so don't need to delete it
 			return toAdd, toDelete
 		}
-		if m.trace["need"] {
-			debugf("NEED: lf:%v gf:%v", lf, gf)
+		if debugNeed {
+			dlog.Printf("need: lf:%v gf:%v", lf, gf)
 		}
 
 		if gf.Flags&protocol.FlagDeleted != 0 {
 			toDelete = append(toDelete, gf)
 		} else {
-			local, remote := BlockDiff(lf.Blocks, gf.Blocks)
+			local, remote := scanner.BlockDiff(lf.Blocks, gf.Blocks)
 			fm := fileMonitor{
 				name:        gf.Name,
 				path:        path.Clean(path.Join(m.dir, gf.Name)),
@@ -865,8 +872,8 @@ func (m *Model) WhoHas(name string) []string {
 
 func (m *Model) deleteLoop() {
 	for file := range m.dq {
-		if m.trace["file"] {
-			debugln("FILE: Delete", file.Name)
+		if debugPull {
+			dlog.Println("delete", file.Name)
 		}
 		path := path.Clean(path.Join(m.dir, file.Name))
 		err := os.Remove(path)
@@ -878,28 +885,29 @@ func (m *Model) deleteLoop() {
 	}
 }
 
-func fileFromFileInfo(f protocol.FileInfo) File {
-	var blocks = make([]Block, len(f.Blocks))
+func fileFromFileInfo(f protocol.FileInfo) scanner.File {
+	var blocks = make([]scanner.Block, len(f.Blocks))
 	var offset int64
 	for i, b := range f.Blocks {
-		blocks[i] = Block{
+		blocks[i] = scanner.Block{
 			Offset: offset,
 			Size:   b.Size,
 			Hash:   b.Hash,
 		}
 		offset += int64(b.Size)
 	}
-	return File{
-		Name:     f.Name,
-		Size:     offset,
-		Flags:    f.Flags,
-		Modified: f.Modified,
-		Version:  f.Version,
-		Blocks:   blocks,
+	return scanner.File{
+		Name:       f.Name,
+		Size:       offset,
+		Flags:      f.Flags &^ protocol.FlagInvalid,
+		Modified:   f.Modified,
+		Version:    f.Version,
+		Blocks:     blocks,
+		Suppressed: f.Flags&protocol.FlagInvalid != 0,
 	}
 }
 
-func fileInfoFromFile(f File) protocol.FileInfo {
+func fileInfoFromFile(f scanner.File) protocol.FileInfo {
 	var blocks = make([]protocol.BlockInfo, len(f.Blocks))
 	for i, b := range f.Blocks {
 		blocks[i] = protocol.BlockInfo{
@@ -907,11 +915,15 @@ func fileInfoFromFile(f File) protocol.FileInfo {
 			Hash: b.Hash,
 		}
 	}
-	return protocol.FileInfo{
+	pf := protocol.FileInfo{
 		Name:     f.Name,
 		Flags:    f.Flags,
 		Modified: f.Modified,
 		Version:  f.Version,
 		Blocks:   blocks,
 	}
+	if f.Suppressed {
+		pf.Flags |= protocol.FlagInvalid
+	}
+	return pf
 }

cmd/syncthing/model_test.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/calmh/syncthing/protocol"
+	"github.com/calmh/syncthing/scanner"
 )
 
 func TestNewModel(t *testing.T) {
@@ -27,27 +28,27 @@ func TestNewModel(t *testing.T) {
 	}
 }
 
-var testDataExpected = map[string]File{
-	"foo": File{
+var testDataExpected = map[string]scanner.File{
+	"foo": scanner.File{
 		Name:     "foo",
 		Flags:    0,
 		Modified: 0,
 		Size:     7,
-		Blocks:   []Block{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
+		Blocks:   []scanner.Block{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
 	},
-	"empty": File{
+	"empty": scanner.File{
 		Name:     "empty",
 		Flags:    0,
 		Modified: 0,
 		Size:     0,
-		Blocks:   []Block{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
+		Blocks:   []scanner.Block{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
 	},
-	"bar": File{
+	"bar": scanner.File{
 		Name:     "bar",
 		Flags:    0,
 		Modified: 0,
 		Size:     10,
-		Blocks:   []Block{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
+		Blocks:   []scanner.Block{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
 	},
 }
 
@@ -63,7 +64,8 @@ func init() {
 
 func TestUpdateLocal(t *testing.T) {
 	m := NewModel("testdata", 1e6)
-	fs, _ := m.Walk(false)
+	w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
+	fs, _ := w.Walk()
 	m.ReplaceLocal(fs)
 
 	if fs, _ := m.NeedFiles(); len(fs) > 0 {
@@ -105,7 +107,8 @@ func TestUpdateLocal(t *testing.T) {
 
 func TestRemoteUpdateExisting(t *testing.T) {
 	m := NewModel("testdata", 1e6)
-	fs, _ := m.Walk(false)
+	w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
+	fs, _ := w.Walk()
 	m.ReplaceLocal(fs)
 
 	newFile := protocol.FileInfo{
@@ -122,7 +125,8 @@ func TestRemoteUpdateExisting(t *testing.T) {
 
 func TestRemoteAddNew(t *testing.T) {
 	m := NewModel("testdata", 1e6)
-	fs, _ := m.Walk(false)
+	w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
+	fs, _ := w.Walk()
 	m.ReplaceLocal(fs)
 
 	newFile := protocol.FileInfo{
@@ -139,7 +143,8 @@ func TestRemoteAddNew(t *testing.T) {
 
 func TestRemoteUpdateOld(t *testing.T) {
 	m := NewModel("testdata", 1e6)
-	fs, _ := m.Walk(false)
+	w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
+	fs, _ := w.Walk()
 	m.ReplaceLocal(fs)
 
 	oldTimeStamp := int64(1234)
@@ -157,7 +162,8 @@ func TestRemoteUpdateOld(t *testing.T) {
 
 func TestRemoteIndexUpdate(t *testing.T) {
 	m := NewModel("testdata", 1e6)
-	fs, _ := m.Walk(false)
+	w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
+	fs, _ := w.Walk()
 	m.ReplaceLocal(fs)
 
 	foo := protocol.FileInfo{
@@ -190,7 +196,8 @@ func TestRemoteIndexUpdate(t *testing.T) {
 
 func TestDelete(t *testing.T) {
 	m := NewModel("testdata", 1e6)
-	fs, _ := m.Walk(false)
+	w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
+	fs, _ := w.Walk()
 	m.ReplaceLocal(fs)
 
 	if l1, l2 := len(m.local), len(fs); l1 != l2 {
@@ -201,10 +208,10 @@ func TestDelete(t *testing.T) {
 	}
 
 	ot := time.Now().Unix()
-	newFile := File{
+	newFile := scanner.File{
 		Name:     "a new file",
 		Modified: ot,
-		Blocks:   []Block{{0, 100, []byte("some hash bytes")}},
+		Blocks:   []scanner.Block{{0, 100, []byte("some hash bytes")}},
 	}
 	m.updateLocal(newFile)
 
@@ -292,7 +299,8 @@ func TestDelete(t *testing.T) {
 
 func TestForgetNode(t *testing.T) {
 	m := NewModel("testdata", 1e6)
-	fs, _ := m.Walk(false)
+	w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
+	fs, _ := w.Walk()
 	m.ReplaceLocal(fs)
 
 	if l1, l2 := len(m.local), len(fs); l1 != l2 {
@@ -345,7 +353,8 @@ func TestForgetNode(t *testing.T) {
 
 func TestRequest(t *testing.T) {
 	m := NewModel("testdata", 1e6)
-	fs, _ := m.Walk(false)
+	w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
+	fs, _ := w.Walk()
 	m.ReplaceLocal(fs)
 
 	bs, err := m.Request("some node", "default", "foo", 0, 6)
@@ -367,7 +376,8 @@ func TestRequest(t *testing.T) {
 
 func TestIgnoreWithUnknownFlags(t *testing.T) {
 	m := NewModel("testdata", 1e6)
-	fs, _ := m.Walk(false)
+	w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
+	fs, _ := w.Walk()
 	m.ReplaceLocal(fs)
 
 	valid := protocol.FileInfo{
@@ -410,7 +420,8 @@ func genFiles(n int) []protocol.FileInfo {
 
 func BenchmarkIndex10000(b *testing.B) {
 	m := NewModel("testdata", 1e6)
-	fs, _ := m.Walk(false)
+	w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
+	fs, _ := w.Walk()
 	m.ReplaceLocal(fs)
 	files := genFiles(10000)
 
@@ -422,7 +433,8 @@ func BenchmarkIndex10000(b *testing.B) {
 
 func BenchmarkIndex00100(b *testing.B) {
 	m := NewModel("testdata", 1e6)
-	fs, _ := m.Walk(false)
+	w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
+	fs, _ := w.Walk()
 	m.ReplaceLocal(fs)
 	files := genFiles(100)
 
@@ -434,7 +446,8 @@ func BenchmarkIndex00100(b *testing.B) {
 
 func BenchmarkIndexUpdate10000f10000(b *testing.B) {
 	m := NewModel("testdata", 1e6)
-	fs, _ := m.Walk(false)
+	w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
+	fs, _ := w.Walk()
 	m.ReplaceLocal(fs)
 	files := genFiles(10000)
 	m.Index("42", files)
@@ -447,7 +460,8 @@ func BenchmarkIndexUpdate10000f10000(b *testing.B) {
 
 func BenchmarkIndexUpdate10000f00100(b *testing.B) {
 	m := NewModel("testdata", 1e6)
-	fs, _ := m.Walk(false)
+	w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
+	fs, _ := w.Walk()
 	m.ReplaceLocal(fs)
 	files := genFiles(10000)
 	m.Index("42", files)
@@ -461,7 +475,8 @@ func BenchmarkIndexUpdate10000f00100(b *testing.B) {
 
 func BenchmarkIndexUpdate10000f00001(b *testing.B) {
 	m := NewModel("testdata", 1e6)
-	fs, _ := m.Walk(false)
+	w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
+	fs, _ := w.Walk()
 	m.ReplaceLocal(fs)
 	files := genFiles(10000)
 	m.Index("42", files)
@@ -506,7 +521,8 @@ func (FakeConnection) Statistics() protocol.Statistics {
 
 func BenchmarkRequest(b *testing.B) {
 	m := NewModel("testdata", 1e6)
-	fs, _ := m.Walk(false)
+	w := scanner.Walker{Dir: "testdata", IgnoreFile: ".stignore", BlockSize: 128 * 1024}
+	fs, _ := w.Walk()
 	m.ReplaceLocal(fs)
 
 	const n = 1000

cmd/syncthing/openurl.go

@@ -0,0 +1,34 @@
+/*
+Copyright 2011 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"os/exec"
+	"runtime"
+)
+
+func openURL(url string) error {
+	if runtime.GOOS == "windows" {
+		return exec.Command("cmd.exe", "/C", "start "+url).Run()
+	}
+
+	if runtime.GOOS == "darwin" {
+		return exec.Command("open", url).Run()
+	}
+
+	return exec.Command("xdg-open", url).Run()
+}

cmd/syncthing/suppressor.go

@@ -1,12 +1,13 @@
 package main
 
 import (
+	"os"
 	"sync"
 	"time"
 )
 
 const (
-	MAX_CHANGE_HISTORY = 4
+	MaxChangeHistory = 4
 )
 
 type change struct {
@@ -45,12 +46,17 @@ func (h changeHistory) bandwidth(t time.Time) int64 {
 
 func (h *changeHistory) append(size int64, t time.Time) {
 	c := change{size, t}
-	if len(h.changes) == MAX_CHANGE_HISTORY {
-		h.changes = h.changes[1:MAX_CHANGE_HISTORY]
+	if len(h.changes) == MaxChangeHistory {
+		h.changes = h.changes[1:MaxChangeHistory]
 	}
 	h.changes = append(h.changes, c)
 }
 
+func (s *suppressor) Suppress(name string, fi os.FileInfo) bool {
+	sup, _ := s.suppress(name, fi.Size(), time.Now())
+	return sup
+}
+
 func (s *suppressor) suppress(name string, size int64, t time.Time) (bool, bool) {
 	s.Lock()
 

cmd/syncthing/suppressor_test.go

@@ -21,7 +21,7 @@ func TestSuppressor(t *testing.T) {
 	// bw is 10000 / 10 = 1000
 	t1 = t0.Add(10 * time.Second)
 	if bw := s.changes["foo"].bandwidth(t1); bw != 1000 {
-		t.Error("Incorrect bw %d", bw)
+		t.Errorf("Incorrect bw %d", bw)
 	}
 	sup, prev = s.suppress("foo", 10000, t1)
 	if sup {
@@ -34,7 +34,7 @@ func TestSuppressor(t *testing.T) {
 	// bw is (10000 + 10000) / 11 = 1818
 	t1 = t0.Add(11 * time.Second)
 	if bw := s.changes["foo"].bandwidth(t1); bw != 1818 {
-		t.Error("Incorrect bw %d", bw)
+		t.Errorf("Incorrect bw %d", bw)
 	}
 	sup, prev = s.suppress("foo", 100500, t1)
 	if sup {
@@ -47,7 +47,7 @@ func TestSuppressor(t *testing.T) {
 	// bw is (10000 + 10000 + 100500) / 12 = 10041
 	t1 = t0.Add(12 * time.Second)
 	if bw := s.changes["foo"].bandwidth(t1); bw != 10041 {
-		t.Error("Incorrect bw %d", bw)
+		t.Errorf("Incorrect bw %d", bw)
 	}
 	sup, prev = s.suppress("foo", 10000000, t1) // value will be ignored
 	if !sup {
@@ -60,7 +60,7 @@ func TestSuppressor(t *testing.T) {
 	// bw is (10000 + 10000 + 100500) / 15 = 8033
 	t1 = t0.Add(15 * time.Second)
 	if bw := s.changes["foo"].bandwidth(t1); bw != 8033 {
-		t.Error("Incorrect bw %d", bw)
+		t.Errorf("Incorrect bw %d", bw)
 	}
 	sup, prev = s.suppress("foo", 10000000, t1)
 	if sup {
@@ -84,29 +84,29 @@ func TestHistory(t *testing.T) {
 		t.Errorf("Incorrect first record size %d", s)
 	}
 
-	for i := 1; i < MAX_CHANGE_HISTORY; i++ {
+	for i := 1; i < MaxChangeHistory; i++ {
 		h.append(int64(40+i), t0.Add(time.Duration(i)*time.Second))
 	}
 
-	if l := len(h.changes); l != MAX_CHANGE_HISTORY {
+	if l := len(h.changes); l != MaxChangeHistory {
 		t.Errorf("Incorrect history length %d", l)
 	}
 	if s := h.changes[0].size; s != 40 {
 		t.Errorf("Incorrect first record size %d", s)
 	}
-	if s := h.changes[MAX_CHANGE_HISTORY-1].size; s != 40+MAX_CHANGE_HISTORY-1 {
+	if s := h.changes[MaxChangeHistory-1].size; s != 40+MaxChangeHistory-1 {
 		t.Errorf("Incorrect last record size %d", s)
 	}
 
 	h.append(999, t0.Add(time.Duration(999)*time.Second))
 
-	if l := len(h.changes); l != MAX_CHANGE_HISTORY {
+	if l := len(h.changes); l != MaxChangeHistory {
 		t.Errorf("Incorrect history length %d", l)
 	}
 	if s := h.changes[0].size; s != 41 {
 		t.Errorf("Incorrect first record size %d", s)
 	}
-	if s := h.changes[MAX_CHANGE_HISTORY-1].size; s != 999 {
+	if s := h.changes[MaxChangeHistory-1].size; s != 999 {
 		t.Errorf("Incorrect last record size %d", s)
 	}
 

cmd/syncthing/tempname.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"fmt"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+type tempNamer struct {
+	prefix string
+}
+
+var defTempNamer = tempNamer{".syncthing"}
+
+func (t tempNamer) IsTemporary(name string) bool {
+	if runtime.GOOS == "windows" {
+		name = filepath.ToSlash(name)
+	}
+	return strings.HasPrefix(path.Base(name), t.prefix)
+}
+
+func (t tempNamer) TempName(name string) string {
+	tdir := path.Dir(name)
+	tname := fmt.Sprintf("%s.%s", t.prefix, path.Base(name))
+	return path.Join(tdir, tname)
+}

cmd/syncthing/tls.go

@@ -25,7 +25,7 @@ func loadCert(dir string) (tls.Certificate, error) {
 	return tls.LoadX509KeyPair(path.Join(dir, "cert.pem"), path.Join(dir, "key.pem"))
 }
 
-func certId(bs []byte) string {
+func certID(bs []byte) string {
 	hf := sha256.New()
 	hf.Write(bs)
 	id := hf.Sum(nil)

cmd/syncthing/usage.go

@@ -22,18 +22,14 @@ func optionTable(w io.Writer, rows [][]string) {
 	tw.Flush()
 }
 
-func usageFor(fs *flag.FlagSet, usage string) func() {
+func usageFor(fs *flag.FlagSet, usage string, extra string) func() {
 	return func() {
 		var b bytes.Buffer
 		b.WriteString("Usage:\n  " + usage + "\n")
 
 		var options [][]string
 		fs.VisitAll(func(f *flag.Flag) {
-			var dash = "-"
-			if len(f.Name) > 1 {
-				dash = "--"
-			}
-			var opt = "  " + dash + f.Name
+			var opt = "  -" + f.Name
 
 			if f.DefValue != "false" {
 				opt += "=" + f.DefValue
@@ -48,5 +44,9 @@ func usageFor(fs *flag.FlagSet, usage string) func() {
 		}
 
 		fmt.Println(b.String())
+
+		if len(extra) > 0 {
+			fmt.Println(extra)
+		}
 	}
 }

discover/debug.go

@@ -0,0 +1,12 @@
+package discover
+
+import (
+	"log"
+	"os"
+	"strings"
+)
+
+var (
+	dlog  = log.New(os.Stderr, "discover: ", log.Lmicroseconds|log.Lshortfile)
+	debug = strings.Contains(os.Getenv("STTRACE"), "discover")
+)

discover/discover.go

@@ -15,7 +15,6 @@ import (
 
 const (
 	AnnouncementPort = 21025
-	Debug            = false
 )
 
 type Discoverer struct {
@@ -34,7 +33,7 @@ type Discoverer struct {
 }
 
 var (
-	ErrIncorrectMagic = errors.New("Incorrect magic number")
+	ErrIncorrectMagic = errors.New("incorrect magic number")
 )
 
 // We tolerate a certain amount of errors because we might be running on
@@ -80,24 +79,75 @@ func (d *Discoverer) sendAnnouncements() {
 	var errCounter = 0
 	var err error
 
+	remote := &net.UDPAddr{
+		IP:   net.IP{255, 255, 255, 255},
+		Port: AnnouncementPort,
+	}
+
 	for errCounter < maxErrors {
-		for _, ipStr := range allBroadcasts() {
-			var addrStr = ipStr + ":21025"
+		intfs, err := net.Interfaces()
+		if err != nil {
+			log.Printf("discover/listInterfaces: %v; no local announcements", err)
+			return
+		}
 
-			remote, err := net.ResolveUDPAddr("udp4", addrStr)
-			if err != nil {
-				log.Printf("discover/external: %v; no external announcements", err)
-				return
-			}
+		for _, intf := range intfs {
+			if intf.Flags&(net.FlagBroadcast|net.FlagLoopback) == net.FlagBroadcast {
+				addrs, err := intf.Addrs()
+				if err != nil {
+					log.Println("discover/listAddrs: warning:", err)
+					errCounter++
+					continue
+				}
 
-			if Debug {
-				fmt.Println("send announcement -> ", remote)
-			}
-			_, _, err = d.conn.WriteMsgUDP(buf, nil, remote)
-			if err != nil {
-				log.Println("discover/write: warning:", err)
-				errCounter++
-			} else {
+				var srcAddr string
+				for _, addr := range addrs {
+					if strings.Contains(addr.String(), ".") {
+						// Found an IPv4 adress
+						parts := strings.Split(addr.String(), "/")
+						srcAddr = parts[0]
+						break
+					}
+				}
+				if len(srcAddr) == 0 {
+					if debug {
+						dlog.Println("no source address found on interface", intf.Name)
+					}
+					continue
+				}
+
+				iaddr, err := net.ResolveUDPAddr("udp4", srcAddr+":0")
+				if err != nil {
+					log.Println("discover/resolve: warning:", err)
+					errCounter++
+					continue
+				}
+
+				conn, err := net.ListenUDP("udp4", iaddr)
+				if err != nil {
+					log.Println("discover/listen: warning:", err)
+					errCounter++
+					continue
+				}
+
+				if debug {
+					dlog.Println("send announcement from", conn.LocalAddr(), "to", remote, "on", intf.Name)
+				}
+
+				_, err = conn.WriteTo(buf, remote)
+				if err != nil {
+					// Some interfaces don't seem to support broadcast even though the flags claims they do, i.e. vmnet
+					conn.Close()
+
+					if debug {
+						log.Println(err)
+					}
+
+					errCounter++
+					continue
+				}
+
+				conn.Close()
 				errCounter = 0
 			}
 		}
@@ -122,10 +172,10 @@ func (d *Discoverer) sendExtAnnouncements() {
 	var errCounter = 0
 
 	for errCounter < maxErrors {
-		if Debug {
-			fmt.Println("send announcement -> ", remote)
+		if debug {
+			dlog.Println("send announcement -> ", remote)
 		}
-		_, _, err = d.conn.WriteMsgUDP(buf, nil, remote)
+		_, err = d.conn.WriteTo(buf, remote)
 		if err != nil {
 			log.Println("discover/write: warning:", err)
 			errCounter++
@@ -134,7 +184,7 @@ func (d *Discoverer) sendExtAnnouncements() {
 		}
 		time.Sleep(d.ExtBroadcastIntv)
 	}
-	log.Println("discover/write: %v: stopping due to too many errors:", remote, err)
+	log.Printf("discover/write: %v: stopping due to too many errors: %v", remote, err)
 }
 
 func (d *Discoverer) recvAnnouncements() {
@@ -149,8 +199,8 @@ func (d *Discoverer) recvAnnouncements() {
 			continue
 		}
 
-		if Debug {
-			log.Printf("read announcement:\n%s", hex.Dump(buf[:n]))
+		if debug {
+			dlog.Printf("read announcement:\n%s", hex.Dump(buf[:n]))
 		}
 
 		var pkt AnnounceV2
@@ -161,8 +211,8 @@ func (d *Discoverer) recvAnnouncements() {
 			continue
 		}
 
-		if Debug {
-			log.Printf("read announcement: %#v", pkt)
+		if debug {
+			dlog.Printf("parsed announcement: %#v", pkt)
 		}
 
 		errCounter = 0
@@ -178,13 +228,12 @@ func (d *Discoverer) recvAnnouncements() {
 				}
 				addrs = append(addrs, nodeAddr)
 			}
-			if Debug {
-				log.Printf("register: %#v", addrs)
+			if debug {
+				dlog.Printf("register: %#v", addrs)
 			}
 			d.registryLock.Lock()
 			_, seen := d.registry[pkt.NodeID]
 			if !seen {
-				fmt.Println("new node seen, forced announce")
 				select {
 				case d.forcedBroadcastTick <- time.Now():
 				}
@@ -237,8 +286,8 @@ func (d *Discoverer) externalLookup(node string) []string {
 		return nil
 	}
 
-	if Debug {
-		log.Printf("read external:\n%s", hex.Dump(buf[:n]))
+	if debug {
+		dlog.Printf("read external:\n%s", hex.Dump(buf[:n]))
 	}
 
 	var pkt AnnounceV2
@@ -248,8 +297,8 @@ func (d *Discoverer) externalLookup(node string) []string {
 		return nil
 	}
 
-	if Debug {
-		log.Printf("read external: %#v", pkt)
+	if debug {
+		dlog.Printf("parsed external: %#v", pkt)
 	}
 
 	var addrs []string
@@ -290,38 +339,3 @@ func ipStr(ip []byte) string {
 	}
 	return strings.Join(ss, s)
 }
-
-func allBroadcasts() []string {
-	var bcasts = make(map[string]bool)
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		return nil
-	}
-
-	for _, addr := range addrs {
-		switch {
-		case strings.HasPrefix(addr.String(), "127."):
-			// Ignore v4 localhost
-
-		case strings.Contains(addr.String(), ":"):
-			// Ignore all v6, because we need link local multicast there which I haven't implemented
-
-		default:
-			if in, ok := addr.(*net.IPNet); ok {
-				il := len(in.IP) - 1
-				ml := len(in.Mask) - 1
-				for i := range in.Mask {
-					in.IP[il-i] = in.IP[il-i] | ^in.Mask[ml-i]
-				}
-				parts := strings.Split(in.String(), "/")
-				bcasts[parts[0]] = true
-			}
-		}
-	}
-
-	var l []string
-	for ip := range bcasts {
-		l = append(l, ip)
-	}
-	return l
-}

discover/doc.go

@@ -0,0 +1,2 @@
+// Package discover implements the node discovery protocol.
+package discover

files/set.go

@@ -0,0 +1,174 @@
+// Package fileset provides a set type to track local/remote files with newness checks.
+package fileset
+
+import "sync"
+
+type File struct {
+	Key      Key
+	Modified int64
+	Flags    uint32
+	Data     interface{}
+}
+
+type Key struct {
+	Name    string
+	Version uint32
+}
+
+type fileRecord struct {
+	Usage int
+	File  File
+}
+
+type bitset uint64
+
+func (a Key) newerThan(b Key) bool {
+	return a.Version > b.Version
+}
+
+type Set struct {
+	mutex              sync.RWMutex
+	files              map[Key]fileRecord
+	remoteKey          [64]map[string]Key
+	globalAvailability map[string]bitset
+	globalKey          map[string]Key
+}
+
+func NewSet() *Set {
+	var m = Set{
+		files:              make(map[Key]fileRecord),
+		globalAvailability: make(map[string]bitset),
+		globalKey:          make(map[string]Key),
+	}
+	return &m
+}
+
+func (m *Set) AddLocal(fs []File) {
+	m.mutex.Lock()
+	m.unlockedAddRemote(0, fs)
+	m.mutex.Unlock()
+}
+
+func (m *Set) SetLocal(fs []File) {
+	m.mutex.Lock()
+	m.unlockedSetRemote(0, fs)
+	m.mutex.Unlock()
+}
+
+func (m *Set) AddRemote(cid uint, fs []File) {
+	if cid < 1 || cid > 63 {
+		panic("Connection ID must be in the range 1 - 63 inclusive")
+	}
+	m.mutex.Lock()
+	m.unlockedAddRemote(cid, fs)
+	m.mutex.Unlock()
+}
+
+func (m *Set) SetRemote(cid uint, fs []File) {
+	if cid < 1 || cid > 63 {
+		panic("Connection ID must be in the range 1 - 63 inclusive")
+	}
+	m.mutex.Lock()
+	m.unlockedSetRemote(cid, fs)
+	m.mutex.Unlock()
+}
+
+func (m *Set) unlockedAddRemote(cid uint, fs []File) {
+	remFiles := m.remoteKey[cid]
+	for _, f := range fs {
+		n := f.Key.Name
+
+		if ck, ok := remFiles[n]; ok && ck == f.Key {
+			// The remote already has exactly this file, skip it
+			continue
+		}
+
+		remFiles[n] = f.Key
+
+		// Keep the block list or increment the usage
+		if br, ok := m.files[f.Key]; !ok {
+			m.files[f.Key] = fileRecord{
+				Usage: 1,
+				File:  f,
+			}
+		} else {
+			br.Usage++
+			m.files[f.Key] = br
+		}
+
+		// Update global view
+		gk, ok := m.globalKey[n]
+		switch {
+		case ok && f.Key == gk:
+			av := m.globalAvailability[n]
+			av |= 1 << cid
+			m.globalAvailability[n] = av
+		case f.Key.newerThan(gk):
+			m.globalKey[n] = f.Key
+			m.globalAvailability[n] = 1 << cid
+		}
+	}
+}
+
+func (m *Set) unlockedSetRemote(cid uint, fs []File) {
+	// Decrement usage for all files belonging to this remote, and remove
+	// those that are no longer needed.
+	for _, fk := range m.remoteKey[cid] {
+		br, ok := m.files[fk]
+		switch {
+		case ok && br.Usage == 1:
+			delete(m.files, fk)
+		case ok && br.Usage > 1:
+			br.Usage--
+			m.files[fk] = br
+		}
+	}
+
+	// Clear existing remote remoteKey
+	m.remoteKey[cid] = make(map[string]Key)
+
+	// Recalculate global based on all remaining remoteKey
+	for n := range m.globalKey {
+		var nk Key    // newest key
+		var na bitset // newest availability
+
+		for i, rem := range m.remoteKey {
+			if rk, ok := rem[n]; ok {
+				switch {
+				case rk == nk:
+					na |= 1 << uint(i)
+				case rk.newerThan(nk):
+					nk = rk
+					na = 1 << uint(i)
+				}
+			}
+		}
+
+		if na != 0 {
+			// Someone had the file
+			m.globalKey[n] = nk
+			m.globalAvailability[n] = na
+		} else {
+			// Noone had the file
+			delete(m.globalKey, n)
+			delete(m.globalAvailability, n)
+		}
+	}
+
+	// Add new remote remoteKey to the mix
+	m.unlockedAddRemote(cid, fs)
+}
+
+func (m *Set) Need(cid uint) []File {
+	var fs []File
+	m.mutex.Lock()
+
+	for name, gk := range m.globalKey {
+		if gk.newerThan(m.remoteKey[cid][name]) {
+			fs = append(fs, m.files[gk].File)
+		}
+	}
+
+	m.mutex.Unlock()
+	return fs
+}

files/set_test.go

@@ -0,0 +1,207 @@
+package fileset
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestGlobalSet(t *testing.T) {
+	m := NewSet()
+
+	local := []File{
+		File{Key{"a", 1000}, 0, 0, nil},
+		File{Key{"b", 1000}, 0, 0, nil},
+		File{Key{"c", 1000}, 0, 0, nil},
+		File{Key{"d", 1000}, 0, 0, nil},
+	}
+
+	remote := []File{
+		File{Key{"a", 1000}, 0, 0, nil},
+		File{Key{"b", 1001}, 0, 0, nil},
+		File{Key{"c", 1002}, 0, 0, nil},
+		File{Key{"e", 1000}, 0, 0, nil},
+	}
+
+	expectedGlobal := map[string]Key{
+		"a": local[0].Key,
+		"b": remote[1].Key,
+		"c": remote[2].Key,
+		"d": local[3].Key,
+		"e": remote[3].Key,
+	}
+
+	m.SetLocal(local)
+	m.SetRemote(1, remote)
+
+	if !reflect.DeepEqual(m.globalKey, expectedGlobal) {
+		t.Errorf("Global incorrect;\n%v !=\n%v", m.globalKey, expectedGlobal)
+	}
+
+	if lb := len(m.files); lb != 7 {
+		t.Errorf("Num files incorrect %d != 7\n%v", lb, m.files)
+	}
+}
+
+func BenchmarkSetLocal10k(b *testing.B) {
+	m := NewSet()
+
+	var local []File
+	for i := 0; i < 10000; i++ {
+		local = append(local, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	var remote []File
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	m.SetRemote(1, remote)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.SetLocal(local)
+	}
+}
+
+func BenchmarkSetLocal10(b *testing.B) {
+	m := NewSet()
+
+	var local []File
+	for i := 0; i < 10; i++ {
+		local = append(local, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	var remote []File
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	m.SetRemote(1, remote)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.SetLocal(local)
+	}
+}
+
+func BenchmarkAddLocal10k(b *testing.B) {
+	m := NewSet()
+
+	var local []File
+	for i := 0; i < 10000; i++ {
+		local = append(local, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	var remote []File
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	m.SetRemote(1, remote)
+	m.SetLocal(local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		for j := range local {
+			local[j].Key.Version++
+		}
+		b.StartTimer()
+		m.AddLocal(local)
+	}
+}
+
+func BenchmarkAddLocal10(b *testing.B) {
+	m := NewSet()
+
+	var local []File
+	for i := 0; i < 10; i++ {
+		local = append(local, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	var remote []File
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	m.SetRemote(1, remote)
+	m.SetLocal(local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for j := range local {
+			local[j].Key.Version++
+		}
+		m.AddLocal(local)
+	}
+}
+
+func TestGlobalReset(t *testing.T) {
+	m := NewSet()
+
+	local := []File{
+		File{Key{"a", 1000}, 0, 0, nil},
+		File{Key{"b", 1000}, 0, 0, nil},
+		File{Key{"c", 1000}, 0, 0, nil},
+		File{Key{"d", 1000}, 0, 0, nil},
+	}
+
+	remote := []File{
+		File{Key{"a", 1000}, 0, 0, nil},
+		File{Key{"b", 1001}, 0, 0, nil},
+		File{Key{"c", 1002}, 0, 0, nil},
+		File{Key{"e", 1000}, 0, 0, nil},
+	}
+
+	expectedGlobalKey := map[string]Key{
+		"a": local[0].Key,
+		"b": local[1].Key,
+		"c": local[2].Key,
+		"d": local[3].Key,
+	}
+
+	m.SetLocal(local)
+	m.SetRemote(1, remote)
+	m.SetRemote(1, nil)
+
+	if !reflect.DeepEqual(m.globalKey, expectedGlobalKey) {
+		t.Errorf("Global incorrect;\n%v !=\n%v", m.globalKey, expectedGlobalKey)
+	}
+
+	if lb := len(m.files); lb != 4 {
+		t.Errorf("Num files incorrect %d != 4\n%v", lb, m.files)
+	}
+}
+
+func TestNeed(t *testing.T) {
+	m := NewSet()
+
+	local := []File{
+		File{Key{"a", 1000}, 0, 0, nil},
+		File{Key{"b", 1000}, 0, 0, nil},
+		File{Key{"c", 1000}, 0, 0, nil},
+		File{Key{"d", 1000}, 0, 0, nil},
+	}
+
+	remote := []File{
+		File{Key{"a", 1000}, 0, 0, nil},
+		File{Key{"b", 1001}, 0, 0, nil},
+		File{Key{"c", 1002}, 0, 0, nil},
+		File{Key{"e", 1000}, 0, 0, nil},
+	}
+
+	shouldNeed := []File{
+		File{Key{"b", 1001}, 0, 0, nil},
+		File{Key{"c", 1002}, 0, 0, nil},
+		File{Key{"e", 1000}, 0, 0, nil},
+	}
+
+	m.SetLocal(local)
+	m.SetRemote(1, remote)
+
+	need := m.Need(0)
+	if !reflect.DeepEqual(need, shouldNeed) {
+		t.Errorf("Need incorrect;\n%v !=\n%v", need, shouldNeed)
+	}
+}

gui/app.js

@@ -32,6 +32,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         {id: 'FollowSymlinks', descr: 'Follow Symlinks', type: 'bool', restart: true},
         {id: 'GlobalAnnEnabled', descr: 'Global Announce', type: 'bool', restart: true},
         {id: 'LocalAnnEnabled', descr: 'Local Announce', type: 'bool', restart: true},
+        {id: 'StartBrowser', descr: 'Start Browser', type: 'bool'},
     ];
 
     function modelGetSucceeded() {
@@ -317,7 +318,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         }
         return errors;
     };
-    
+
     $scope.clearErrors = function () {
         $scope.seenError = $scope.errors[$scope.errors.length - 1].Time;
     };

integration/.gitignore

@@ -1,5 +1,8 @@
-files-*
-conf-*
+s1
+s2
+s3
 md5-*
 genfiles
 md5r
+json
+*.idx.gz

integration/genfiles.go

@@ -8,6 +8,7 @@ import (
 	mr "math/rand"
 	"os"
 	"path"
+	"time"
 )
 
 func name() string {
@@ -38,5 +39,10 @@ func main() {
 		rand.Reader.Read(b)
 		p1 := path.Join(p0, n)
 		ioutil.WriteFile(p1, b, 0644)
+
+		os.Chmod(p1, os.FileMode(mr.Intn(0777)|0400))
+
+		t := time.Now().Add(-time.Duration(mr.Intn(30*86400)) * time.Second)
+		os.Chtimes(p1, t, t)
 	}
 }

integration/h1/cert.pem

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID3jCCAkigAwIBAgIBADALBgkqhkiG9w0BAQUwFDESMBAGA1UEAxMJc3luY3Ro
+aW5nMB4XDTE0MDMxNDA3MDA1M1oXDTQ5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJ
+c3luY3RoaW5nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEArDOcd5ft
+R7SnalxF1ckU3lDQpgfMIPhFDU//4dvdSSFevrMuVDTbUYhyCfGtg/g+F5TmKhZg
+E2peYhllITupz5MP7OHGaO2GHf2XnUDD4QUO3E+KVAUw7dyFSwy09esqApVLzH3+
+ov+QXyyzmRWPsJe9u18BHU1Hob/RmBhS9m2CAJgzN6EJ8KGjApiW3iR8lD/hjVyi
+IVde8IRD6qYHEJYiPJuziTVcQpCblVYxTz3ScmmT190/O9UvViIpcOPQdwgOdewP
+NNMK35c9Edt0AH5flYp6jgrja9NkLQJ3+KOiro6yl9IUS5w87GMxI8qzI8SgCAZZ
+pYSoLbu1FJPvxV4p5eHwuprBCwmFYZWw6Y7rqH0sN52C+3TeObJCMNP9ilPadqRI
++G0Q99TCaloeR022x33r/8D8SIn3FP35zrlFM+DvqlxoS6glbNb/Bj3p9vN0XONO
+RCuynOGe9F/4h/DaNnrbrRWqJOxBsZTsbbcJaKATfWU/Z9GcC+pUpPRhAgMBAAGj
+PzA9MA4GA1UdDwEB/wQEAwIAoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH
+AwIwDAYDVR0TAQH/BAIwADALBgkqhkiG9w0BAQUDggGBAFF8dklGoC43fMrUZfb4
+6areRWG8quO6cSX6ATzRQVJ8WJ5VcC7OJk8/FeiYA+wcvUJ/1Zm/VHMYugtOz5M8
+CrWAF1r9D3Xfe5D8qfrEOYG2XjxD2nFHCnkbY4fP+SMSuXaDs7ixQnzw0UFh1wsV
+9Jy/QrgXFAIFZtu1Nz+rrvoAgw24gkDhY3557MbmYfmfPsJ8cw+WJ845sxGMPFF2
+c+5EN0jiSm0AwZK11BMJda36ke829UZctDkopbGEg1peydDR5LiyhiTAPtWn7uT/
+PkzHYLuaECAkVbWC3bZLocMGOP6F1pG+BMr00NJgVy05ASQzi4FPjcZQNNY8s69R
+ZgoCIBaJZq3ti1EsZQ1H0Ynm2c2NMVKdj4czoy8a9ZC+DCuhG7EV5Foh20VhCWgA
+RfPhlHVJthuimsWBx39X85gjSBR017uk0AxOJa6pzh/b/RPCRtUfX8EArInS3XCf
+RvRtdrnBZNI3tiREopZGt0SzgDZUs4uDVBUX8HnHzyFJrg==
+-----END CERTIFICATE-----

integration/h1/config.xml

@@ -0,0 +1,30 @@
+<configuration version="1">
+    <repository directory="s1">
+        <node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1">
+            <address>127.0.0.1:22001</address>
+        </node>
+        <node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ" name="s2">
+            <address>127.0.0.1:22002</address>
+        </node>
+        <node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3">
+            <address>127.0.0.1:22003</address>
+        </node>
+    </repository>
+    <options>
+        <listenAddress>127.0.0.1:22001</listenAddress>
+        <readOnly>false</readOnly>
+        <allowDelete>true</allowDelete>
+        <followSymlinks>true</followSymlinks>
+        <guiEnabled>true</guiEnabled>
+        <guiAddress>127.0.0.1:8081</guiAddress>
+        <globalAnnounceServer>announce.syncthing.net:22025</globalAnnounceServer>
+        <globalAnnounceEnabled>false</globalAnnounceEnabled>
+        <localAnnounceEnabled>false</localAnnounceEnabled>
+        <parallelRequests>16</parallelRequests>
+        <maxSendKbps>0</maxSendKbps>
+        <rescanIntervalS>60</rescanIntervalS>
+        <reconnectionIntervalS>5</reconnectionIntervalS>
+        <maxChangeKbps>1000</maxChangeKbps>
+        <startBrowser>false</startBrowser>
+    </options>
+</configuration>

integration/h1/key.pem

@@ -0,0 +1,39 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIG5AIBAAKCAYEArDOcd5ftR7SnalxF1ckU3lDQpgfMIPhFDU//4dvdSSFevrMu
+VDTbUYhyCfGtg/g+F5TmKhZgE2peYhllITupz5MP7OHGaO2GHf2XnUDD4QUO3E+K
+VAUw7dyFSwy09esqApVLzH3+ov+QXyyzmRWPsJe9u18BHU1Hob/RmBhS9m2CAJgz
+N6EJ8KGjApiW3iR8lD/hjVyiIVde8IRD6qYHEJYiPJuziTVcQpCblVYxTz3ScmmT
+190/O9UvViIpcOPQdwgOdewPNNMK35c9Edt0AH5flYp6jgrja9NkLQJ3+KOiro6y
+l9IUS5w87GMxI8qzI8SgCAZZpYSoLbu1FJPvxV4p5eHwuprBCwmFYZWw6Y7rqH0s
+N52C+3TeObJCMNP9ilPadqRI+G0Q99TCaloeR022x33r/8D8SIn3FP35zrlFM+Dv
+qlxoS6glbNb/Bj3p9vN0XONORCuynOGe9F/4h/DaNnrbrRWqJOxBsZTsbbcJaKAT
+fWU/Z9GcC+pUpPRhAgMBAAECggGAL8+Unc/c3Y/W+7zq1tShqqgdhjub/XtxEKUp
+kngNFITjXWc6cb7LNfQAVap4Vq/R7ZI15XGY80sRMYODhJqgJzXZshdtkyx/lEwY
+kFyvBgb1fU3IRlO6phAYIiJBDBZi75ysEvbYgEEcwJAUvWgzIQDAeQmDsbMHNG2h
+r+zw++Kjua6IaeWYcOsv60Safsr6m96wrSMPENrFTVor0TaPt5c3okRIsMvT9ddY
+mzn3Lt0nVQTjO4f+SoqCPhP2FZXqksfKlZlKlr6BLxXGt6b49OrLSXM5eQXIcIZn
+ZDRsO24X5z8156qPgM9cA8oNEjuSdnArUTreBOsTwNoSpf24Qadsv/uTZlaHM19V
+q6zQvkjH3ERcOpixmg48TKdIj8cPYxezvcbNqSbZmdyQuaVlgDbUxwYI8A4IhhWl
+6xhwpX3qPDgw/QHIEngFIWfiIfCk11EPY0SN4cGO6f1rLYug8kqxMPuIQ5Jz9Hhx
+eFSRnr/fWoJcVYG6bMDKn9YWObQBAoHBAM8NahsLbjl8mdT43LH1Od1tDmDch+0Y
+JM7TgiIN/GM3piZSpGMOFqToLAqvY+Gf3l4sPgNs10cqdPAEpMk8MJ/IXGmbKq38
+iVmMaqHTQorCxyUbc54q9AbFU4HKv//F6ZN6K1wSaJt2RBeZpYI+MyBXr5baFiBZ
+ddXtXlqoEcCFyNR0DhlXrlZPs+cnyM2ZDp++lpn9Wfy+zkv36+NWpAkXVnARjxdF
+l6M+L7OlurYAWiyJE4uHUjawAM82i5+w8QKBwQDU6RCN6/AMmVrYqPy+7QcnAq67
+tPDv25gzVExeMKLBAMoz1TkMS+jIF1NMp3cYg5GbLqvx8Qd27fjFbWe/GPeZvlgL
+qdQI/T8J60dHAySMeOFOB2QWXhI1kwh0b2X0SDkTgfdJBKGdrKVcLTuLyVE24exu
+yRc8cXpYwBtVkXNBYFd7XEM+tC4b1khO23OJXHJUen9+hgsmn8/zUjASAoq3+Zly
+J+OHwwXcDcTFLeok3kX3A9NuqIV/Fa9DOGYlenECgcEAvO1onDTZ5uqjE4nhFyDE
+JB+WtxuDi/wz2eV1IM3SNlZY7S8LgLciQmb3iOhxIzdVGGkWTNnLtcwv17LlCho5
+5BJXAKXtU8TTLzrJMdArL6J7RIi//tsCwAreH9h5SVG1yDP5zJGfkftgNoikVSuc
+Sy63sdZdyjbXJtTo+5/QUvPARNuA4e73zRn89jd/Kts2VNz7XpemvND+PKOEQnSU
+SRdab/gVsQ53RyU/MZVPwTKhFXIeu3pGsk/27RzAWn6BAoHBAMIRYwaKDffd/SHJ
+/v+lHEThvBXa21c26ae36hhc6q1UI/tVGrfrpVZldIdFilgs7RbvVsmksvIj/gMv
+M0bL4j0gdC7FcUF0XPaUoBbJdZIZSP0P3ZpJyv1MdYN0WxFsl6IBcD79WrdXPC8m
+B8XmDgIhsppU77onkaa+DOxVNSJdR8BpG95W7ERxcN14SPrm6ku4kOfqFNXzC+C1
+hJ2V9Y22lLiqRUplaLzpS/eTX36VoF6E/T87mtt5D5UNHoaA8QKBwH5sRqZXoatU
+X+vw1MHU5eptMwG7LXR0gw2xmvG3cCN4hbnnBp5YaXlWPiIMmaWhpvschgBIo1TP
+qGWUpMEETGES18NenLBym+tWIXlfuyZH3B4NUi4kItiZaKb09LzmTjFvzdfQzun4
+HzIeigTNBDHdS0rdicNIn83QLZ4pJaOZJHq79+mFYkp+9It7UUoWsws6DGl/qX8o
+0cj4NmJB6QiJa1QCzrGkaajbtThbFoQal9Twk2h3jHgJzX3FbwCpLw==
+-----END RSA PRIVATE KEY-----

integration/h2/cert.pem

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID3jCCAkigAwIBAgIBADALBgkqhkiG9w0BAQUwFDESMBAGA1UEAxMJc3luY3Ro
+aW5nMB4XDTE0MDMxNDA3MDEwNFoXDTQ5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJ
+c3luY3RoaW5nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAsIV0syyR
+O56BvIOro4bIqB6iFJsNc4zX8MiM4QPTWgqGlYwsKSVmNppTdlACZCJIqyzoscrF
+qJPto8/e2Fc3oaTdEREGIs7cmc7LSXfot/mAgPpy71SVWtb7xNmXro2JJPZjRBCS
+pl1ulPug+/8w7fSKQdLMjh4Hp2YlwVBfVu0bYEEW+7Vl9PZVTv+NbTqXYvYVc9R6
+QFIbN/njWAuo2wpjJlY7vqNnSYZyskAaaAC17fFJkVQKKblTeTk1C9PxTmVTB1j9
+yOoD3+V/6IrTYKXdTHGJ1MqdieTHj1jHXe5TOeSB+Hjgq4tr25mPfQ4ixXqDqIcx
+5390DAjInuSKNUJ5pqiFrVe9eIDmySZCg5/JIL3c8phy6g1bxiJN14+Dn0om/0+9
+UrHK8LVzWMmtFRVycWVUYmARWFY3EE10k0RXU2HtzmjfnBkRrl13b0ExizlA1qJ3
+3ngxF5rNEDSMpwf4og5uYOjRUPYuvCL9XtQKr254NFO/sg/qqPV4hFWTAgMBAAGj
+PzA9MA4GA1UdDwEB/wQEAwIAoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH
+AwIwDAYDVR0TAQH/BAIwADALBgkqhkiG9w0BAQUDggGBAAZSU08zAzyuGqKqqU/c
+Pr+xML8oKiJqko5pb3ETDQC+uVw+qUHwiGYsvHI1cih4ix+tKvf+Yaiizp/35VkP
+qwls3a4ljq1Ww0Sf7J87QX0DumYpBGOfoCpmV4MacyjLhpLRKRGZHwIbOeFsmEu9
+oO38co+GvDy4CiAt3tuOdjBNs0gNOAdTTxqgm97raB9oXeg2i4Fb4MCT4UBUdXLM
+ZNLCifza+PWkBxmfBORvlKGeJBruLpXHBWnWEigZSLXIFjn3JJUy4fKd+/JMp063
+8Pjo6zUOckBCH8Lv90vzfrmdlQK555jWpcebN0l9neESEXw19l0OlqkJGVTr6JKq
+w5kjiL4eP7kpKKwCezhDSX3jf4P36wdF8MpOUBxVqfM+Oh5tHIcZctnurhYV7rXs
+jR70FMqWjHBmwemsXGrObNVt8c75yB+19U6DAulr2RhRw5GD74U1znP00eGZ8TJf
+RN1FYilUPCawMYeQoB8WIn9So7zIm0MfOl4KXNWDX02+Kw==
+-----END CERTIFICATE-----

integration/h2/config.xml

@@ -0,0 +1,30 @@
+<configuration version="1">
+    <repository directory="s2">
+        <node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1">
+            <address>127.0.0.1:22001</address>
+        </node>
+        <node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ" name="s2">
+            <address>127.0.0.1:22002</address>
+        </node>
+        <node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3">
+            <address>127.0.0.1:22003</address>
+        </node>
+    </repository>
+    <options>
+        <listenAddress>127.0.0.2:22002</listenAddress>
+        <readOnly>false</readOnly>
+        <allowDelete>true</allowDelete>
+        <followSymlinks>true</followSymlinks>
+        <guiEnabled>true</guiEnabled>
+        <guiAddress>127.0.0.1:8082</guiAddress>
+        <globalAnnounceServer>announce.syncthing.net:22025</globalAnnounceServer>
+        <globalAnnounceEnabled>false</globalAnnounceEnabled>
+        <localAnnounceEnabled>false</localAnnounceEnabled>
+        <parallelRequests>16</parallelRequests>
+        <maxSendKbps>0</maxSendKbps>
+        <rescanIntervalS>60</rescanIntervalS>
+        <reconnectionIntervalS>5</reconnectionIntervalS>
+        <maxChangeKbps>1000</maxChangeKbps>
+        <startBrowser>false</startBrowser>
+    </options>
+</configuration>

integration/h2/key.pem

@@ -0,0 +1,39 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIG5AIBAAKCAYEAsIV0syyRO56BvIOro4bIqB6iFJsNc4zX8MiM4QPTWgqGlYws
+KSVmNppTdlACZCJIqyzoscrFqJPto8/e2Fc3oaTdEREGIs7cmc7LSXfot/mAgPpy
+71SVWtb7xNmXro2JJPZjRBCSpl1ulPug+/8w7fSKQdLMjh4Hp2YlwVBfVu0bYEEW
++7Vl9PZVTv+NbTqXYvYVc9R6QFIbN/njWAuo2wpjJlY7vqNnSYZyskAaaAC17fFJ
+kVQKKblTeTk1C9PxTmVTB1j9yOoD3+V/6IrTYKXdTHGJ1MqdieTHj1jHXe5TOeSB
++Hjgq4tr25mPfQ4ixXqDqIcx5390DAjInuSKNUJ5pqiFrVe9eIDmySZCg5/JIL3c
+8phy6g1bxiJN14+Dn0om/0+9UrHK8LVzWMmtFRVycWVUYmARWFY3EE10k0RXU2Ht
+zmjfnBkRrl13b0ExizlA1qJ33ngxF5rNEDSMpwf4og5uYOjRUPYuvCL9XtQKr254
+NFO/sg/qqPV4hFWTAgMBAAECggGAH6SMuuGuVyWe1BA2YGX06k4zd8Yjryb8Pql0
+t5Fb/bQNVBmAgQ+3NuqLM5Y8F38dz7GJNPXIYOPDoa3NoLJhwpQvHLQUiYDTgq7T
+OiRIj1ImevhqSgS7kUEgeLUYv62XfAy+1qCx6Siuff5taT7hooZHkm0bRg6UCKoC
+8phZvtdaJPMGD7EAydyuhi7BR2dNY+wBBHZ+Q7F0N6CP5GSSrFE8XM7wfsgD5+Y2
+AUYEdchK1JCAQ5DxEXGrSPu8SpZ/SuhMjLc3/JDwB8SZPT0C1jX7YMeUiPONy9VK
+J6Fdnl0FMhS9VJHocL4o5IU9OLoahAcpq/Z25arm9z7yyxUoO4nVUAl3H9N7+N7A
+cwpbSgMld15bQ9iPV8MCB/eVKzfgLbWuhpZr6h6oJF9pgIq9DDCK/mc9KYzSGd1J
+dOVuizi0dMYS+iOJRFR3kIrNW7dGCecniigZfrrprqqkycl7823VTi0zIU4CHbDm
+ypu/b8sbs+h6mHN71muWAlmChz3hAoHBAN5Cm3ZZeQJj/p7Kb3sn6WAXlRqnnDz9
+fJDaa3788o4VQ3ie4odDNzALF7bHhYnfovXWrh/4XGkjiW98GPczpEFEdYF9gCGO
+mAaHV/unvtjbGF7Wk3xjgaXwPeKXGU8vZrQ4y41u5eZWpA1fwSK3T+AQ79t4R2jr
+kRgFz7iIJ8iQGleI+F9X4PRjhoOSsdaUkJRB6pxvxcsiYIKxDi7VScTx8iD0pgwn
+tgcQ0do1A0ZQsnJMtBnfIj1/J0sSMHEE9QKBwQDLUVyLmVjv0apqDxnNCw0laIFm
+ofp7S/q4pXfDDg3SqrM05Wgm4CHijzKzoqvFLILQvI004LShRcNXTMAsAbbIRzcY
+YbEOYytHB+k9WfEjAFJkNM4qB4w8erELKwnvjflodLgBw9k97cybhYIZCnwWIIHp
+SwXPT9AI5Ck8E6wifo1nWjpgMZtg5PaH6yfGa+o+ahmetKoU+4ENzVeU95XuKbVa
+x/6UW+wNbPqo3oEfV/K25U6WGHoGfX2X8wn/m2cCgcEAnlABXi5i9GH3ZnG5MJcA
+M3L4wNCsiADirmb19LEFsFDTC2LY5hHpiG4OSSIbK1bBQ6zTwG/umvE2HtPdEI+X
+KuoxbLfRAZYJEXVsJROZ6+s7k6nxycMzANh7rB+GZpHT7QEbdDWOyh/ioKgY8Lpz
+yZ0mzEQDUWehpOPWzpElDUYfjURB7d+xm0Ic+TEPPVH7Ha9KBn3S/FsTNWQaPx+r
+eP4BQpoggD30+VlwsKXcHES0ppeeHWODhxxAB8f/+zDVAoHBALJY3GVYTruPn30J
+YgiK+S0nTttImwAs1fHCtBtV6KozMp/j3Ei9svuZwU/yEdsUAGw5+WO4+Lm/CGs7
+2BbCKiPk1F9+0mFcfEoCloZKr0uUrLFZ4L7dgBZNSaASUNTiJTWLrR1fPuEkB6ck
+pcpxeAew3ERYmvAPgt1JxyH737Mib8eJTkuzOCj2r4rqrClR4Fh/mZmtwMRHGh2R
+UpJJ3CreS0cmyBo7yAS+4+HdzEZCT5Y/73+aWO/4hIMVnl+pYQKBwCpUb85zm5zg
+UnZ8nBS22FLGTcvBs8hbyXUtioSNadNteuqk6jsN2F+Pwsh6eHbVHW4Lu9j6Gn+J
+S1ss/ztgGkErvQF/9DpxMeYt01FpvZaUJthThQVQ9xvr9i7utgthtdspNvQ0fux1
+9Xg2fhLnDz707PUt7OhmVW7d+XOfoc19mYZlN0IOHsqMUMphIW97Lp5QWlZXxr23
+Zrv2j5mTvv3Fq2TRDNfz5dwijFMvv7kpGfHA1950ZIbobQvYYsoC7A==
+-----END RSA PRIVATE KEY-----

integration/h3/cert.pem

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID3jCCAkigAwIBAgIBADALBgkqhkiG9w0BAQUwFDESMBAGA1UEAxMJc3luY3Ro
+aW5nMB4XDTE0MDMxNDA3MDExMVoXDTQ5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJ
+c3luY3RoaW5nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAypziXAfA
+GG5mEwTIo+dUt3a4SxKnCVWfTXq7RF5ukCdz3jRRRoqy2ujbW5oe7qmqELTUzkJU
+eJkuCyvLIhVsKgt2r1peThy5+3/kUst9Faitnke1/jYtWwSf7WmK9TQBN/tFxdgR
+SM0LpdvsIJQqB5eodR51KrsjkJcKzEHeE2IQA8EKD1LcfXz8p22zdaZ4CNhz7mgF
+ghkRT6NINDZiMtm6R4qXqYyW1MNlYLw+DqXvE2eCQMQOcqI7SSZTeS3eSTP5NQum
+AmaLKn3JZQ9G6Ldn0VSWSPlXqVSaFTI9LnYICTDa4Nj4+L6idwzjmBkxPr8vBjlj
+15eo6xcU4fKqh2/xHIcZYpas1bwo5ljGdcfRm4L1zpxDY+nJFIgRv0Ndpltve+T8
+o3qiqFvIMiphFUtdb2pzgaTi8FW9SHMRFiuj6sNLI0Yb9u8BoN/QsWZnFh6fQoHW
+PKtkJrPZGw4GMMTHeEQe/eLW9VAH7ywPyc2hSzSri44aR44s4ErSV+XfAgMBAAGj
+PzA9MA4GA1UdDwEB/wQEAwIAoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH
+AwIwDAYDVR0TAQH/BAIwADALBgkqhkiG9w0BAQUDggGBAJshUcNfAveVE115S/NQ
+h+HfqahB3Gw69KmSWgFNXAEyXwOftmz9p2gL6DwfbU1SkbakU/YLJ3/mP4Q69EmX
+axzr108hUJ2qxH091IfKPCuTDtLsFY7g0PI+SBzADFSkoFe79aF/PSjaZk6fjvEl
+44PXzhxqqnSS+9lXQk+DIS6/jSVuX1XYvkkdHRXd9d7nw42sONDdSX8oQAUvGOeR
+CBleWB4AqS6HH2zFVIzRvjDsSg2wUXQjcqMqef7zBMpuWKTIQf+uaXWsEuPK2a3V
+IPwreWGwkxqGTToLagqOdAhgN+owvTJS9PaqxFqlxp8ypIYBnZ5VL5G5p115C17/
+xVENRxX0/oB99L5HGRhAuIGE4LhXq858DzmTZFcwm3fHragPKS/MMI71Z6U5RH2p
+VouosWzxAEx3Piq+ccenPLt6VPO5iFn+ppoBfwAFtNKyTrXcTQm33oJWFz+NSMAl
+vCq5H4c9x5CoWDpX2bdjpnZ3xcSj84fPYpkKBI52eZAvcw==
+-----END CERTIFICATE-----

integration/h3/config.xml

@@ -0,0 +1,30 @@
+<configuration version="1">
+    <repository directory="s3">
+        <node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1">
+            <address>127.0.0.1:22001</address>
+        </node>
+        <node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ" name="s2">
+            <address>127.0.0.1:22002</address>
+        </node>
+        <node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3">
+            <address>127.0.0.1:22003</address>
+        </node>
+    </repository>
+    <options>
+        <listenAddress>127.0.0.1:22003</listenAddress>
+        <readOnly>false</readOnly>
+        <allowDelete>true</allowDelete>
+        <followSymlinks>true</followSymlinks>
+        <guiEnabled>true</guiEnabled>
+        <guiAddress>127.0.0.1:8083</guiAddress>
+        <globalAnnounceServer>announce.syncthing.net:22025</globalAnnounceServer>
+        <globalAnnounceEnabled>false</globalAnnounceEnabled>
+        <localAnnounceEnabled>false</localAnnounceEnabled>
+        <parallelRequests>16</parallelRequests>
+        <maxSendKbps>0</maxSendKbps>
+        <rescanIntervalS>60</rescanIntervalS>
+        <reconnectionIntervalS>5</reconnectionIntervalS>
+        <maxChangeKbps>1000</maxChangeKbps>
+        <startBrowser>false</startBrowser>
+    </options>
+</configuration>

integration/h3/key.pem

@@ -0,0 +1,39 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIG5AIBAAKCAYEAypziXAfAGG5mEwTIo+dUt3a4SxKnCVWfTXq7RF5ukCdz3jRR
+Roqy2ujbW5oe7qmqELTUzkJUeJkuCyvLIhVsKgt2r1peThy5+3/kUst9Faitnke1
+/jYtWwSf7WmK9TQBN/tFxdgRSM0LpdvsIJQqB5eodR51KrsjkJcKzEHeE2IQA8EK
+D1LcfXz8p22zdaZ4CNhz7mgFghkRT6NINDZiMtm6R4qXqYyW1MNlYLw+DqXvE2eC
+QMQOcqI7SSZTeS3eSTP5NQumAmaLKn3JZQ9G6Ldn0VSWSPlXqVSaFTI9LnYICTDa
+4Nj4+L6idwzjmBkxPr8vBjlj15eo6xcU4fKqh2/xHIcZYpas1bwo5ljGdcfRm4L1
+zpxDY+nJFIgRv0Ndpltve+T8o3qiqFvIMiphFUtdb2pzgaTi8FW9SHMRFiuj6sNL
+I0Yb9u8BoN/QsWZnFh6fQoHWPKtkJrPZGw4GMMTHeEQe/eLW9VAH7ywPyc2hSzSr
+i44aR44s4ErSV+XfAgMBAAECggGBALolpuXsjPUlQIyKoZfMag3gefMnIOW3j5NM
+hg6LP8MbLB3jLSTFOwtaUmZ3U6HrqP6OVNFnKVpfSWkkBA29ZtG+FH2IZgoX5FsH
+JgtXPwWOImy/75mtxr/PoOsrQ2qCK/h116WsHD0pfWEVi3xnA7JUCIYJxJXMtyEZ
+U+dTQKfIOXRpf0eS1lZIZYuhgvY9Shy+WMyZLy5Wv4vONQEbUd7sIHOoBizUqKKz
+HkngyJcGpn7KY2YDek6hdByBpME5fFRo1DmXPeE+pTycDBP3C55JTB6v6kGkqxAl
+hTFK+x8tU9N+d+Wmuvfdm4MdQ1CnLjv+TL01OTYJx4nTDRk8PkVxVfrXdSY/LFib
+4mhghDPglvPHyFljbwkdiah60dAd2dIUhZwax79oxliyV6Ivf3teB5bvvrWIsMCX
+harESDSC6sgkHRGb05u1uYOPpsnAVOmi6CVgsKQtTmRZ6tqBA/O97RBWj92ROJi+
+0PqFs+cLXvpd4u36sI0MeWYO7Eko2QKBwQDo1Mc/fUsquniFrkjjXnGC8Q+tUpRC
+/TS2vjP5zFlgIOD8fSOoNF4VwFapCoG8l5NsOZrFOYvzHKZRtuBHLL0xdAOuxzge
+72mJYDsfO+vwOtg8xMrdtlJqDOnQWU2NVgOB+xSQKILEOm422BRyXbOM3hylsL7Y
+GPSTdqst9jcLODd6VFY4xq9VDebO63z5Ku62K+BerUJsUfrs64ES6U//Pivq4c06
+57BQq1AbVXqd8q9aInOsSQ1nJwg28PuBREUCgcEA3sZQRJSpktsdsnlvk4Nvf+KK
+OdYEAa2hPcflWDn/ZiEzaP0NR5s6NFSYhzGWVAX39lPUuHJ9MZ17CO4GqMojbXJc
+hneWi7+SWNqZn1rxdCl8FU7wJccX479G45lEL0aIN91i+9K0ZFGZg198EQcpX4kt
+s8J3O3n81rLnHg4bgMhJ6BGRIAxtYwmdOSfTmdyewVJfNUookswjC0sD5JDL/jd1
+Yi4CeaFOqRFVY+4ge+whYz9OcfBBj4kBl1OYoq3TAoHAZe35CJd+l8coykVhjYCk
+KxIDrfpQA/+72yDrujk3C3l5ZbAXMyUx503b4odCAuFM3f1d/2fRF+579ZwdFavF
+a+gBULvQmuJvDoA9gdAG25W3Yus4cNXHwLvEhL0D0ZNNV9MmznGdxfBandH9KZBR
+8aKvYe49rndGY2R5TlbTBCtpRjmzwYlh6td6Ky42+RJNjR0qTeiGAsvlEWGMkU3p
+ArIiIeMWqOoTa02EdWL3mjxLfidFArC8mGMjGoJnaNENAoHBAIQ9ZnJ/aPXS+hry
+uQfw0qQwXuscHr68SeW5nmuz6ea/OJxO8q4Z+AAOY4iFJ/5ymJHxi2l+FND58YoI
+eY2CiGs0orXzkTsdmgsCoISW4JOa+JxRgn56Y7T+217JoU8K6Ft5IIPpvMl8Ist4
+R9Z30Nh0PtvhSRPWQv9TrZwKtcrJmg2XN/W1Ss2qbFj8SkCgVODfO8MPZWxWn0rZ
+2dK5HU6nrxd7xl5bIa18q5qpRUEql1spvjAmdVR1+KrSpd2TnwKBwDGptuY2og9y
+PeEipbGrxww44DMFp6026xn0r8NXSlylD8Rihm9z2wQ0vFPjG1VHjupcQYI5bQLB
+piYt1PyPzCs27WrnXU6eEFjhVP4CzZHFKHkPeW6uBt7XbHpR14hxJ54FLN33Rewr
+Vt6xys+K3OHFzgDfBnhtvROYHtNQENiyJ3rMwRZcxXROmR/zj0cVl1R8wdE3XCwA
+rFPm/n3hzKv2OG3KX0KHh1bH0eXPVhIEmPcgz8RgL3uPGkqh8H+sgQ==
+-----END RSA PRIVATE KEY-----

integration/json.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strconv"
+	"strings"
+)
+
+func main() {
+	log.SetFlags(0)
+	flag.Parse()
+	path := strings.Split(flag.Arg(0), "/")
+
+	var obj map[string]interface{}
+	dec := json.NewDecoder(os.Stdin)
+	dec.UseNumber()
+	dec.Decode(&obj)
+
+	var v interface{} = obj
+	for _, p := range path {
+		switch tv := v.(type) {
+		case map[string]interface{}:
+			v = tv[p]
+		case []interface{}:
+			i, err := strconv.Atoi(p)
+			if err != nil {
+				log.Fatal(err)
+			}
+			v = tv[i]
+		default:
+			return // Silence is golden
+		}
+	}
+	fmt.Println(v)
+}

integration/md5r.go

@@ -9,7 +9,10 @@ import (
 	"path/filepath"
 )
 
+var long bool
+
 func main() {
+	flag.BoolVar(&long, "l", false, "Long output")
 	flag.Parse()
 	args := flag.Args()
 
@@ -37,7 +40,12 @@ func walker(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
-		fmt.Printf("%s  %s\n", sum, path)
+		if long {
+			fmt.Printf("%s  %s 0%03o %d\n", sum, path, info.Mode(), info.ModTime().Unix())
+
+		} else {
+			fmt.Printf("%s  %s\n", sum, path)
+		}
 	}
 
 	return nil

integration/test.sh

@@ -1,74 +1,94 @@
 #!/bin/bash
 
-rm -rf files-* conf-* md5-*
+export STNORESTART=1
 
-extraopts=""
-p=$(pwd)
+id1=I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA
+id2=JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ
+id3=373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA
 
 go build genfiles.go
 go build md5r.go
+go build json.go
 
-echo "Setting up (keys)..."
-i1=$(syncthing --home conf-1 2>&1 | awk '/My ID/ {print $7}')
-echo $i1
-i2=$(syncthing --home conf-2 2>&1 | awk '/My ID/ {print $7}')
-echo $i2
-i3=$(syncthing --home conf-3 2>&1 | awk '/My ID/ {print $7}')
-echo $i3
-
-echo "Setting up (files)..."
-for i in 1 2 3 ; do
-	cat >conf-$i/syncthing.ini <<EOT
-[repository]
-dir = $p/files-$i
-
-[nodes]
-$i1 = 127.0.0.1:22001
-$i2 = 127.0.0.1:22002
-$i3 = 127.0.0.1:22003
-
-[settings]
-gui-enabled = false
-listen-address = :2200$i
-EOT
-
-	mkdir files-$i
-	pushd files-$i >/dev/null
-	../genfiles -maxexp 21 -files 400
-	touch empty-$i
-	../md5r > ../md5-$i
-	popd >/dev/null
-done
-
-echo "Starting..."
-for i in 1 2 3 ; do
-	sleep 1
-	syncthing --home conf-$i $extraopts &
-done
-
-cat md5-* | sort > md5-tot
-while true ; do
-	read
-	echo Verifying...
-
-	conv=0
+testConvergence() {
+	echo "Starting..."
 	for i in 1 2 3 ; do
-		pushd files-$i >/dev/null
-		../md5r | sort > ../md5-$i
-		popd >/dev/null
-		if ! cmp md5-$i md5-tot >/dev/null ; then
-			echo $i unconverged
-		else
-			conv=$((conv + 1))
-			echo $i converged
+		sleep 1
+		syncthing -home "h$i" &
+	done
+
+	while true ; do
+		sleep 5
+		s1comp=$(curl -s "http://localhost:8082/rest/connections" | ./json "$id1/Completion")
+		s2comp=$(curl -s "http://localhost:8083/rest/connections" | ./json "$id2/Completion")
+		s3comp=$(curl -s "http://localhost:8081/rest/connections" | ./json "$id3/Completion")
+		s1comp=${s1comp:-0}
+		s2comp=${s2comp:-0}
+		s3comp=${s3comp:-0}
+		tot=$(($s1comp + $s2comp + $s3comp))
+		echo $tot / 300
+		if [[ $tot == 300 ]] ; then
+			echo "Stopping..."
+			pkill syncthing
+			break
 		fi
 	done
 
-	if [[ $conv == 3 ]] ; then
-		kill %1
-		kill %2
-		kill %3
-		exit
-	fi
+	echo "Verifying..."
+	cat md5-* | sort | uniq > md5-tot
+
+	for i in 1 2 3 ; do
+		pushd "s$i" >/dev/null
+		../md5r -l | sort > ../md5-$i
+		popd >/dev/null
+		if ! cmp "md5-$i" md5-tot >/dev/null ; then
+			echo "Fail: instance $i unconverged"
+			diff -u md5-tot "md5-$i"
+			exit
+		else
+			echo "OK: instance $i converged"
+		fi
+	done
+}
+
+echo "Setting up files..."
+for i in 1 2 3 ; do
+	rm -f h$i/*.idx.gz
+	rm -rf "s$i"
+	mkdir "s$i"
+	pushd "s$i" >/dev/null
+	echo "  $i: random nonoverlapping"
+	../genfiles -maxexp 22 -files 600
+	echo "  $i: empty file"
+	touch "empty-$i"
+	echo "  $i: common file"
+	dd if=/dev/urandom of=common bs=1000 count=1000 2>/dev/null
+	popd >/dev/null
 done
 
+# instance 1 common file should be the newest, the other should disappear
+sleep 2
+touch "s1/common"
+
+echo "MD5-summing..."
+for i in 1 2 3 ; do
+	pushd "s$i" >/dev/null
+	../md5r -l > ../md5-$i
+	popd >/dev/null
+done
+grep -v common md5-2 > t ; mv t md5-2
+grep -v common md5-3 > t ; mv t md5-3
+
+testConvergence
+
+echo "Add and remove random files..."
+for i in 1 2 3 ; do
+	pushd "s$i" >/dev/null
+	rm -rf */?[02468ace]
+	../genfiles -maxexp 22 -files 600
+	../md5r -l > ../md5-$i
+	popd >/dev/null
+done
+
+testConvergence
+

protocol/common_test.go

@@ -46,7 +46,6 @@ func (e *ErrPipe) Write(data []byte) (int, error) {
 		e.PipeWriter.CloseWithError(e.err)
 		e.closed = true
 		return n, e.err
-	} else {
-		return e.PipeWriter.Write(data)
 	}
+	return e.PipeWriter.Write(data)
 }

protocol/doc.go

@@ -0,0 +1,2 @@
+// Package protocol implements the Block Exchange Protocol.
+package protocol

protocol/protocol.go

@@ -26,12 +26,13 @@ const (
 )
 
 const (
-	FlagDeleted = 1 << 12
-	FlagInvalid = 1 << 13
+	FlagDeleted uint32 = 1 << 12
+	FlagInvalid        = 1 << 13
 )
 
 var (
-	ErrClusterHash = fmt.Errorf("Configuration error: mismatched cluster hash")
+	ErrClusterHash = fmt.Errorf("configuration error: mismatched cluster hash")
+	ErrClosed      = errors.New("connection closed")
 )
 
 type Model interface {
@@ -56,7 +57,7 @@ type Connection struct {
 	xw          *xdr.Writer
 	closed      bool
 	awaiting    map[int]chan asyncResult
-	nextId      int
+	nextID      int
 	indexSent   map[string]map[string][2]int64
 	peerOptions map[string]string
 	myOptions   map[string]string
@@ -68,8 +69,6 @@ type Connection struct {
 	statisticsLock sync.Mutex
 }
 
-var ErrClosed = errors.New("Connection closed")
-
 type asyncResult struct {
 	val []byte
 	err error
@@ -105,7 +104,7 @@ func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver M
 		c.myOptions = options
 		go func() {
 			c.Lock()
-			header{0, c.nextId, messageTypeOptions}.encodeXDR(c.xw)
+			header{0, c.nextID, messageTypeOptions}.encodeXDR(c.xw)
 			var om OptionsMessage
 			for k, v := range options {
 				om.Options = append(om.Options, Option{k, v})
@@ -118,7 +117,7 @@ func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver M
 			if err != nil {
 				log.Println("Warning: Write error during initial handshake:", err)
 			}
-			c.nextId++
+			c.nextID++
 			c.Unlock()
 		}()
 	}
@@ -155,12 +154,12 @@ func (c *Connection) Index(repo string, idx []FileInfo) {
 		idx = diff
 	}
 
-	header{0, c.nextId, msgType}.encodeXDR(c.xw)
+	header{0, c.nextID, msgType}.encodeXDR(c.xw)
 	_, err := IndexMessage{repo, idx}.encodeXDR(c.xw)
 	if err == nil {
 		err = c.flush()
 	}
-	c.nextId = (c.nextId + 1) & 0xfff
+	c.nextID = (c.nextID + 1) & 0xfff
 	c.hasSentIndex = true
 	c.Unlock()
 
@@ -178,8 +177,8 @@ func (c *Connection) Request(repo string, name string, offset int64, size int) (
 		return nil, ErrClosed
 	}
 	rc := make(chan asyncResult)
-	c.awaiting[c.nextId] = rc
-	header{0, c.nextId, messageTypeRequest}.encodeXDR(c.xw)
+	c.awaiting[c.nextID] = rc
+	header{0, c.nextID, messageTypeRequest}.encodeXDR(c.xw)
 	_, err := RequestMessage{repo, name, uint64(offset), uint32(size)}.encodeXDR(c.xw)
 	if err == nil {
 		err = c.flush()
@@ -189,7 +188,7 @@ func (c *Connection) Request(repo string, name string, offset int64, size int) (
 		c.close(err)
 		return nil, err
 	}
-	c.nextId = (c.nextId + 1) & 0xfff
+	c.nextID = (c.nextID + 1) & 0xfff
 	c.Unlock()
 
 	res, ok := <-rc
@@ -206,8 +205,8 @@ func (c *Connection) ping() bool {
 		return false
 	}
 	rc := make(chan asyncResult, 1)
-	c.awaiting[c.nextId] = rc
-	header{0, c.nextId, messageTypePing}.encodeXDR(c.xw)
+	c.awaiting[c.nextID] = rc
+	header{0, c.nextID, messageTypePing}.encodeXDR(c.xw)
 	err := c.flush()
 	if err != nil {
 		c.Unlock()
@@ -218,7 +217,7 @@ func (c *Connection) ping() bool {
 		c.close(c.xw.Error())
 		return false
 	}
-	c.nextId = (c.nextId + 1) & 0xfff
+	c.nextID = (c.nextID + 1) & 0xfff
 	c.Unlock()
 
 	res, ok := <-rc
@@ -268,7 +267,7 @@ loop:
 			break loop
 		}
 		if hdr.version != 0 {
-			c.close(fmt.Errorf("Protocol error: %s: unknown message version %#x", c.ID, hdr.version))
+			c.close(fmt.Errorf("protocol error: %s: unknown message version %#x", c.id, hdr.version))
 			break loop
 		}
 
@@ -371,7 +370,7 @@ loop:
 			}
 
 		default:
-			c.close(fmt.Errorf("Protocol error: %s: unknown message type %#x", c.ID, hdr.msgType))
+			c.close(fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType))
 			break loop
 		}
 	}
@@ -410,10 +409,10 @@ func (c *Connection) pingerLoop() {
 			select {
 			case ok := <-rc:
 				if !ok {
-					c.close(fmt.Errorf("Ping failure"))
+					c.close(fmt.Errorf("ping failure"))
 				}
 			case <-time.After(pingTimeout):
-				c.close(fmt.Errorf("Ping timeout"))
+				c.close(fmt.Errorf("ping timeout"))
 			}
 		}
 	}

protocol/protocol_test.go

@@ -38,7 +38,7 @@ func TestPing(t *testing.T) {
 }
 
 func TestPingErr(t *testing.T) {
-	e := errors.New("Something broke")
+	e := errors.New("something broke")
 
 	for i := 0; i < 12; i++ {
 		for j := 0; j < 12; j++ {
@@ -64,7 +64,7 @@ func TestPingErr(t *testing.T) {
 }
 
 func TestRequestResponseErr(t *testing.T) {
-	e := errors.New("Something broke")
+	e := errors.New("something broke")
 
 	var pass bool
 	for i := 0; i < 48; i++ {
@@ -99,16 +99,16 @@ func TestRequestResponseErr(t *testing.T) {
 				t.Errorf("Incorrect response data %q", string(d))
 			}
 			if m0.repo != "default" {
-				t.Error("Incorrect repo %q", m0.repo)
+				t.Errorf("Incorrect repo %q", m0.repo)
 			}
 			if m0.name != "tn" {
-				t.Error("Incorrect name %q", m0.name)
+				t.Errorf("Incorrect name %q", m0.name)
 			}
 			if m0.offset != 1234 {
-				t.Error("Incorrect offset %d", m0.offset)
+				t.Errorf("Incorrect offset %d", m0.offset)
 			}
 			if m0.size != 5678 {
-				t.Error("Incorrect size %d", m0.size)
+				t.Errorf("Incorrect size %d", m0.size)
 			}
 			t.Logf("Pass at %d+%d bytes", i, j)
 			pass = true

scanner/blocks.go

@@ -1,4 +1,4 @@
-package main
+package scanner
 
 import (
 	"bytes"
@@ -17,7 +17,7 @@ func Blocks(r io.Reader, blocksize int) ([]Block, error) {
 	var blocks []Block
 	var offset int64
 	for {
-		lr := &io.LimitedReader{r, int64(blocksize)}
+		lr := &io.LimitedReader{R: r, N: int64(blocksize)}
 		hf := sha256.New()
 		n, err := io.Copy(hf, lr)
 		if err != nil {

scanner/blocks_test.go

@@ -1,4 +1,4 @@
-package main
+package scanner
 
 import (
 	"bytes"

scanner/debug.go

@@ -0,0 +1,12 @@
+package scanner
+
+import (
+	"log"
+	"os"
+	"strings"
+)
+
+var (
+	dlog  = log.New(os.Stderr, "scanner: ", log.Lmicroseconds|log.Lshortfile)
+	debug = strings.Contains(os.Getenv("STTRACE"), "scanner")
+)

scanner/doc.go

@@ -0,0 +1,2 @@
+// Package scanner implements a file system scanner and hasher.
+package scanner

scanner/file.go

@@ -0,0 +1,26 @@
+package scanner
+
+import "fmt"
+
+type File struct {
+	Name       string
+	Flags      uint32
+	Modified   int64
+	Version    uint32
+	Size       int64
+	Blocks     []Block
+	Suppressed bool
+}
+
+func (f File) String() string {
+	return fmt.Sprintf("File{Name:%q, Flags:0x%x, Modified:%d, Version:%d, Size:%d, NumBlocks:%d}",
+		f.Name, f.Flags, f.Modified, f.Version, f.Size, len(f.Blocks))
+}
+
+func (f File) Equals(o File) bool {
+	return f.Modified == o.Modified && f.Version == o.Version
+}
+
+func (f File) NewerThan(o File) bool {
+	return f.Modified > o.Modified || (f.Modified == o.Modified && f.Version > o.Version)
+}

scanner/testdata/.stignore

@@ -0,0 +1,2 @@
+.*
+quux

scanner/testdata/bar

@@ -0,0 +1 @@
+foobarbaz

scanner/testdata/baz/quux

@@ -0,0 +1 @@
+baazquux

scanner/testdata/foo

@@ -0,0 +1 @@
+foobar

scanner/walk.go

@@ -0,0 +1,262 @@
+package scanner
+
+import (
+	"bytes"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+type Walker struct {
+	// Dir is the base directory for the walk
+	Dir string
+	// If FollowSymlinks is true, symbolic links directly under Dir will be followed.
+	// Symbolic links at deeper levels are never followed regardless of this flag.
+	FollowSymlinks bool
+	// BlockSize controls the size of the block used when hashing.
+	BlockSize int
+	// If IgnoreFile is not empty, it is the name used for the file that holds ignore patterns.
+	IgnoreFile string
+	// If TempNamer is not nil, it is used to ignore tempory files when walking.
+	TempNamer TempNamer
+	// If CurrentFiler is not nil, it is queried for the current file before rescanning.
+	CurrentFiler CurrentFiler
+	// If Suppressor is not nil, it is queried for supression of modified files.
+	// Suppressed files will be returned with empty metadata and the Suppressed flag set.
+	// Requires CurrentFiler to be set.
+	Suppressor Suppressor
+
+	suppressed map[string]bool // file name -> suppression status
+}
+
+type TempNamer interface {
+	// Temporary returns a temporary name for the filed referred to by path.
+	TempName(path string) string
+	// IsTemporary returns true if path refers to the name of temporary file.
+	IsTemporary(path string) bool
+}
+
+type Suppressor interface {
+	// Supress returns true if the update to the named file should be ignored.
+	Suppress(name string, fi os.FileInfo) bool
+}
+
+type CurrentFiler interface {
+	// CurrentFile returns the file as seen at last scan.
+	CurrentFile(name string) File
+}
+
+// Walk returns the list of files found in the local repository by scanning the
+// file system. Files are blockwise hashed.
+func (w *Walker) Walk() (files []File, ignore map[string][]string) {
+	w.lazyInit()
+
+	if debug {
+		dlog.Println("Walk", w.Dir, w.FollowSymlinks, w.BlockSize, w.IgnoreFile)
+	}
+	t0 := time.Now()
+
+	ignore = make(map[string][]string)
+	hashFiles := w.walkAndHashFiles(&files, ignore)
+
+	filepath.Walk(w.Dir, w.loadIgnoreFiles(w.Dir, ignore))
+	filepath.Walk(w.Dir, hashFiles)
+
+	if w.FollowSymlinks {
+		d, err := os.Open(w.Dir)
+		if err != nil {
+			return
+		}
+		defer d.Close()
+
+		fis, err := d.Readdir(-1)
+		if err != nil {
+			return
+		}
+
+		for _, info := range fis {
+			if info.Mode()&os.ModeSymlink != 0 {
+				dir := path.Join(w.Dir, info.Name()) + "/"
+				filepath.Walk(dir, w.loadIgnoreFiles(dir, ignore))
+				filepath.Walk(dir, hashFiles)
+			}
+		}
+	}
+
+	if debug {
+		t1 := time.Now()
+		d := t1.Sub(t0).Seconds()
+		dlog.Printf("Walk in %.02f ms, %.0f files/s", d*1000, float64(len(files))/d)
+	}
+	return
+}
+
+// CleanTempFiles removes all files that match the temporary filename pattern.
+func (w *Walker) CleanTempFiles() {
+	filepath.Walk(w.Dir, w.cleanTempFile)
+}
+
+func (w *Walker) lazyInit() {
+	if w.suppressed == nil {
+		w.suppressed = make(map[string]bool)
+	}
+}
+
+func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.WalkFunc {
+	return func(p string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+
+		rn, err := filepath.Rel(dir, p)
+		if err != nil {
+			return nil
+		}
+
+		if pn, sn := path.Split(rn); sn == w.IgnoreFile {
+			pn := strings.Trim(pn, "/")
+			bs, _ := ioutil.ReadFile(p)
+			lines := bytes.Split(bs, []byte("\n"))
+			var patterns []string
+			for _, line := range lines {
+				if len(line) > 0 {
+					patterns = append(patterns, string(line))
+				}
+			}
+			ign[pn] = patterns
+		}
+
+		return nil
+	}
+}
+
+func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
+	return func(p string, info os.FileInfo, err error) error {
+		if err != nil {
+			if debug {
+				dlog.Println("error:", p, info, err)
+			}
+			return nil
+		}
+
+		rn, err := filepath.Rel(w.Dir, p)
+		if err != nil {
+			if debug {
+				dlog.Println("rel error:", p, err)
+			}
+			return nil
+		}
+
+		if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
+			if debug {
+				dlog.Println("temporary:", rn)
+			}
+			return nil
+		}
+
+		if _, sn := path.Split(rn); sn == w.IgnoreFile {
+			if debug {
+				dlog.Println("ignorefile:", rn)
+			}
+			return nil
+		}
+
+		if rn != "." && w.ignoreFile(ign, rn) {
+			if debug {
+				dlog.Println("ignored:", rn)
+			}
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+
+		if info.Mode()&os.ModeType == 0 {
+			if w.CurrentFiler != nil {
+				cf := w.CurrentFiler.CurrentFile(rn)
+				if cf.Modified == info.ModTime().Unix() {
+					if debug {
+						dlog.Println("unchanged:", rn)
+					}
+					*res = append(*res, cf)
+					return nil
+				}
+
+				if w.Suppressor != nil && w.Suppressor.Suppress(rn, info) {
+					if debug {
+						dlog.Println("suppressed:", rn)
+					}
+					if !w.suppressed[rn] {
+						w.suppressed[rn] = true
+						log.Printf("INFO: Changes to %q are being temporarily suppressed because it changes too frequently.", p)
+					}
+					cf.Suppressed = true
+					*res = append(*res, cf)
+				} else if w.suppressed[rn] {
+					log.Printf("INFO: Changes to %q are no longer suppressed.", p)
+					delete(w.suppressed, rn)
+				}
+			}
+
+			fd, err := os.Open(p)
+			if err != nil {
+				if debug {
+					dlog.Println("open:", p, err)
+				}
+				return nil
+			}
+			defer fd.Close()
+
+			t0 := time.Now()
+			blocks, err := Blocks(fd, w.BlockSize)
+			if err != nil {
+				if debug {
+					dlog.Println("hash error:", rn, err)
+				}
+				return nil
+			}
+			if debug {
+				t1 := time.Now()
+				dlog.Println("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s")
+			}
+			f := File{
+				Name:     rn,
+				Size:     info.Size(),
+				Flags:    uint32(info.Mode()),
+				Modified: info.ModTime().Unix(),
+				Blocks:   blocks,
+			}
+			*res = append(*res, f)
+		}
+
+		return nil
+	}
+}
+
+func (w *Walker) cleanTempFile(path string, info os.FileInfo, err error) error {
+	if err != nil {
+		return err
+	}
+	if info.Mode()&os.ModeType == 0 && w.TempNamer.IsTemporary(path) {
+		os.Remove(path)
+	}
+	return nil
+}
+
+func (w *Walker) ignoreFile(patterns map[string][]string, file string) bool {
+	first, last := path.Split(file)
+	for prefix, pats := range patterns {
+		if len(prefix) == 0 || prefix == first || strings.HasPrefix(first, prefix+"/") {
+			for _, pattern := range pats {
+				if match, _ := path.Match(pattern, last); match {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}

scanner/walk_test.go

@@ -1,4 +1,4 @@
-package main
+package scanner
 
 import (
 	"fmt"
@@ -22,8 +22,12 @@ var correctIgnores = map[string][]string{
 }
 
 func TestWalk(t *testing.T) {
-	m := NewModel("testdata", 1e6)
-	files, ignores := m.Walk(false)
+	w := Walker{
+		Dir:        "testdata",
+		BlockSize:  128 * 1024,
+		IgnoreFile: ".stignore",
+	}
+	files, ignores := w.Walk()
 
 	if l1, l2 := len(files), len(testdata); l1 != l2 {
 		t.Fatalf("Incorrect number of walked files %d != %d", l1, l2)
@@ -75,8 +79,9 @@ func TestIgnore(t *testing.T) {
 		{"foo/bazz/quux", false},
 	}
 
+	w := Walker{}
 	for i, tc := range tests {
-		if r := ignoreFile(patterns, tc.f); r != tc.r {
+		if r := w.ignoreFile(patterns, tc.f); r != tc.r {
 			t.Errorf("Incorrect ignoreFile() #%d; E: %v, A: %v", i, tc.r, r)
 		}
 	}

walk.go

@@ -1,238 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"path"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/calmh/syncthing/protocol"
-)
-
-const BlockSize = 128 * 1024
-
-type File struct {
-	Name     string
-	Flags    uint32
-	Modified int64
-	Version  uint32
-	Size     int64
-	Blocks   []Block
-}
-
-func (f File) String() string {
-	return fmt.Sprintf("File{Name:%q, Flags:0x%x, Modified:%d, Version:%d, Size:%d, NumBlocks:%d}",
-		f.Name, f.Flags, f.Modified, f.Version, f.Size, len(f.Blocks))
-}
-
-func (f File) Equals(o File) bool {
-	return f.Modified == o.Modified && f.Version == o.Version
-}
-
-func (f File) NewerThan(o File) bool {
-	return f.Modified > o.Modified || (f.Modified == o.Modified && f.Version > o.Version)
-}
-
-func isTempName(name string) bool {
-	return strings.HasPrefix(path.Base(name), ".syncthing.")
-}
-
-func tempName(name string, modified int64) string {
-	tdir := path.Dir(name)
-	tname := fmt.Sprintf(".syncthing.%s.%d", path.Base(name), modified)
-	return path.Join(tdir, tname)
-}
-
-func (m *Model) loadIgnoreFiles(ign map[string][]string) filepath.WalkFunc {
-	return func(p string, info os.FileInfo, err error) error {
-		if err != nil {
-			return nil
-		}
-
-		rn, err := filepath.Rel(m.dir, p)
-		if err != nil {
-			return nil
-		}
-
-		if pn, sn := path.Split(rn); sn == ".stignore" {
-			pn := strings.Trim(pn, "/")
-			bs, _ := ioutil.ReadFile(p)
-			lines := bytes.Split(bs, []byte("\n"))
-			var patterns []string
-			for _, line := range lines {
-				if len(line) > 0 {
-					patterns = append(patterns, string(line))
-				}
-			}
-			ign[pn] = patterns
-		}
-
-		return nil
-	}
-}
-
-func (m *Model) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
-	return func(p string, info os.FileInfo, err error) error {
-		if err != nil {
-			if m.trace["file"] {
-				log.Printf("FILE: %q: %v", p, err)
-			}
-			return nil
-		}
-
-		if isTempName(p) {
-			return nil
-		}
-
-		rn, err := filepath.Rel(m.dir, p)
-		if err != nil {
-			return nil
-		}
-
-		if _, sn := path.Split(rn); sn == ".stignore" {
-			// We never sync the .stignore files
-			return nil
-		}
-
-		if ignoreFile(ign, rn) {
-			if m.trace["file"] {
-				log.Println("FILE: IGNORE:", rn)
-			}
-			return nil
-		}
-
-		if info.Mode()&os.ModeType == 0 {
-			modified := info.ModTime().Unix()
-
-			m.lmut.RLock()
-			lf, ok := m.local[rn]
-			m.lmut.RUnlock()
-
-			if ok && lf.Modified == modified {
-				if nf := uint32(info.Mode()); nf != lf.Flags {
-					lf.Flags = nf
-					lf.Version++
-				}
-				*res = append(*res, lf)
-			} else {
-				if cur, prev := m.sup.suppress(rn, info.Size(), time.Now()); cur {
-					if m.trace["file"] {
-						log.Printf("FILE: SUPPRESS: %q change bw over threshold", rn)
-					}
-					if !prev {
-						log.Printf("INFO: Changes to %q are being temporarily suppressed because it changes too frequently.", rn)
-					}
-
-					if ok {
-						lf.Flags = protocol.FlagInvalid
-						lf.Version++
-						*res = append(*res, lf)
-					}
-					return nil
-				} else if prev && !cur {
-					log.Printf("INFO: Changes to %q are no longer suppressed.", rn)
-				}
-
-				if m.trace["file"] {
-					log.Printf("FILE: Hash %q", p)
-				}
-				fd, err := os.Open(p)
-				if err != nil {
-					if m.trace["file"] {
-						log.Printf("FILE: %q: %v", p, err)
-					}
-					return nil
-				}
-				defer fd.Close()
-
-				blocks, err := Blocks(fd, BlockSize)
-				if err != nil {
-					if m.trace["file"] {
-						log.Printf("FILE: %q: %v", p, err)
-					}
-					return nil
-				}
-				f := File{
-					Name:     rn,
-					Size:     info.Size(),
-					Flags:    uint32(info.Mode()),
-					Modified: modified,
-					Blocks:   blocks,
-				}
-				*res = append(*res, f)
-			}
-		}
-
-		return nil
-	}
-}
-
-// Walk returns the list of files found in the local repository by scanning the
-// file system. Files are blockwise hashed.
-func (m *Model) Walk(followSymlinks bool) (files []File, ignore map[string][]string) {
-	ignore = make(map[string][]string)
-
-	hashFiles := m.walkAndHashFiles(&files, ignore)
-
-	filepath.Walk(m.dir, m.loadIgnoreFiles(ignore))
-	filepath.Walk(m.dir, hashFiles)
-
-	if followSymlinks {
-		d, err := os.Open(m.dir)
-		if err != nil {
-			return
-		}
-		defer d.Close()
-
-		fis, err := d.Readdir(-1)
-		if err != nil {
-			return
-		}
-
-		for _, info := range fis {
-			if info.Mode()&os.ModeSymlink != 0 {
-				dir := path.Join(m.dir, info.Name()) + "/"
-				filepath.Walk(dir, m.loadIgnoreFiles(ignore))
-				filepath.Walk(dir, hashFiles)
-			}
-		}
-	}
-
-	return
-}
-
-func (m *Model) cleanTempFile(path string, info os.FileInfo, err error) error {
-	if err != nil {
-		return err
-	}
-	if info.Mode()&os.ModeType == 0 && isTempName(path) {
-		if m.trace["file"] {
-			log.Printf("FILE: Remove %q", path)
-		}
-		os.Remove(path)
-	}
-	return nil
-}
-
-func (m *Model) cleanTempFiles() {
-	filepath.Walk(m.dir, m.cleanTempFile)
-}
-
-func ignoreFile(patterns map[string][]string, file string) bool {
-	first, last := path.Split(file)
-	for prefix, pats := range patterns {
-		if len(prefix) == 0 || prefix == first || strings.HasPrefix(first, prefix+"/") {
-			for _, pattern := range pats {
-				if match, _ := path.Match(pattern, last); match {
-					return true
-				}
-			}
-		}
-	}
-	return false
-}

xdr/doc.go

@@ -0,0 +1,2 @@
+// Package xdr implements an XDR (RFC 4506) encoder/decoder.
+package xdr

xdr/reader.go

@@ -5,7 +5,7 @@ import (
 	"io"
 )
 
-var ErrElementSizeExceeded = errors.New("Element size exceeded")
+var ErrElementSizeExceeded = errors.New("element size exceeded")
 
 type Reader struct {
 	r   io.Reader