Actually announce listen port locally

Change default ListenAddress to "0.0.0.0:22000"

CONTRIBUTORS

@@ -2,4 +2,6 @@ Aaron Bieber <qbit@deftly.net>
 Andrew Dunham <andrew@du.nham.ca>
 Brandon Philips <brandon@ifup.org>
 James Patterson <jamespatterson@operamail.com>
+Jens Diemer <github.com@jensdiemer.de>
 Philippe Schommers <philippe@schommers.be>
+Veeti Paananen <veeti.paananen@rojekti.fi>

README.md

@@ -5,8 +5,8 @@ This is the `syncthing` project. The following are the project goals:
 
  1. Define a protocol for synchronization of a file repository between a
     number of collaborating nodes. The protocol should be well defined,
-    unambigous, easily understood, free to use, efficient, secure and
-    languange neutral. This is the [Block Exchange
+    unambiguous, easily understood, free to use, efficient, secure and
+    language neutral. This is the [Block Exchange
     Protocol](https://github.com/calmh/syncthing/blob/master/protocol/PROTOCOL.md).
 
  2. Provide the reference implementation to demonstrate the usability of

beacon/beacon.go

@@ -0,0 +1,127 @@
+package beacon
+
+import "net"
+
+type recv struct {
+	data []byte
+	src  net.Addr
+}
+
+type dst struct {
+	intf string
+	conn *net.UDPConn
+}
+
+type Beacon struct {
+	conn   *net.UDPConn
+	port   int
+	conns  []dst
+	inbox  chan []byte
+	outbox chan recv
+}
+
+func New(port int) (*Beacon, error) {
+	conn, err := net.ListenUDP("udp", &net.UDPAddr{Port: port})
+	if err != nil {
+		return nil, err
+	}
+	b := &Beacon{
+		conn:   conn,
+		port:   port,
+		inbox:  make(chan []byte),
+		outbox: make(chan recv, 16),
+	}
+
+	go b.reader()
+	go b.writer()
+
+	return b, nil
+}
+
+func (b *Beacon) Send(data []byte) {
+	b.inbox <- data
+}
+
+func (b *Beacon) Recv() ([]byte, net.Addr) {
+	recv := <-b.outbox
+	return recv.data, recv.src
+}
+
+func (b *Beacon) reader() {
+	var bs = make([]byte, 65536)
+	for {
+		n, addr, err := b.conn.ReadFrom(bs)
+		if err != nil {
+			l.Warnln("Beacon read:", err)
+			return
+		}
+		if debug {
+			l.Debugf("recv %d bytes from %s", n, addr)
+		}
+		select {
+		case b.outbox <- recv{bs[:n], addr}:
+		default:
+			if debug {
+				l.Debugln("dropping message")
+			}
+		}
+	}
+}
+
+func (b *Beacon) writer() {
+	for bs := range b.inbox {
+
+		addrs, err := net.InterfaceAddrs()
+		if err != nil {
+			l.Warnln("Beacon: interface addresses:", err)
+			continue
+		}
+
+		var dsts []net.IP
+		for _, addr := range addrs {
+			if iaddr, ok := addr.(*net.IPNet); ok && iaddr.IP.IsGlobalUnicast() {
+				baddr := bcast(iaddr)
+				dsts = append(dsts, baddr.IP)
+			}
+		}
+
+		if len(dsts) == 0 {
+			// Fall back to the general IPv4 broadcast address
+			dsts = append(dsts, net.IP{0xff, 0xff, 0xff, 0xff})
+		}
+
+		if debug {
+			l.Debugln("addresses:", dsts)
+		}
+
+		for _, ip := range dsts {
+			dst := &net.UDPAddr{IP: ip, Port: b.port}
+
+			_, err := b.conn.WriteTo(bs, dst)
+			if err != nil {
+				if debug {
+					l.Debugln(err)
+				}
+				return
+			}
+			if debug {
+				l.Debugf("sent %d bytes to %s", len(bs), dst)
+			}
+		}
+	}
+}
+
+func bcast(ip *net.IPNet) *net.IPNet {
+	var bc = &net.IPNet{}
+	bc.IP = make([]byte, len(ip.IP))
+	copy(bc.IP, ip.IP)
+	bc.Mask = ip.Mask
+
+	offset := len(bc.IP) - len(bc.Mask)
+	for i := range bc.IP {
+		if i-offset > 0 {
+			bc.IP[i] = ip.IP[i] | ^ip.Mask[i-offset]
+		}
+	}
+	return bc
+}

beacon/cmd/mctest/main.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"encoding/binary"
+	"log"
+	"time"
+
+	"github.com/calmh/syncthing/beacon"
+)
+
+func main() {
+	b, err := beacon.NewBeacon(21025)
+	if err != nil {
+		log.Fatal(err)
+	}
+	go func() {
+		for {
+			bs, addr := b.Recv()
+			log.Printf("Received %d bytes from %s: %x %x", len(bs), addr, bs[:8], bs[8:])
+		}
+	}()
+	go func() {
+		bs := [16]byte{}
+		binary.BigEndian.PutUint64(bs[:], uint64(time.Now().UnixNano()))
+		log.Printf("My ID: %x", bs[:8])
+		for {
+			binary.BigEndian.PutUint64(bs[8:], uint64(time.Now().UnixNano()))
+			b.Send(bs[:])
+			log.Printf("Sent %d bytes", len(bs[:]))
+			time.Sleep(10 * time.Second)
+		}
+	}()
+	select {}
+}

beacon/debug.go

@@ -0,0 +1,13 @@
+package beacon
+
+import (
+	"os"
+	"strings"
+
+	"github.com/calmh/syncthing/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "beacon") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

beacon/doc.go

@@ -0,0 +1,2 @@
+// Package beacon implements an UDP broadcast beacon
+package beacon

build.sh

@@ -4,32 +4,34 @@ export COPYFILE_DISABLE=true
 
 distFiles=(README.md LICENSE) # apart from the binary itself
 version=$(git describe --always --dirty)
-date=$(date +%s)
+date=$(git show -s --format=%ct)
 user=$(whoami)
 host=$(hostname)
 host=${host%%.*}
 ldflags="-w -X main.Version $version -X main.BuildStamp $date -X main.BuildUser $user -X main.BuildHost $host"
 
+check() {
+	if ! command -v godep >/dev/null ; then
+		echo "Error: no godep. Try \"$0 setup\"."
+		exit 1
+	fi
+}
+
 build() {
+	check
+
 	go vet ./... || exit 1
 
-	if command -v godep >/dev/null ; then
-		godep=godep
-	else
-		echo "Warning: no godep, using \"go get\" instead."
-		echo "Try \"go get github.com/tools/godep\"."
-		go get -d ./cmd/syncthing
-		godep=
-	fi
-	${godep} go build $* -ldflags "$ldflags" ./cmd/syncthing
-	${godep} go build -ldflags "$ldflags" ./cmd/stcli
+	godep go build $* -ldflags "$ldflags" ./cmd/syncthing
 }
 
 assets() {
+	check
 	godep go run cmd/assets/assets.go gui > auto/gui.files.go
 }
 
 test() {
+	check
 	godep go test -cpu=1,2,4 ./...
 }
 
@@ -64,7 +66,15 @@ zipDist() {
 }
 
 deps() {
-	godep save ./cmd/syncthing ./cmd/assets ./cmd/stcli ./discover/cmd/discosrv
+	check
+	godep save ./cmd/syncthing ./cmd/assets ./discover/cmd/discosrv
+}
+
+setup() {
+	echo Installing godep...
+	go get -u github.com/tools/godep
+	echo Installing go vet...
+	go get -u code.google.com/p/go.tools/cmd/vet
 }
 
 case "$1" in
@@ -77,6 +87,10 @@ case "$1" in
 		build -race
 		;;
 
+	guidev)
+		build -tags guidev
+		;;
+
 	test)
 		test
 		;;
@@ -98,7 +112,7 @@ case "$1" in
 		test || exit 1
 		assets
 
-		for os in darwin-amd64 linux-386 linux-amd64 freebsd-amd64 windows-amd64 ; do
+		for os in darwin-amd64 linux-386 linux-amd64 freebsd-amd64 windows-amd64 windows-386 ; do
 			export GOOS=${os%-*}
 			export GOARCH=${os#*-}
 
@@ -128,6 +142,10 @@ case "$1" in
 		build
 		tarDist "syncthing-linux-armv6-$version"
 
+		export GOARM=5
+		build
+		tarDist "syncthing-linux-armv5-$version"
+
 		;;
 
 	upload)
@@ -146,6 +164,10 @@ case "$1" in
 		assets
 		;;
 
+	setup)
+		setup
+		;;
+
 	*)
 		echo "Unknown build parameter $1"
 		;;

cmd/syncthing/debug.go

@@ -1,15 +1,10 @@
 package main
 
 import (
-	"log"
 	"os"
 	"strings"
 )
 
 var (
-	dlog      = log.New(os.Stderr, "main: ", log.Lmicroseconds|log.Lshortfile)
-	debugNet  = strings.Contains(os.Getenv("STTRACE"), "net")
-	debugIdx  = strings.Contains(os.Getenv("STTRACE"), "idx")
-	debugNeed = strings.Contains(os.Getenv("STTRACE"), "need")
-	debugPull = strings.Contains(os.Getenv("STTRACE"), "pull")
+	debugNet = strings.Contains(os.Getenv("STTRACE"), "net") || os.Getenv("STTRACE") == "all"
 )

cmd/syncthing/gui.go

@@ -14,6 +14,9 @@ import (
 	"time"
 
 	"code.google.com/p/go.crypto/bcrypt"
+	"github.com/calmh/syncthing/config"
+	"github.com/calmh/syncthing/logger"
+	"github.com/calmh/syncthing/model"
 	"github.com/codegangsta/martini"
 )
 
@@ -34,8 +37,12 @@ const (
 	unchangedPassword = "--password-unchanged--"
 )
 
-func startGUI(cfg GUIConfiguration, m *Model) error {
-	l, err := net.Listen("tcp", cfg.Address)
+func init() {
+	l.AddHandler(logger.LevelWarn, showGuiError)
+}
+
+func startGUI(cfg config.GUIConfiguration, m *model.Model) error {
+	listener, err := net.Listen("tcp", cfg.Address)
 	if err != nil {
 		return err
 	}
@@ -49,12 +56,15 @@ func startGUI(cfg GUIConfiguration, m *Model) error {
 	router.Get("/rest/config/sync", restGetConfigInSync)
 	router.Get("/rest/system", restGetSystem)
 	router.Get("/rest/errors", restGetErrors)
+	router.Get("/rest/discovery", restGetDiscovery)
 
 	router.Post("/rest/config", restPostConfig)
 	router.Post("/rest/restart", restPostRestart)
 	router.Post("/rest/reset", restPostReset)
+	router.Post("/rest/shutdown", restPostShutdown)
 	router.Post("/rest/error", restPostError)
 	router.Post("/rest/error/clear", restClearErrors)
+	router.Post("/rest/discovery/hint", restPostDiscoveryHint)
 
 	mr := martini.New()
 	if len(cfg.User) > 0 && len(cfg.Password) > 0 {
@@ -66,7 +76,7 @@ func startGUI(cfg GUIConfiguration, m *Model) error {
 	mr.Action(router.Handle)
 	mr.Map(m)
 
-	go http.Serve(l, mr)
+	go http.Serve(listener, mr)
 
 	return nil
 }
@@ -86,7 +96,7 @@ func restGetVersion() string {
 	return Version
 }
 
-func restGetModel(m *Model, w http.ResponseWriter, r *http.Request) {
+func restGetModel(m *model.Model, w http.ResponseWriter, r *http.Request) {
 	var qs = r.URL.Query()
 	var repo = qs.Get("repo")
 	var res = make(map[string]interface{})
@@ -115,7 +125,7 @@ func restGetModel(m *Model, w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(res)
 }
 
-func restGetConnections(m *Model, w http.ResponseWriter) {
+func restGetConnections(m *model.Model, w http.ResponseWriter) {
 	var res = m.ConnectionStats()
 	w.Header().Set("Content-Type", "application/json")
 	json.NewEncoder(w).Encode(res)
@@ -133,14 +143,14 @@ func restPostConfig(req *http.Request) {
 	var prevPassHash = cfg.GUI.Password
 	err := json.NewDecoder(req.Body).Decode(&cfg)
 	if err != nil {
-		warnln(err)
+		l.Warnln(err)
 	} else {
 		if cfg.GUI.Password == "" {
 			// Leave it empty
 		} else if cfg.GUI.Password != unchangedPassword {
 			hash, err := bcrypt.GenerateFromPassword([]byte(cfg.GUI.Password), 0)
 			if err != nil {
-				warnln(err)
+				l.Warnln(err)
 			} else {
 				cfg.GUI.Password = string(hash)
 			}
@@ -156,15 +166,28 @@ func restGetConfigInSync(w http.ResponseWriter) {
 	json.NewEncoder(w).Encode(map[string]bool{"configInSync": configInSync})
 }
 
-func restPostRestart(req *http.Request) {
+func restPostRestart(w http.ResponseWriter) {
+	flushResponse(`{"ok": "restarting"}`, w)
 	go restart()
 }
 
-func restPostReset(req *http.Request) {
+func restPostReset(w http.ResponseWriter) {
+	flushResponse(`{"ok": "resetting repos"}`, w)
 	resetRepositories()
 	go restart()
 }
 
+func restPostShutdown(w http.ResponseWriter) {
+	flushResponse(`{"ok": "shutting down"}`, w)
+	go shutdown()
+}
+
+func flushResponse(s string, w http.ResponseWriter) {
+	w.Write([]byte(s + "\n"))
+	f := w.(http.Flusher)
+	f.Flush()
+}
+
 var cpuUsagePercent [10]float64 // The last ten seconds
 var cpuUsageLock sync.RWMutex
 
@@ -201,7 +224,7 @@ func restGetErrors(w http.ResponseWriter) {
 func restPostError(req *http.Request) {
 	bs, _ := ioutil.ReadAll(req.Body)
 	req.Body.Close()
-	showGuiError(string(bs))
+	showGuiError(0, string(bs))
 }
 
 func restClearErrors() {
@@ -210,7 +233,7 @@ func restClearErrors() {
 	guiErrorsMut.Unlock()
 }
 
-func showGuiError(err string) {
+func showGuiError(l logger.LogLevel, err string) {
 	guiErrorsMut.Lock()
 	guiErrors = append(guiErrors, guiError{time.Now(), err})
 	if len(guiErrors) > 5 {
@@ -219,6 +242,19 @@ func showGuiError(err string) {
 	guiErrorsMut.Unlock()
 }
 
+func restPostDiscoveryHint(r *http.Request) {
+	var qs = r.URL.Query()
+	var node = qs.Get("node")
+	var addr = qs.Get("addr")
+	if len(node) != 0 && len(addr) != 0 && discoverer != nil {
+		discoverer.Hint(node, []string{addr})
+	}
+}
+
+func restGetDiscovery(w http.ResponseWriter) {
+	json.NewEncoder(w).Encode(discoverer.All())
+}
+
 func basic(username string, passhash string) http.HandlerFunc {
 	return func(res http.ResponseWriter, req *http.Request) {
 		error := func() {

cmd/syncthing/gui_solaris.go

@@ -75,7 +75,7 @@ func trackCPUUsage() {
 	for _ = range time.NewTicker(time.Second).C {
 		err := solarisPrusage(pid, &rusage)
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 			continue
 		}
 		curTime := time.Now().UnixNano()

cmd/syncthing/logger.go

@@ -1,64 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-	"os"
-)
-
-var logger *log.Logger
-
-func init() {
-	log.SetOutput(os.Stderr)
-	logger = log.New(os.Stderr, "", log.Flags())
-}
-
-func infoln(vals ...interface{}) {
-	s := fmt.Sprintln(vals...)
-	logger.Output(2, "INFO: "+s)
-}
-
-func infof(format string, vals ...interface{}) {
-	s := fmt.Sprintf(format, vals...)
-	logger.Output(2, "INFO: "+s)
-}
-
-func okln(vals ...interface{}) {
-	s := fmt.Sprintln(vals...)
-	logger.Output(2, "OK: "+s)
-}
-
-func okf(format string, vals ...interface{}) {
-	s := fmt.Sprintf(format, vals...)
-	logger.Output(2, "OK: "+s)
-}
-
-func warnln(vals ...interface{}) {
-	s := fmt.Sprintln(vals...)
-	showGuiError(s)
-	logger.Output(2, "WARNING: "+s)
-}
-
-func warnf(format string, vals ...interface{}) {
-	s := fmt.Sprintf(format, vals...)
-	showGuiError(s)
-	logger.Output(2, "WARNING: "+s)
-}
-
-func fatalln(vals ...interface{}) {
-	s := fmt.Sprintln(vals...)
-	logger.Output(2, "FATAL: "+s)
-	os.Exit(3)
-}
-
-func fatalf(format string, vals ...interface{}) {
-	s := fmt.Sprintf(format, vals...)
-	logger.Output(2, "FATAL: "+s)
-	os.Exit(3)
-}
-
-func fatalErr(err error) {
-	if err != nil {
-		fatalf(err.Error())
-	}
-}

cmd/syncthing/main.go

@@ -20,14 +20,15 @@ import (
 	"strings"
 	"time"
 
+	"github.com/calmh/syncthing/config"
 	"github.com/calmh/syncthing/discover"
+	"github.com/calmh/syncthing/logger"
+	"github.com/calmh/syncthing/model"
 	"github.com/calmh/syncthing/protocol"
 	"github.com/calmh/syncthing/upnp"
 	"github.com/juju/ratelimit"
 )
 
-const BlockSize = 128 * 1024
-
 var (
 	Version     = "unknown-dev"
 	BuildStamp  = "0"
@@ -37,16 +38,22 @@ var (
 	LongVersion string
 )
 
+var l = logger.DefaultLogger
+
 func init() {
 	stamp, _ := strconv.Atoi(BuildStamp)
 	BuildDate = time.Unix(int64(stamp), 0)
 
-	date := BuildDate.UTC().Format(time.RFC3339)
+	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
 	LongVersion = fmt.Sprintf("syncthing %s (%s %s-%s) %s@%s %s", Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
+
+	if os.Getenv("STTRACE") != "" {
+		l.SetFlags(log.Ltime | log.Ldate | log.Lmicroseconds | log.Lshortfile)
+	}
 }
 
 var (
-	cfg        Configuration
+	cfg        config.Configuration
 	myID       string
 	confDir    string
 	rateBucket *ratelimit.Bucket
@@ -67,15 +74,13 @@ const (
 
  STTRACE       A comma separated string of facilities to trace. The valid
                facility strings:
-               - "discover" (the node discovery package)
-               - "files"    (file set store)
-               - "idx"      (index sending and receiving)
-               - "mc"       (multicast beacon)
-               - "need"     (file need calculations)
-               - "net"      (connecting and disconnecting, network messages)
-               - "pull"     (file pull activity)
-               - "scanner"  (the file change scanner)
-               - "upnp"     (the upnp port mapper)
+               - "beacon"   (the beacon package)
+               - "discover" (the discover package)
+               - "files"    (the files package)
+               - "net"      (the main packge; connections & network messages)
+               - "scanner"  (the scanner package)
+               - "upnp"     (the upnp package)
+               - "all"      (all of the above)
 
  STCPUPROFILE  Write CPU profile to the specified file.`
 )
@@ -104,7 +109,7 @@ func main() {
 	if doUpgrade {
 		err := upgrade()
 		if err != nil {
-			fatalln(err)
+			l.Fatalln(err)
 		}
 		return
 	}
@@ -130,7 +135,7 @@ func main() {
 		if _, err := os.Stat(oldDefault); err == nil {
 			os.MkdirAll(filepath.Dir(confDir), 0700)
 			if err := os.Rename(oldDefault, confDir); err == nil {
-				infoln("Moved config dir", oldDefault, "to", confDir)
+				l.Infoln("Moved config dir", oldDefault, "to", confDir)
 			}
 		}
 	}
@@ -142,15 +147,14 @@ func main() {
 	if err != nil {
 		newCertificate(confDir)
 		cert, err = loadCert(confDir)
-		fatalErr(err)
+		l.FatalErr(err)
 	}
 
 	myID = certID(cert.Certificate[0])
-	log.SetPrefix("[" + myID[0:5] + "] ")
-	logger.SetPrefix("[" + myID[0:5] + "] ")
+	l.SetPrefix(fmt.Sprintf("[%s] ", myID[:5]))
 
-	infoln(LongVersion)
-	infoln("My ID:", myID)
+	l.Infoln(LongVersion)
+	l.Infoln("My ID:", myID)
 
 	// Prepare to be able to save configuration
 
@@ -163,26 +167,26 @@ func main() {
 	cf, err := os.Open(cfgFile)
 	if err == nil {
 		// Read config.xml
-		cfg, err = readConfigXML(cf, myID)
+		cfg, err = config.Load(cf, myID)
 		if err != nil {
-			fatalln(err)
+			l.Fatalln(err)
 		}
 		cf.Close()
 	} else {
-		infoln("No config file; starting with empty defaults")
+		l.Infoln("No config file; starting with empty defaults")
 		name, _ := os.Hostname()
 		defaultRepo := filepath.Join(getHomeDir(), "Sync")
 		ensureDir(defaultRepo, 0755)
 
-		cfg, err = readConfigXML(nil, myID)
-		cfg.Repositories = []RepositoryConfiguration{
+		cfg, err = config.Load(nil, myID)
+		cfg.Repositories = []config.RepositoryConfiguration{
 			{
 				ID:        "default",
 				Directory: defaultRepo,
-				Nodes:     []NodeConfiguration{{NodeID: myID}},
+				Nodes:     []config.NodeConfiguration{{NodeID: myID}},
 			},
 		}
-		cfg.Nodes = []NodeConfiguration{
+		cfg.Nodes = []config.NodeConfiguration{
 			{
 				NodeID:    myID,
 				Addresses: []string{"dynamic"},
@@ -190,8 +194,16 @@ func main() {
 			},
 		}
 
+		port, err := getFreePort("127.0.0.1", 8080)
+		l.FatalErr(err)
+		cfg.GUI.Address = fmt.Sprintf("127.0.0.1:%d", port)
+
+		port, err = getFreePort("", 22000)
+		l.FatalErr(err)
+		cfg.Options.ListenAddress = []string{fmt.Sprintf(":%d", port)}
+
 		saveConfig()
-		infof("Edit %s to taste or use the GUI\n", cfgFile)
+		l.Infof("Edit %s to taste or use the GUI\n", cfgFile)
 	}
 
 	if reset {
@@ -201,10 +213,10 @@ func main() {
 
 	if profiler := os.Getenv("STPROFILER"); len(profiler) > 0 {
 		go func() {
-			dlog.Println("Starting profiler on", profiler)
+			l.Debugln("Starting profiler on", profiler)
 			err := http.ListenAndServe(profiler, nil)
 			if err != nil {
-				dlog.Fatal(err)
+				l.Fatalln(err)
 			}
 		}()
 	}
@@ -229,7 +241,7 @@ func main() {
 		rateBucket = ratelimit.NewBucketWithRate(float64(1000*cfg.Options.MaxSendKbps), int64(5*1000*cfg.Options.MaxSendKbps))
 	}
 
-	m := NewModel(cfg.Options.MaxChangeKbps * 1000)
+	m := model.NewModel(confDir, &cfg, "syncthing", Version)
 
 	for _, repo := range cfg.Repositories {
 		if repo.Invalid != "" {
@@ -243,7 +255,7 @@ func main() {
 	if cfg.GUI.Enabled && cfg.GUI.Address != "" {
 		addr, err := net.ResolveTCPAddr("tcp", cfg.GUI.Address)
 		if err != nil {
-			fatalf("Cannot start GUI on %q: %v", cfg.GUI.Address, err)
+			l.Fatalf("Cannot start GUI on %q: %v", cfg.GUI.Address, err)
 		} else {
 			var hostOpen, hostShow string
 			switch {
@@ -258,10 +270,10 @@ func main() {
 				hostShow = hostOpen
 			}
 
-			infof("Starting web GUI on http://%s:%d/", hostShow, addr.Port)
+			l.Infof("Starting web GUI on http://%s:%d/", hostShow, addr.Port)
 			err := startGUI(cfg.GUI, m)
 			if err != nil {
-				fatalln("Cannot start GUI:", err)
+				l.Fatalln("Cannot start GUI:", err)
 			}
 			if cfg.Options.StartBrowser && len(os.Getenv("STRESTART")) == 0 {
 				openURL(fmt.Sprintf("http://%s:%d", hostOpen, addr.Port))
@@ -272,7 +284,7 @@ func main() {
 	// Walk the repository and update the local model before establishing any
 	// connections to other nodes.
 
-	infoln("Populating repository index")
+	l.Infoln("Populating repository index")
 	m.LoadIndexes(confDir)
 
 	for _, repo := range cfg.Repositories {
@@ -288,8 +300,8 @@ func main() {
 
 		if files, _, _ := m.LocalSize(repo.ID); files > 0 {
 			if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
-				warnf("Configured repository %q has index but directory %q is missing; not starting.", repo.ID, repo.Directory)
-				fatalf("Ensure that directory is present or remove repository from configuration.")
+				l.Warnf("Configured repository %q has index but directory %q is missing; not starting.", repo.ID, repo.Directory)
+				l.Fatalf("Ensure that directory is present or remove repository from configuration.")
 			}
 		}
 
@@ -297,6 +309,7 @@ func main() {
 		ensureDir(dir, -1)
 	}
 
+	m.CleanRepos()
 	m.ScanRepos()
 	m.SaveIndexes(confDir)
 
@@ -322,10 +335,10 @@ func main() {
 		// Routine to pull blocks from other nodes to synchronize the local
 		// repository. Does not run when we are in read only (publish only) mode.
 		if repo.ReadOnly {
-			okf("Ready to synchronize %s (read only; no external updates accepted)", repo.ID)
+			l.Okf("Ready to synchronize %s (read only; no external updates accepted)", repo.ID)
 			m.StartRepoRO(repo.ID)
 		} else {
-			okf("Ready to synchronize %s (read-write)", repo.ID)
+			l.Okf("Ready to synchronize %s (read-write)", repo.ID)
 			m.StartRepoRW(repo.ID, cfg.Options.ParallelRequests)
 		}
 	}
@@ -347,7 +360,7 @@ func setupUPnP() int {
 	if len(cfg.Options.ListenAddress) == 1 {
 		_, portStr, err := net.SplitHostPort(cfg.Options.ListenAddress[0])
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 		} else {
 			// Set up incoming port forwarding, if necessary and possible
 			port, _ := strconv.Atoi(portStr)
@@ -358,19 +371,19 @@ func setupUPnP() int {
 					err := igd.AddPortMapping(upnp.TCP, r, port, "syncthing", 0)
 					if err == nil {
 						externalPort = r
-						infoln("Created UPnP port mapping - external port", externalPort)
+						l.Infoln("Created UPnP port mapping - external port", externalPort)
 						break
 					}
 				}
 				if externalPort == 0 {
-					warnln("Failed to create UPnP port mapping")
+					l.Warnln("Failed to create UPnP port mapping")
 				}
 			} else {
-				infof("No UPnP IGD device found, no port mapping created (%v)", err)
+				l.Infof("No UPnP IGD device found, no port mapping created (%v)", err)
 			}
 		}
 	} else {
-		warnln("Multiple listening addresses; not attempting UPnP port mapping")
+		l.Warnln("Multiple listening addresses; not attempting UPnP port mapping")
 	}
 	return externalPort
 }
@@ -379,7 +392,7 @@ func resetRepositories() {
 	suffix := fmt.Sprintf(".syncthing-reset-%d", time.Now().UnixNano())
 	for _, repo := range cfg.Repositories {
 		if _, err := os.Stat(repo.Directory); err == nil {
-			infof("Reset: Moving %s -> %s", repo.Directory, repo.Directory+suffix)
+			l.Infof("Reset: Moving %s -> %s", repo.Directory, repo.Directory+suffix)
 			os.Rename(repo.Directory, repo.Directory+suffix)
 		}
 	}
@@ -388,17 +401,17 @@ func resetRepositories() {
 	idxs, err := filepath.Glob(pat)
 	if err == nil {
 		for _, idx := range idxs {
-			infof("Reset: Removing %s", idx)
+			l.Infof("Reset: Removing %s", idx)
 			os.Remove(idx)
 		}
 	}
 }
 
 func restart() {
-	infoln("Restarting")
+	l.Infoln("Restarting")
 	if os.Getenv("SMF_FMRI") != "" || os.Getenv("STNORESTART") != "" {
 		// Solaris SMF
-		infoln("Service manager detected; exit instead of restart")
+		l.Infoln("Service manager detected; exit instead of restart")
 		stop <- true
 		return
 	}
@@ -409,7 +422,7 @@ func restart() {
 	}
 	pgm, err := exec.LookPath(os.Args[0])
 	if err != nil {
-		warnln(err)
+		l.Warnln(err)
 		return
 	}
 	proc, err := os.StartProcess(pgm, os.Args, &os.ProcAttr{
@@ -417,38 +430,42 @@ func restart() {
 		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
 	})
 	if err != nil {
-		fatalln(err)
+		l.Fatalln(err)
 	}
 	proc.Release()
 	stop <- true
 }
 
+func shutdown() {
+	stop <- true
+}
+
 var saveConfigCh = make(chan struct{})
 
 func saveConfigLoop(cfgFile string) {
 	for _ = range saveConfigCh {
 		fd, err := os.Create(cfgFile + ".tmp")
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 			continue
 		}
 
-		err = writeConfigXML(fd, cfg)
+		err = config.Save(fd, cfg)
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 			fd.Close()
 			continue
 		}
 
 		err = fd.Close()
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 			continue
 		}
 
-		err = Rename(cfgFile+".tmp", cfgFile)
+		err = model.Rename(cfgFile+".tmp", cfgFile)
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 		}
 	}
 }
@@ -457,7 +474,7 @@ func saveConfig() {
 	saveConfigCh <- struct{}{}
 }
 
-func listenConnect(myID string, m *Model, tlsCfg *tls.Config) {
+func listenConnect(myID string, m *model.Model, tlsCfg *tls.Config) {
 	var conns = make(chan *tls.Conn)
 
 	// Listen
@@ -465,26 +482,26 @@ func listenConnect(myID string, m *Model, tlsCfg *tls.Config) {
 		addr := addr
 		go func() {
 			if debugNet {
-				dlog.Println("listening on", addr)
+				l.Debugln("listening on", addr)
 			}
-			l, err := tls.Listen("tcp", addr, tlsCfg)
-			fatalErr(err)
+			listener, err := tls.Listen("tcp", addr, tlsCfg)
+			l.FatalErr(err)
 
 			for {
-				conn, err := l.Accept()
+				conn, err := listener.Accept()
 				if err != nil {
-					warnln(err)
+					l.Warnln(err)
 					continue
 				}
 
 				if debugNet {
-					dlog.Println("connect from", conn.RemoteAddr())
+					l.Debugln("connect from", conn.RemoteAddr())
 				}
 
 				tc := conn.(*tls.Conn)
 				err = tc.Handshake()
 				if err != nil {
-					warnln(err)
+					l.Warnln(err)
 					tc.Close()
 					continue
 				}
@@ -531,12 +548,12 @@ func listenConnect(myID string, m *Model, tlsCfg *tls.Config) {
 						addr = net.JoinHostPort(host, "22000")
 					}
 					if debugNet {
-						dlog.Println("dial", nodeCfg.NodeID, addr)
+						l.Debugln("dial", nodeCfg.NodeID, addr)
 					}
 					conn, err := tls.Dial("tcp", addr, tlsCfg)
 					if err != nil {
 						if debugNet {
-							dlog.Println(err)
+							l.Debugln(err)
 						}
 						continue
 					}
@@ -553,21 +570,21 @@ func listenConnect(myID string, m *Model, tlsCfg *tls.Config) {
 next:
 	for conn := range conns {
 		certs := conn.ConnectionState().PeerCertificates
-		if l := len(certs); l != 1 {
-			warnf("Got peer certificate list of length %d != 1; protocol error", l)
+		if cl := len(certs); cl != 1 {
+			l.Warnf("Got peer certificate list of length %d != 1; protocol error", cl)
 			conn.Close()
 			continue
 		}
 		remoteID := certID(certs[0].Raw)
 
 		if remoteID == myID {
-			warnf("Connected to myself (%s) - should not happen", remoteID)
+			l.Warnf("Connected to myself (%s) - should not happen", remoteID)
 			conn.Close()
 			continue
 		}
 
 		if m.ConnectedTo(remoteID) {
-			warnf("Connected to already connected node (%s)", remoteID)
+			l.Warnf("Connected to already connected node (%s)", remoteID)
 			conn.Close()
 			continue
 		}
@@ -590,17 +607,17 @@ next:
 func discovery(extPort int) *discover.Discoverer {
 	disc, err := discover.NewDiscoverer(myID, cfg.Options.ListenAddress)
 	if err != nil {
-		warnf("No discovery possible (%v)", err)
+		l.Warnf("No discovery possible (%v)", err)
 		return nil
 	}
 
 	if cfg.Options.LocalAnnEnabled {
-		infoln("Sending local discovery announcements")
+		l.Infoln("Sending local discovery announcements")
 		disc.StartLocal()
 	}
 
 	if cfg.Options.GlobalAnnEnabled {
-		infoln("Sending global discovery announcements")
+		l.Infoln("Sending global discovery announcements")
 		disc.StartGlobal(cfg.Options.GlobalAnnServer, uint16(extPort))
 	}
 
@@ -611,10 +628,10 @@ func ensureDir(dir string, mode int) {
 	fi, err := os.Stat(dir)
 	if os.IsNotExist(err) {
 		err := os.MkdirAll(dir, 0700)
-		fatalErr(err)
+		l.FatalErr(err)
 	} else if mode >= 0 && err == nil && int(fi.Mode()&0777) != mode {
 		err := os.Chmod(dir, os.FileMode(mode))
-		fatalErr(err)
+		l.FatalErr(err)
 	}
 }
 
@@ -657,8 +674,40 @@ func getHomeDir() string {
 	}
 
 	if home == "" {
-		fatalln("No home directory found - set $HOME (or the platform equivalent).")
+		l.Fatalln("No home directory found - set $HOME (or the platform equivalent).")
 	}
 
 	return home
 }
+
+// getFreePort returns a free TCP port fort listening on. The ports given are
+// tried in succession and the first to succeed is returned. If none succeed,
+// a random high port is returned.
+func getFreePort(host string, ports ...int) (int, error) {
+	for _, port := range ports {
+		c, err := net.Listen("tcp", fmt.Sprintf("%s:%d", host, port))
+		if err == nil {
+			c.Close()
+			return port, nil
+		}
+	}
+
+	c, err := net.Listen("tcp", host+":0")
+	if err != nil {
+		return 0, err
+	}
+	addr := c.Addr().String()
+	c.Close()
+
+	_, portstr, err := net.SplitHostPort(addr)
+	if err != nil {
+		return 0, err
+	}
+
+	port, err := strconv.Atoi(portstr)
+	if err != nil {
+		return 0, err
+	}
+
+	return port, nil
+}

cmd/syncthing/tls.go

@@ -41,10 +41,10 @@ func certSeed(bs []byte) int64 {
 }
 
 func newCertificate(dir string) {
-	infoln("Generating RSA certificate and key...")
+	l.Infoln("Generating RSA certificate and key...")
 
 	priv, err := rsa.GenerateKey(rand.Reader, tlsRSABits)
-	fatalErr(err)
+	l.FatalErr(err)
 
 	notBefore := time.Now()
 	notAfter := time.Date(2049, 12, 31, 23, 59, 59, 0, time.UTC)
@@ -63,17 +63,17 @@ func newCertificate(dir string) {
 	}
 
 	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
-	fatalErr(err)
+	l.FatalErr(err)
 
 	certOut, err := os.Create(filepath.Join(dir, "cert.pem"))
-	fatalErr(err)
+	l.FatalErr(err)
 	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
 	certOut.Close()
-	okln("Created RSA certificate file")
+	l.Okln("Created RSA certificate file")
 
 	keyOut, err := os.OpenFile(filepath.Join(dir, "key.pem"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
-	fatalErr(err)
+	l.FatalErr(err)
 	pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
 	keyOut.Close()
-	okln("Created RSA key file")
+	l.Okln("Created RSA key file")
 }

cmd/syncthing/upgrade_unix.go

@@ -51,12 +51,12 @@ func upgrade() error {
 	rel := rels[0]
 
 	if rel.Tag > Version {
-		infof("Attempting upgrade to %s...", rel.Tag)
+		l.Infof("Attempting upgrade to %s...", rel.Tag)
 	} else if rel.Tag == Version {
-		okf("Already running the latest version, %s. Not upgrading.", Version)
+		l.Okf("Already running the latest version, %s. Not upgrading.", Version)
 		return nil
 	} else {
-		okf("Current version %s is newer than latest release %s. Not upgrading.", Version, rel.Tag)
+		l.Okf("Current version %s is newer than latest release %s. Not upgrading.", Version, rel.Tag)
 		return nil
 	}
 
@@ -64,7 +64,7 @@ func upgrade() error {
 	for _, asset := range rel.Assets {
 		if strings.HasPrefix(asset.Name, expectedRelease) {
 			if strings.HasSuffix(asset.Name, ".tar.gz") {
-				infof("Downloading %s...", asset.Name)
+				l.Infof("Downloading %s...", asset.Name)
 				fname, err := readTarGZ(asset.URL, filepath.Dir(path))
 				if err != nil {
 					return err
@@ -80,8 +80,8 @@ func upgrade() error {
 					return err
 				}
 
-				okf("Upgraded %q to %s.", path, rel.Tag)
-				okf("Previous version saved in %q.", old)
+				l.Okf("Upgraded %q to %s.", path, rel.Tag)
+				l.Okf("Previous version saved in %q.", old)
 
 				return nil
 			}

cmd/syncthing/usage.go

@@ -32,7 +32,7 @@ func usageFor(fs *flag.FlagSet, usage string, extra string) func() {
 			var opt = "  -" + f.Name
 
 			if f.DefValue != "false" {
-				opt += "=" + f.DefValue
+				opt += "=" + fmt.Sprintf("%q", f.DefValue)
 			}
 
 			options = append(options, []string{opt, f.Usage})

config/config.go

@@ -1,7 +1,9 @@
-package main
+// Package config implements reading and writing of the syncthing configuration file.
+package config
 
 import (
 	"encoding/xml"
+	"fmt"
 	"io"
 	"os"
 	"reflect"
@@ -9,8 +11,11 @@ import (
 	"strconv"
 
 	"code.google.com/p/go.crypto/bcrypt"
+	"github.com/calmh/syncthing/logger"
 )
 
+var l = logger.DefaultLogger
+
 type Configuration struct {
 	Version      int                       `xml:"version,attr" default:"2"`
 	Repositories []RepositoryConfiguration `xml:"repository"`
@@ -45,7 +50,7 @@ type NodeConfiguration struct {
 }
 
 type OptionsConfiguration struct {
-	ListenAddress      []string `xml:"listenAddress" default:":22000"`
+	ListenAddress      []string `xml:"listenAddress" default:"0.0.0.0:22000"`
 	GlobalAnnServer    string   `xml:"globalAnnounceServer" default:"announce.syncthing.net:22025"`
 	GlobalAnnEnabled   bool     `xml:"globalAnnounceEnabled" default:"true"`
 	LocalAnnEnabled    bool     `xml:"localAnnounceEnabled" default:"true"`
@@ -53,13 +58,13 @@ type OptionsConfiguration struct {
 	MaxSendKbps        int      `xml:"maxSendKbps"`
 	RescanIntervalS    int      `xml:"rescanIntervalS" default:"60"`
 	ReconnectIntervalS int      `xml:"reconnectionIntervalS" default:"60"`
-	MaxChangeKbps      int      `xml:"maxChangeKbps" default:"1000"`
+	MaxChangeKbps      int      `xml:"maxChangeKbps" default:"10000"`
 	StartBrowser       bool     `xml:"startBrowser" default:"true"`
 	UPnPEnabled        bool     `xml:"upnpEnabled" default:"true"`
 
-	Deprecated_ReadOnly   bool   `xml:"readOnly,omitempty"`
-	Deprecated_GUIEnabled bool   `xml:"guiEnabled,omitempty"`
-	Deprecated_GUIAddress string `xml:"guiAddress,omitempty"`
+	Deprecated_ReadOnly   bool   `xml:"readOnly,omitempty" json:"-"`
+	Deprecated_GUIEnabled bool   `xml:"guiEnabled,omitempty" json:"-"`
+	Deprecated_GUIAddress string `xml:"guiAddress,omitempty" json:"-"`
 }
 
 type GUIConfiguration struct {
@@ -130,7 +135,7 @@ func fillNilSlices(data interface{}) error {
 	return nil
 }
 
-func writeConfigXML(wr io.Writer, cfg Configuration) error {
+func Save(wr io.Writer, cfg Configuration) error {
 	e := xml.NewEncoder(wr)
 	e.Indent("", "    ")
 	err := e.Encode(cfg)
@@ -155,7 +160,7 @@ func uniqueStrings(ss []string) []string {
 	return us
 }
 
-func readConfigXML(rd io.Reader, myID string) (Configuration, error) {
+func Load(rd io.Reader, myID string) (Configuration, error) {
 	var cfg Configuration
 
 	setDefaults(&cfg)
@@ -171,19 +176,37 @@ func readConfigXML(rd io.Reader, myID string) (Configuration, error) {
 
 	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
 
-	// Check for missing or duplicate repository ID:s
+	// Initialize an empty slice for repositories if the config has none
+	if cfg.Repositories == nil {
+		cfg.Repositories = []RepositoryConfiguration{}
+	}
+
+	// Check for missing, bad or duplicate repository ID:s
 	var seenRepos = map[string]*RepositoryConfiguration{}
+	var uniqueCounter int
 	for i := range cfg.Repositories {
 		repo := &cfg.Repositories[i]
 
+		if len(repo.Directory) == 0 {
+			repo.Invalid = "empty directory"
+			continue
+		}
+
 		if repo.ID == "" {
 			repo.ID = "default"
 		}
 
 		if seen, ok := seenRepos[repo.ID]; ok {
+			l.Warnf("Multiple repositories with ID %q; disabling", repo.ID)
+
 			seen.Invalid = "duplicate repository ID"
+			if seen.ID == repo.ID {
+				uniqueCounter++
+				seen.ID = fmt.Sprintf("%s~%d", repo.ID, uniqueCounter)
+			}
 			repo.Invalid = "duplicate repository ID"
-			warnf("Multiple repositories with ID %q; disabling", repo.ID)
+			uniqueCounter++
+			repo.ID = fmt.Sprintf("%s~%d", repo.ID, uniqueCounter)
 		} else {
 			seenRepos[repo.ID] = repo
 		}
@@ -198,7 +221,7 @@ func readConfigXML(rd io.Reader, myID string) (Configuration, error) {
 	if len(cfg.GUI.Password) > 0 && cfg.GUI.Password[0] != '$' {
 		hash, err := bcrypt.GenerateFromPassword([]byte(cfg.GUI.Password), 0)
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 		} else {
 			cfg.GUI.Password = string(hash)
 		}

config/config_test.go

@@ -1,4 +1,4 @@
-package main
+package config
 
 import (
 	"bytes"
@@ -10,7 +10,7 @@ import (
 
 func TestDefaultValues(t *testing.T) {
 	expected := OptionsConfiguration{
-		ListenAddress:      []string{":22000"},
+		ListenAddress:      []string{"0.0.0.0:22000"},
 		GlobalAnnServer:    "announce.syncthing.net:22025",
 		GlobalAnnEnabled:   true,
 		LocalAnnEnabled:    true,
@@ -18,12 +18,12 @@ func TestDefaultValues(t *testing.T) {
 		MaxSendKbps:        0,
 		RescanIntervalS:    60,
 		ReconnectIntervalS: 60,
-		MaxChangeKbps:      1000,
+		MaxChangeKbps:      10000,
 		StartBrowser:       true,
 		UPnPEnabled:        true,
 	}
 
-	cfg, err := readConfigXML(bytes.NewReader(nil), "nodeID")
+	cfg, err := Load(bytes.NewReader(nil), "nodeID")
 	if err != io.EOF {
 		t.Error(err)
 	}
@@ -66,7 +66,7 @@ func TestNodeConfig(t *testing.T) {
 `)
 
 	for i, data := range [][]byte{v1data, v2data} {
-		cfg, err := readConfigXML(bytes.NewReader(data), "node1")
+		cfg, err := Load(bytes.NewReader(data), "node1")
 		if err != nil {
 			t.Error(err)
 		}
@@ -121,7 +121,7 @@ func TestNoListenAddress(t *testing.T) {
 </configuration>
 `)
 
-	cfg, err := readConfigXML(bytes.NewReader(data), "nodeID")
+	cfg, err := Load(bytes.NewReader(data), "nodeID")
 	if err != nil {
 		t.Error(err)
 	}
@@ -170,7 +170,7 @@ func TestOverriddenValues(t *testing.T) {
 		UPnPEnabled:        false,
 	}
 
-	cfg, err := readConfigXML(bytes.NewReader(data), "nodeID")
+	cfg, err := Load(bytes.NewReader(data), "nodeID")
 	if err != nil {
 		t.Error(err)
 	}
@@ -215,7 +215,7 @@ func TestNodeAddresses(t *testing.T) {
 		},
 	}
 
-	cfg, err := readConfigXML(bytes.NewReader(data), "n4")
+	cfg, err := Load(bytes.NewReader(data), "n4")
 	if err != nil {
 		t.Error(err)
 	}

discover/PROTOCOL.md

@@ -36,12 +36,28 @@ The Announcement packet has the following structure:
      0                   1                   2                   3
      0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |                   Magic Number (0x029E4C77)                   |
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |                       Length of Node ID                       |
+    |                      Magic (0x029E4C77)                       |
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
     /                                                               /
-    \                   Node ID (variable length)                   \
+    \                        Node Structure                         \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                     Number of Extra Nodes                     |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                 Zero or more Node Structures                  \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+    Node Structure:
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                         Length of ID                          |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                     ID (variable length)                      \
     /                                                               /
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
     |                      Number of Addresses                      |
@@ -62,29 +78,37 @@ The Announcement packet has the following structure:
     \                     IP (variable length)                      \
     /                                                               /
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |          Port Number          |            0x0000             |
+    |             Port              |            0x0000             |
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 This is the XDR encoding of:
 
     struct Announcement {
-        unsigned int MagicNumber;
-        string NodeID<>;
+        unsigned int Magic;
+        Node This;
+        Node Extra<>;
+    }
+
+    struct Node {
+        string ID<>;
         Address Addresses<>;
     }
 
     struct Address {
         opaque IP<>;
-        unsigned short PortNumber;
+        unsigned short Port;
     }
 
-NodeID is padded to a multiple of 32 bits and all fields are in sent in
-network (big endian) byte order. In the Address structure, the IP field
-can be of three differnt kinds;
+The first Node structure contains information about the sending node.
+The following zero or more Extra nodes contain information about other
+nodes known to the sending node.
+
+In the Address structure, the IP field can be of three differnt kinds;
 
  - A zero length indicates that the IP address should be taken from the
    source address of the announcement packet, be it IPv4 or IPv6. The
-   source address must be a valid unicast address.
+   source address must be a valid unicast address. This is only valid
+   in the first node structure, not in the list of extras.
 
  - A four byte length indicates that the address is an IPv4 unicast
    address.

discover/debug.go

@@ -1,12 +1,13 @@
 package discover
 
 import (
-	"log"
 	"os"
 	"strings"
+
+	"github.com/calmh/syncthing/logger"
 )
 
 var (
-	dlog  = log.New(os.Stderr, "discover: ", log.Lmicroseconds|log.Lshortfile)
-	debug = strings.Contains(os.Getenv("STTRACE"), "discover")
+	debug = strings.Contains(os.Getenv("STTRACE"), "discover") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
 )

discover/discover.go

@@ -4,13 +4,13 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"log"
+	"io"
 	"net"
 	"sync"
 	"time"
 
+	"github.com/calmh/syncthing/beacon"
 	"github.com/calmh/syncthing/buffers"
-	"github.com/calmh/syncthing/mc"
 )
 
 const (
@@ -22,7 +22,7 @@ type Discoverer struct {
 	listenAddrs      []string
 	localBcastIntv   time.Duration
 	globalBcastIntv  time.Duration
-	beacon           *mc.Beacon
+	beacon           *beacon.Beacon
 	registry         map[string][]string
 	registryLock     sync.RWMutex
 	extServer        string
@@ -43,12 +43,16 @@ var (
 const maxErrors = 30
 
 func NewDiscoverer(id string, addresses []string) (*Discoverer, error) {
+	b, err := beacon.New(21025)
+	if err != nil {
+		return nil, err
+	}
 	disc := &Discoverer{
 		myID:            id,
 		listenAddrs:     addresses,
 		localBcastIntv:  30 * time.Second,
 		globalBcastIntv: 1800 * time.Second,
-		beacon:          mc.NewBeacon("239.21.0.25", 21025),
+		beacon:          b,
 		registry:        make(map[string][]string),
 	}
 
@@ -89,15 +93,35 @@ func (d *Discoverer) Lookup(node string) []string {
 	return nil
 }
 
+func (d *Discoverer) Hint(node string, addrs []string) {
+	resAddrs := resolveAddrs(addrs)
+	d.registerNode(nil, Node{
+		ID:        node,
+		Addresses: resAddrs,
+	})
+}
+
+func (d *Discoverer) All() map[string][]string {
+	d.registryLock.RLock()
+	nodes := make(map[string][]string, len(d.registry))
+	for node, addrs := range d.registry {
+		addrsCopy := make([]string, len(addrs))
+		copy(addrsCopy, addrs)
+		nodes[node] = addrsCopy
+	}
+	d.registryLock.RUnlock()
+	return nodes
+}
+
 func (d *Discoverer) announcementPkt() []byte {
 	var addrs []Address
 	for _, astr := range d.listenAddrs {
 		addr, err := net.ResolveTCPAddr("tcp", astr)
 		if err != nil {
-			log.Printf("discover/announcement: %v: not announcing %s", err, astr)
+			l.Warnln("%v: not announcing %s", err, astr)
 			continue
 		} else if debug {
-			dlog.Printf("announcing %s: %#v", astr, addr)
+			l.Debugf("discover: announcing %s: %#v", astr, addr)
 		}
 		if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
 			addrs = append(addrs, Address{Port: uint16(addr.Port)})
@@ -108,18 +132,34 @@ func (d *Discoverer) announcementPkt() []byte {
 		}
 	}
 	var pkt = AnnounceV2{
-		Magic:     AnnouncementMagicV2,
-		NodeID:    d.myID,
-		Addresses: addrs,
+		Magic: AnnouncementMagicV2,
+		This:  Node{d.myID, addrs},
 	}
 	return pkt.MarshalXDR()
 }
 
 func (d *Discoverer) sendLocalAnnouncements() {
-	var buf = d.announcementPkt()
+	var addrs = resolveAddrs(d.listenAddrs)
+
+	var pkt = AnnounceV2{
+		Magic: AnnouncementMagicV2,
+		This:  Node{d.myID, addrs},
+	}
 
 	for {
-		d.beacon.Send(buf)
+		pkt.Extra = nil
+		d.registryLock.RLock()
+		for node, addrs := range d.registry {
+			if len(pkt.Extra) == 16 {
+				break
+			}
+
+			anode := Node{node, resolveAddrs(addrs)}
+			pkt.Extra = append(pkt.Extra, anode)
+		}
+		d.registryLock.RUnlock()
+
+		d.beacon.Send(pkt.MarshalXDR())
 
 		select {
 		case <-d.localBcastTick:
@@ -131,22 +171,21 @@ func (d *Discoverer) sendLocalAnnouncements() {
 func (d *Discoverer) sendExternalAnnouncements() {
 	remote, err := net.ResolveUDPAddr("udp", d.extServer)
 	if err != nil {
-		log.Printf("discover/external: %v; no external announcements", err)
+		l.Warnf("Global discovery: %v; no external announcements", err)
 		return
 	}
 
 	conn, err := net.ListenUDP("udp", nil)
 	if err != nil {
-		log.Printf("discover/external: %v; no external announcements", err)
+		l.Warnf("Global discovery: %v; no external announcements", err)
 		return
 	}
 
 	var buf []byte
 	if d.extPort != 0 {
 		var pkt = AnnounceV2{
-			Magic:     AnnouncementMagicV2,
-			NodeID:    d.myID,
-			Addresses: []Address{{Port: d.extPort}},
+			Magic: AnnouncementMagicV2,
+			This:  Node{d.myID, []Address{{Port: d.extPort}}},
 		}
 		buf = pkt.MarshalXDR()
 	} else {
@@ -158,12 +197,14 @@ func (d *Discoverer) sendExternalAnnouncements() {
 		var ok bool
 
 		if debug {
-			dlog.Printf("send announcement -> %v\n%s", remote, hex.Dump(buf))
+			l.Debugf("discover: send announcement -> %v\n%s", remote, hex.Dump(buf))
 		}
 
 		_, err = conn.WriteTo(buf, remote)
 		if err != nil {
-			log.Println("discover/write: warning:", err)
+			if debug {
+				l.Debugln("discover: warning:", err)
+			}
 			errCounter++
 			ok = false
 		} else {
@@ -174,7 +215,7 @@ func (d *Discoverer) sendExternalAnnouncements() {
 			time.Sleep(1 * time.Second)
 			res := d.externalLookup(d.myID)
 			if debug {
-				dlog.Println("external lookup check:", res)
+				l.Debugln("discover: external lookup check:", res)
 			}
 			ok = len(res) > 0
 
@@ -190,7 +231,7 @@ func (d *Discoverer) sendExternalAnnouncements() {
 			time.Sleep(60 * time.Second)
 		}
 	}
-	log.Printf("discover/write: %v: stopping due to too many errors: %v", remote, err)
+	l.Warnf("Global discovery: %v: stopping due to too many errors: %v", remote, err)
 }
 
 func (d *Discoverer) recvAnnouncements() {
@@ -198,77 +239,105 @@ func (d *Discoverer) recvAnnouncements() {
 		buf, addr := d.beacon.Recv()
 
 		if debug {
-			dlog.Printf("read announcement:\n%s", hex.Dump(buf))
+			l.Debugf("discover: read announcement:\n%s", hex.Dump(buf))
 		}
 
 		var pkt AnnounceV2
 		err := pkt.UnmarshalXDR(buf)
-		if err != nil {
+		if err != nil && err != io.EOF {
 			continue
 		}
 
 		if debug {
-			dlog.Printf("parsed announcement: %#v", pkt)
+			l.Debugf("discover: parsed announcement: %#v", pkt)
 		}
 
-		if pkt.NodeID != d.myID {
-			var addrs []string
-			for _, a := range pkt.Addresses {
-				var nodeAddr string
-				if len(a.IP) > 0 {
-					nodeAddr = fmt.Sprintf("%s:%d", net.IP(a.IP), a.Port)
-				} else {
-					ua := addr.(*net.UDPAddr)
-					ua.Port = int(a.Port)
-					nodeAddr = ua.String()
-				}
-				addrs = append(addrs, nodeAddr)
-			}
-			if debug {
-				dlog.Printf("register: %#v", addrs)
-			}
-			d.registryLock.Lock()
-			_, seen := d.registry[pkt.NodeID]
-			if !seen {
-				select {
-				case d.forcedBcastTick <- time.Now():
+		var newNode bool
+		if pkt.This.ID != d.myID {
+			n := d.registerNode(addr, pkt.This)
+			newNode = newNode || n
+			for _, node := range pkt.Extra {
+				if node.ID != d.myID {
+					n := d.registerNode(nil, node)
+					newNode = newNode || n
 				}
 			}
-			d.registry[pkt.NodeID] = addrs
-			d.registryLock.Unlock()
+		}
+
+		if newNode {
+			select {
+			case d.forcedBcastTick <- time.Now():
+			}
 		}
 	}
 }
 
+func (d *Discoverer) registerNode(addr net.Addr, node Node) bool {
+	var addrs []string
+	for _, a := range node.Addresses {
+		var nodeAddr string
+		if len(a.IP) > 0 {
+			nodeAddr = fmt.Sprintf("%s:%d", net.IP(a.IP), a.Port)
+			addrs = append(addrs, nodeAddr)
+		} else if addr != nil {
+			ua := addr.(*net.UDPAddr)
+			ua.Port = int(a.Port)
+			nodeAddr = ua.String()
+			addrs = append(addrs, nodeAddr)
+		}
+	}
+	if len(addrs) == 0 {
+		if debug {
+			l.Debugln("discover: no valid address for", node.ID)
+		}
+	}
+	if debug {
+		l.Debugf("discover: register: %s -> %#v", node.ID, addrs)
+	}
+	d.registryLock.Lock()
+	_, seen := d.registry[node.ID]
+	d.registry[node.ID] = addrs
+	d.registryLock.Unlock()
+	return !seen
+}
+
 func (d *Discoverer) externalLookup(node string) []string {
 	extIP, err := net.ResolveUDPAddr("udp", d.extServer)
 	if err != nil {
-		log.Printf("discover/external: %v; no external lookup", err)
+		if debug {
+			l.Debugf("discover: %v; no external lookup", err)
+		}
 		return nil
 	}
 
 	conn, err := net.DialUDP("udp", nil, extIP)
 	if err != nil {
-		log.Printf("discover/external: %v; no external lookup", err)
+		if debug {
+			l.Debugf("discover: %v; no external lookup", err)
+		}
 		return nil
 	}
 	defer conn.Close()
 
 	err = conn.SetDeadline(time.Now().Add(5 * time.Second))
 	if err != nil {
-		log.Printf("discover/external: %v; no external lookup", err)
+		if debug {
+			l.Debugf("discover: %v; no external lookup", err)
+		}
 		return nil
 	}
 
 	buf := QueryV2{QueryMagicV2, node}.MarshalXDR()
 	_, err = conn.Write(buf)
 	if err != nil {
-		log.Printf("discover/external: %v; no external lookup", err)
+		if debug {
+			l.Debugf("discover: %v; no external lookup", err)
+		}
 		return nil
 	}
 	buffers.Put(buf)
 
-	buf = buffers.Get(256)
+	buf = buffers.Get(2048)
 	defer buffers.Put(buf)
 
 	n, err := conn.Read(buf)
@@ -277,29 +346,61 @@ func (d *Discoverer) externalLookup(node string) []string {
 			// Expected if the server doesn't know about requested node ID
 			return nil
 		}
-		log.Printf("discover/external/read: %v; no external lookup", err)
+		if debug {
+			l.Debugf("discover: %v; no external lookup", err)
+		}
 		return nil
 	}
 
 	if debug {
-		dlog.Printf("read external:\n%s", hex.Dump(buf[:n]))
+		l.Debugf("discover: read external:\n%s", hex.Dump(buf[:n]))
 	}
 
 	var pkt AnnounceV2
 	err = pkt.UnmarshalXDR(buf[:n])
-	if err != nil {
-		log.Println("discover/external/decode:", err)
+	if err != nil && err != io.EOF {
+		if debug {
+			l.Debugln("discover:", err)
+		}
 		return nil
 	}
 
 	if debug {
-		dlog.Printf("parsed external: %#v", pkt)
+		l.Debugf("discover: parsed external: %#v", pkt)
 	}
 
 	var addrs []string
-	for _, a := range pkt.Addresses {
+	for _, a := range pkt.This.Addresses {
 		nodeAddr := fmt.Sprintf("%s:%d", net.IP(a.IP), a.Port)
 		addrs = append(addrs, nodeAddr)
 	}
 	return addrs
 }
+
+func addrToAddr(addr *net.TCPAddr) Address {
+	if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
+		return Address{Port: uint16(addr.Port)}
+	} else if bs := addr.IP.To4(); bs != nil {
+		return Address{IP: bs, Port: uint16(addr.Port)}
+	} else if bs := addr.IP.To16(); bs != nil {
+		return Address{IP: bs, Port: uint16(addr.Port)}
+	}
+	return Address{}
+}
+
+func resolveAddrs(addrs []string) []Address {
+	var raddrs []Address
+	for _, addrStr := range addrs {
+		addrRes, err := net.ResolveTCPAddr("tcp", addrStr)
+		if err != nil {
+			continue
+		}
+		addr := addrToAddr(addrRes)
+		if len(addr.IP) > 0 {
+			raddrs = append(raddrs, addr)
+		} else {
+			raddrs = append(raddrs, Address{Port: addr.Port})
+		}
+	}
+	return raddrs
+}

discover/packets.go

@@ -11,8 +11,13 @@ type QueryV2 struct {
 }
 
 type AnnounceV2 struct {
-	Magic     uint32
-	NodeID    string    // max:64
+	Magic uint32
+	This  Node
+	Extra []Node // max:16
+}
+
+type Node struct {
+	ID        string    // max:64
 	Addresses []Address // max:16
 }
 

discover/packets_xdr.go

@@ -59,16 +59,13 @@ func (o AnnounceV2) MarshalXDR() []byte {
 
 func (o AnnounceV2) encodeXDR(xw *xdr.Writer) (int, error) {
 	xw.WriteUint32(o.Magic)
-	if len(o.NodeID) > 64 {
+	o.This.encodeXDR(xw)
+	if len(o.Extra) > 16 {
 		return xw.Tot(), xdr.ErrElementSizeExceeded
 	}
-	xw.WriteString(o.NodeID)
-	if len(o.Addresses) > 16 {
-		return xw.Tot(), xdr.ErrElementSizeExceeded
-	}
-	xw.WriteUint32(uint32(len(o.Addresses)))
-	for i := range o.Addresses {
-		o.Addresses[i].encodeXDR(xw)
+	xw.WriteUint32(uint32(len(o.Extra)))
+	for i := range o.Extra {
+		o.Extra[i].encodeXDR(xw)
 	}
 	return xw.Tot(), xw.Error()
 }
@@ -86,7 +83,58 @@ func (o *AnnounceV2) UnmarshalXDR(bs []byte) error {
 
 func (o *AnnounceV2) decodeXDR(xr *xdr.Reader) error {
 	o.Magic = xr.ReadUint32()
-	o.NodeID = xr.ReadStringMax(64)
+	(&o.This).decodeXDR(xr)
+	_ExtraSize := int(xr.ReadUint32())
+	if _ExtraSize > 16 {
+		return xdr.ErrElementSizeExceeded
+	}
+	o.Extra = make([]Node, _ExtraSize)
+	for i := range o.Extra {
+		(&o.Extra[i]).decodeXDR(xr)
+	}
+	return xr.Error()
+}
+
+func (o Node) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o Node) MarshalXDR() []byte {
+	var buf bytes.Buffer
+	var xw = xdr.NewWriter(&buf)
+	o.encodeXDR(xw)
+	return buf.Bytes()
+}
+
+func (o Node) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.ID) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.ID)
+	if len(o.Addresses) > 16 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteUint32(uint32(len(o.Addresses)))
+	for i := range o.Addresses {
+		o.Addresses[i].encodeXDR(xw)
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Node) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *Node) UnmarshalXDR(bs []byte) error {
+	var buf = bytes.NewBuffer(bs)
+	var xr = xdr.NewReader(buf)
+	return o.decodeXDR(xr)
+}
+
+func (o *Node) decodeXDR(xr *xdr.Reader) error {
+	o.ID = xr.ReadStringMax(64)
 	_AddressesSize := int(xr.ReadUint32())
 	if _AddressesSize > 16 {
 		return xdr.ErrElementSizeExceeded

files/debug.go

@@ -1,12 +1,13 @@
 package files
 
 import (
-	"log"
 	"os"
 	"strings"
+
+	"github.com/calmh/syncthing/logger"
 )
 
 var (
-	dlog  = log.New(os.Stderr, "files: ", log.Lmicroseconds|log.Lshortfile)
-	debug = strings.Contains(os.Getenv("STTRACE"), "files")
+	debug = strings.Contains(os.Getenv("STTRACE"), "files") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
 )

files/set.go

@@ -38,7 +38,7 @@ func NewSet() *Set {
 
 func (m *Set) Replace(id uint, fs []scanner.File) {
 	if debug {
-		dlog.Printf("Replace(%d, [%d])", id, len(fs))
+		l.Debugf("Replace(%d, [%d])", id, len(fs))
 	}
 	if id > 63 {
 		panic("Connection ID must be in the range 0 - 63 inclusive")
@@ -54,7 +54,7 @@ func (m *Set) Replace(id uint, fs []scanner.File) {
 
 func (m *Set) ReplaceWithDelete(id uint, fs []scanner.File) {
 	if debug {
-		dlog.Printf("ReplaceWithDelete(%d, [%d])", id, len(fs))
+		l.Debugf("ReplaceWithDelete(%d, [%d])", id, len(fs))
 	}
 	if id > 63 {
 		panic("Connection ID must be in the range 0 - 63 inclusive")
@@ -84,7 +84,7 @@ func (m *Set) ReplaceWithDelete(id uint, fs []scanner.File) {
 				}
 				fs = append(fs, cf)
 				if debug {
-					dlog.Println("deleted:", ck.Name)
+					l.Debugln("deleted:", ck.Name)
 				}
 			}
 		}
@@ -96,7 +96,7 @@ func (m *Set) ReplaceWithDelete(id uint, fs []scanner.File) {
 
 func (m *Set) Update(id uint, fs []scanner.File) {
 	if debug {
-		dlog.Printf("Update(%d, [%d])", id, len(fs))
+		l.Debugf("Update(%d, [%d])", id, len(fs))
 	}
 	m.Lock()
 	m.update(id, fs)
@@ -106,7 +106,7 @@ func (m *Set) Update(id uint, fs []scanner.File) {
 
 func (m *Set) Need(id uint) []scanner.File {
 	if debug {
-		dlog.Printf("Need(%d)", id)
+		l.Debugf("Need(%d)", id)
 	}
 	m.Lock()
 	var fs = make([]scanner.File, 0, len(m.globalKey)/2) // Just a guess, but avoids too many reallocations
@@ -130,7 +130,7 @@ func (m *Set) Need(id uint) []scanner.File {
 
 func (m *Set) Have(id uint) []scanner.File {
 	if debug {
-		dlog.Printf("Have(%d)", id)
+		l.Debugf("Have(%d)", id)
 	}
 	var fs = make([]scanner.File, 0, len(m.remoteKey[id]))
 	m.Lock()
@@ -143,7 +143,7 @@ func (m *Set) Have(id uint) []scanner.File {
 
 func (m *Set) Global() []scanner.File {
 	if debug {
-		dlog.Printf("Global()")
+		l.Debugf("Global()")
 	}
 	m.Lock()
 	var fs = make([]scanner.File, 0, len(m.globalKey))
@@ -160,7 +160,7 @@ func (m *Set) Get(id uint, file string) scanner.File {
 	m.Lock()
 	defer m.Unlock()
 	if debug {
-		dlog.Printf("Get(%d, %q)", id, file)
+		l.Debugf("Get(%d, %q)", id, file)
 	}
 	return m.files[m.remoteKey[id][file]].File
 }
@@ -169,7 +169,7 @@ func (m *Set) GetGlobal(file string) scanner.File {
 	m.Lock()
 	defer m.Unlock()
 	if debug {
-		dlog.Printf("GetGlobal(%q)", file)
+		l.Debugf("GetGlobal(%q)", file)
 	}
 	return m.files[m.globalKey[file]].File
 }
@@ -179,7 +179,7 @@ func (m *Set) Availability(name string) bitset {
 	defer m.Unlock()
 	av := m.globalAvailability[name]
 	if debug {
-		dlog.Printf("Availability(%q) = %0x", name, av)
+		l.Debugf("Availability(%q) = %0x", name, av)
 	}
 	return av
 }
@@ -188,7 +188,7 @@ func (m *Set) Changes(id uint) uint64 {
 	m.Lock()
 	defer m.Unlock()
 	if debug {
-		dlog.Printf("Changes(%d)", id)
+		l.Debugf("Changes(%d)", id)
 	}
 	return m.changes[id]
 }

gui/angular.min.js

@@ -200,4 +200,3 @@ isolateScope:Ea.isolateScope,controller:Ea.controller,injector:Ea.injector,inher
 $$csp:Tb});Ta=Uc(Z);try{Ta("ngLocale")}catch(c){Ta("ngLocale",[]).provider("$locale",rd)}Ta("ng",["ngLocale"],["$provide",function(a){a.provider({$$sanitizeUri:Bd});a.provider("$compile",ic).directive({a:Wd,input:Lc,textarea:Lc,form:Xd,script:De,select:Ge,style:Ie,option:He,ngBind:he,ngBindHtml:je,ngBindTemplate:ie,ngClass:ke,ngClassEven:me,ngClassOdd:le,ngCloak:ne,ngController:oe,ngForm:Yd,ngHide:xe,ngIf:pe,ngInclude:qe,ngInit:se,ngNonBindable:te,ngPluralize:ue,ngRepeat:ve,ngShow:we,ngStyle:ye,ngSwitch:ze,
 ngSwitchWhen:Ae,ngSwitchDefault:Be,ngOptions:Fe,ngTransclude:Ce,ngModel:ce,ngList:ee,ngChange:de,required:Mc,ngRequired:Mc,ngValue:ge}).directive({ngInclude:re}).directive(Nb).directive(Nc);a.provider({$anchorScroll:cd,$animate:Td,$browser:ed,$cacheFactory:fd,$controller:id,$document:jd,$exceptionHandler:kd,$filter:Ac,$interpolate:pd,$interval:qd,$http:ld,$httpBackend:nd,$location:td,$log:ud,$parse:xd,$rootScope:Ad,$q:yd,$sce:Ed,$sceDelegate:Dd,$sniffer:Fd,$templateCache:gd,$timeout:Gd,$window:Hd})}])})(Na);
 A(Q).ready(function(){Sc(Q,Yb)})})(window,document);!angular.$$csp()&&angular.element(document).find("head").prepend('<style type="text/css">@charset "UTF-8";[ng\\:cloak],[ng-cloak],[data-ng-cloak],[x-ng-cloak],.ng-cloak,.x-ng-cloak,.ng-hide{display:none !important;}ng\\:form{display:block;}</style>');
-//# sourceMappingURL=angular.min.js.map

gui/app.js

@@ -19,7 +19,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
     $scope.errors = [];
     $scope.seenError = '';
     $scope.model = {};
-    $scope.repos = [];
+    $scope.repos = {};
 
     // Strings before bools look better
     $scope.settings = [
@@ -44,10 +44,12 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
 
     function getSucceeded() {
         if (!getOK) {
+            $scope.init();
             $('#networkError').modal('hide');
             getOK = true;
         }
         if (restarting) {
+            $scope.init();
             $('#restarting').modal('hide');
             restarting = false;
         }
@@ -63,18 +65,6 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         }
     }
 
-    function nodeCompare(a, b) {
-        if (typeof a.Name !== 'undefined' && typeof b.Name !== 'undefined') {
-            if (a.Name < b.Name)
-                return -1;
-            return a.Name > b.Name;
-        }
-        if (a.NodeID < b.NodeID) {
-            return -1;
-        }
-        return a.NodeID > b.NodeID;
-    }
-
     $scope.refresh = function () {
         $http.get(urlbase + '/system').success(function (data) {
             getSucceeded();
@@ -82,9 +72,9 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         }).error(function () {
             getFailed();
         });
-        $scope.repos.forEach(function (repo) {
-            $http.get(urlbase + '/model?repo=' + encodeURIComponent(repo.ID)).success(function (data) {
-                $scope.model[repo.ID] = data;
+        Object.keys($scope.repos).forEach(function (id) {
+            $http.get(urlbase + '/model?repo=' + encodeURIComponent(id)).success(function (data) {
+                $scope.model[id] = data;
             });
         });
         $http.get(urlbase + '/connections').success(function (data) {
@@ -142,7 +132,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         }
 
         if ($scope.model[repo].invalid !== '') {
-            return 'text-warning';
+            return 'text-danger';
         }
 
         var state = '' + $scope.model[repo].state;
@@ -240,6 +230,18 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         return nodeCfg.NodeID.substr(0, 6);
     };
 
+    $scope.thisNodeName = function () {
+        var nodes = $scope.thisNode();
+        if (typeof nodes === 'undefined' || nodes.length != 1) {
+            return "(unknown node)";
+        }
+        var nodeCfg = nodes[0];
+        if (nodeCfg.Name) {
+            return nodeCfg.Name;
+        }
+        return nodeCfg.NodeID.substr(0, 6);
+    };
+
     $scope.editSettings = function () {
         $('#settings').modal({backdrop: 'static', keyboard: true});
     }
@@ -263,6 +265,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         $scope.editingExisting = true;
         $scope.editingSelf = (nodeCfg.NodeID == $scope.myID);
         $scope.currentNode.AddressesStr = nodeCfg.Addresses.join(', ');
+        $scope.nodeEditor.$setPristine();
         $('#editNode').modal({backdrop: 'static', keyboard: true});
     };
 
@@ -270,6 +273,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         $scope.currentNode = {AddressesStr: 'dynamic'};
         $scope.editingExisting = false;
         $scope.editingSelf = false;
+        $scope.nodeEditor.$setPristine();
         $('#editNode').modal({backdrop: 'static', keyboard: true});
     };
 
@@ -284,8 +288,8 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         });
         $scope.config.Nodes = $scope.nodes;
 
-        for (var i = 0; i < $scope.repos.length; i++) {
-            $scope.repos[i].Nodes = $scope.repos[i].Nodes.filter(function (n) {
+        for (var id in repos) {
+            $scope.repos[id].Nodes = $scope.repos[id].Nodes.filter(function (n) {
                 return n.NodeID !== $scope.currentNode.NodeID;
             });
         }
@@ -300,6 +304,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         $scope.configInSync = false;
         $('#editNode').modal('hide');
         nodeCfg = $scope.currentNode;
+        nodeCfg.NodeID = nodeCfg.NodeID.replace(/ /g, '').trim();
         nodeCfg.Addresses = nodeCfg.AddressesStr.split(',').map(function (x) { return x.trim(); });
 
         done = false;
@@ -357,18 +362,24 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         return str;
     };
 
+    $scope.repoList = function () {
+        return repoList($scope.repos);
+    }
+
     $scope.editRepo = function (nodeCfg) {
         $scope.currentRepo = $.extend({selectedNodes: {}}, nodeCfg);
         $scope.currentRepo.Nodes.forEach(function (n) {
             $scope.currentRepo.selectedNodes[n.NodeID] = true;
         });
         $scope.editingExisting = true;
+        $scope.repoEditor.$setPristine();
         $('#editRepo').modal({backdrop: 'static', keyboard: true});
     };
 
     $scope.addRepo = function () {
         $scope.currentRepo = {selectedNodes: {}};
         $scope.editingExisting = false;
+        $scope.repoEditor.$setPristine();
         $('#editRepo').modal({backdrop: 'static', keyboard: true});
     };
 
@@ -387,20 +398,8 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         }
         delete repoCfg.selectedNodes;
 
-        done = false;
-        for (i = 0; i < $scope.repos.length; i++) {
-            if ($scope.repos[i].ID === repoCfg.ID) {
-                $scope.repos[i] = repoCfg;
-                done = true;
-                break;
-            }
-        }
-
-        if (!done) {
-            $scope.repos.push(repoCfg);
-        }
-
-        $scope.config.Repositories = $scope.repos;
+        $scope.repos[repoCfg.ID] = repoCfg;
+        $scope.config.Repositories = repoList($scope.repos);
 
         $http.post(urlbase + '/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
     };
@@ -411,45 +410,80 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
             return;
         }
 
-        $scope.repos = $scope.repos.filter(function (r) {
-            return r.ID !== $scope.currentRepo.ID;
-        });
-
-        $scope.config.Repositories = $scope.repos;
+        delete $scope.repos[$scope.currentRepo.ID];
+        $scope.config.Repositories = repoList($scope.repos);
 
         $scope.configInSync = false;
         $http.post(urlbase + '/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
     };
 
-    $http.get(urlbase + '/version').success(function (data) {
-        $scope.version = data;
-    });
+    $scope.init = function() {
+        $http.get(urlbase + '/version').success(function (data) {
+            $scope.version = data;
+        });
 
-    $http.get(urlbase + '/system').success(function (data) {
-        $scope.system = data;
-        $scope.myID = data.myID;
-    });
+        $http.get(urlbase + '/system').success(function (data) {
+            $scope.system = data;
+            $scope.myID = data.myID;
+        });
 
-    $http.get(urlbase + '/config').success(function (data) {
-        $scope.config = data;
-        $scope.config.Options.ListenStr = $scope.config.Options.ListenAddress.join(', ');
+        $http.get(urlbase + '/config').success(function (data) {
+            $scope.config = data;
+            $scope.config.Options.ListenStr = $scope.config.Options.ListenAddress.join(', ');
 
-        var nodes = $scope.config.Nodes;
-        nodes.sort(nodeCompare);
-        $scope.nodes = nodes;
+            $scope.nodes = $scope.config.Nodes;
+            $scope.nodes.sort(nodeCompare);
 
-        $scope.repos = $scope.config.Repositories;
+            $scope.repos = repoMap($scope.config.Repositories);
 
-        $scope.refresh();
-    });
+            $scope.refresh();
+        });
 
-    $http.get(urlbase + '/config/sync').success(function (data) {
-        $scope.configInSync = data.configInSync;
-    });
+        $http.get(urlbase + '/config/sync').success(function (data) {
+            $scope.configInSync = data.configInSync;
+        });
+    };
 
+    $scope.init();
     setInterval($scope.refresh, 10000);
 });
 
+function nodeCompare(a, b) {
+    if (typeof a.Name !== 'undefined' && typeof b.Name !== 'undefined') {
+        if (a.Name < b.Name)
+            return -1;
+        return a.Name > b.Name;
+    }
+    if (a.NodeID < b.NodeID) {
+        return -1;
+    }
+    return a.NodeID > b.NodeID;
+}
+
+function repoCompare(a, b) {
+    if (a.Directory < b.Directory) {
+        return -1;
+    }
+    return a.Directory > b.Directory;
+}
+
+function repoMap(l) {
+    var m = {};
+    l.forEach(function (r) {
+        m[r.ID] = r;
+    });
+    return m;
+}
+
+function repoList(m) {
+    var l = [];
+    for (var id in m) {
+        l.push(m[id])
+    }
+    l.sort(repoCompare);
+    return l;
+}
+
 function decimals(val, num) {
     var digits, decs;
 
@@ -525,6 +559,17 @@ syncthing.filter('alwaysNumber', function () {
     };
 });
 
+syncthing.filter('chunkID', function () {
+    return function (input) {
+        if (input === undefined)
+            return "";
+        var parts = input.match(/.{1,6}/g);
+        if (!parts)
+            return "";
+        return parts.join(' ');
+    }
+});
+
 syncthing.directive('optionEditor', function () {
     return {
         restrict: 'C',
@@ -536,3 +581,24 @@ syncthing.directive('optionEditor', function () {
         template: '<input type="text" ng-model="config.Options[setting.id]"></input>',
     };
 });
+
+syncthing.directive('uniqueRepo', function() {
+    return {
+        require: 'ngModel',
+        link: function(scope, elm, attrs, ctrl) {
+            ctrl.$parsers.unshift(function(viewValue) {
+                if (scope.editingExisting) {
+                    // we shouldn't validate
+                    ctrl.$setValidity('uniqueRepo', true);
+                } else if (scope.repos[viewValue]) {
+                    // the repo exists already
+                    ctrl.$setValidity('uniqueRepo', false);
+                } else {
+                    // the repo is unique
+                    ctrl.$setValidity('uniqueRepo', true);
+                }
+                return viewValue;
+            });
+        }
+    };
+});

gui/index.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html lang="en" ng-app="syncthing">
+<html lang="en" ng-app="syncthing" ng-controller="SyncthingCtrl" class="ng-cloak">
 <head>
   <meta charset="utf-8">
   <meta http-equiv="X-UA-Compatible" content="IE=edge">
@@ -8,7 +8,7 @@
   <meta name="author" content="">
   <link rel="shortcut icon" href="favicon.png">
 
-  <title>syncthing</title>
+  <title>Syncthing | {{thisNodeName()}}</title>
   <link href="bootstrap/css/bootstrap.min.css" rel="stylesheet">
   <style type="text/css">
     body {
@@ -91,13 +91,13 @@
   </style>
 </head>
 
-<body ng-controller="SyncthingCtrl" class="ng-cloak">
+<body>
 
   <!-- Top bar -->
 
   <nav class="navbar navbar-top navbar-default" role="navigation">
     <div class="container">
-      <span class="navbar-brand"><img class="logo" src="st-logo-128.png" width="32" height="32"> Syncthing</span>
+      <span class="navbar-brand"><img class="logo" src="st-logo-128.png" width="32" height="32" /> Syncthing<small> | {{thisNodeName()}}</small></span>
       <button type="button" class="btn btn-default btn-sm pull-right navbar-btn" ng-click="editSettings()"><span class="glyphicon glyphicon-cog"></span> Settings</button>
     </div>
   </nav>
@@ -131,10 +131,10 @@
         <div class="panel panel-default">
           <div class="panel-heading"><h3 class="panel-title">Repositories</h3></div>
           <div class="panel-body">
-            <ul class="list-unstyled" ng-repeat="repo in repos">
+            <ul class="list-unstyled" ng-repeat="repo in repoList()">
               <li>
                 <span class="text-monospace">{{repo.Directory}}</span>
-                <span ng-if="repo.Invalid" class="label label-danger">Invalid: {{repo.Invalid}}</span>
+                <span ng-if="model[repo.ID].invalid" class="label label-danger">{{model[repo.ID].invalid}}</span>
                 <ul class="list-no-bullet">
                   <li>
                     <div class="li-column" title="Repository ID">
@@ -299,7 +299,7 @@
       <ul class="nav navbar-nav navbar-right">
         <li><a class="navbar-link" href="http://discourse.syncthing.net/">Support / Forum</a></li>
         <li><a class="navbar-link hidden-sm" href="https://github.com/calmh/syncthing/releases">Latest Release</a></li>
-        <li><a class="navbar-link" href="https://github.com/calmh/syncthing/wiki">Documentation</a></li>
+        <li><a class="navbar-link" href="http://discourse.syncthing.net/category/documentation">Documentation</a></li>
         <li><a class="navbar-link hidden-sm" href="https://github.com/calmh/syncthing/issues">Bugs</a></li>
         <li><a class="navbar-link hidden-sm" href="https://github.com/calmh/syncthing">Source Code</a></li>
       </ul>
@@ -358,12 +358,15 @@
           <h4 ng-show="editingExisting" class="modal-title">Edit Node</h4>
         </div>
         <div class="modal-body">
-          <form role="form">
-            <div class="form-group">
+          <form role="form" name="nodeEditor">
+            <div class="form-group" ng-class="{'has-error': nodeEditor.nodeID.$invalid && nodeEditor.nodeID.$dirty}">
               <label for="nodeID">Node ID</label>
-              <input ng-if="!editingExisting" id="nodeID" class="form-control" type="text" ng-model="currentNode.NodeID"></input>
-              <div ng-if="editingExisting" class="well well-sm">{{currentNode.NodeID}}</div>
-              <p class="help-block">The node ID can be found in the "Add Node" dialog on the other node.</p>
+              <input ng-if="!editingExisting" name="nodeID" id="nodeID" class="form-control" type="text" ng-model="currentNode.NodeID" required></input>
+              <div ng-if="editingExisting" class="well well-sm">{{currentNode.NodeID | chunkID}}</div>
+              <p class="help-block">
+                <span ng-if="nodeEditor.nodeID.$valid || nodeEditor.nodeID.$pristine">The node ID to enter here can be found in the "Add Node" dialog on the other node. Spaces are ignored.</span>
+                <span ng-if="nodeEditor.nodeID.$error.required && nodeEditor.nodeID.$dirty">The node ID cannot be blank.</span>
+              </p>
             </div>
             <div class="form-group">
               <label for="name">Name</label>
@@ -378,11 +381,11 @@
           </form>
           <div ng-show="!editingExisting">
             When adding a new node, keep in mind that <em>this node</em> must be added on the other side too. The Node ID of this node is:
-            <pre>{{myID}}</pre>
+            <pre>{{myID | chunkID}}</pre>
           </div>
         </div>
         <div class="modal-footer">
-          <button type="button" class="btn btn-primary" ng-click="saveNode()">Save</button>
+          <button type="button" class="btn btn-primary" ng-click="saveNode()" ng-disabled="nodeEditor.$invalid">Save</button>
           <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
           <button ng-if="editingExisting && !editingSelf" type="button" class="btn btn-danger pull-left" ng-click="deleteNode()">Delete</button>
         </div>
@@ -401,23 +404,31 @@
           <h4 ng-show="editingExisting" class="modal-title">Edit Repository</h4>
         </div>
         <div class="modal-body">
-          <form role="form">
-            <div class="form-group">
+          <form role="form" name="repoEditor">
+            <div class="form-group" ng-class="{'has-error': repoEditor.repoID.$invalid && repoEditor.repoID.$dirty}">
               <label for="repoID">Repository ID</label>
-              <input placeholder="documents" ng-disabled="editingExisting" id="repoID" class="form-control" type="text" ng-model="currentRepo.ID"></input>
-              <p class="help-block">Short, unique identifier for the repository. Must be the same on all cluster nodes.</p>
+              <input name="repoID" placeholder="documents" ng-disabled="editingExisting" id="repoID" class="form-control" type="text" ng-model="currentRepo.ID" required unique-repo></input>
+              <p class="help-block">
+              <span ng-if="repoEditor.repoID.$valid || repoEditor.repoID.$pristine">Short identifier for the repository. Must be the same on all cluster nodes.</span>
+              <span ng-if="repoEditor.repoID.$error.uniqueRepo">The repository ID must be unique.</span>
+              <span ng-if="repoEditor.repoID.$error.required && repoEditor.repoID.$dirty">The repository ID cannot be blank.</span>
+              </p>
             </div>
-            <div class="form-group">
+            <div class="form-group" ng-class="{'has-error': repoEditor.repoPath.$invalid && repoEditor.repoPath.$dirty}">
               <label for="repoPath">Repository Path</label>
-              <input placeholder="~/Documents" id="repoPath" class="form-control" type="text" ng-model="currentRepo.Directory"></input>
-              <p class="help-block">Path to the repository on the local computer. Will be created if it does not exist.</p>
+              <input name="repoPath" placeholder="~/Documents" id="repoPath" class="form-control" type="text" ng-model="currentRepo.Directory" required></input>
+              <p class="help-block">
+              <span ng-if="repoEditor.repoPath.$valid || repoEditor.repoPath.$pristine">Path to the repository on the local computer. Will be created if it does not exist.</span>
+              <span ng-if="repoEditor.repoPath.$error.required && repoEditor.repoPath.$dirty">The repository path cannot be blank.</span>
+              </p>
             </div>
             <div class="form-group">
               <div class="checkbox">
                 <label>
-                  <input type="checkbox" ng-model="currentRepo.ReadOnly"> Read Only
+                  <input type="checkbox" ng-model="currentRepo.ReadOnly"> Repository Master
                 </label>
               </div>
+              <p class="help-block">Files are protected from changes made on other nodes, but changes made on <em>this</em> node will be sent to the rest of the cluster.</p>
             </div>
             <div class="form-group">
               <label for="nodes">Nodes</label>
@@ -434,7 +445,7 @@
           </div>
         </div>
         <div class="modal-footer">
-          <button type="button" class="btn btn-primary" ng-click="saveRepo()">Save</button>
+          <button type="button" class="btn btn-primary" ng-click="saveRepo()" ng-disabled="repoEditor.$invalid">Save</button>
           <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
           <button ng-if="editingExisting" type="button" class="btn btn-danger pull-left" ng-click="deleteRepo()">Delete</button>
         </div>

integration/h1/config.xml

@@ -9,6 +9,9 @@
         <node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3">
             <address>127.0.0.1:22003</address>
         </node>
+        <node id="EJHMPAQOGCVORISB4IS3SYYVJXTKJGLTU66DIQPGJ5D2GXGQ3OWQ" name="s4">
+            <address>127.0.0.1:22004</address>
+        </node>
     </repository>
     <repository id="s12" directory="s12-1">
         <node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1">
@@ -32,7 +35,7 @@
         <maxSendKbps>0</maxSendKbps>
         <rescanIntervalS>10</rescanIntervalS>
         <reconnectionIntervalS>5</reconnectionIntervalS>
-        <maxChangeKbps>1000</maxChangeKbps>
+        <maxChangeKbps>10000</maxChangeKbps>
         <startBrowser>false</startBrowser>
     </options>
 </configuration>

integration/h2/config.xml

@@ -40,7 +40,7 @@
         <maxSendKbps>0</maxSendKbps>
         <rescanIntervalS>15</rescanIntervalS>
         <reconnectionIntervalS>5</reconnectionIntervalS>
-        <maxChangeKbps>1000</maxChangeKbps>
+        <maxChangeKbps>10000</maxChangeKbps>
         <startBrowser>false</startBrowser>
     </options>
 </configuration>

integration/h3/config.xml

@@ -32,7 +32,7 @@
         <maxSendKbps>0</maxSendKbps>
         <rescanIntervalS>20</rescanIntervalS>
         <reconnectionIntervalS>5</reconnectionIntervalS>
-        <maxChangeKbps>1000</maxChangeKbps>
+        <maxChangeKbps>10000</maxChangeKbps>
         <startBrowser>false</startBrowser>
     </options>
 </configuration>

integration/h4/cert.pem

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID3jCCAkigAwIBAgIBADALBgkqhkiG9w0BAQswFDESMBAGA1UEAxMJc3luY3Ro
+aW5nMB4XDTE0MDUxMDAwNTM0N1oXDTQ5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJ
+c3luY3RoaW5nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA9MRyBtAr
+Sjt29azNoCWxx5xZF3RodBcQu+wv5sRR8lWozrr4brfUJLslcQHowqaAprOU1NP+
+BH12P5CSymsUrwAmCwSQ54CimXrNi5RiNMl7dtInJksk4Kp6nJgfyR7TqeQgqxtv
++skVWdJY7ptxqpVuDfkf1JnNr68dbANw8hEJpPaGm3qOt81YvSg37R75HiOCzv+h
+FcSjKpPyFMvPARMCOHuZS0fYRJtI5nwmR0mWtKfnH/2204YNiQUne/8h2fgtkpxy
+OjxKOs2KJxbmpV6Uur/YyGyinb5+Aa0df3KCBuZmE+i/AsZcTsk0fgefe+bshWG/
+hzrNfV0wsX3TYjYOSBJ04+f/uQW00G1GGSxPwTsShGqVuwfJkTqkjAXX5wcH+PgJ
+ewG/dyMzKklMg19Y65WkhpWa/19o2KSZNw6TO8YM1arwT0STcMc+4fdrVB09lX6q
+NJA8UL8hUX+jbKBzatDY64h1d9E8PE0ODHYgYFO2Ko7e2GnWCQeijGmnAgMBAAGj
+PzA9MA4GA1UdDwEB/wQEAwIAoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH
+AwIwDAYDVR0TAQH/BAIwADALBgkqhkiG9w0BAQsDggGBANFiHcATP5Lm11o65wbh
+sKk7yteTapRohMoLNdW44YNyM8ZkELnrdNY8pe3CWSGy3spBH01+4jbUT+gSltQr
+KTLVxSZ7f91696Og5ag4BQCeFY6ghKD/G9+PlBSj6yb3Y98NZsx8huLfylH+XuJw
+2gP5Nqov4uXaKgYylx2gdaeCb2M+wM/br1DO2HCPCmgbZE5g8RM5JxzojGn/41Le
+IbCd39zdI6NKj9c7T1Bxmt20uzca4nRgXVVzJymedEoF+//sBRk6PQzqgjgn/r3S
+h9vrqo5j8ly/+ojFjBaVY7gq2XHM6/q0LTjeKkv2MUQw+vEEZX65GpBOgBZ8U0Wb
+/NMUUhhDjGE/0G6TCJgq/HdkjmsNaWjO5sWjhnwXNImYXBdH4OenhXIrHcLhcnxN
+2n5sPkDc6n0LVVV7VAjBPXcTmu2uOSK02yqNZLLWJygp1Wl6lbiqLS3bJgYrUv2m
+YkRaR+IqVPw5EPs/QlH0qLBeCyIasaSWUVZeitVwRmqIUA==
+-----END CERTIFICATE-----

integration/h4/config.xml

@@ -0,0 +1,36 @@
+<configuration version="2">
+    <repository id="unique" directory="s4" ro="false">
+        <node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1"></node>
+        <node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ" name="s2"></node>
+        <node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3"></node>
+        <node id="EJHMPAQOGCVORISB4IS3SYYVJXTKJGLTU66DIQPGJ5D2GXGQ3OWQ" name="s4"></node>
+    </repository>
+    <node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1">
+        <address>127.0.0.1:22001</address>
+    </node>
+    <node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ" name="s2">
+        <address>127.0.0.1:22002</address>
+    </node>
+    <node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3">
+        <address>127.0.0.1:22003</address>
+    </node>
+    <node id="EJHMPAQOGCVORISB4IS3SYYVJXTKJGLTU66DIQPGJ5D2GXGQ3OWQ" name="s4">
+        <address>dynamic</address>
+    </node>
+    <gui enabled="true">
+        <address>127.0.0.1:8084</address>
+    </gui>
+    <options>
+        <listenAddress>:22004</listenAddress>
+        <globalAnnounceServer>announce.syncthing.net:22025</globalAnnounceServer>
+        <globalAnnounceEnabled>false</globalAnnounceEnabled>
+        <localAnnounceEnabled>true</localAnnounceEnabled>
+        <parallelRequests>16</parallelRequests>
+        <maxSendKbps>0</maxSendKbps>
+        <rescanIntervalS>60</rescanIntervalS>
+        <reconnectionIntervalS>10</reconnectionIntervalS>
+        <maxChangeKbps>10000</maxChangeKbps>
+        <startBrowser>false</startBrowser>
+        <upnpEnabled>false</upnpEnabled>
+    </options>
+</configuration>

integration/h4/key.pem

@@ -0,0 +1,39 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIG5AIBAAKCAYEA9MRyBtArSjt29azNoCWxx5xZF3RodBcQu+wv5sRR8lWozrr4
+brfUJLslcQHowqaAprOU1NP+BH12P5CSymsUrwAmCwSQ54CimXrNi5RiNMl7dtIn
+Jksk4Kp6nJgfyR7TqeQgqxtv+skVWdJY7ptxqpVuDfkf1JnNr68dbANw8hEJpPaG
+m3qOt81YvSg37R75HiOCzv+hFcSjKpPyFMvPARMCOHuZS0fYRJtI5nwmR0mWtKfn
+H/2204YNiQUne/8h2fgtkpxyOjxKOs2KJxbmpV6Uur/YyGyinb5+Aa0df3KCBuZm
+E+i/AsZcTsk0fgefe+bshWG/hzrNfV0wsX3TYjYOSBJ04+f/uQW00G1GGSxPwTsS
+hGqVuwfJkTqkjAXX5wcH+PgJewG/dyMzKklMg19Y65WkhpWa/19o2KSZNw6TO8YM
+1arwT0STcMc+4fdrVB09lX6qNJA8UL8hUX+jbKBzatDY64h1d9E8PE0ODHYgYFO2
+Ko7e2GnWCQeijGmnAgMBAAECggGBAIjKaLdqC2d3CCqQonJH3q0hsaCsC9wlL9L2
+UmbzfKCkQq0WTNUDo2nLtUcMvBpclzWS0zCGMUYtH7Kyh3bclTigKqKpsJnQiA6i
+VNEW4jOCDp//HqYGBNwSKmftlIX/1mbx+VfnA5PyYR5LsivXb5TX4iOpAKL+Obdf
+dF/zJGIEJ5GrvNqTicMq3dcI7Qh18N9pFSe+MTZLKK0Y9Yetx0hgaTNL0AYEZtcg
+uYMmCvZ4J+Namo6EanKYTmQvHzvq/tZVMvud9Gcr6uKKtVBcgex9S/R7IicaKg78
+oDTgH0nDrpI55pZCX8vuVGk8nVTXXLTsMR1XojOpiYjS6ucfTkPEw3fOW/YRhHg5
+93TrdDiWkqSWube5LNUF87q65t/aw/y2EH2aTNqcPD5OQ+EZRS8OGYPqOrJ4Ycbp
+j6CMSE+LX2IDMQyJ+9J0vPHtFsAviBKQkPoQ1L6mvhJuw6ksy34NQGykNDHz7nQK
+SeqvCJ6XCtaWNkq+00lC3UFaGsjuUQKBwQD8+y370co5G7G5GDLbLE3i+pguUN7L
+5YfDj5qqsM9hOJNqeKAHrKFP2ii0F9WxGw/ruY0k8k7zUt6LepgwkCI5BYfckRKJ
+g8YsNTizjqPLRGtiqL9Garjo+xPxFGj+TkTg9fYD4xTWFa1I15zzCu7Ye7xObeEH
+LRtcm3R4fU54JDrKtKDccoQmTEAzsxRdNXi9ifc7qgjGBH9W02guuGPY4ltT1aZR
+bcO5vpi44Fnl2h6d7N6iwCtFJ0CaT1pAZ4UCgcEA97Asf5DTDWKByZBhk+VvuT1b
+6nMYjqKxDNMmCaomCmk8Mif0w9SEJmAg0b/gbs/H6T78a+9WjbN5q9xHcDU91uax
+TdCenTq7H981AjgUG7OA7XwYn+AKy+hGSnsTJglMJzJm6TGt+Sq0oO9EahBRDlsP
+PiQRot2gyQfubwcl3rhdErRwaCM92BUyPkC2fy2OppAeZOOxxuzxrvHflDOuDGCZ
+KPCmy6U9HV0JOAO2FSNJeZdNLBixXa1Pk8TgbLY7AoHBAPG7lhn9Qg3Fz9H9NINH
+13jfWdFQB0SwJEWTEAiwgMj2ha6Eau5KX63s2V4VNGVSZakqmZtHSneppOuEjq5A
+2+K+zS7PFPaACzos9OxmjU7rJu2UL4m66sv9NvXzOcxev+RyQs0+DKfw+K8VEG0Q
+8l+8BJiw2AjCalXYWbfUjMmyXNdbOCbN6kaqL+L26KuUL7Z1gd/qPw3wODmgMvoJ
+yabxzLDUA2PlzdPMMyTdhCllfkILmEXN+MrQkiOhVa0a/QKBwGZjAhH9ePD4fnQm
+5d8wIb3uGlfRGh6kLBIEGp42IqF9HPASykBFUhdW91odOhY0eAv4CHpJpnrO7QXY
++gLtT1HNbQ+gpGCUTZQAPbZcHhvRWQNSoA8+mtftfVj+hUzc3Qj68cWFzsfIGoDI
+R3ycoBUSGTvzxwKPIQ7Y43wr9UCa74Zy5mB16POw12MadxYda/F4c8f6w5taiRFr
+VKO7tT/Skp101U4rURcZRV1NU3BrdMz5eWI4FuGFafbIlIj7zwKBwHCt3VQt+JmZ
+OhCJR+8Q+jT0JvnMu1zi4CcMRiT8FbNdZDY/3B0wG4ySTNrEikFzIjihF4zIp2nv
+nD3qKQs+THl51GA8AnP9bNk7hknD7rXUuScndccTW58+PGrjqfwJp/1MEeOJQpoX
+0JML1w+dIKHzsKN0X6UL7Gyq8m+0SJKmQQguan3d3M8CMpnW0srgqOfJ+q1+bz8b
+6FuJeijoaN8+zyKkN+9R91Erw5pk+7vJRzEpDtkhprEE5tLNDKrXJw==
+-----END RSA PRIVATE KEY-----

integration/test.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-export STNORESTART=1
-
 iterations=${1:-5}
 
 id1=I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA
@@ -14,7 +12,7 @@ go build json.go
 
 start() {
 	echo "Starting..."
-	for i in 1 2 3 ; do
+	for i in 1 2 3 4 ; do
 		STPROFILER=":909$i" syncthing -home "h$i" &
 	done
 }
@@ -137,4 +135,6 @@ for ((t = 1; t <= $iterations; t++)) ; do
 	testConvergence
 done
 
-pkill syncthing
+for i in 1 2 3 4 ; do
+	curl -X POST "http://localhost:808$i/rest/shutdown"
+done

lamport/clock.go

@@ -1,3 +1,4 @@
+// Package lamport implements a simple Lamport Clock for versioning
 package lamport
 
 import "sync"

logger/logger.go

@@ -0,0 +1,143 @@
+// Package logger implements a standardized logger with callback functionality
+package logger
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"sync"
+)
+
+type LogLevel int
+
+const (
+	LevelDebug LogLevel = iota
+	LevelInfo
+	LevelOK
+	LevelWarn
+	LevelFatal
+	NumLevels
+)
+
+type MessageHandler func(l LogLevel, msg string)
+
+type Logger struct {
+	logger   *log.Logger
+	handlers [NumLevels][]MessageHandler
+	mut      sync.Mutex
+}
+
+var DefaultLogger = New()
+
+func New() *Logger {
+	return &Logger{
+		logger: log.New(os.Stderr, "", log.Ltime),
+	}
+}
+
+func (l *Logger) AddHandler(level LogLevel, h MessageHandler) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	l.handlers[level] = append(l.handlers[level], h)
+}
+
+func (l *Logger) SetFlags(flag int) {
+	l.logger.SetFlags(flag)
+}
+
+func (l *Logger) SetPrefix(prefix string) {
+	l.logger.SetPrefix(prefix)
+}
+
+func (l *Logger) callHandlers(level LogLevel, s string) {
+	for _, h := range l.handlers[level] {
+		h(level, s)
+	}
+}
+
+func (l *Logger) Debugln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "DEBUG: "+s)
+	l.callHandlers(LevelDebug, s)
+}
+
+func (l *Logger) Debugf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "DEBUG: "+s)
+	l.callHandlers(LevelDebug, s)
+}
+func (l *Logger) Infoln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "INFO: "+s)
+	l.callHandlers(LevelInfo, s)
+}
+
+func (l *Logger) Infof(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "INFO: "+s)
+	l.callHandlers(LevelInfo, s)
+}
+
+func (l *Logger) Okln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "OK: "+s)
+	l.callHandlers(LevelOK, s)
+}
+
+func (l *Logger) Okf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "OK: "+s)
+	l.callHandlers(LevelOK, s)
+}
+
+func (l *Logger) Warnln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "WARNING: "+s)
+	l.callHandlers(LevelWarn, s)
+}
+
+func (l *Logger) Warnf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "WARNING: "+s)
+	l.callHandlers(LevelWarn, s)
+}
+
+func (l *Logger) Fatalln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "FATAL: "+s)
+	l.callHandlers(LevelFatal, s)
+	os.Exit(3)
+}
+
+func (l *Logger) Fatalf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "FATAL: "+s)
+	l.callHandlers(LevelFatal, s)
+	os.Exit(3)
+}
+
+func (l *Logger) FatalErr(err error) {
+	if err != nil {
+		l.Fatalf(err.Error())
+	}
+}

mc/beacon.go

@@ -1,103 +0,0 @@
-package mc
-
-import (
-	"log"
-	"net"
-)
-
-type recv struct {
-	data []byte
-	src  net.Addr
-}
-
-type Beacon struct {
-	group  string
-	port   int
-	conns  []*net.UDPConn
-	inbox  chan []byte
-	outbox chan recv
-}
-
-func NewBeacon(group string, port int) *Beacon {
-	b := &Beacon{
-		group:  group,
-		port:   port,
-		inbox:  make(chan []byte),
-		outbox: make(chan recv),
-	}
-	go b.run()
-	return b
-}
-
-func (b *Beacon) Send(data []byte) {
-	b.inbox <- data
-}
-
-func (b *Beacon) Recv() ([]byte, net.Addr) {
-	recv := <-b.outbox
-	return recv.data, recv.src
-}
-
-func (b *Beacon) run() {
-	group := &net.UDPAddr{IP: net.ParseIP(b.group), Port: b.port}
-
-	intfs, err := net.Interfaces()
-	if err != nil {
-		log.Fatal(err)
-	}
-	if debug {
-		dlog.Printf("trying %d interfaces", len(intfs))
-	}
-
-	for _, intf := range intfs {
-		intf := intf
-
-		if debug {
-			dlog.Printf("trying interface %q", intf.Name)
-		}
-		conn, err := net.ListenMulticastUDP("udp4", &intf, group)
-		if err != nil {
-			if debug {
-				dlog.Printf("failed to listen for multicast group on %q: %v", intf.Name, err)
-			}
-		} else {
-			b.conns = append(b.conns, conn)
-			if debug {
-				dlog.Printf("listening for multicast group on %q", intf.Name)
-			}
-		}
-	}
-
-	for _, conn := range b.conns {
-		conn := conn
-		go func() {
-			for {
-				var bs = make([]byte, 1500)
-				n, addr, err := conn.ReadFrom(bs)
-				if err != nil {
-					dlog.Println(err)
-					return
-				}
-				if debug {
-					dlog.Printf("recv %d bytes from %s on %v", n, addr, conn)
-				}
-				b.outbox <- recv{bs[:n], addr}
-			}
-		}()
-	}
-
-	go func() {
-		for bs := range b.inbox {
-			for _, conn := range b.conns {
-				_, err := conn.WriteTo(bs, group)
-				if err != nil {
-					dlog.Println(err)
-					return
-				}
-				if debug {
-					dlog.Printf("sent %d bytes to %s on %v", len(bs), group, conn)
-				}
-			}
-		}
-	}()
-}

mc/debug.go

@@ -1,12 +0,0 @@
-package mc
-
-import (
-	"log"
-	"os"
-	"strings"
-)
-
-var (
-	dlog  = log.New(os.Stderr, "mc: ", log.Lmicroseconds|log.Lshortfile)
-	debug = strings.Contains(os.Getenv("STTRACE"), "mc")
-)

model/blockqueue.go

@@ -1,4 +1,4 @@
-package main
+package model
 
 import (
 	"sync/atomic"

model/debug.go

@@ -0,0 +1,13 @@
+package model
+
+import (
+	"os"
+	"strings"
+
+	"github.com/calmh/syncthing/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "model") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

model/doc.go

@@ -0,0 +1,2 @@
+// Package model implements repository abstraction and file pulling mechanisms
+package model

model/model.go

@@ -1,4 +1,4 @@
-package main
+package model
 
 import (
 	"compress/gzip"
@@ -14,6 +14,7 @@ import (
 
 	"github.com/calmh/syncthing/buffers"
 	"github.com/calmh/syncthing/cid"
+	"github.com/calmh/syncthing/config"
 	"github.com/calmh/syncthing/files"
 	"github.com/calmh/syncthing/lamport"
 	"github.com/calmh/syncthing/protocol"
@@ -30,12 +31,19 @@ const (
 )
 
 type Model struct {
-	repoDirs  map[string]string     // repo -> dir
-	repoFiles map[string]*files.Set // repo -> files
-	repoNodes map[string][]string   // repo -> nodeIDs
-	nodeRepos map[string][]string   // nodeID -> repos
-	repoState map[string]repoState  // repo -> state
-	rmut      sync.RWMutex          // protects the above
+	indexDir string
+	cfg      *config.Configuration
+
+	clientName    string
+	clientVersion string
+
+	repoDirs   map[string]string      // repo -> dir
+	repoFiles  map[string]*files.Set  // repo -> files
+	repoNodes  map[string][]string    // repo -> nodeIDs
+	nodeRepos  map[string][]string    // nodeID -> repos
+	repoState  map[string]repoState   // repo -> state
+	suppressor map[string]*suppressor // repo -> suppressor
+	rmut       sync.RWMutex           // protects the above
 
 	cm *cid.Map
 
@@ -58,18 +66,23 @@ var (
 // NewModel creates and starts a new model. The model starts in read-only mode,
 // where it sends index information to connected peers and responds to requests
 // for file data without altering the local repository in any way.
-func NewModel(maxChangeBw int) *Model {
+func NewModel(indexDir string, cfg *config.Configuration, clientName, clientVersion string) *Model {
 	m := &Model{
-		repoDirs:  make(map[string]string),
-		repoFiles: make(map[string]*files.Set),
-		repoNodes: make(map[string][]string),
-		nodeRepos: make(map[string][]string),
-		repoState: make(map[string]repoState),
-		cm:        cid.NewMap(),
-		protoConn: make(map[string]protocol.Connection),
-		rawConn:   make(map[string]io.Closer),
-		nodeVer:   make(map[string]string),
-		sup:       suppressor{threshold: int64(maxChangeBw)},
+		indexDir:      indexDir,
+		cfg:           cfg,
+		clientName:    clientName,
+		clientVersion: clientVersion,
+		repoDirs:      make(map[string]string),
+		repoFiles:     make(map[string]*files.Set),
+		repoNodes:     make(map[string][]string),
+		nodeRepos:     make(map[string][]string),
+		repoState:     make(map[string]repoState),
+		suppressor:    make(map[string]*suppressor),
+		cm:            cid.NewMap(),
+		protoConn:     make(map[string]protocol.Connection),
+		rawConn:       make(map[string]io.Closer),
+		nodeVer:       make(map[string]string),
+		sup:           suppressor{threshold: int64(cfg.Options.MaxChangeKbps)},
 	}
 
 	go m.broadcastIndexLoop()
@@ -86,7 +99,7 @@ func (m *Model) StartRepoRW(repo string, threads int) {
 	if dir, ok := m.repoDirs[repo]; !ok {
 		panic("cannot start without repo")
 	} else {
-		newPuller(repo, dir, m, threads)
+		newPuller(repo, dir, m, threads, m.cfg)
 	}
 }
 
@@ -213,8 +226,8 @@ func (m *Model) NeedFilesRepo(repo string) []scanner.File {
 // Index is called when a new node is connected and we receive their full index.
 // Implements the protocol.Model interface.
 func (m *Model) Index(nodeID string, repo string, fs []protocol.FileInfo) {
-	if debugNet {
-		dlog.Printf("IDX(in): %s / %q: %d files", nodeID, repo, len(fs))
+	if debug {
+		l.Debugf("IDX(in): %s / %q: %d files", nodeID, repo, len(fs))
 	}
 
 	var files = make([]scanner.File, len(fs))
@@ -228,7 +241,7 @@ func (m *Model) Index(nodeID string, repo string, fs []protocol.FileInfo) {
 	if r, ok := m.repoFiles[repo]; ok {
 		r.Replace(id, files)
 	} else {
-		warnf("Index from %s for nonexistant repo %q; dropping", nodeID, repo)
+		l.Warnf("Index from %s for nonexistant repo %q; dropping", nodeID, repo)
 	}
 	m.rmut.RUnlock()
 }
@@ -236,8 +249,8 @@ func (m *Model) Index(nodeID string, repo string, fs []protocol.FileInfo) {
 // IndexUpdate is called for incremental updates to connected nodes' indexes.
 // Implements the protocol.Model interface.
 func (m *Model) IndexUpdate(nodeID string, repo string, fs []protocol.FileInfo) {
-	if debugNet {
-		dlog.Printf("IDXUP(in): %s / %q: %d files", nodeID, repo, len(fs))
+	if debug {
+		l.Debugf("IDXUP(in): %s / %q: %d files", nodeID, repo, len(fs))
 	}
 
 	var files = make([]scanner.File, len(fs))
@@ -251,20 +264,20 @@ func (m *Model) IndexUpdate(nodeID string, repo string, fs []protocol.FileInfo)
 	if r, ok := m.repoFiles[repo]; ok {
 		r.Update(id, files)
 	} else {
-		warnf("Index update from %s for nonexistant repo %q; dropping", nodeID, repo)
+		l.Warnf("Index update from %s for nonexistant repo %q; dropping", nodeID, repo)
 	}
 	m.rmut.RUnlock()
 }
 
 func (m *Model) ClusterConfig(nodeID string, config protocol.ClusterConfigMessage) {
 	compErr := compareClusterConfig(m.clusterConfig(nodeID), config)
-	if debugNet {
-		dlog.Printf("ClusterConfig: %s: %#v", nodeID, config)
-		dlog.Printf("  ... compare: %s: %v", nodeID, compErr)
+	if debug {
+		l.Debugf("ClusterConfig: %s: %#v", nodeID, config)
+		l.Debugf("  ... compare: %s: %v", nodeID, compErr)
 	}
 
 	if compErr != nil {
-		warnf("%s: %v", nodeID, compErr)
+		l.Warnf("%s: %v", nodeID, compErr)
 		m.Close(nodeID, compErr)
 	}
 
@@ -280,14 +293,14 @@ func (m *Model) ClusterConfig(nodeID string, config protocol.ClusterConfigMessag
 // Close removes the peer from the model and closes the underlying connection if possible.
 // Implements the protocol.Model interface.
 func (m *Model) Close(node string, err error) {
-	if debugNet {
-		dlog.Printf("%s: %v", node, err)
+	if debug {
+		l.Debugf("%s: %v", node, err)
 	}
 
 	if err != io.EOF {
-		warnf("Connection to %s closed: %v", node, err)
+		l.Warnf("Connection to %s closed: %v", node, err)
 	} else if _, ok := err.(ClusterConfigMismatch); ok {
-		warnf("Connection to %s closed: %v", node, err)
+		l.Warnf("Connection to %s closed: %v", node, err)
 	}
 
 	cid := m.cm.Get(node)
@@ -318,22 +331,24 @@ func (m *Model) Request(nodeID, repo, name string, offset int64, size int) ([]by
 	m.rmut.RUnlock()
 
 	if !ok {
-		warnf("Request from %s for file %s in nonexistent repo %q", nodeID, name, repo)
+		l.Warnf("Request from %s for file %s in nonexistent repo %q", nodeID, name, repo)
 		return nil, ErrNoSuchFile
 	}
 
 	lf := r.Get(cid.LocalID, name)
-	if offset > lf.Size {
-		warnf("SECURITY (nonexistent file) REQ(in): %s: %q o=%d s=%d", nodeID, name, offset, size)
-		return nil, ErrNoSuchFile
-	}
-
-	if lf.Suppressed {
+	if lf.Suppressed || lf.Flags&protocol.FlagDeleted != 0 {
 		return nil, ErrInvalid
 	}
 
-	if debugNet && nodeID != "<local>" {
-		dlog.Printf("REQ(in): %s: %q / %q o=%d s=%d", nodeID, repo, name, offset, size)
+	if offset > lf.Size {
+		if debug {
+			l.Debugf("REQ(in; nonexistent): %s: %q o=%d s=%d", nodeID, name, offset, size)
+		}
+		return nil, ErrNoSuchFile
+	}
+
+	if debug && nodeID != "<local>" {
+		l.Debugf("REQ(in): %s: %q / %q o=%d s=%d", nodeID, repo, name, offset, size)
 	}
 	m.rmut.RLock()
 	fn := filepath.Join(m.repoDirs[repo], name)
@@ -423,16 +438,18 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection)
 	cm := m.clusterConfig(nodeID)
 	protoConn.ClusterConfig(cm)
 
+	var idxToSend = make(map[string][]protocol.FileInfo)
+
+	m.rmut.RLock()
+	for _, repo := range m.nodeRepos[nodeID] {
+		idxToSend[repo] = m.protocolIndex(repo)
+	}
+	m.rmut.RUnlock()
+
 	go func() {
-		m.rmut.RLock()
-		repos := m.nodeRepos[nodeID]
-		m.rmut.RUnlock()
-		for _, repo := range repos {
-			m.rmut.RLock()
-			idx := m.protocolIndex(repo)
-			m.rmut.RUnlock()
-			if debugNet {
-				dlog.Printf("IDX(out/initial): %s: %q: %d files", nodeID, repo, len(idx))
+		for repo, idx := range idxToSend {
+			if debug {
+				l.Debugf("IDX(out/initial): %s: %q: %d files", nodeID, repo, len(idx))
 			}
 			protoConn.Index(repo, idx)
 		}
@@ -447,12 +464,12 @@ func (m *Model) protocolIndex(repo string) []protocol.FileInfo {
 
 	for _, f := range fs {
 		mf := fileInfoFromFile(f)
-		if debugIdx {
+		if debug {
 			var flagComment string
 			if mf.Flags&protocol.FlagDeleted != 0 {
 				flagComment = " (deleted)"
 			}
-			dlog.Printf("IDX(out): %q/%q m=%d f=%o%s v=%d (%d blocks)", repo, mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks))
+			l.Debugf("IDX(out): %q/%q m=%d f=%o%s v=%d (%d blocks)", repo, mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks))
 		}
 		index = append(index, mf)
 	}
@@ -475,8 +492,8 @@ func (m *Model) requestGlobal(nodeID, repo, name string, offset int64, size int,
 		return nil, fmt.Errorf("requestGlobal: no such node: %s", nodeID)
 	}
 
-	if debugNet {
-		dlog.Printf("REQ(out): %s: %q / %q o=%d s=%d h=%x", nodeID, repo, name, offset, size, hash)
+	if debug {
+		l.Debugf("REQ(out): %s: %q / %q o=%d s=%d h=%x", nodeID, repo, name, offset, size, hash)
 	}
 
 	return nc.Request(repo, name, offset, size)
@@ -498,14 +515,14 @@ func (m *Model) broadcastIndexLoop() {
 			lastChange[repo] = c
 
 			idx := m.protocolIndex(repo)
-			m.saveIndex(repo, confDir, idx)
+			m.saveIndex(repo, m.indexDir, idx)
 
 			var indexWg sync.WaitGroup
 			for _, nodeID := range m.repoNodes[repo] {
 				if conn, ok := m.protoConn[nodeID]; ok {
 					indexWg.Add(1)
-					if debugNet {
-						dlog.Printf("IDX(out/loop): %s: %d files", nodeID, len(idx))
+					if debug {
+						l.Debugf("IDX(out/loop): %s: %d files", nodeID, len(idx))
 					}
 					go func() {
 						conn.Index(repo, idx)
@@ -522,7 +539,7 @@ func (m *Model) broadcastIndexLoop() {
 	}
 }
 
-func (m *Model) AddRepo(id, dir string, nodes []NodeConfiguration) {
+func (m *Model) AddRepo(id, dir string, nodes []config.NodeConfiguration) {
 	if m.started {
 		panic("cannot add repo to started model")
 	}
@@ -533,6 +550,7 @@ func (m *Model) AddRepo(id, dir string, nodes []NodeConfiguration) {
 	m.rmut.Lock()
 	m.repoDirs[id] = dir
 	m.repoFiles[id] = files.NewSet()
+	m.suppressor[id] = &suppressor{threshold: int64(m.cfg.Options.MaxChangeKbps)}
 
 	m.repoNodes[id] = make([]string, len(nodes))
 	for i, node := range nodes {
@@ -552,27 +570,60 @@ func (m *Model) ScanRepos() {
 	}
 	m.rmut.RUnlock()
 
+	var wg sync.WaitGroup
+	wg.Add(len(repos))
 	for _, repo := range repos {
-		m.ScanRepo(repo)
+		repo := repo
+		go func() {
+			m.ScanRepo(repo)
+			wg.Done()
+		}()
 	}
+	wg.Wait()
 }
 
-func (m *Model) ScanRepo(repo string) {
-	sup := &suppressor{threshold: int64(cfg.Options.MaxChangeKbps)}
+func (m *Model) CleanRepos() {
+	m.rmut.RLock()
+	var dirs = make([]string, 0, len(m.repoDirs))
+	for _, dir := range m.repoDirs {
+		dirs = append(dirs, dir)
+	}
+	m.rmut.RUnlock()
+
+	var wg sync.WaitGroup
+	wg.Add(len(dirs))
+	for _, dir := range dirs {
+		w := &scanner.Walker{
+			Dir:       dir,
+			TempNamer: defTempNamer,
+		}
+		go func() {
+			w.CleanTempFiles()
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+}
+
+func (m *Model) ScanRepo(repo string) error {
 	m.rmut.RLock()
 	w := &scanner.Walker{
 		Dir:          m.repoDirs[repo],
 		IgnoreFile:   ".stignore",
-		BlockSize:    BlockSize,
+		BlockSize:    scanner.StandardBlockSize,
 		TempNamer:    defTempNamer,
-		Suppressor:   sup,
+		Suppressor:   m.suppressor[repo],
 		CurrentFiler: cFiler{m, repo},
 	}
 	m.rmut.RUnlock()
 	m.setState(repo, RepoScanning)
-	fs, _ := w.Walk()
+	fs, _, err := w.Walk()
+	if err != nil {
+		return err
+	}
 	m.ReplaceLocal(repo, fs)
 	m.setState(repo, RepoIdle)
+	return nil
 }
 
 func (m *Model) SaveIndexes(dir string) {
@@ -644,8 +695,8 @@ func (m *Model) loadIndex(repo string, dir string) []protocol.FileInfo {
 // clusterConfig returns a ClusterConfigMessage that is correct for the given peer node
 func (m *Model) clusterConfig(node string) protocol.ClusterConfigMessage {
 	cm := protocol.ClusterConfigMessage{
-		ClientName:    "syncthing",
-		ClientVersion: Version,
+		ClientName:    m.clientName,
+		ClientVersion: m.clientVersion,
 	}
 
 	m.rmut.RLock()

model/model_test.go

@@ -1,4 +1,4 @@
-package main
+package model
 
 import (
 	"bytes"
@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/calmh/syncthing/cid"
+	"github.com/calmh/syncthing/config"
 	"github.com/calmh/syncthing/protocol"
 	"github.com/calmh/syncthing/scanner"
 )
@@ -47,7 +48,7 @@ func init() {
 }
 
 func TestRequest(t *testing.T) {
-	m := NewModel(1e6)
+	m := NewModel("/tmp", &config.Configuration{}, "syncthing", "dev")
 	m.AddRepo("default", "testdata", nil)
 	m.ScanRepo("default")
 
@@ -83,7 +84,7 @@ func genFiles(n int) []protocol.FileInfo {
 }
 
 func BenchmarkIndex10000(b *testing.B) {
-	m := NewModel(1e6)
+	m := NewModel("/tmp", nil, "syncthing", "dev")
 	m.AddRepo("default", "testdata", nil)
 	m.ScanRepo("default")
 	files := genFiles(10000)
@@ -95,7 +96,7 @@ func BenchmarkIndex10000(b *testing.B) {
 }
 
 func BenchmarkIndex00100(b *testing.B) {
-	m := NewModel(1e6)
+	m := NewModel("/tmp", nil, "syncthing", "dev")
 	m.AddRepo("default", "testdata", nil)
 	m.ScanRepo("default")
 	files := genFiles(100)
@@ -107,7 +108,7 @@ func BenchmarkIndex00100(b *testing.B) {
 }
 
 func BenchmarkIndexUpdate10000f10000(b *testing.B) {
-	m := NewModel(1e6)
+	m := NewModel("/tmp", nil, "syncthing", "dev")
 	m.AddRepo("default", "testdata", nil)
 	m.ScanRepo("default")
 	files := genFiles(10000)
@@ -120,7 +121,7 @@ func BenchmarkIndexUpdate10000f10000(b *testing.B) {
 }
 
 func BenchmarkIndexUpdate10000f00100(b *testing.B) {
-	m := NewModel(1e6)
+	m := NewModel("/tmp", nil, "syncthing", "dev")
 	m.AddRepo("default", "testdata", nil)
 	m.ScanRepo("default")
 	files := genFiles(10000)
@@ -134,7 +135,7 @@ func BenchmarkIndexUpdate10000f00100(b *testing.B) {
 }
 
 func BenchmarkIndexUpdate10000f00001(b *testing.B) {
-	m := NewModel(1e6)
+	m := NewModel("/tmp", nil, "syncthing", "dev")
 	m.AddRepo("default", "testdata", nil)
 	m.ScanRepo("default")
 	files := genFiles(10000)
@@ -181,7 +182,7 @@ func (FakeConnection) Statistics() protocol.Statistics {
 }
 
 func BenchmarkRequest(b *testing.B) {
-	m := NewModel(1e6)
+	m := NewModel("/tmp", nil, "syncthing", "dev")
 	m.AddRepo("default", "testdata", nil)
 	m.ScanRepo("default")
 

model/puller.go

@@ -1,4 +1,4 @@
-package main
+package model
 
 import (
 	"bytes"
@@ -9,6 +9,7 @@ import (
 
 	"github.com/calmh/syncthing/buffers"
 	"github.com/calmh/syncthing/cid"
+	"github.com/calmh/syncthing/config"
 	"github.com/calmh/syncthing/protocol"
 	"github.com/calmh/syncthing/scanner"
 )
@@ -61,6 +62,7 @@ func (m activityMap) decrease(node string) {
 var errNoNode = errors.New("no available source node")
 
 type puller struct {
+	cfg               *config.Configuration
 	repo              string
 	dir               string
 	bq                *blockQueue
@@ -72,8 +74,9 @@ type puller struct {
 	requestResults    chan requestResult
 }
 
-func newPuller(repo, dir string, model *Model, slots int) *puller {
+func newPuller(repo, dir string, model *Model, slots int, cfg *config.Configuration) *puller {
 	p := &puller{
+		cfg:               cfg,
 		repo:              repo,
 		dir:               dir,
 		bq:                newBlockQueue(),
@@ -90,14 +93,14 @@ func newPuller(repo, dir string, model *Model, slots int) *puller {
 		for i := 0; i < slots; i++ {
 			p.requestSlots <- true
 		}
-		if debugPull {
-			dlog.Printf("starting puller; repo %q dir %q slots %d", repo, dir, slots)
+		if debug {
+			l.Debugf("starting puller; repo %q dir %q slots %d", repo, dir, slots)
 		}
 		go p.run()
 	} else {
 		// Read only
-		if debugPull {
-			dlog.Printf("starting puller; repo %q dir %q (read only)", repo, dir)
+		if debug {
+			l.Debugf("starting puller; repo %q dir %q (read only)", repo, dir)
 		}
 		go p.runRO()
 	}
@@ -110,14 +113,14 @@ func (p *puller) run() {
 		for {
 			<-p.requestSlots
 			b := p.bq.get()
-			if debugPull {
-				dlog.Printf("filler: queueing %q / %q offset %d copy %d", p.repo, b.file.Name, b.block.Offset, len(b.copy))
+			if debug {
+				l.Debugf("filler: queueing %q / %q offset %d copy %d", p.repo, b.file.Name, b.block.Offset, len(b.copy))
 			}
 			p.blocks <- b
 		}
 	}()
 
-	walkTicker := time.Tick(time.Duration(cfg.Options.RescanIntervalS) * time.Second)
+	walkTicker := time.Tick(time.Duration(p.cfg.Options.RescanIntervalS) * time.Second)
 	timeout := time.Tick(5 * time.Second)
 	changed := true
 
@@ -145,11 +148,11 @@ func (p *puller) run() {
 					// Nothing more to do for the moment
 					break pull
 				}
-				if debugPull {
-					dlog.Printf("%q: idle but have %d open files", p.repo, len(p.openFiles))
+				if debug {
+					l.Debugf("%q: idle but have %d open files", p.repo, len(p.openFiles))
 					i := 5
 					for _, f := range p.openFiles {
-						dlog.Printf("  %v", f)
+						l.Debugf("  %v", f)
 						i--
 						if i == 0 {
 							break
@@ -170,10 +173,14 @@ func (p *puller) run() {
 		// Do a rescan if it's time for it
 		select {
 		case <-walkTicker:
-			if debugPull {
-				dlog.Printf("%q: time for rescan", p.repo)
+			if debug {
+				l.Debugf("%q: time for rescan", p.repo)
+			}
+			err := p.model.ScanRepo(p.repo)
+			if err != nil {
+				invalidateRepo(p.cfg, p.repo, err)
+				return
 			}
-			p.model.ScanRepo(p.repo)
 
 		default:
 		}
@@ -184,13 +191,17 @@ func (p *puller) run() {
 }
 
 func (p *puller) runRO() {
-	walkTicker := time.Tick(time.Duration(cfg.Options.RescanIntervalS) * time.Second)
+	walkTicker := time.Tick(time.Duration(p.cfg.Options.RescanIntervalS) * time.Second)
 
 	for _ = range walkTicker {
-		if debugPull {
-			dlog.Printf("%q: time for rescan", p.repo)
+		if debug {
+			l.Debugf("%q: time for rescan", p.repo)
+		}
+		err := p.model.ScanRepo(p.repo)
+		if err != nil {
+			invalidateRepo(p.cfg, p.repo, err)
+			return
 		}
-		p.model.ScanRepo(p.repo)
 	}
 }
 
@@ -217,8 +228,8 @@ func (p *puller) fixupDirectories() {
 		}
 
 		if cur.Flags&protocol.FlagDeleted != 0 {
-			if debugPull {
-				dlog.Printf("queue delete dir: %v", cur)
+			if debug {
+				l.Debugf("queue delete dir: %v", cur)
 			}
 
 			// We queue the directories to delete since we walk the
@@ -231,16 +242,16 @@ func (p *puller) fixupDirectories() {
 
 		if cur.Flags&uint32(os.ModePerm) != uint32(info.Mode()&os.ModePerm) {
 			os.Chmod(path, os.FileMode(cur.Flags)&os.ModePerm)
-			if debugPull {
-				dlog.Printf("restored dir flags: %o -> %v", info.Mode()&os.ModePerm, cur)
+			if debug {
+				l.Debugf("restored dir flags: %o -> %v", info.Mode()&os.ModePerm, cur)
 			}
 		}
 
 		if cur.Modified != info.ModTime().Unix() {
 			t := time.Unix(cur.Modified, 0)
 			os.Chtimes(path, t, t)
-			if debugPull {
-				dlog.Printf("restored dir modtime: %d -> %v", info.ModTime().Unix(), cur)
+			if debug {
+				l.Debugf("restored dir modtime: %d -> %v", info.ModTime().Unix(), cur)
 			}
 		}
 
@@ -249,12 +260,12 @@ func (p *puller) fixupDirectories() {
 
 	// Delete any queued directories
 	for i := len(deleteDirs) - 1; i >= 0; i-- {
-		if debugPull {
-			dlog.Println("delete dir:", deleteDirs[i])
+		if debug {
+			l.Debugln("delete dir:", deleteDirs[i])
 		}
 		err := os.Remove(deleteDirs[i])
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 		}
 	}
 }
@@ -275,8 +286,8 @@ func (p *puller) handleRequestResult(res requestResult) {
 	of.outstanding--
 	p.openFiles[f.Name] = of
 
-	if debugPull {
-		dlog.Printf("pull: wrote %q / %q offset %d outstanding %d done %v", p.repo, f.Name, res.offset, of.outstanding, of.done)
+	if debug {
+		l.Debugf("pull: wrote %q / %q offset %d outstanding %d done %v", p.repo, f.Name, res.offset, of.outstanding, of.done)
 	}
 
 	if of.done && of.outstanding == 0 {
@@ -305,8 +316,8 @@ func (p *puller) handleBlock(b bqBlock) bool {
 	of.done = b.last
 
 	if !ok {
-		if debugPull {
-			dlog.Printf("pull: %q: opening file %q", p.repo, f.Name)
+		if debug {
+			l.Debugf("pull: %q: opening file %q", p.repo, f.Name)
 		}
 
 		of.availability = uint64(p.model.repoFiles[p.repo].Availability(f.Name))
@@ -319,13 +330,13 @@ func (p *puller) handleBlock(b bqBlock) bool {
 			err = os.MkdirAll(dirName, 0777)
 		}
 		if err != nil {
-			dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, err)
+			l.Debugf("pull: error: %q / %q: %v", p.repo, f.Name, err)
 		}
 
 		of.file, of.err = os.Create(of.temp)
 		if of.err != nil {
-			if debugPull {
-				dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, of.err)
+			if debug {
+				l.Debugf("pull: error: %q / %q: %v", p.repo, f.Name, of.err)
 			}
 			if !b.last {
 				p.openFiles[f.Name] = of
@@ -337,11 +348,10 @@ func (p *puller) handleBlock(b bqBlock) bool {
 
 	if of.err != nil {
 		// We have already failed this file.
-		if debugPull {
-			dlog.Printf("pull: error: %q / %q has already failed: %v", p.repo, f.Name, of.err)
+		if debug {
+			l.Debugf("pull: error: %q / %q has already failed: %v", p.repo, f.Name, of.err)
 		}
 		if b.last {
-			dlog.Printf("pull: removing failed file %q / %q", p.repo, f.Name)
 			delete(p.openFiles, f.Name)
 		}
 
@@ -369,15 +379,15 @@ func (p *puller) handleCopyBlock(b bqBlock) {
 	f := b.file
 	of := p.openFiles[f.Name]
 
-	if debugPull {
-		dlog.Printf("pull: copying %d blocks for %q / %q", len(b.copy), p.repo, f.Name)
+	if debug {
+		l.Debugf("pull: copying %d blocks for %q / %q", len(b.copy), p.repo, f.Name)
 	}
 
 	var exfd *os.File
 	exfd, of.err = os.Open(of.filepath)
 	if of.err != nil {
-		if debugPull {
-			dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, of.err)
+		if debug {
+			l.Debugf("pull: error: %q / %q: %v", p.repo, f.Name, of.err)
 		}
 		of.file.Close()
 		of.file = nil
@@ -395,8 +405,8 @@ func (p *puller) handleCopyBlock(b bqBlock) {
 		}
 		buffers.Put(bs)
 		if of.err != nil {
-			if debugPull {
-				dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, of.err)
+			if debug {
+				l.Debugf("pull: error: %q / %q: %v", p.repo, f.Name, of.err)
 			}
 			exfd.Close()
 			of.file.Close()
@@ -438,8 +448,8 @@ func (p *puller) handleRequestBlock(b bqBlock) bool {
 	p.openFiles[f.Name] = of
 
 	go func(node string, b bqBlock) {
-		if debugPull {
-			dlog.Printf("pull: requesting %q / %q offset %d size %d from %q outstanding %d", p.repo, f.Name, b.block.Offset, b.block.Size, node, of.outstanding)
+		if debug {
+			l.Debugf("pull: requesting %q / %q offset %d size %d from %q outstanding %d", p.repo, f.Name, b.block.Offset, b.block.Size, node, of.outstanding)
 		}
 
 		bs, err := p.model.requestGlobal(node, p.repo, f.Name, b.block.Offset, int(b.block.Size), nil)
@@ -467,14 +477,14 @@ func (p *puller) handleEmptyBlock(b bqBlock) {
 	}
 
 	if f.Flags&protocol.FlagDeleted != 0 {
-		if debugPull {
-			dlog.Printf("pull: delete %q", f.Name)
+		if debug {
+			l.Debugf("pull: delete %q", f.Name)
 		}
 		os.Remove(of.temp)
 		os.Remove(of.filepath)
 	} else {
-		if debugPull {
-			dlog.Printf("pull: no blocks to fetch and nothing to copy for %q / %q", p.repo, f.Name)
+		if debug {
+			l.Debugf("pull: no blocks to fetch and nothing to copy for %q / %q", p.repo, f.Name)
 		}
 		t := time.Unix(f.Modified, 0)
 		os.Chtimes(of.temp, t, t)
@@ -491,8 +501,8 @@ func (p *puller) queueNeededBlocks() {
 	for _, f := range p.model.NeedFilesRepo(p.repo) {
 		lf := p.model.CurrentRepoFile(p.repo, f.Name)
 		have, need := scanner.BlockDiff(lf.Blocks, f.Blocks)
-		if debugNeed {
-			dlog.Printf("need:\n  local: %v\n  global: %v\n  haveBlocks: %v\n  needBlocks: %v", lf, f, have, need)
+		if debug {
+			l.Debugf("need:\n  local: %v\n  global: %v\n  haveBlocks: %v\n  needBlocks: %v", lf, f, have, need)
 		}
 		queued++
 		p.bq.put(bqAdd{
@@ -501,14 +511,14 @@ func (p *puller) queueNeededBlocks() {
 			need: need,
 		})
 	}
-	if debugPull && queued > 0 {
-		dlog.Printf("%q: queued %d blocks", p.repo, queued)
+	if debug && queued > 0 {
+		l.Debugf("%q: queued %d blocks", p.repo, queued)
 	}
 }
 
 func (p *puller) closeFile(f scanner.File) {
-	if debugPull {
-		dlog.Printf("pull: closing %q / %q", p.repo, f.Name)
+	if debug {
+		l.Debugf("pull: closing %q / %q", p.repo, f.Name)
 	}
 
 	of := p.openFiles[f.Name]
@@ -519,24 +529,24 @@ func (p *puller) closeFile(f scanner.File) {
 
 	fd, err := os.Open(of.temp)
 	if err != nil {
-		if debugPull {
-			dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, err)
+		if debug {
+			l.Debugf("pull: error: %q / %q: %v", p.repo, f.Name, err)
 		}
 		return
 	}
-	hb, _ := scanner.Blocks(fd, BlockSize)
+	hb, _ := scanner.Blocks(fd, scanner.StandardBlockSize)
 	fd.Close()
 
 	if l0, l1 := len(hb), len(f.Blocks); l0 != l1 {
-		if debugPull {
-			dlog.Printf("pull: %q / %q: nblocks %d != %d", p.repo, f.Name, l0, l1)
+		if debug {
+			l.Debugf("pull: %q / %q: nblocks %d != %d", p.repo, f.Name, l0, l1)
 		}
 		return
 	}
 
 	for i := range hb {
 		if bytes.Compare(hb[i].Hash, f.Blocks[i].Hash) != 0 {
-			dlog.Printf("pull: %q / %q: block %d hash mismatch", p.repo, f.Name, i)
+			l.Debugf("pull: %q / %q: block %d hash mismatch", p.repo, f.Name, i)
 			return
 		}
 	}
@@ -545,12 +555,22 @@ func (p *puller) closeFile(f scanner.File) {
 	os.Chtimes(of.temp, t, t)
 	os.Chmod(of.temp, os.FileMode(f.Flags&0777))
 	defTempNamer.Show(of.temp)
-	if debugPull {
-		dlog.Printf("pull: rename %q / %q: %q", p.repo, f.Name, of.filepath)
+	if debug {
+		l.Debugf("pull: rename %q / %q: %q", p.repo, f.Name, of.filepath)
 	}
 	if err := Rename(of.temp, of.filepath); err == nil {
 		p.model.updateLocal(p.repo, f)
 	} else {
-		dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, err)
+		l.Debugf("pull: error: %q / %q: %v", p.repo, f.Name, err)
+	}
+}
+
+func invalidateRepo(cfg *config.Configuration, repoID string, err error) {
+	for i := range cfg.Repositories {
+		repo := &cfg.Repositories[i]
+		if repo.ID == repoID {
+			repo.Invalid = err.Error()
+			return
+		}
 	}
 }

model/suppressor.go

@@ -1,4 +1,4 @@
-package main
+package model
 
 import (
 	"os"
@@ -52,9 +52,8 @@ func (h *changeHistory) append(size int64, t time.Time) {
 	h.changes = append(h.changes, c)
 }
 
-func (s *suppressor) Suppress(name string, fi os.FileInfo) bool {
-	sup, _ := s.suppress(name, fi.Size(), time.Now())
-	return sup
+func (s *suppressor) Suppress(name string, fi os.FileInfo) (cur, prev bool) {
+	return s.suppress(name, fi.Size(), time.Now())
 }
 
 func (s *suppressor) suppress(name string, size int64, t time.Time) (bool, bool) {

model/suppressor_test.go

@@ -1,4 +1,4 @@
-package main
+package model
 
 import (
 	"testing"

model/tempname.go

@@ -1,6 +1,6 @@
 // +build !windows
 
-package main
+package model
 
 import (
 	"fmt"

model/tempname_windows.go

@@ -1,6 +1,6 @@
 // +build windows
 
-package main
+package model
 
 import (
 	"fmt"

model/util.go

@@ -1,4 +1,4 @@
-package main
+package model
 
 import (
 	"fmt"
@@ -14,7 +14,7 @@ func Rename(from, to string) error {
 	if runtime.GOOS == "windows" {
 		err := os.Remove(to)
 		if err != nil && !os.IsNotExist(err) {
-			warnln(err)
+			l.Warnln(err)
 		}
 	}
 	return os.Rename(from, to)

model/util_test.go

@@ -1,4 +1,4 @@
-package main
+package model
 
 import (
 	"testing"

protocol/PROTOCOL.md

@@ -38,13 +38,13 @@ level protocols providing compression, encryption and authentication.
     |-----------------------------|
     v             ...             v
 
-Compression is started directly after a successfull TLS handshake,
+Compression is started directly after a successful TLS handshake,
 before the first message is sent. The compression is flushed at each
 message boundary. Compression SHALL use the DEFLATE format as specified
 in RFC 1951.
 
 The encryption and authentication layer SHALL use TLS 1.2 or a higher
-revision. A strong cipher suite SHALL be used, with "string cipher
+revision. A strong cipher suite SHALL be used, with "strong cipher
 suite" being defined as being without known weaknesses and providing
 Perfect Forward Secrecy (PFS). Examples of strong cipher suites are
 given at the end of this document. This is not to be taken as an
@@ -82,7 +82,7 @@ For BEP v1 the Version field is set to zero. Future versions with
 incompatible message formats will increment the Version field. A message
 with an unknown version is a protocol error and MUST result in the
 connection being terminated. A client supporting multiple versions MAY
-retry with a different protcol version upon disconnection.
+retry with a different protocol version upon disconnection.
 
 The Type field indicates the type of data following the message header
 and is one of the integers defined below. A message of an unknown type
@@ -262,7 +262,7 @@ pairs, both of string type. Key ID:s are implementation specific. An
 implementation MUST ignore unknown keys. An implementation MAY impose
 limits on the length keys and values. The options list may be used to
 inform nodes of relevant local configuration options such as rate
-limiting or make recommendations about request parallellism, node
+limiting or make recommendations about request parallelism, node
 priorities, etc. An empty options list is valid for nodes not having any
 such information to share. Nodes MAY NOT make any assumptions about
 peers acting in a specific manner as a result of sent options.
@@ -392,7 +392,7 @@ The Flags field is made up of the following single bit flags:
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
  - The lower 12 bits hold the common Unix permission and mode bits. An
-   implemention MAY ignore or interpret these as is suitable on the host
+   implementation MAY ignore or interpret these as is suitable on the host
    operating system.
 
  - Bit 19 ("D") is set when the file has been deleted. The block list
@@ -575,7 +575,7 @@ restrictive than the following:
 ### Index and Index Update Messages
 
  - Repository: 64 bytes
- - Number of Files: 100.000
+ - Number of Files: 1.000.000
  - Name: 1024 bytes
  - Number of Blocks: 100.000
  - Hash: 64 bytes

protocol/message_types.go

@@ -2,7 +2,7 @@ package protocol
 
 type IndexMessage struct {
 	Repository string     // max:64
-	Files      []FileInfo // max:100000
+	Files      []FileInfo // max:1000000
 }
 
 type FileInfo struct {

protocol/message_xdr.go

@@ -24,7 +24,7 @@ func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 		return xw.Tot(), xdr.ErrElementSizeExceeded
 	}
 	xw.WriteString(o.Repository)
-	if len(o.Files) > 100000 {
+	if len(o.Files) > 1000000 {
 		return xw.Tot(), xdr.ErrElementSizeExceeded
 	}
 	xw.WriteUint32(uint32(len(o.Files)))
@@ -48,7 +48,7 @@ func (o *IndexMessage) UnmarshalXDR(bs []byte) error {
 func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error {
 	o.Repository = xr.ReadStringMax(64)
 	_FilesSize := int(xr.ReadUint32())
-	if _FilesSize > 100000 {
+	if _FilesSize > 1000000 {
 		return xdr.ErrElementSizeExceeded
 	}
 	o.Files = make([]FileInfo, _FilesSize)

protocol/protocol.go

@@ -9,7 +9,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/calmh/syncthing/buffers"
 	"github.com/calmh/syncthing/xdr"
 )
 
@@ -64,24 +63,26 @@ type Connection interface {
 }
 
 type rawConnection struct {
-	sync.RWMutex
+	id       string
+	receiver Model
+
+	reader io.ReadCloser
+	cr     *countingReader
+	xr     *xdr.Reader
+	writer io.WriteCloser
+
+	cw   *countingWriter
+	wb   *bufio.Writer
+	xw   *xdr.Writer
+	wmut sync.Mutex
 
-	id        string
-	receiver  Model
-	reader    io.ReadCloser
-	cr        *countingReader
-	xr        *xdr.Reader
-	writer    io.WriteCloser
-	cw        *countingWriter
-	wb        *bufio.Writer
-	xw        *xdr.Writer
-	closed    chan struct{}
-	awaiting  map[int]chan asyncResult
-	nextID    int
 	indexSent map[string]map[string][2]int64
+	awaiting  []chan asyncResult
+	imut      sync.Mutex
 
-	hasSentIndex  bool
-	hasRecvdIndex bool
+	nextID chan int
+	outbox chan []encodable
+	closed chan struct{}
 }
 
 type asyncResult struct {
@@ -90,7 +91,7 @@ type asyncResult struct {
 }
 
 const (
-	pingTimeout  = 2 * time.Minute
+	pingTimeout  = 4 * time.Minute
 	pingIdleTime = 5 * time.Minute
 )
 
@@ -115,13 +116,17 @@ func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver M
 		cw:        cw,
 		wb:        wb,
 		xw:        xdr.NewWriter(wb),
-		closed:    make(chan struct{}),
-		awaiting:  make(map[int]chan asyncResult),
+		awaiting:  make([]chan asyncResult, 0x1000),
 		indexSent: make(map[string]map[string][2]int64),
+		outbox:    make(chan []encodable),
+		nextID:    make(chan int),
+		closed:    make(chan struct{}),
 	}
 
 	go c.readerLoop()
+	go c.writerLoop()
 	go c.pingerLoop()
+	go c.idGenerator()
 
 	return wireFormatConnection{&c}
 }
@@ -132,11 +137,7 @@ func (c *rawConnection) ID() string {
 
 // Index writes the list of file information to the connected peer node
 func (c *rawConnection) Index(repo string, idx []FileInfo) {
-	c.Lock()
-	if c.isClosed() {
-		c.Unlock()
-		return
-	}
+	c.imut.Lock()
 	var msgType int
 	if c.indexSent[repo] == nil {
 		// This is the first time we send an index.
@@ -158,46 +159,33 @@ func (c *rawConnection) Index(repo string, idx []FileInfo) {
 		}
 		idx = diff
 	}
+	c.imut.Unlock()
 
-	header{0, c.nextID, msgType}.encodeXDR(c.xw)
-	_, err := IndexMessage{repo, idx}.encodeXDR(c.xw)
-	if err == nil {
-		err = c.flush()
-	}
-	c.nextID = (c.nextID + 1) & 0xfff
-	c.hasSentIndex = true
-	c.Unlock()
-
-	if err != nil {
-		c.close(err)
-		return
-	}
+	c.send(header{0, -1, msgType}, IndexMessage{repo, idx})
 }
 
 // Request returns the bytes for the specified block after fetching them from the connected peer.
 func (c *rawConnection) Request(repo string, name string, offset int64, size int) ([]byte, error) {
-	c.Lock()
-	if c.isClosed() {
-		c.Unlock()
+	var id int
+	select {
+	case id = <-c.nextID:
+	case <-c.closed:
 		return nil, ErrClosed
 	}
-	rc := make(chan asyncResult)
-	if _, ok := c.awaiting[c.nextID]; ok {
+
+	c.imut.Lock()
+	if ch := c.awaiting[id]; ch != nil {
 		panic("id taken")
 	}
-	c.awaiting[c.nextID] = rc
-	header{0, c.nextID, messageTypeRequest}.encodeXDR(c.xw)
-	_, err := RequestMessage{repo, name, uint64(offset), uint32(size)}.encodeXDR(c.xw)
-	if err == nil {
-		err = c.flush()
+	rc := make(chan asyncResult)
+	c.awaiting[id] = rc
+	c.imut.Unlock()
+
+	ok := c.send(header{0, id, messageTypeRequest},
+		RequestMessage{repo, name, uint64(offset), uint32(size)})
+	if !ok {
+		return nil, ErrClosed
 	}
-	if err != nil {
-		c.Unlock()
-		c.close(err)
-		return nil, err
-	}
-	c.nextID = (c.nextID + 1) & 0xfff
-	c.Unlock()
 
 	res, ok := <-rc
 	if !ok {
@@ -208,225 +196,276 @@ func (c *rawConnection) Request(repo string, name string, offset int64, size int
 
 // ClusterConfig send the cluster configuration message to the peer and returns any error
 func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) {
-	c.Lock()
-	defer c.Unlock()
-
-	if c.isClosed() {
-		return
-	}
-
-	header{0, c.nextID, messageTypeClusterConfig}.encodeXDR(c.xw)
-	c.nextID = (c.nextID + 1) & 0xfff
-
-	_, err := config.encodeXDR(c.xw)
-	if err == nil {
-		err = c.flush()
-	}
-	if err != nil {
-		c.close(err)
-	}
+	c.send(header{0, -1, messageTypeClusterConfig}, config)
 }
 
 func (c *rawConnection) ping() bool {
-	c.Lock()
-	if c.isClosed() {
-		c.Unlock()
+	var id int
+	select {
+	case id = <-c.nextID:
+	case <-c.closed:
 		return false
 	}
+
 	rc := make(chan asyncResult, 1)
-	c.awaiting[c.nextID] = rc
-	header{0, c.nextID, messageTypePing}.encodeXDR(c.xw)
-	err := c.flush()
-	if err != nil {
-		c.Unlock()
-		c.close(err)
-		return false
-	} else if c.xw.Error() != nil {
-		c.Unlock()
-		c.close(c.xw.Error())
+	c.imut.Lock()
+	c.awaiting[id] = rc
+	c.imut.Unlock()
+
+	ok := c.send(header{0, id, messageTypePing})
+	if !ok {
 		return false
 	}
-	c.nextID = (c.nextID + 1) & 0xfff
-	c.Unlock()
 
 	res, ok := <-rc
 	return ok && res.err == nil
 }
 
+func (c *rawConnection) readerLoop() (err error) {
+	defer func() {
+		c.close(err)
+	}()
+
+	for {
+		select {
+		case <-c.closed:
+			return ErrClosed
+		default:
+		}
+
+		var hdr header
+		hdr.decodeXDR(c.xr)
+		if err := c.xr.Error(); err != nil {
+			return err
+		}
+		if hdr.version != 0 {
+			return fmt.Errorf("protocol error: %s: unknown message version %#x", c.id, hdr.version)
+		}
+
+		switch hdr.msgType {
+		case messageTypeIndex:
+			if err := c.handleIndex(); err != nil {
+				return err
+			}
+
+		case messageTypeIndexUpdate:
+			if err := c.handleIndexUpdate(); err != nil {
+				return err
+			}
+
+		case messageTypeRequest:
+			if err := c.handleRequest(hdr); err != nil {
+				return err
+			}
+
+		case messageTypeResponse:
+			if err := c.handleResponse(hdr); err != nil {
+				return err
+			}
+
+		case messageTypePing:
+			c.send(header{0, hdr.msgID, messageTypePong})
+
+		case messageTypePong:
+			c.handlePong(hdr)
+
+		case messageTypeClusterConfig:
+			if err := c.handleClusterConfig(); err != nil {
+				return err
+			}
+
+		default:
+			return fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType)
+		}
+	}
+}
+
+func (c *rawConnection) handleIndex() error {
+	var im IndexMessage
+	im.decodeXDR(c.xr)
+	if err := c.xr.Error(); err != nil {
+		return err
+	} else {
+
+		// We run this (and the corresponding one for update, below)
+		// in a separate goroutine to avoid blocking the read loop.
+		// There is otherwise a potential deadlock where both sides
+		// has the model locked because it's sending a large index
+		// update and can't receive the large index update from the
+		// other side.
+
+		go c.receiver.Index(c.id, im.Repository, im.Files)
+	}
+	return nil
+}
+
+func (c *rawConnection) handleIndexUpdate() error {
+	var im IndexMessage
+	im.decodeXDR(c.xr)
+	if err := c.xr.Error(); err != nil {
+		return err
+	} else {
+		go c.receiver.IndexUpdate(c.id, im.Repository, im.Files)
+	}
+	return nil
+}
+
+func (c *rawConnection) handleRequest(hdr header) error {
+	var req RequestMessage
+	req.decodeXDR(c.xr)
+	if err := c.xr.Error(); err != nil {
+		return err
+	}
+	go c.processRequest(hdr.msgID, req)
+	return nil
+}
+
+func (c *rawConnection) handleResponse(hdr header) error {
+	data := c.xr.ReadBytesMax(256 * 1024) // Sufficiently larger than max expected block size
+
+	if err := c.xr.Error(); err != nil {
+		return err
+	}
+
+	go func(hdr header, err error) {
+		c.imut.Lock()
+		rc := c.awaiting[hdr.msgID]
+		c.awaiting[hdr.msgID] = nil
+		c.imut.Unlock()
+
+		if rc != nil {
+			rc <- asyncResult{data, err}
+			close(rc)
+		}
+	}(hdr, c.xr.Error())
+
+	return nil
+}
+
+func (c *rawConnection) handlePong(hdr header) {
+	c.imut.Lock()
+	if rc := c.awaiting[hdr.msgID]; rc != nil {
+		go func() {
+			rc <- asyncResult{}
+			close(rc)
+		}()
+
+		c.awaiting[hdr.msgID] = nil
+	}
+	c.imut.Unlock()
+}
+
+func (c *rawConnection) handleClusterConfig() error {
+	var cm ClusterConfigMessage
+	cm.decodeXDR(c.xr)
+	if err := c.xr.Error(); err != nil {
+		return err
+	} else {
+		go c.receiver.ClusterConfig(c.id, cm)
+	}
+	return nil
+}
+
+type encodable interface {
+	encodeXDR(*xdr.Writer) (int, error)
+}
+type encodableBytes []byte
+
+func (e encodableBytes) encodeXDR(xw *xdr.Writer) (int, error) {
+	return xw.WriteBytes(e)
+}
+
+func (c *rawConnection) send(h header, es ...encodable) bool {
+	if h.msgID < 0 {
+		select {
+		case id := <-c.nextID:
+			h.msgID = id
+		case <-c.closed:
+			return false
+		}
+	}
+	msg := append([]encodable{h}, es...)
+
+	select {
+	case c.outbox <- msg:
+		return true
+	case <-c.closed:
+		return false
+	}
+}
+
+func (c *rawConnection) writerLoop() {
+	var err error
+	for es := range c.outbox {
+		c.wmut.Lock()
+		for _, e := range es {
+			e.encodeXDR(c.xw)
+		}
+
+		if err = c.flush(); err != nil {
+			c.wmut.Unlock()
+			c.close(err)
+			return
+		}
+		c.wmut.Unlock()
+	}
+}
+
 type flusher interface {
 	Flush() error
 }
 
 func (c *rawConnection) flush() error {
-	c.wb.Flush()
+	if err := c.xw.Error(); err != nil {
+		return err
+	}
+
+	if err := c.wb.Flush(); err != nil {
+		return err
+	}
+
 	if f, ok := c.writer.(flusher); ok {
 		return f.Flush()
 	}
+
 	return nil
 }
 
 func (c *rawConnection) close(err error) {
-	c.Lock()
+	c.imut.Lock()
+	c.wmut.Lock()
+	defer c.imut.Unlock()
+	defer c.wmut.Unlock()
+
 	select {
 	case <-c.closed:
-		c.Unlock()
 		return
 	default:
-	}
-	close(c.closed)
-	for _, ch := range c.awaiting {
-		close(ch)
-	}
-	c.awaiting = nil
-	c.writer.Close()
-	c.reader.Close()
-	c.Unlock()
+		close(c.closed)
 
-	c.receiver.Close(c.id, err)
-}
+		for i, ch := range c.awaiting {
+			if ch != nil {
+				close(ch)
+				c.awaiting[i] = nil
+			}
+		}
 
-func (c *rawConnection) isClosed() bool {
-	select {
-	case <-c.closed:
-		return true
-	default:
-		return false
+		c.writer.Close()
+		c.reader.Close()
+
+		go c.receiver.Close(c.id, err)
 	}
 }
 
-func (c *rawConnection) readerLoop() {
-loop:
-	for !c.isClosed() {
-		var hdr header
-		hdr.decodeXDR(c.xr)
-		if c.xr.Error() != nil {
-			c.close(c.xr.Error())
-			break loop
+func (c *rawConnection) idGenerator() {
+	nextID := 0
+	for {
+		nextID = (nextID + 1) & 0xfff
+		select {
+		case c.nextID <- nextID:
+		case <-c.closed:
+			return
 		}
-		if hdr.version != 0 {
-			c.close(fmt.Errorf("protocol error: %s: unknown message version %#x", c.id, hdr.version))
-			break loop
-		}
-
-		switch hdr.msgType {
-		case messageTypeIndex:
-			var im IndexMessage
-			im.decodeXDR(c.xr)
-			if c.xr.Error() != nil {
-				c.close(c.xr.Error())
-				break loop
-			} else {
-
-				// We run this (and the corresponding one for update, below)
-				// in a separate goroutine to avoid blocking the read loop.
-				// There is otherwise a potential deadlock where both sides
-				// has the model locked because it's sending a large index
-				// update and can't receive the large index update from the
-				// other side.
-
-				go c.receiver.Index(c.id, im.Repository, im.Files)
-			}
-			c.Lock()
-			c.hasRecvdIndex = true
-			c.Unlock()
-
-		case messageTypeIndexUpdate:
-			var im IndexMessage
-			im.decodeXDR(c.xr)
-			if c.xr.Error() != nil {
-				c.close(c.xr.Error())
-				break loop
-			} else {
-				go c.receiver.IndexUpdate(c.id, im.Repository, im.Files)
-			}
-
-		case messageTypeRequest:
-			var req RequestMessage
-			req.decodeXDR(c.xr)
-			if c.xr.Error() != nil {
-				c.close(c.xr.Error())
-				break loop
-			}
-			go c.processRequest(hdr.msgID, req)
-
-		case messageTypeResponse:
-			data := c.xr.ReadBytesMax(256 * 1024) // Sufficiently larger than max expected block size
-
-			if c.xr.Error() != nil {
-				c.close(c.xr.Error())
-				break loop
-			}
-
-			go func(hdr header, err error) {
-				c.Lock()
-				rc, ok := c.awaiting[hdr.msgID]
-				delete(c.awaiting, hdr.msgID)
-				c.Unlock()
-
-				if ok {
-					rc <- asyncResult{data, err}
-					close(rc)
-				}
-			}(hdr, c.xr.Error())
-
-		case messageTypePing:
-			c.Lock()
-			header{0, hdr.msgID, messageTypePong}.encodeXDR(c.xw)
-			err := c.flush()
-			c.Unlock()
-			if err != nil {
-				c.close(err)
-				break loop
-			} else if c.xw.Error() != nil {
-				c.close(c.xw.Error())
-				break loop
-			}
-
-		case messageTypePong:
-			c.RLock()
-			rc, ok := c.awaiting[hdr.msgID]
-			c.RUnlock()
-
-			if ok {
-				rc <- asyncResult{}
-				close(rc)
-
-				c.Lock()
-				delete(c.awaiting, hdr.msgID)
-				c.Unlock()
-			}
-
-		case messageTypeClusterConfig:
-			var cm ClusterConfigMessage
-			cm.decodeXDR(c.xr)
-			if c.xr.Error() != nil {
-				c.close(c.xr.Error())
-				break loop
-			} else {
-				go c.receiver.ClusterConfig(c.id, cm)
-			}
-
-		default:
-			c.close(fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType))
-			break loop
-		}
-	}
-}
-
-func (c *rawConnection) processRequest(msgID int, req RequestMessage) {
-	data, _ := c.receiver.Request(c.id, req.Repository, req.Name, int64(req.Offset), int(req.Size))
-
-	c.Lock()
-	header{0, msgID, messageTypeResponse}.encodeXDR(c.xw)
-	_, err := c.xw.WriteBytes(data)
-	if err == nil {
-		err = c.flush()
-	}
-	c.Unlock()
-
-	buffers.Put(data)
-	if err != nil {
-		c.close(err)
 	}
 }
 
@@ -436,29 +475,33 @@ func (c *rawConnection) pingerLoop() {
 	for {
 		select {
 		case <-ticker:
-			c.RLock()
-			ready := c.hasRecvdIndex && c.hasSentIndex
-			c.RUnlock()
-
-			if ready {
-				go func() {
-					rc <- c.ping()
-				}()
-				select {
-				case ok := <-rc:
-					if !ok {
-						c.close(fmt.Errorf("ping failure"))
-					}
-				case <-time.After(pingTimeout):
-					c.close(fmt.Errorf("ping timeout"))
+			go func() {
+				rc <- c.ping()
+			}()
+			select {
+			case ok := <-rc:
+				if !ok {
+					c.close(fmt.Errorf("ping failure"))
 				}
+			case <-time.After(pingTimeout):
+				c.close(fmt.Errorf("ping timeout"))
+			case <-c.closed:
+				return
 			}
+
 		case <-c.closed:
 			return
 		}
 	}
 }
 
+func (c *rawConnection) processRequest(msgID int, req RequestMessage) {
+	data, _ := c.receiver.Request(c.id, req.Repository, req.Name, int64(req.Offset), int(req.Size))
+
+	c.send(header{0, msgID, messageTypeResponse},
+		encodableBytes(data))
+}
+
 type Statistics struct {
 	At            time.Time
 	InBytesTotal  int

protocol/protocol_test.go

@@ -174,9 +174,7 @@ func TestClose(t *testing.T) {
 
 	c0.close(nil)
 
-	if !c0.isClosed() {
-		t.Fatal("Connection should be closed")
-	}
+	<-c0.closed
 	if !m0.isClosed() {
 		t.Fatal("Connection should be closed")
 	}

scanner/blocks.go

@@ -6,6 +6,8 @@ import (
 	"io"
 )
 
+const StandardBlockSize = 128 * 1024
+
 type Block struct {
 	Offset int64
 	Size   uint32

scanner/debug.go

@@ -1,12 +1,13 @@
 package scanner
 
 import (
-	"log"
 	"os"
 	"strings"
+
+	"github.com/calmh/syncthing/logger"
 )
 
 var (
-	dlog  = log.New(os.Stderr, "scanner: ", log.Lmicroseconds|log.Lshortfile)
-	debug = strings.Contains(os.Getenv("STTRACE"), "scanner")
+	debug = strings.Contains(os.Getenv("STTRACE"), "scanner") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
 )

scanner/walk.go

@@ -2,8 +2,8 @@ package scanner
 
 import (
 	"bytes"
+	"errors"
 	"io/ioutil"
-	"log"
 	"os"
 	"path/filepath"
 	"strings"
@@ -28,8 +28,6 @@ type Walker struct {
 	// Suppressed files will be returned with empty metadata and the Suppressed flag set.
 	// Requires CurrentFiler to be set.
 	Suppressor Suppressor
-
-	suppressed map[string]bool // file name -> suppression status
 }
 
 type TempNamer interface {
@@ -41,7 +39,7 @@ type TempNamer interface {
 
 type Suppressor interface {
 	// Supress returns true if the update to the named file should be ignored.
-	Suppress(name string, fi os.FileInfo) bool
+	Suppress(name string, fi os.FileInfo) (bool, bool)
 }
 
 type CurrentFiler interface {
@@ -51,12 +49,16 @@ type CurrentFiler interface {
 
 // Walk returns the list of files found in the local repository by scanning the
 // file system. Files are blockwise hashed.
-func (w *Walker) Walk() (files []File, ignore map[string][]string) {
-	w.lazyInit()
-
+func (w *Walker) Walk() (files []File, ignore map[string][]string, err error) {
 	if debug {
-		dlog.Println("Walk", w.Dir, w.BlockSize, w.IgnoreFile)
+		l.Debugln("Walk", w.Dir, w.BlockSize, w.IgnoreFile)
 	}
+
+	err = checkDir(w.Dir)
+	if err != nil {
+		return
+	}
+
 	t0 := time.Now()
 
 	ignore = make(map[string][]string)
@@ -68,8 +70,10 @@ func (w *Walker) Walk() (files []File, ignore map[string][]string) {
 	if debug {
 		t1 := time.Now()
 		d := t1.Sub(t0).Seconds()
-		dlog.Printf("Walk in %.02f ms, %.0f files/s", d*1000, float64(len(files))/d)
+		l.Debugf("Walk in %.02f ms, %.0f files/s", d*1000, float64(len(files))/d)
 	}
+
+	err = checkDir(w.Dir)
 	return
 }
 
@@ -78,12 +82,6 @@ func (w *Walker) CleanTempFiles() {
 	filepath.Walk(w.Dir, w.cleanTempFile)
 }
 
-func (w *Walker) lazyInit() {
-	if w.suppressed == nil {
-		w.suppressed = make(map[string]bool)
-	}
-}
-
 func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.WalkFunc {
 	return func(p string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -116,7 +114,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 	return func(p string, info os.FileInfo, err error) error {
 		if err != nil {
 			if debug {
-				dlog.Println("error:", p, info, err)
+				l.Debugln("error:", p, info, err)
 			}
 			return nil
 		}
@@ -124,7 +122,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 		rn, err := filepath.Rel(w.Dir, p)
 		if err != nil {
 			if debug {
-				dlog.Println("rel error:", p, err)
+				l.Debugln("rel error:", p, err)
 			}
 			return nil
 		}
@@ -136,7 +134,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 		if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
 			// A temporary file
 			if debug {
-				dlog.Println("temporary:", rn)
+				l.Debugln("temporary:", rn)
 			}
 			return nil
 		}
@@ -144,7 +142,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 		if _, sn := filepath.Split(rn); sn == w.IgnoreFile {
 			// An ignore-file; these are ignored themselves
 			if debug {
-				dlog.Println("ignorefile:", rn)
+				l.Debugln("ignorefile:", rn)
 			}
 			return nil
 		}
@@ -152,7 +150,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 		if w.ignoreFile(ign, rn) {
 			// An ignored file
 			if debug {
-				dlog.Println("ignored:", rn)
+				l.Debugln("ignored:", rn)
 			}
 			if info.IsDir() {
 				return filepath.SkipDir
@@ -165,7 +163,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 				cf := w.CurrentFiler.CurrentFile(rn)
 				if cf.Modified == info.ModTime().Unix() && cf.Flags == uint32(info.Mode()&os.ModePerm|protocol.FlagDirectory) {
 					if debug {
-						dlog.Println("unchanged:", cf)
+						l.Debugln("unchanged:", cf)
 					}
 					*res = append(*res, cf)
 				} else {
@@ -176,7 +174,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 						Modified: info.ModTime().Unix(),
 					}
 					if debug {
-						dlog.Println("dir:", cf, f)
+						l.Debugln("dir:", cf, f)
 					}
 					*res = append(*res, f)
 				}
@@ -189,34 +187,32 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 				cf := w.CurrentFiler.CurrentFile(rn)
 				if cf.Flags&protocol.FlagDeleted == 0 && cf.Modified == info.ModTime().Unix() {
 					if debug {
-						dlog.Println("unchanged:", cf)
+						l.Debugln("unchanged:", cf)
 					}
 					*res = append(*res, cf)
 					return nil
 				}
 
-				if w.Suppressor != nil && w.Suppressor.Suppress(rn, info) {
-					if !w.suppressed[rn] {
-						w.suppressed[rn] = true
-						log.Printf("INFO: Changes to %q are being temporarily suppressed because it changes too frequently.", p)
+				if w.Suppressor != nil {
+					if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev {
+						l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p)
 						cf.Suppressed = true
 						cf.Version++
+						if debug {
+							l.Debugln("suppressed:", cf)
+						}
+						*res = append(*res, cf)
+						return nil
+					} else if prev && !cur {
+						l.Infof("Changes to %q are no longer suppressed.", p)
 					}
-					if debug {
-						dlog.Println("suppressed:", cf)
-					}
-					*res = append(*res, cf)
-					return nil
-				} else if w.suppressed[rn] {
-					log.Printf("INFO: Changes to %q are no longer suppressed.", p)
-					delete(w.suppressed, rn)
 				}
 			}
 
 			fd, err := os.Open(p)
 			if err != nil {
 				if debug {
-					dlog.Println("open:", p, err)
+					l.Debugln("open:", p, err)
 				}
 				return nil
 			}
@@ -226,13 +222,13 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 			blocks, err := Blocks(fd, w.BlockSize)
 			if err != nil {
 				if debug {
-					dlog.Println("hash error:", rn, err)
+					l.Debugln("hash error:", rn, err)
 				}
 				return nil
 			}
 			if debug {
 				t1 := time.Now()
-				dlog.Println("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s")
+				l.Debugln("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s")
 			}
 			f := File{
 				Name:     rn,
@@ -272,3 +268,12 @@ func (w *Walker) ignoreFile(patterns map[string][]string, file string) bool {
 	}
 	return false
 }
+
+func checkDir(dir string) error {
+	if info, err := os.Stat(dir); err != nil {
+		return err
+	} else if !info.IsDir() {
+		return errors.New(dir + ": not a directory")
+	}
+	return nil
+}

scanner/walk_test.go

@@ -27,7 +27,11 @@ func TestWalk(t *testing.T) {
 		BlockSize:  128 * 1024,
 		IgnoreFile: ".stignore",
 	}
-	files, ignores := w.Walk()
+	files, ignores, err := w.Walk()
+
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	if l1, l2 := len(files), len(testdata); l1 != l2 {
 		t.Fatalf("Incorrect number of walked files %d != %d", l1, l2)
@@ -54,6 +58,30 @@ func TestWalk(t *testing.T) {
 	}
 }
 
+func TestWalkError(t *testing.T) {
+	w := Walker{
+		Dir:        "testdata-missing",
+		BlockSize:  128 * 1024,
+		IgnoreFile: ".stignore",
+	}
+	_, _, err := w.Walk()
+
+	if err == nil {
+		t.Error("no error from missing directory")
+	}
+
+	w = Walker{
+		Dir:        "testdata/bar",
+		BlockSize:  128 * 1024,
+		IgnoreFile: ".stignore",
+	}
+	_, _, err = w.Walk()
+
+	if err == nil {
+		t.Error("no error from non-directory")
+	}
+}
+
 func TestIgnore(t *testing.T) {
 	var patterns = map[string][]string{
 		"":        {"t2"},

upnp/debug.go

@@ -1,12 +1,13 @@
 package upnp
 
 import (
-	"log"
 	"os"
 	"strings"
+
+	"github.com/calmh/syncthing/logger"
 )
 
 var (
-	dlog  = log.New(os.Stderr, "upnp: ", log.Lmicroseconds|log.Lshortfile)
-	debug = strings.Contains(os.Getenv("STTRACE"), "upnp")
+	debug = strings.Contains(os.Getenv("STTRACE"), "upnp") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
 )

upnp/upnp.go

@@ -81,7 +81,7 @@ Mx: 3
 	}
 
 	if debug {
-		dlog.Println(string(resp[:n]))
+		l.Debugln(string(resp[:n]))
 	}
 
 	reader := bufio.NewReader(bytes.NewBuffer(resp[:n]))
@@ -225,8 +225,8 @@ func soapRequest(url, function, message string) error {
 	req.Header.Set("Pragma", "no-cache")
 
 	if debug {
-		dlog.Println(req.Header.Get("SOAPAction"))
-		dlog.Println(body)
+		l.Debugln(req.Header.Get("SOAPAction"))
+		l.Debugln(body)
 	}
 
 	r, err := http.DefaultClient.Do(req)
@@ -236,7 +236,7 @@ func soapRequest(url, function, message string) error {
 
 	if debug {
 		resp, _ := ioutil.ReadAll(r.Body)
-		dlog.Println(string(resp))
+		l.Debugln(string(resp))
 	}
 
 	r.Body.Close()

xdr/reader.go

@@ -72,20 +72,25 @@ func (r *Reader) ReadUint16() uint16 {
 }
 
 func (r *Reader) ReadUint32() uint32 {
+	var n int
 	if r.err != nil {
 		return 0
 	}
-	_, r.err = io.ReadFull(r.r, r.b[:4])
-	r.tot += 4
+	n, r.err = io.ReadFull(r.r, r.b[:4])
+	if n < 4 {
+		return 0
+	}
+	r.tot += n
 	return uint32(r.b[3]) | uint32(r.b[2])<<8 | uint32(r.b[1])<<16 | uint32(r.b[0])<<24
 }
 
 func (r *Reader) ReadUint64() uint64 {
+	var n int
 	if r.err != nil {
 		return 0
 	}
-	_, r.err = io.ReadFull(r.r, r.b[:8])
-	r.tot += 8
+	n, r.err = io.ReadFull(r.r, r.b[:8])
+	r.tot += n
 	return uint64(r.b[7]) | uint64(r.b[6])<<8 | uint64(r.b[5])<<16 | uint64(r.b[4])<<24 |
 		uint64(r.b[3])<<32 | uint64(r.b[2])<<40 | uint64(r.b[1])<<48 | uint64(r.b[0])<<56
 }