Show node name in title/header (fixes #221)

Change default ListenAddress to "0.0.0.0:22000"

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 Please do contribute! If you want to contribute but are unsure where to
 start, the [Contributions Needed
-page](https://github.com/calmh/syncthing/wiki/Contributions-Needed)
+topic](http://discourse.syncthing.net/t/contributions-needed/49)
 lists areas in need of attention.
 
 ## Licensing
@@ -15,7 +15,8 @@ will ensure that you are added to the CONTRIBUTORS file.
 
 ## Building
 
-[See the wiki](https://github.com/calmh/syncthing/wiki/Building)
+[See the
+documentation](http://discourse.syncthing.net/t/building-syncthing/44)
 
 ## Branches
 
@@ -46,7 +47,7 @@ Yes please!
 
 ## Documentation
 
-[Hack it here](https://github.com/calmh/syncthing/wiki)
+[Over here!](http://discourse.syncthing.net/category/documentation)
 
 ## License
 

CONTRIBUTORS

@@ -1,4 +1,7 @@
 Aaron Bieber <qbit@deftly.net>
+Andrew Dunham <andrew@du.nham.ca>
 Brandon Philips <brandon@ifup.org>
 James Patterson <jamespatterson@operamail.com>
+Jens Diemer <github.com@jensdiemer.de>
 Philippe Schommers <philippe@schommers.be>
+Veeti Paananen <veeti.paananen@rojekti.fi>

Godeps/Godeps.json

@@ -8,6 +8,11 @@
 		"./discover/cmd/discosrv"
 	],
 	"Deps": [
+		{
+			"ImportPath": "bitbucket.org/kardianos/osext",
+			"Comment": "null-9",
+			"Rev": "364fb577de68fb646c4cb39cc0e09c887ee16376"
+		},
 		{
 			"ImportPath": "code.google.com/p/go.crypto/bcrypt",
 			"Comment": "null-185",

Godeps/_workspace/src/bitbucket.org/kardianos/osext/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Daniel Theophanes
+
+This software is provided 'as-is', without any express or implied
+warranty. In no event will the authors be held liable for any damages
+arising from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose,
+including commercial applications, and to alter it and redistribute it
+freely, subject to the following restrictions:
+
+   1. The origin of this software must not be misrepresented; you must not
+   claim that you wrote the original software. If you use this software
+   in a product, an acknowledgment in the product documentation would be
+   appreciated but is not required.
+
+   2. Altered source versions must be plainly marked as such, and must not be
+   misrepresented as being the original software.
+
+   3. This notice may not be removed or altered from any source
+   distribution.

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext.go

@@ -0,0 +1,32 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Extensions to the standard "os" package.
+package osext
+
+import "path/filepath"
+
+// Executable returns an absolute path that can be used to
+// re-invoke the current program.
+// It may not be valid after the current program exits.
+func Executable() (string, error) {
+	p, err := executable()
+	return filepath.Clean(p), err
+}
+
+// Returns same path as Executable, returns just the folder
+// path. Excludes the executable name.
+func ExecutableFolder() (string, error) {
+	p, err := Executable()
+	if err != nil {
+		return "", err
+	}
+	folder, _ := filepath.Split(p)
+	return folder, nil
+}
+
+// Depricated. Same as Executable().
+func GetExePath() (exePath string, err error) {
+	return Executable()
+}

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext_plan9.go

@@ -0,0 +1,16 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package osext
+
+import "syscall"
+
+func executable() (string, error) {
+	f, err := Open("/proc/" + itoa(Getpid()) + "/text")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	return syscall.Fd2path(int(f.Fd()))
+}

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext_procfs.go

@@ -0,0 +1,25 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux netbsd openbsd
+
+package osext
+
+import (
+	"errors"
+	"os"
+	"runtime"
+)
+
+func executable() (string, error) {
+	switch runtime.GOOS {
+	case "linux":
+		return os.Readlink("/proc/self/exe")
+	case "netbsd":
+		return os.Readlink("/proc/curproc/exe")
+	case "openbsd":
+		return os.Readlink("/proc/curproc/file")
+	}
+	return "", errors.New("ExecPath not implemented for " + runtime.GOOS)
+}

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext_sysctl.go

@@ -0,0 +1,82 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin freebsd
+
+package osext
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"syscall"
+	"unsafe"
+)
+
+var startUpcwd, getwdError = os.Getwd()
+
+func executable() (string, error) {
+	var mib [4]int32
+	switch runtime.GOOS {
+	case "freebsd":
+		mib = [4]int32{1 /* CTL_KERN */, 14 /* KERN_PROC */, 12 /* KERN_PROC_PATHNAME */, -1}
+	case "darwin":
+		mib = [4]int32{1 /* CTL_KERN */, 38 /* KERN_PROCARGS */, int32(os.Getpid()), -1}
+	}
+
+	n := uintptr(0)
+	// get length
+	_, _, err := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0)
+	if err != 0 {
+		return "", err
+	}
+	if n == 0 { // shouldn't happen
+		return "", nil
+	}
+	buf := make([]byte, n)
+	_, _, err = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0)
+	if err != 0 {
+		return "", err
+	}
+	if n == 0 { // shouldn't happen
+		return "", nil
+	}
+	for i, v := range buf {
+		if v == 0 {
+			buf = buf[:i]
+			break
+		}
+	}
+	var strpath string
+	if buf[0] != '/' {
+		var e error
+		if strpath, e = getAbs(buf); e != nil {
+			return strpath, e
+		}
+	} else {
+		strpath = string(buf)
+	}
+	// darwin KERN_PROCARGS may return the path to a symlink rather than the
+	// actual executable
+	if runtime.GOOS == "darwin" {
+		if strpath, err := filepath.EvalSymlinks(strpath); err != nil {
+			return strpath, err
+		}
+	}
+	return strpath, nil
+}
+
+func getAbs(buf []byte) (string, error) {
+	if getwdError != nil {
+		return string(buf), getwdError
+	} else {
+		if buf[0] == '.' {
+			buf = buf[1:]
+		}
+		if startUpcwd[len(startUpcwd)-1] != '/' && buf[0] != '/' {
+			return startUpcwd + "/" + string(buf), nil
+		}
+		return startUpcwd + string(buf), nil
+	}
+}

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext_test.go

@@ -0,0 +1,79 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin linux freebsd netbsd windows
+
+package osext
+
+import (
+	"fmt"
+	"os"
+	oexec "os/exec"
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+const execPath_EnvVar = "OSTEST_OUTPUT_EXECPATH"
+
+func TestExecPath(t *testing.T) {
+	ep, err := Executable()
+	if err != nil {
+		t.Fatalf("ExecPath failed: %v", err)
+	}
+	// we want fn to be of the form "dir/prog"
+	dir := filepath.Dir(filepath.Dir(ep))
+	fn, err := filepath.Rel(dir, ep)
+	if err != nil {
+		t.Fatalf("filepath.Rel: %v", err)
+	}
+	cmd := &oexec.Cmd{}
+	// make child start with a relative program path
+	cmd.Dir = dir
+	cmd.Path = fn
+	// forge argv[0] for child, so that we can verify we could correctly
+	// get real path of the executable without influenced by argv[0].
+	cmd.Args = []string{"-", "-test.run=XXXX"}
+	cmd.Env = []string{fmt.Sprintf("%s=1", execPath_EnvVar)}
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("exec(self) failed: %v", err)
+	}
+	outs := string(out)
+	if !filepath.IsAbs(outs) {
+		t.Fatalf("Child returned %q, want an absolute path", out)
+	}
+	if !sameFile(outs, ep) {
+		t.Fatalf("Child returned %q, not the same file as %q", out, ep)
+	}
+}
+
+func sameFile(fn1, fn2 string) bool {
+	fi1, err := os.Stat(fn1)
+	if err != nil {
+		return false
+	}
+	fi2, err := os.Stat(fn2)
+	if err != nil {
+		return false
+	}
+	return os.SameFile(fi1, fi2)
+}
+
+func init() {
+	if e := os.Getenv(execPath_EnvVar); e != "" {
+		// first chdir to another path
+		dir := "/"
+		if runtime.GOOS == "windows" {
+			dir = filepath.VolumeName(".")
+		}
+		os.Chdir(dir)
+		if ep, err := Executable(); err != nil {
+			fmt.Fprint(os.Stderr, "ERROR: ", err)
+		} else {
+			fmt.Fprint(os.Stderr, ep)
+		}
+		os.Exit(0)
+	}
+}

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext_windows.go

@@ -0,0 +1,34 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package osext
+
+import (
+	"syscall"
+	"unicode/utf16"
+	"unsafe"
+)
+
+var (
+	kernel                = syscall.MustLoadDLL("kernel32.dll")
+	getModuleFileNameProc = kernel.MustFindProc("GetModuleFileNameW")
+)
+
+// GetModuleFileName() with hModule = NULL
+func executable() (exePath string, err error) {
+	return getModuleFileName()
+}
+
+func getModuleFileName() (string, error) {
+	var n uint32
+	b := make([]uint16, syscall.MAX_PATH)
+	size := uint32(len(b))
+
+	r0, _, e1 := getModuleFileNameProc.Call(0, uintptr(unsafe.Pointer(&b[0])), uintptr(size))
+	n = uint32(r0)
+	if n == 0 {
+		return "", e1
+	}
+	return string(utf16.Decode(b[0:n])), nil
+}

README.md

@@ -1,12 +1,12 @@
-syncthing [![Build Status](https://drone.io/github.com/calmh/syncthing/status.png)](https://drone.io/github.com/calmh/syncthing/latest)
+syncthing
 =========
 
 This is the `syncthing` project. The following are the project goals:
 
  1. Define a protocol for synchronization of a file repository between a
     number of collaborating nodes. The protocol should be well defined,
-    unambigous, easily understood, free to use, efficient, secure and
-    languange neutral. This is the [Block Exchange
+    unambiguous, easily understood, free to use, efficient, secure and
+    language neutral. This is the [Block Exchange
     Protocol](https://github.com/calmh/syncthing/blob/master/protocol/PROTOCOL.md).
 
  2. Provide the reference implementation to demonstrate the usability of
@@ -25,6 +25,11 @@ making sure large swarms of selfish agents behave and somehow work
 towards a common goal. Here we have a much smaller swarm of cooperative
 agents and a simpler approach will suffice.
 
+Getting Started
+---------------
+
+Take a look at the [getting started guide](http://discourse.syncthing.net/t/getting-started/46).
+
 Signed Releases
 ---------------
 
@@ -35,8 +40,9 @@ normal release bundle as `syncthing.asc` or `syncthing.exe.asc`.
 Documentation
 =============
 
-The syncthing documentation is kept on the
-[GitHub Wiki](https://github.com/calmh/syncthing/wiki).
+The [syncthing
+documentation](http://discourse.syncthing.net/category/documentation) is
+on the discourse site.
 
 License
 =======

beacon/beacon.go

@@ -0,0 +1,127 @@
+package beacon
+
+import "net"
+
+type recv struct {
+	data []byte
+	src  net.Addr
+}
+
+type dst struct {
+	intf string
+	conn *net.UDPConn
+}
+
+type Beacon struct {
+	conn   *net.UDPConn
+	port   int
+	conns  []dst
+	inbox  chan []byte
+	outbox chan recv
+}
+
+func New(port int) (*Beacon, error) {
+	conn, err := net.ListenUDP("udp", &net.UDPAddr{Port: port})
+	if err != nil {
+		return nil, err
+	}
+	b := &Beacon{
+		conn:   conn,
+		port:   port,
+		inbox:  make(chan []byte),
+		outbox: make(chan recv, 16),
+	}
+
+	go b.reader()
+	go b.writer()
+
+	return b, nil
+}
+
+func (b *Beacon) Send(data []byte) {
+	b.inbox <- data
+}
+
+func (b *Beacon) Recv() ([]byte, net.Addr) {
+	recv := <-b.outbox
+	return recv.data, recv.src
+}
+
+func (b *Beacon) reader() {
+	var bs = make([]byte, 65536)
+	for {
+		n, addr, err := b.conn.ReadFrom(bs)
+		if err != nil {
+			l.Warnln("Beacon read:", err)
+			return
+		}
+		if debug {
+			l.Debugf("recv %d bytes from %s", n, addr)
+		}
+		select {
+		case b.outbox <- recv{bs[:n], addr}:
+		default:
+			if debug {
+				l.Debugln("dropping message")
+			}
+		}
+	}
+}
+
+func (b *Beacon) writer() {
+	for bs := range b.inbox {
+
+		addrs, err := net.InterfaceAddrs()
+		if err != nil {
+			l.Warnln("Beacon: interface addresses:", err)
+			continue
+		}
+
+		var dsts []net.IP
+		for _, addr := range addrs {
+			if iaddr, ok := addr.(*net.IPNet); ok && iaddr.IP.IsGlobalUnicast() {
+				baddr := bcast(iaddr)
+				dsts = append(dsts, baddr.IP)
+			}
+		}
+
+		if len(dsts) == 0 {
+			// Fall back to the general IPv4 broadcast address
+			dsts = append(dsts, net.IP{0xff, 0xff, 0xff, 0xff})
+		}
+
+		if debug {
+			l.Debugln("addresses:", dsts)
+		}
+
+		for _, ip := range dsts {
+			dst := &net.UDPAddr{IP: ip, Port: b.port}
+
+			_, err := b.conn.WriteTo(bs, dst)
+			if err != nil {
+				if debug {
+					l.Debugln(err)
+				}
+				return
+			}
+			if debug {
+				l.Debugf("sent %d bytes to %s", len(bs), dst)
+			}
+		}
+	}
+}
+
+func bcast(ip *net.IPNet) *net.IPNet {
+	var bc = &net.IPNet{}
+	bc.IP = make([]byte, len(ip.IP))
+	copy(bc.IP, ip.IP)
+	bc.Mask = ip.Mask
+
+	offset := len(bc.IP) - len(bc.Mask)
+	for i := range bc.IP {
+		if i-offset > 0 {
+			bc.IP[i] = ip.IP[i] | ^ip.Mask[i-offset]
+		}
+	}
+	return bc
+}

beacon/cmd/mctest/main.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"encoding/binary"
+	"log"
+	"time"
+
+	"github.com/calmh/syncthing/beacon"
+)
+
+func main() {
+	b, err := beacon.NewBeacon(21025)
+	if err != nil {
+		log.Fatal(err)
+	}
+	go func() {
+		for {
+			bs, addr := b.Recv()
+			log.Printf("Received %d bytes from %s: %x %x", len(bs), addr, bs[:8], bs[8:])
+		}
+	}()
+	go func() {
+		bs := [16]byte{}
+		binary.BigEndian.PutUint64(bs[:], uint64(time.Now().UnixNano()))
+		log.Printf("My ID: %x", bs[:8])
+		for {
+			binary.BigEndian.PutUint64(bs[8:], uint64(time.Now().UnixNano()))
+			b.Send(bs[:])
+			log.Printf("Sent %d bytes", len(bs[:]))
+			time.Sleep(10 * time.Second)
+		}
+	}()
+	select {}
+}

beacon/debug.go

@@ -0,0 +1,13 @@
+package beacon
+
+import (
+	"os"
+	"strings"
+
+	"github.com/calmh/syncthing/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "beacon") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

beacon/doc.go

@@ -0,0 +1,2 @@
+// Package beacon implements an UDP broadcast beacon
+package beacon

build.sh

@@ -4,30 +4,34 @@ export COPYFILE_DISABLE=true
 
 distFiles=(README.md LICENSE) # apart from the binary itself
 version=$(git describe --always --dirty)
-date=$(date +%s)
+date=$(git show -s --format=%ct)
 user=$(whoami)
 host=$(hostname)
 host=${host%%.*}
 ldflags="-w -X main.Version $version -X main.BuildStamp $date -X main.BuildUser $user -X main.BuildHost $host"
 
-build() {
-	if command -v godep >/dev/null ; then
-		godep=godep
-	else
-		echo "Warning: no godep, using \"go get\" instead."
-		echo "Try \"go get github.com/tools/godep\"."
-		go get -d ./cmd/syncthing
-		godep=
+check() {
+	if ! command -v godep >/dev/null ; then
+		echo "Error: no godep. Try \"$0 setup\"."
+		exit 1
 	fi
-	${godep} go build $* -ldflags "$ldflags" ./cmd/syncthing
-	${godep} go build -ldflags "$ldflags" ./cmd/stcli
+}
+
+build() {
+	check
+
+	go vet ./... || exit 1
+
+	godep go build $* -ldflags "$ldflags" ./cmd/syncthing
 }
 
 assets() {
+	check
 	godep go run cmd/assets/assets.go gui > auto/gui.files.go
 }
 
 test() {
+	check
 	godep go test -cpu=1,2,4 ./...
 }
 
@@ -62,7 +66,15 @@ zipDist() {
 }
 
 deps() {
-	godep save ./cmd/syncthing ./cmd/assets ./cmd/stcli ./discover/cmd/discosrv
+	check
+	godep save ./cmd/syncthing ./cmd/assets ./discover/cmd/discosrv
+}
+
+setup() {
+	echo Installing godep...
+	go get -u github.com/tools/godep
+	echo Installing go vet...
+	go get -u code.google.com/p/go.tools/cmd/vet
 }
 
 case "$1" in
@@ -75,6 +87,10 @@ case "$1" in
 		build -race
 		;;
 
+	guidev)
+		build -tags guidev
+		;;
+
 	test)
 		test
 		;;
@@ -96,7 +112,7 @@ case "$1" in
 		test || exit 1
 		assets
 
-		for os in darwin-amd64 linux-386 linux-amd64 freebsd-amd64 windows-amd64 ; do
+		for os in darwin-amd64 linux-386 linux-amd64 freebsd-amd64 windows-amd64 windows-386 ; do
 			export GOOS=${os%-*}
 			export GOARCH=${os#*-}
 
@@ -126,6 +142,10 @@ case "$1" in
 		build
 		tarDist "syncthing-linux-armv6-$version"
 
+		export GOARM=5
+		build
+		tarDist "syncthing-linux-armv5-$version"
+
 		;;
 
 	upload)
@@ -144,6 +164,10 @@ case "$1" in
 		assets
 		;;
 
+	setup)
+		setup
+		;;
+
 	*)
 		echo "Unknown build parameter $1"
 		;;

cmd/stcli/logger.go

@@ -1,72 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-	"os"
-)
-
-var logger *log.Logger
-
-func init() {
-	log.SetOutput(os.Stderr)
-	logger = log.New(os.Stderr, "", log.Flags())
-}
-
-func debugln(vals ...interface{}) {
-	s := fmt.Sprintln(vals...)
-	logger.Output(2, "DEBUG: "+s)
-}
-
-func debugf(format string, vals ...interface{}) {
-	s := fmt.Sprintf(format, vals...)
-	logger.Output(2, "DEBUG: "+s)
-}
-
-func infoln(vals ...interface{}) {
-	s := fmt.Sprintln(vals...)
-	logger.Output(2, "INFO: "+s)
-}
-
-func infof(format string, vals ...interface{}) {
-	s := fmt.Sprintf(format, vals...)
-	logger.Output(2, "INFO: "+s)
-}
-
-func okln(vals ...interface{}) {
-	s := fmt.Sprintln(vals...)
-	logger.Output(2, "OK: "+s)
-}
-
-func okf(format string, vals ...interface{}) {
-	s := fmt.Sprintf(format, vals...)
-	logger.Output(2, "OK: "+s)
-}
-
-func warnln(vals ...interface{}) {
-	s := fmt.Sprintln(vals...)
-	logger.Output(2, "WARNING: "+s)
-}
-
-func warnf(format string, vals ...interface{}) {
-	s := fmt.Sprintf(format, vals...)
-	logger.Output(2, "WARNING: "+s)
-}
-
-func fatalln(vals ...interface{}) {
-	s := fmt.Sprintln(vals...)
-	logger.Output(2, "FATAL: "+s)
-	os.Exit(3)
-}
-
-func fatalf(format string, vals ...interface{}) {
-	s := fmt.Sprintf(format, vals...)
-	logger.Output(2, "FATAL: "+s)
-	os.Exit(3)
-}
-
-func fatalErr(err error) {
-	if err != nil {
-		fatalf(err.Error())
-	}
-}

cmd/stcli/tls.go

@@ -1,24 +1,11 @@
 package main
 
 import (
-	"crypto/rand"
-	"crypto/rsa"
 	"crypto/sha256"
 	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
 	"encoding/base32"
-	"encoding/pem"
-	"math/big"
-	"os"
 	"path/filepath"
 	"strings"
-	"time"
-)
-
-const (
-	tlsRSABits = 3072
-	tlsName    = "syncthing"
 )
 
 func loadCert(dir string) (tls.Certificate, error) {
@@ -31,41 +18,3 @@ func certID(bs []byte) string {
 	id := hf.Sum(nil)
 	return strings.Trim(base32.StdEncoding.EncodeToString(id), "=")
 }
-
-func newCertificate(dir string) {
-	infoln("Generating RSA certificate and key...")
-
-	priv, err := rsa.GenerateKey(rand.Reader, tlsRSABits)
-	fatalErr(err)
-
-	notBefore := time.Now()
-	notAfter := time.Date(2049, 12, 31, 23, 59, 59, 0, time.UTC)
-
-	template := x509.Certificate{
-		SerialNumber: new(big.Int).SetInt64(0),
-		Subject: pkix.Name{
-			CommonName: tlsName,
-		},
-		NotBefore: notBefore,
-		NotAfter:  notAfter,
-
-		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
-		BasicConstraintsValid: true,
-	}
-
-	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
-	fatalErr(err)
-
-	certOut, err := os.Create(filepath.Join(dir, "cert.pem"))
-	fatalErr(err)
-	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
-	certOut.Close()
-	okln("Created RSA certificate file")
-
-	keyOut, err := os.OpenFile(filepath.Join(dir, "key.pem"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
-	fatalErr(err)
-	pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
-	keyOut.Close()
-	okln("Created RSA key file")
-}

cmd/syncthing/debug.go

@@ -1,15 +1,10 @@
 package main
 
 import (
-	"log"
 	"os"
 	"strings"
 )
 
 var (
-	dlog      = log.New(os.Stderr, "main: ", log.Lmicroseconds|log.Lshortfile)
-	debugNet  = strings.Contains(os.Getenv("STTRACE"), "net")
-	debugIdx  = strings.Contains(os.Getenv("STTRACE"), "idx")
-	debugNeed = strings.Contains(os.Getenv("STTRACE"), "need")
-	debugPull = strings.Contains(os.Getenv("STTRACE"), "pull")
+	debugNet = strings.Contains(os.Getenv("STTRACE"), "net") || os.Getenv("STTRACE") == "all"
 )

cmd/syncthing/gui.go

@@ -5,14 +5,18 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"io/ioutil"
+	"log"
 	"math/rand"
+	"net"
 	"net/http"
 	"runtime"
 	"sync"
 	"time"
 
 	"code.google.com/p/go.crypto/bcrypt"
-	"github.com/calmh/syncthing/scanner"
+	"github.com/calmh/syncthing/config"
+	"github.com/calmh/syncthing/logger"
+	"github.com/calmh/syncthing/model"
 	"github.com/codegangsta/martini"
 )
 
@@ -25,13 +29,24 @@ var (
 	configInSync = true
 	guiErrors    = []guiError{}
 	guiErrorsMut sync.Mutex
+	static       = embeddedStatic()
+	staticFunc   = static.(func(http.ResponseWriter, *http.Request, *log.Logger))
 )
 
 const (
 	unchangedPassword = "--password-unchanged--"
 )
 
-func startGUI(cfg GUIConfiguration, m *Model) {
+func init() {
+	l.AddHandler(logger.LevelWarn, showGuiError)
+}
+
+func startGUI(cfg config.GUIConfiguration, m *model.Model) error {
+	listener, err := net.Listen("tcp", cfg.Address)
+	if err != nil {
+		return err
+	}
+
 	router := martini.NewRouter()
 	router.Get("/", getRoot)
 	router.Get("/rest/version", restGetVersion)
@@ -41,32 +56,34 @@ func startGUI(cfg GUIConfiguration, m *Model) {
 	router.Get("/rest/config/sync", restGetConfigInSync)
 	router.Get("/rest/system", restGetSystem)
 	router.Get("/rest/errors", restGetErrors)
+	router.Get("/rest/discovery", restGetDiscovery)
 
 	router.Post("/rest/config", restPostConfig)
 	router.Post("/rest/restart", restPostRestart)
 	router.Post("/rest/reset", restPostReset)
+	router.Post("/rest/shutdown", restPostShutdown)
 	router.Post("/rest/error", restPostError)
 	router.Post("/rest/error/clear", restClearErrors)
+	router.Post("/rest/discovery/hint", restPostDiscoveryHint)
 
-	go func() {
-		mr := martini.New()
-		if len(cfg.User) > 0 && len(cfg.Password) > 0 {
-			mr.Use(basic(cfg.User, cfg.Password))
-		}
-		mr.Use(embeddedStatic())
-		mr.Use(martini.Recovery())
-		mr.Use(restMiddleware)
-		mr.Action(router.Handle)
-		mr.Map(m)
-		err := http.ListenAndServe(cfg.Address, mr)
-		if err != nil {
-			warnln("GUI not possible:", err)
-		}
-	}()
+	mr := martini.New()
+	if len(cfg.User) > 0 && len(cfg.Password) > 0 {
+		mr.Use(basic(cfg.User, cfg.Password))
+	}
+	mr.Use(static)
+	mr.Use(martini.Recovery())
+	mr.Use(restMiddleware)
+	mr.Action(router.Handle)
+	mr.Map(m)
+
+	go http.Serve(listener, mr)
+
+	return nil
 }
 
 func getRoot(w http.ResponseWriter, r *http.Request) {
-	http.Redirect(w, r, "/index.html", 302)
+	r.URL.Path = "/index.html"
+	staticFunc(w, r, nil)
 }
 
 func restMiddleware(w http.ResponseWriter, r *http.Request) {
@@ -79,11 +96,18 @@ func restGetVersion() string {
 	return Version
 }
 
-func restGetModel(m *Model, w http.ResponseWriter, r *http.Request) {
+func restGetModel(m *model.Model, w http.ResponseWriter, r *http.Request) {
 	var qs = r.URL.Query()
 	var repo = qs.Get("repo")
 	var res = make(map[string]interface{})
 
+	for _, cr := range cfg.Repositories {
+		if cr.ID == repo {
+			res["invalid"] = cr.Invalid
+			break
+		}
+	}
+
 	globalFiles, globalDeleted, globalBytes := m.GlobalSize(repo)
 	res["globalFiles"], res["globalDeleted"], res["globalBytes"] = globalFiles, globalDeleted, globalBytes
 
@@ -101,7 +125,7 @@ func restGetModel(m *Model, w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(res)
 }
 
-func restGetConnections(m *Model, w http.ResponseWriter) {
+func restGetConnections(m *model.Model, w http.ResponseWriter) {
 	var res = m.ConnectionStats()
 	w.Header().Set("Content-Type", "application/json")
 	json.NewEncoder(w).Encode(res)
@@ -119,14 +143,14 @@ func restPostConfig(req *http.Request) {
 	var prevPassHash = cfg.GUI.Password
 	err := json.NewDecoder(req.Body).Decode(&cfg)
 	if err != nil {
-		warnln(err)
+		l.Warnln(err)
 	} else {
 		if cfg.GUI.Password == "" {
 			// Leave it empty
 		} else if cfg.GUI.Password != unchangedPassword {
 			hash, err := bcrypt.GenerateFromPassword([]byte(cfg.GUI.Password), 0)
 			if err != nil {
-				warnln(err)
+				l.Warnln(err)
 			} else {
 				cfg.GUI.Password = string(hash)
 			}
@@ -142,30 +166,26 @@ func restGetConfigInSync(w http.ResponseWriter) {
 	json.NewEncoder(w).Encode(map[string]bool{"configInSync": configInSync})
 }
 
-func restPostRestart(req *http.Request) {
+func restPostRestart(w http.ResponseWriter) {
+	flushResponse(`{"ok": "restarting"}`, w)
 	go restart()
 }
 
-func restPostReset(req *http.Request) {
+func restPostReset(w http.ResponseWriter) {
+	flushResponse(`{"ok": "resetting repos"}`, w)
 	resetRepositories()
 	go restart()
 }
 
-type guiFile scanner.File
+func restPostShutdown(w http.ResponseWriter) {
+	flushResponse(`{"ok": "shutting down"}`, w)
+	go shutdown()
+}
 
-func (f guiFile) MarshalJSON() ([]byte, error) {
-	type t struct {
-		Name     string
-		Size     int64
-		Modified int64
-		Flags    uint32
-	}
-	return json.Marshal(t{
-		Name:     f.Name,
-		Size:     scanner.File(f).Size,
-		Modified: f.Modified,
-		Flags:    f.Flags,
-	})
+func flushResponse(s string, w http.ResponseWriter) {
+	w.Write([]byte(s + "\n"))
+	f := w.(http.Flusher)
+	f.Flush()
 }
 
 var cpuUsagePercent [10]float64 // The last ten seconds
@@ -180,7 +200,7 @@ func restGetSystem(w http.ResponseWriter) {
 	res["goroutines"] = runtime.NumGoroutine()
 	res["alloc"] = m.Alloc
 	res["sys"] = m.Sys
-	if discoverer != nil {
+	if cfg.Options.GlobalAnnEnabled && discoverer != nil {
 		res["extAnnounceOK"] = discoverer.ExtAnnounceOK()
 	}
 	cpuUsageLock.RLock()
@@ -204,7 +224,7 @@ func restGetErrors(w http.ResponseWriter) {
 func restPostError(req *http.Request) {
 	bs, _ := ioutil.ReadAll(req.Body)
 	req.Body.Close()
-	showGuiError(string(bs))
+	showGuiError(0, string(bs))
 }
 
 func restClearErrors() {
@@ -213,7 +233,7 @@ func restClearErrors() {
 	guiErrorsMut.Unlock()
 }
 
-func showGuiError(err string) {
+func showGuiError(l logger.LogLevel, err string) {
 	guiErrorsMut.Lock()
 	guiErrors = append(guiErrors, guiError{time.Now(), err})
 	if len(guiErrors) > 5 {
@@ -222,6 +242,19 @@ func showGuiError(err string) {
 	guiErrorsMut.Unlock()
 }
 
+func restPostDiscoveryHint(r *http.Request) {
+	var qs = r.URL.Query()
+	var node = qs.Get("node")
+	var addr = qs.Get("addr")
+	if len(node) != 0 && len(addr) != 0 && discoverer != nil {
+		discoverer.Hint(node, []string{addr})
+	}
+}
+
+func restGetDiscovery(w http.ResponseWriter) {
+	json.NewEncoder(w).Encode(discoverer.All())
+}
+
 func basic(username string, passhash string) http.HandlerFunc {
 	return func(res http.ResponseWriter, req *http.Request) {
 		error := func() {

cmd/syncthing/gui_embedded.go

@@ -32,7 +32,7 @@ func embeddedStatic() interface{} {
 		if len(mtype) != 0 {
 			res.Header().Set("Content-Type", mtype)
 		}
-		res.Header().Set("Content-Size", fmt.Sprintf("%d", len(bs)))
+		res.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs)))
 		res.Header().Set("Last-Modified", modt)
 
 		res.Write(bs)

cmd/syncthing/gui_solaris.go

@@ -75,7 +75,7 @@ func trackCPUUsage() {
 	for _ = range time.NewTicker(time.Second).C {
 		err := solarisPrusage(pid, &rusage)
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 			continue
 		}
 		curTime := time.Now().UnixNano()

cmd/syncthing/logger.go

@@ -1,64 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-	"os"
-)
-
-var logger *log.Logger
-
-func init() {
-	log.SetOutput(os.Stderr)
-	logger = log.New(os.Stderr, "", log.Flags())
-}
-
-func infoln(vals ...interface{}) {
-	s := fmt.Sprintln(vals...)
-	logger.Output(2, "INFO: "+s)
-}
-
-func infof(format string, vals ...interface{}) {
-	s := fmt.Sprintf(format, vals...)
-	logger.Output(2, "INFO: "+s)
-}
-
-func okln(vals ...interface{}) {
-	s := fmt.Sprintln(vals...)
-	logger.Output(2, "OK: "+s)
-}
-
-func okf(format string, vals ...interface{}) {
-	s := fmt.Sprintf(format, vals...)
-	logger.Output(2, "OK: "+s)
-}
-
-func warnln(vals ...interface{}) {
-	s := fmt.Sprintln(vals...)
-	showGuiError(s)
-	logger.Output(2, "WARNING: "+s)
-}
-
-func warnf(format string, vals ...interface{}) {
-	s := fmt.Sprintf(format, vals...)
-	showGuiError(s)
-	logger.Output(2, "WARNING: "+s)
-}
-
-func fatalln(vals ...interface{}) {
-	s := fmt.Sprintln(vals...)
-	logger.Output(2, "FATAL: "+s)
-	os.Exit(3)
-}
-
-func fatalf(format string, vals ...interface{}) {
-	s := fmt.Sprintf(format, vals...)
-	logger.Output(2, "FATAL: "+s)
-	os.Exit(3)
-}
-
-func fatalErr(err error) {
-	if err != nil {
-		fatalf(err.Error())
-	}
-}

cmd/syncthing/main.go

@@ -20,14 +20,15 @@ import (
 	"strings"
 	"time"
 
+	"github.com/calmh/syncthing/config"
 	"github.com/calmh/syncthing/discover"
+	"github.com/calmh/syncthing/logger"
+	"github.com/calmh/syncthing/model"
 	"github.com/calmh/syncthing/protocol"
 	"github.com/calmh/syncthing/upnp"
 	"github.com/juju/ratelimit"
 )
 
-const BlockSize = 128 * 1024
-
 var (
 	Version     = "unknown-dev"
 	BuildStamp  = "0"
@@ -37,16 +38,22 @@ var (
 	LongVersion string
 )
 
+var l = logger.DefaultLogger
+
 func init() {
 	stamp, _ := strconv.Atoi(BuildStamp)
 	BuildDate = time.Unix(int64(stamp), 0)
 
-	date := BuildDate.UTC().Format(time.RFC3339)
+	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
 	LongVersion = fmt.Sprintf("syncthing %s (%s %s-%s) %s@%s %s", Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
+
+	if os.Getenv("STTRACE") != "" {
+		l.SetFlags(log.Ltime | log.Ldate | log.Lmicroseconds | log.Lshortfile)
+	}
 }
 
 var (
-	cfg        Configuration
+	cfg        config.Configuration
 	myID       string
 	confDir    string
 	rateBucket *ratelimit.Bucket
@@ -67,15 +74,13 @@ const (
 
  STTRACE       A comma separated string of facilities to trace. The valid
                facility strings:
-               - "discover" (the node discovery package)
-               - "files"    (file set store)
-               - "idx"      (index sending and receiving)
-               - "mc"       (multicast beacon)
-               - "need"     (file need calculations)
-               - "net"      (connecting and disconnecting, network messages)
-               - "pull"     (file pull activity)
-               - "scanner"  (the file change scanner)
-               - "upnp"     (the upnp port mapper)
+               - "beacon"   (the beacon package)
+               - "discover" (the discover package)
+               - "files"    (the files package)
+               - "net"      (the main packge; connections & network messages)
+               - "scanner"  (the scanner package)
+               - "upnp"     (the upnp package)
+               - "all"      (all of the above)
 
  STCPUPROFILE  Write CPU profile to the specified file.`
 )
@@ -83,9 +88,11 @@ const (
 func main() {
 	var reset bool
 	var showVersion bool
+	var doUpgrade bool
 	flag.StringVar(&confDir, "home", getDefaultConfDir(), "Set configuration directory")
 	flag.BoolVar(&reset, "reset", false, "Prepare to resync from cluster")
 	flag.BoolVar(&showVersion, "version", false, "Show version")
+	flag.BoolVar(&doUpgrade, "upgrade", false, "Perform upgrade")
 	flag.Usage = usageFor(flag.CommandLine, usage, extraUsage)
 	flag.Parse()
 
@@ -99,6 +106,14 @@ func main() {
 		return
 	}
 
+	if doUpgrade {
+		err := upgrade()
+		if err != nil {
+			l.Fatalln(err)
+		}
+		return
+	}
+
 	if len(os.Getenv("GOGC")) == 0 {
 		debug.SetGCPercent(25)
 	}
@@ -120,7 +135,7 @@ func main() {
 		if _, err := os.Stat(oldDefault); err == nil {
 			os.MkdirAll(filepath.Dir(confDir), 0700)
 			if err := os.Rename(oldDefault, confDir); err == nil {
-				infoln("Moved config dir", oldDefault, "to", confDir)
+				l.Infoln("Moved config dir", oldDefault, "to", confDir)
 			}
 		}
 	}
@@ -132,15 +147,14 @@ func main() {
 	if err != nil {
 		newCertificate(confDir)
 		cert, err = loadCert(confDir)
-		fatalErr(err)
+		l.FatalErr(err)
 	}
 
 	myID = certID(cert.Certificate[0])
-	log.SetPrefix("[" + myID[0:5] + "] ")
-	logger.SetPrefix("[" + myID[0:5] + "] ")
+	l.SetPrefix(fmt.Sprintf("[%s] ", myID[:5]))
 
-	infoln(LongVersion)
-	infoln("My ID:", myID)
+	l.Infoln(LongVersion)
+	l.Infoln("My ID:", myID)
 
 	// Prepare to be able to save configuration
 
@@ -153,26 +167,26 @@ func main() {
 	cf, err := os.Open(cfgFile)
 	if err == nil {
 		// Read config.xml
-		cfg, err = readConfigXML(cf, myID)
+		cfg, err = config.Load(cf, myID)
 		if err != nil {
-			fatalln(err)
+			l.Fatalln(err)
 		}
 		cf.Close()
 	} else {
-		infoln("No config file; starting with empty defaults")
+		l.Infoln("No config file; starting with empty defaults")
 		name, _ := os.Hostname()
 		defaultRepo := filepath.Join(getHomeDir(), "Sync")
 		ensureDir(defaultRepo, 0755)
 
-		cfg, err = readConfigXML(nil, myID)
-		cfg.Repositories = []RepositoryConfiguration{
+		cfg, err = config.Load(nil, myID)
+		cfg.Repositories = []config.RepositoryConfiguration{
 			{
 				ID:        "default",
 				Directory: defaultRepo,
-				Nodes:     []NodeConfiguration{{NodeID: myID}},
+				Nodes:     []config.NodeConfiguration{{NodeID: myID}},
 			},
 		}
-		cfg.Nodes = []NodeConfiguration{
+		cfg.Nodes = []config.NodeConfiguration{
 			{
 				NodeID:    myID,
 				Addresses: []string{"dynamic"},
@@ -180,8 +194,16 @@ func main() {
 			},
 		}
 
+		port, err := getFreePort("127.0.0.1", 8080)
+		l.FatalErr(err)
+		cfg.GUI.Address = fmt.Sprintf("127.0.0.1:%d", port)
+
+		port, err = getFreePort("", 22000)
+		l.FatalErr(err)
+		cfg.Options.ListenAddress = []string{fmt.Sprintf(":%d", port)}
+
 		saveConfig()
-		infof("Edit %s to taste or use the GUI\n", cfgFile)
+		l.Infof("Edit %s to taste or use the GUI\n", cfgFile)
 	}
 
 	if reset {
@@ -191,10 +213,10 @@ func main() {
 
 	if profiler := os.Getenv("STPROFILER"); len(profiler) > 0 {
 		go func() {
-			dlog.Println("Starting profiler on", profiler)
+			l.Debugln("Starting profiler on", profiler)
 			err := http.ListenAndServe(profiler, nil)
 			if err != nil {
-				dlog.Fatal(err)
+				l.Fatalln(err)
 			}
 		}()
 	}
@@ -219,9 +241,12 @@ func main() {
 		rateBucket = ratelimit.NewBucketWithRate(float64(1000*cfg.Options.MaxSendKbps), int64(5*1000*cfg.Options.MaxSendKbps))
 	}
 
-	m := NewModel(cfg.Options.MaxChangeKbps * 1000)
+	m := model.NewModel(confDir, &cfg, "syncthing", Version)
 
 	for _, repo := range cfg.Repositories {
+		if repo.Invalid != "" {
+			continue
+		}
 		dir := expandTilde(repo.Directory)
 		m.AddRepo(repo.ID, dir, repo.Nodes)
 	}
@@ -230,7 +255,7 @@ func main() {
 	if cfg.GUI.Enabled && cfg.GUI.Address != "" {
 		addr, err := net.ResolveTCPAddr("tcp", cfg.GUI.Address)
 		if err != nil {
-			warnf("Cannot start GUI on %q: %v", cfg.GUI.Address, err)
+			l.Fatalf("Cannot start GUI on %q: %v", cfg.GUI.Address, err)
 		} else {
 			var hostOpen, hostShow string
 			switch {
@@ -245,8 +270,11 @@ func main() {
 				hostShow = hostOpen
 			}
 
-			infof("Starting web GUI on http://%s:%d/", hostShow, addr.Port)
-			startGUI(cfg.GUI, m)
+			l.Infof("Starting web GUI on http://%s:%d/", hostShow, addr.Port)
+			err := startGUI(cfg.GUI, m)
+			if err != nil {
+				l.Fatalln("Cannot start GUI:", err)
+			}
 			if cfg.Options.StartBrowser && len(os.Getenv("STRESTART")) == 0 {
 				openURL(fmt.Sprintf("http://%s:%d", hostOpen, addr.Port))
 			}
@@ -256,10 +284,14 @@ func main() {
 	// Walk the repository and update the local model before establishing any
 	// connections to other nodes.
 
-	infoln("Populating repository index")
+	l.Infoln("Populating repository index")
 	m.LoadIndexes(confDir)
 
 	for _, repo := range cfg.Repositories {
+		if repo.Invalid != "" {
+			continue
+		}
+
 		dir := expandTilde(repo.Directory)
 
 		// Safety check. If the cached index contains files but the repository
@@ -268,8 +300,8 @@ func main() {
 
 		if files, _, _ := m.LocalSize(repo.ID); files > 0 {
 			if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
-				warnf("Configured repository %q has index but directory %q is missing; not starting.", repo.ID, repo.Directory)
-				fatalf("Ensure that directory is present or remove repository from configuration.")
+				l.Warnf("Configured repository %q has index but directory %q is missing; not starting.", repo.ID, repo.Directory)
+				l.Fatalf("Ensure that directory is present or remove repository from configuration.")
 			}
 		}
 
@@ -277,6 +309,7 @@ func main() {
 		ensureDir(dir, -1)
 	}
 
+	m.CleanRepos()
 	m.ScanRepos()
 	m.SaveIndexes(confDir)
 
@@ -295,13 +328,17 @@ func main() {
 	go listenConnect(myID, m, tlsCfg)
 
 	for _, repo := range cfg.Repositories {
+		if repo.Invalid != "" {
+			continue
+		}
+
 		// Routine to pull blocks from other nodes to synchronize the local
 		// repository. Does not run when we are in read only (publish only) mode.
 		if repo.ReadOnly {
-			okf("Ready to synchronize %s (read only; no external updates accepted)", repo.ID)
+			l.Okf("Ready to synchronize %s (read only; no external updates accepted)", repo.ID)
 			m.StartRepoRO(repo.ID)
 		} else {
-			okf("Ready to synchronize %s (read-write)", repo.ID)
+			l.Okf("Ready to synchronize %s (read-write)", repo.ID)
 			m.StartRepoRW(repo.ID, cfg.Options.ParallelRequests)
 		}
 	}
@@ -323,7 +360,7 @@ func setupUPnP() int {
 	if len(cfg.Options.ListenAddress) == 1 {
 		_, portStr, err := net.SplitHostPort(cfg.Options.ListenAddress[0])
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 		} else {
 			// Set up incoming port forwarding, if necessary and possible
 			port, _ := strconv.Atoi(portStr)
@@ -334,19 +371,19 @@ func setupUPnP() int {
 					err := igd.AddPortMapping(upnp.TCP, r, port, "syncthing", 0)
 					if err == nil {
 						externalPort = r
-						infoln("Created UPnP port mapping - external port", externalPort)
+						l.Infoln("Created UPnP port mapping - external port", externalPort)
 						break
 					}
 				}
 				if externalPort == 0 {
-					warnln("Failed to create UPnP port mapping")
+					l.Warnln("Failed to create UPnP port mapping")
 				}
 			} else {
-				infof("No UPnP IGD device found, no port mapping created (%v)", err)
+				l.Infof("No UPnP IGD device found, no port mapping created (%v)", err)
 			}
 		}
 	} else {
-		warnln("Multiple listening addresses; not attempting UPnP port mapping")
+		l.Warnln("Multiple listening addresses; not attempting UPnP port mapping")
 	}
 	return externalPort
 }
@@ -355,7 +392,7 @@ func resetRepositories() {
 	suffix := fmt.Sprintf(".syncthing-reset-%d", time.Now().UnixNano())
 	for _, repo := range cfg.Repositories {
 		if _, err := os.Stat(repo.Directory); err == nil {
-			infof("Reset: Moving %s -> %s", repo.Directory, repo.Directory+suffix)
+			l.Infof("Reset: Moving %s -> %s", repo.Directory, repo.Directory+suffix)
 			os.Rename(repo.Directory, repo.Directory+suffix)
 		}
 	}
@@ -364,17 +401,17 @@ func resetRepositories() {
 	idxs, err := filepath.Glob(pat)
 	if err == nil {
 		for _, idx := range idxs {
-			infof("Reset: Removing %s", idx)
+			l.Infof("Reset: Removing %s", idx)
 			os.Remove(idx)
 		}
 	}
 }
 
 func restart() {
-	infoln("Restarting")
+	l.Infoln("Restarting")
 	if os.Getenv("SMF_FMRI") != "" || os.Getenv("STNORESTART") != "" {
 		// Solaris SMF
-		infoln("Service manager detected; exit instead of restart")
+		l.Infoln("Service manager detected; exit instead of restart")
 		stop <- true
 		return
 	}
@@ -385,7 +422,7 @@ func restart() {
 	}
 	pgm, err := exec.LookPath(os.Args[0])
 	if err != nil {
-		warnln(err)
+		l.Warnln(err)
 		return
 	}
 	proc, err := os.StartProcess(pgm, os.Args, &os.ProcAttr{
@@ -393,38 +430,42 @@ func restart() {
 		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr},
 	})
 	if err != nil {
-		fatalln(err)
+		l.Fatalln(err)
 	}
 	proc.Release()
 	stop <- true
 }
 
+func shutdown() {
+	stop <- true
+}
+
 var saveConfigCh = make(chan struct{})
 
 func saveConfigLoop(cfgFile string) {
 	for _ = range saveConfigCh {
 		fd, err := os.Create(cfgFile + ".tmp")
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 			continue
 		}
 
-		err = writeConfigXML(fd, cfg)
+		err = config.Save(fd, cfg)
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 			fd.Close()
 			continue
 		}
 
 		err = fd.Close()
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 			continue
 		}
 
-		err = Rename(cfgFile+".tmp", cfgFile)
+		err = model.Rename(cfgFile+".tmp", cfgFile)
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 		}
 	}
 }
@@ -433,7 +474,7 @@ func saveConfig() {
 	saveConfigCh <- struct{}{}
 }
 
-func listenConnect(myID string, m *Model, tlsCfg *tls.Config) {
+func listenConnect(myID string, m *model.Model, tlsCfg *tls.Config) {
 	var conns = make(chan *tls.Conn)
 
 	// Listen
@@ -441,26 +482,26 @@ func listenConnect(myID string, m *Model, tlsCfg *tls.Config) {
 		addr := addr
 		go func() {
 			if debugNet {
-				dlog.Println("listening on", addr)
+				l.Debugln("listening on", addr)
 			}
-			l, err := tls.Listen("tcp", addr, tlsCfg)
-			fatalErr(err)
+			listener, err := tls.Listen("tcp", addr, tlsCfg)
+			l.FatalErr(err)
 
 			for {
-				conn, err := l.Accept()
+				conn, err := listener.Accept()
 				if err != nil {
-					warnln(err)
+					l.Warnln(err)
 					continue
 				}
 
 				if debugNet {
-					dlog.Println("connect from", conn.RemoteAddr())
+					l.Debugln("connect from", conn.RemoteAddr())
 				}
 
 				tc := conn.(*tls.Conn)
 				err = tc.Handshake()
 				if err != nil {
-					warnln(err)
+					l.Warnln(err)
 					tc.Close()
 					continue
 				}
@@ -507,12 +548,12 @@ func listenConnect(myID string, m *Model, tlsCfg *tls.Config) {
 						addr = net.JoinHostPort(host, "22000")
 					}
 					if debugNet {
-						dlog.Println("dial", nodeCfg.NodeID, addr)
+						l.Debugln("dial", nodeCfg.NodeID, addr)
 					}
 					conn, err := tls.Dial("tcp", addr, tlsCfg)
 					if err != nil {
 						if debugNet {
-							dlog.Println(err)
+							l.Debugln(err)
 						}
 						continue
 					}
@@ -529,21 +570,21 @@ func listenConnect(myID string, m *Model, tlsCfg *tls.Config) {
 next:
 	for conn := range conns {
 		certs := conn.ConnectionState().PeerCertificates
-		if l := len(certs); l != 1 {
-			warnf("Got peer certificate list of length %d != 1; protocol error", l)
+		if cl := len(certs); cl != 1 {
+			l.Warnf("Got peer certificate list of length %d != 1; protocol error", cl)
 			conn.Close()
 			continue
 		}
 		remoteID := certID(certs[0].Raw)
 
 		if remoteID == myID {
-			warnf("Connected to myself (%s) - should not happen", remoteID)
+			l.Warnf("Connected to myself (%s) - should not happen", remoteID)
 			conn.Close()
 			continue
 		}
 
 		if m.ConnectedTo(remoteID) {
-			warnf("Connected to already connected node (%s)", remoteID)
+			l.Warnf("Connected to already connected node (%s)", remoteID)
 			conn.Close()
 			continue
 		}
@@ -566,17 +607,17 @@ next:
 func discovery(extPort int) *discover.Discoverer {
 	disc, err := discover.NewDiscoverer(myID, cfg.Options.ListenAddress)
 	if err != nil {
-		warnf("No discovery possible (%v)", err)
+		l.Warnf("No discovery possible (%v)", err)
 		return nil
 	}
 
 	if cfg.Options.LocalAnnEnabled {
-		infoln("Sending local discovery announcements")
+		l.Infoln("Sending local discovery announcements")
 		disc.StartLocal()
 	}
 
 	if cfg.Options.GlobalAnnEnabled {
-		infoln("Sending global discovery announcements")
+		l.Infoln("Sending global discovery announcements")
 		disc.StartGlobal(cfg.Options.GlobalAnnServer, uint16(extPort))
 	}
 
@@ -587,10 +628,10 @@ func ensureDir(dir string, mode int) {
 	fi, err := os.Stat(dir)
 	if os.IsNotExist(err) {
 		err := os.MkdirAll(dir, 0700)
-		fatalErr(err)
+		l.FatalErr(err)
 	} else if mode >= 0 && err == nil && int(fi.Mode()&0777) != mode {
 		err := os.Chmod(dir, os.FileMode(mode))
-		fatalErr(err)
+		l.FatalErr(err)
 	}
 }
 
@@ -633,8 +674,40 @@ func getHomeDir() string {
 	}
 
 	if home == "" {
-		fatalln("No home directory found - set $HOME (or the platform equivalent).")
+		l.Fatalln("No home directory found - set $HOME (or the platform equivalent).")
 	}
 
 	return home
 }
+
+// getFreePort returns a free TCP port fort listening on. The ports given are
+// tried in succession and the first to succeed is returned. If none succeed,
+// a random high port is returned.
+func getFreePort(host string, ports ...int) (int, error) {
+	for _, port := range ports {
+		c, err := net.Listen("tcp", fmt.Sprintf("%s:%d", host, port))
+		if err == nil {
+			c.Close()
+			return port, nil
+		}
+	}
+
+	c, err := net.Listen("tcp", host+":0")
+	if err != nil {
+		return 0, err
+	}
+	addr := c.Addr().String()
+	c.Close()
+
+	_, portstr, err := net.SplitHostPort(addr)
+	if err != nil {
+		return 0, err
+	}
+
+	port, err := strconv.Atoi(portstr)
+	if err != nil {
+		return 0, err
+	}
+
+	return port, nil
+}

cmd/syncthing/openurl.go

@@ -1,34 +0,0 @@
-/*
-Copyright 2011 Google Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"os/exec"
-	"runtime"
-)
-
-func openURL(url string) error {
-	if runtime.GOOS == "windows" {
-		return exec.Command("cmd.exe", "/C", "start "+url).Run()
-	}
-
-	if runtime.GOOS == "darwin" {
-		return exec.Command("open", url).Run()
-	}
-
-	return exec.Command("xdg-open", url).Run()
-}

cmd/syncthing/openurl_unix.go

@@ -0,0 +1,23 @@
+// +build !windows
+
+package main
+
+import (
+	"os/exec"
+	"runtime"
+	"syscall"
+)
+
+func openURL(url string) error {
+	switch runtime.GOOS {
+	case "darwin":
+		return exec.Command("open", url).Run()
+
+	default:
+		cmd := exec.Command("xdg-open", url)
+		cmd.SysProcAttr = &syscall.SysProcAttr{
+			Setpgid: true,
+		}
+		return cmd.Run()
+	}
+}

cmd/syncthing/openurl_windows.go

@@ -0,0 +1,9 @@
+// +build windows
+
+package main
+
+import "os/exec"
+
+func openURL(url string) error {
+	return exec.Command("cmd.exe", "/C", "start "+url).Run()
+}

cmd/syncthing/tls.go

@@ -41,10 +41,10 @@ func certSeed(bs []byte) int64 {
 }
 
 func newCertificate(dir string) {
-	infoln("Generating RSA certificate and key...")
+	l.Infoln("Generating RSA certificate and key...")
 
 	priv, err := rsa.GenerateKey(rand.Reader, tlsRSABits)
-	fatalErr(err)
+	l.FatalErr(err)
 
 	notBefore := time.Now()
 	notAfter := time.Date(2049, 12, 31, 23, 59, 59, 0, time.UTC)
@@ -63,17 +63,17 @@ func newCertificate(dir string) {
 	}
 
 	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
-	fatalErr(err)
+	l.FatalErr(err)
 
 	certOut, err := os.Create(filepath.Join(dir, "cert.pem"))
-	fatalErr(err)
+	l.FatalErr(err)
 	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
 	certOut.Close()
-	okln("Created RSA certificate file")
+	l.Okln("Created RSA certificate file")
 
 	keyOut, err := os.OpenFile(filepath.Join(dir, "key.pem"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
-	fatalErr(err)
+	l.FatalErr(err)
 	pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
 	keyOut.Close()
-	okln("Created RSA key file")
+	l.Okln("Created RSA key file")
 }

cmd/syncthing/upgrade_unix.go

@@ -0,0 +1,146 @@
+// +build !windows
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"bitbucket.org/kardianos/osext"
+)
+
+type githubRelease struct {
+	Tag      string        `json:"tag_name"`
+	Prelease bool          `json:"prerelease"`
+	Assets   []githubAsset `json:"assets"`
+}
+
+type githubAsset struct {
+	URL  string `json:"url"`
+	Name string `json:"name"`
+}
+
+func upgrade() error {
+	path, err := osext.Executable()
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.Get("https://api.github.com/repos/calmh/syncthing/releases?per_page=1")
+	if err != nil {
+		return err
+	}
+
+	var rels []githubRelease
+	json.NewDecoder(resp.Body).Decode(&rels)
+	resp.Body.Close()
+
+	if len(rels) != 1 {
+		return fmt.Errorf("Unexpected number of releases: %d", len(rels))
+	}
+	rel := rels[0]
+
+	if rel.Tag > Version {
+		l.Infof("Attempting upgrade to %s...", rel.Tag)
+	} else if rel.Tag == Version {
+		l.Okf("Already running the latest version, %s. Not upgrading.", Version)
+		return nil
+	} else {
+		l.Okf("Current version %s is newer than latest release %s. Not upgrading.", Version, rel.Tag)
+		return nil
+	}
+
+	expectedRelease := fmt.Sprintf("syncthing-%s-%s-%s.", runtime.GOOS, runtime.GOARCH, rel.Tag)
+	for _, asset := range rel.Assets {
+		if strings.HasPrefix(asset.Name, expectedRelease) {
+			if strings.HasSuffix(asset.Name, ".tar.gz") {
+				l.Infof("Downloading %s...", asset.Name)
+				fname, err := readTarGZ(asset.URL, filepath.Dir(path))
+				if err != nil {
+					return err
+				}
+
+				old := path + "." + Version
+				err = os.Rename(path, old)
+				if err != nil {
+					return err
+				}
+				err = os.Rename(fname, path)
+				if err != nil {
+					return err
+				}
+
+				l.Okf("Upgraded %q to %s.", path, rel.Tag)
+				l.Okf("Previous version saved in %q.", old)
+
+				return nil
+			}
+		}
+	}
+
+	return nil
+}
+
+func readTarGZ(url string, dir string) (string, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return "", err
+	}
+
+	req.Header.Add("Accept", "application/octet-stream")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	gr, err := gzip.NewReader(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	tr := tar.NewReader(gr)
+	if err != nil {
+		return "", err
+	}
+
+	// Iterate through the files in the archive.
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+
+		if path.Base(hdr.Name) == "syncthing" {
+			of, err := ioutil.TempFile(dir, "syncthing")
+			if err != nil {
+				return "", err
+			}
+			io.Copy(of, tr)
+			err = of.Close()
+			if err != nil {
+				os.Remove(of.Name())
+				return "", err
+			}
+
+			os.Chmod(of.Name(), os.FileMode(hdr.Mode))
+			return of.Name(), nil
+		}
+	}
+
+	return "", fmt.Errorf("No upgrade found")
+}

cmd/syncthing/upgrade_windows.go

@@ -0,0 +1,9 @@
+// +build windows
+
+package main
+
+import "errors"
+
+func upgrade() error {
+	return errors.New("Upgrade currently unsupported on Windows")
+}

cmd/syncthing/usage.go

@@ -32,7 +32,7 @@ func usageFor(fs *flag.FlagSet, usage string, extra string) func() {
 			var opt = "  -" + f.Name
 
 			if f.DefValue != "false" {
-				opt += "=" + f.DefValue
+				opt += "=" + fmt.Sprintf("%q", f.DefValue)
 			}
 
 			options = append(options, []string{opt, f.Usage})

config/config.go

@@ -1,7 +1,9 @@
-package main
+// Package config implements reading and writing of the syncthing configuration file.
+package config
 
 import (
 	"encoding/xml"
+	"fmt"
 	"io"
 	"os"
 	"reflect"
@@ -9,8 +11,11 @@ import (
 	"strconv"
 
 	"code.google.com/p/go.crypto/bcrypt"
+	"github.com/calmh/syncthing/logger"
 )
 
+var l = logger.DefaultLogger
+
 type Configuration struct {
 	Version      int                       `xml:"version,attr" default:"2"`
 	Repositories []RepositoryConfiguration `xml:"repository"`
@@ -25,6 +30,7 @@ type RepositoryConfiguration struct {
 	Directory string              `xml:"directory,attr"`
 	Nodes     []NodeConfiguration `xml:"node"`
 	ReadOnly  bool                `xml:"ro,attr"`
+	Invalid   string              `xml:"-"` // Set at runtime when there is an error, not saved
 	nodeIDs   []string
 }
 
@@ -44,7 +50,7 @@ type NodeConfiguration struct {
 }
 
 type OptionsConfiguration struct {
-	ListenAddress      []string `xml:"listenAddress" default:":22000"`
+	ListenAddress      []string `xml:"listenAddress" default:"0.0.0.0:22000"`
 	GlobalAnnServer    string   `xml:"globalAnnounceServer" default:"announce.syncthing.net:22025"`
 	GlobalAnnEnabled   bool     `xml:"globalAnnounceEnabled" default:"true"`
 	LocalAnnEnabled    bool     `xml:"localAnnounceEnabled" default:"true"`
@@ -52,13 +58,13 @@ type OptionsConfiguration struct {
 	MaxSendKbps        int      `xml:"maxSendKbps"`
 	RescanIntervalS    int      `xml:"rescanIntervalS" default:"60"`
 	ReconnectIntervalS int      `xml:"reconnectionIntervalS" default:"60"`
-	MaxChangeKbps      int      `xml:"maxChangeKbps" default:"1000"`
+	MaxChangeKbps      int      `xml:"maxChangeKbps" default:"10000"`
 	StartBrowser       bool     `xml:"startBrowser" default:"true"`
 	UPnPEnabled        bool     `xml:"upnpEnabled" default:"true"`
 
-	Deprecated_ReadOnly   bool   `xml:"readOnly,omitempty"`
-	Deprecated_GUIEnabled bool   `xml:"guiEnabled,omitempty"`
-	Deprecated_GUIAddress string `xml:"guiAddress,omitempty"`
+	Deprecated_ReadOnly   bool   `xml:"readOnly,omitempty" json:"-"`
+	Deprecated_GUIEnabled bool   `xml:"guiEnabled,omitempty" json:"-"`
+	Deprecated_GUIAddress string `xml:"guiAddress,omitempty" json:"-"`
 }
 
 type GUIConfiguration struct {
@@ -129,7 +135,7 @@ func fillNilSlices(data interface{}) error {
 	return nil
 }
 
-func writeConfigXML(wr io.Writer, cfg Configuration) error {
+func Save(wr io.Writer, cfg Configuration) error {
 	e := xml.NewEncoder(wr)
 	e.Indent("", "    ")
 	err := e.Encode(cfg)
@@ -154,7 +160,7 @@ func uniqueStrings(ss []string) []string {
 	return us
 }
 
-func readConfigXML(rd io.Reader, myID string) (Configuration, error) {
+func Load(rd io.Reader, myID string) (Configuration, error) {
 	var cfg Configuration
 
 	setDefaults(&cfg)
@@ -170,18 +176,40 @@ func readConfigXML(rd io.Reader, myID string) (Configuration, error) {
 
 	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
 
-	// Check for missing or duplicate repository ID:s
-	var seenRepos = map[string]bool{}
+	// Initialize an empty slice for repositories if the config has none
+	if cfg.Repositories == nil {
+		cfg.Repositories = []RepositoryConfiguration{}
+	}
+
+	// Check for missing, bad or duplicate repository ID:s
+	var seenRepos = map[string]*RepositoryConfiguration{}
+	var uniqueCounter int
 	for i := range cfg.Repositories {
-		if cfg.Repositories[i].ID == "" {
-			cfg.Repositories[i].ID = "default"
+		repo := &cfg.Repositories[i]
+
+		if len(repo.Directory) == 0 {
+			repo.Invalid = "empty directory"
+			continue
 		}
 
-		id := cfg.Repositories[i].ID
-		if seenRepos[id] {
-			panic("duplicate repository ID " + id)
+		if repo.ID == "" {
+			repo.ID = "default"
+		}
+
+		if seen, ok := seenRepos[repo.ID]; ok {
+			l.Warnf("Multiple repositories with ID %q; disabling", repo.ID)
+
+			seen.Invalid = "duplicate repository ID"
+			if seen.ID == repo.ID {
+				uniqueCounter++
+				seen.ID = fmt.Sprintf("%s~%d", repo.ID, uniqueCounter)
+			}
+			repo.Invalid = "duplicate repository ID"
+			uniqueCounter++
+			repo.ID = fmt.Sprintf("%s~%d", repo.ID, uniqueCounter)
+		} else {
+			seenRepos[repo.ID] = repo
 		}
-		seenRepos[id] = true
 	}
 
 	// Upgrade to v2 configuration if appropriate
@@ -193,7 +221,7 @@ func readConfigXML(rd io.Reader, myID string) (Configuration, error) {
 	if len(cfg.GUI.Password) > 0 && cfg.GUI.Password[0] != '$' {
 		hash, err := bcrypt.GenerateFromPassword([]byte(cfg.GUI.Password), 0)
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 		} else {
 			cfg.GUI.Password = string(hash)
 		}

config/config_test.go

@@ -1,4 +1,4 @@
-package main
+package config
 
 import (
 	"bytes"
@@ -10,7 +10,7 @@ import (
 
 func TestDefaultValues(t *testing.T) {
 	expected := OptionsConfiguration{
-		ListenAddress:      []string{":22000"},
+		ListenAddress:      []string{"0.0.0.0:22000"},
 		GlobalAnnServer:    "announce.syncthing.net:22025",
 		GlobalAnnEnabled:   true,
 		LocalAnnEnabled:    true,
@@ -18,12 +18,12 @@ func TestDefaultValues(t *testing.T) {
 		MaxSendKbps:        0,
 		RescanIntervalS:    60,
 		ReconnectIntervalS: 60,
-		MaxChangeKbps:      1000,
+		MaxChangeKbps:      10000,
 		StartBrowser:       true,
 		UPnPEnabled:        true,
 	}
 
-	cfg, err := readConfigXML(bytes.NewReader(nil), "nodeID")
+	cfg, err := Load(bytes.NewReader(nil), "nodeID")
 	if err != io.EOF {
 		t.Error(err)
 	}
@@ -66,7 +66,7 @@ func TestNodeConfig(t *testing.T) {
 `)
 
 	for i, data := range [][]byte{v1data, v2data} {
-		cfg, err := readConfigXML(bytes.NewReader(data), "node1")
+		cfg, err := Load(bytes.NewReader(data), "node1")
 		if err != nil {
 			t.Error(err)
 		}
@@ -121,7 +121,7 @@ func TestNoListenAddress(t *testing.T) {
 </configuration>
 `)
 
-	cfg, err := readConfigXML(bytes.NewReader(data), "nodeID")
+	cfg, err := Load(bytes.NewReader(data), "nodeID")
 	if err != nil {
 		t.Error(err)
 	}
@@ -170,7 +170,7 @@ func TestOverriddenValues(t *testing.T) {
 		UPnPEnabled:        false,
 	}
 
-	cfg, err := readConfigXML(bytes.NewReader(data), "nodeID")
+	cfg, err := Load(bytes.NewReader(data), "nodeID")
 	if err != nil {
 		t.Error(err)
 	}
@@ -215,7 +215,7 @@ func TestNodeAddresses(t *testing.T) {
 		},
 	}
 
-	cfg, err := readConfigXML(bytes.NewReader(data), "n4")
+	cfg, err := Load(bytes.NewReader(data), "n4")
 	if err != nil {
 		t.Error(err)
 	}

discover/PROTOCOL.md

@@ -36,12 +36,28 @@ The Announcement packet has the following structure:
      0                   1                   2                   3
      0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |                   Magic Number (0x029E4C77)                   |
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |                       Length of Node ID                       |
+    |                      Magic (0x029E4C77)                       |
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
     /                                                               /
-    \                   Node ID (variable length)                   \
+    \                        Node Structure                         \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                     Number of Extra Nodes                     |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                 Zero or more Node Structures                  \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+    Node Structure:
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                         Length of ID                          |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                     ID (variable length)                      \
     /                                                               /
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
     |                      Number of Addresses                      |
@@ -62,29 +78,37 @@ The Announcement packet has the following structure:
     \                     IP (variable length)                      \
     /                                                               /
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |          Port Number          |            0x0000             |
+    |             Port              |            0x0000             |
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 This is the XDR encoding of:
 
     struct Announcement {
-        unsigned int MagicNumber;
-        string NodeID<>;
+        unsigned int Magic;
+        Node This;
+        Node Extra<>;
+    }
+
+    struct Node {
+        string ID<>;
         Address Addresses<>;
     }
 
     struct Address {
         opaque IP<>;
-        unsigned short PortNumber;
+        unsigned short Port;
     }
 
-NodeID is padded to a multiple of 32 bits and all fields are in sent in
-network (big endian) byte order. In the Address structure, the IP field
-can be of three differnt kinds;
+The first Node structure contains information about the sending node.
+The following zero or more Extra nodes contain information about other
+nodes known to the sending node.
+
+In the Address structure, the IP field can be of three differnt kinds;
 
  - A zero length indicates that the IP address should be taken from the
    source address of the announcement packet, be it IPv4 or IPv6. The
-   source address must be a valid unicast address.
+   source address must be a valid unicast address. This is only valid
+   in the first node structure, not in the list of extras.
 
  - A four byte length indicates that the address is an IPv4 unicast
    address.

discover/debug.go

@@ -1,12 +1,13 @@
 package discover
 
 import (
-	"log"
 	"os"
 	"strings"
+
+	"github.com/calmh/syncthing/logger"
 )
 
 var (
-	dlog  = log.New(os.Stderr, "discover: ", log.Lmicroseconds|log.Lshortfile)
-	debug = strings.Contains(os.Getenv("STTRACE"), "discover")
+	debug = strings.Contains(os.Getenv("STTRACE"), "discover") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
 )

discover/discover.go

@@ -4,14 +4,13 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"log"
+	"io"
 	"net"
-	"strings"
 	"sync"
 	"time"
 
+	"github.com/calmh/syncthing/beacon"
 	"github.com/calmh/syncthing/buffers"
-	"github.com/calmh/syncthing/mc"
 )
 
 const (
@@ -23,7 +22,7 @@ type Discoverer struct {
 	listenAddrs      []string
 	localBcastIntv   time.Duration
 	globalBcastIntv  time.Duration
-	beacon           *mc.Beacon
+	beacon           *beacon.Beacon
 	registry         map[string][]string
 	registryLock     sync.RWMutex
 	extServer        string
@@ -44,12 +43,16 @@ var (
 const maxErrors = 30
 
 func NewDiscoverer(id string, addresses []string) (*Discoverer, error) {
+	b, err := beacon.New(21025)
+	if err != nil {
+		return nil, err
+	}
 	disc := &Discoverer{
 		myID:            id,
 		listenAddrs:     addresses,
 		localBcastIntv:  30 * time.Second,
 		globalBcastIntv: 1800 * time.Second,
-		beacon:          mc.NewBeacon("239.21.0.25", 21025),
+		beacon:          b,
 		registry:        make(map[string][]string),
 	}
 
@@ -76,15 +79,49 @@ func (d *Discoverer) ExtAnnounceOK() bool {
 	return d.extAnnounceOK
 }
 
+func (d *Discoverer) Lookup(node string) []string {
+	d.registryLock.Lock()
+	addr, ok := d.registry[node]
+	d.registryLock.Unlock()
+
+	if ok {
+		return addr
+	} else if len(d.extServer) != 0 {
+		// We might want to cache this, but not permanently so it needs some intelligence
+		return d.externalLookup(node)
+	}
+	return nil
+}
+
+func (d *Discoverer) Hint(node string, addrs []string) {
+	resAddrs := resolveAddrs(addrs)
+	d.registerNode(nil, Node{
+		ID:        node,
+		Addresses: resAddrs,
+	})
+}
+
+func (d *Discoverer) All() map[string][]string {
+	d.registryLock.RLock()
+	nodes := make(map[string][]string, len(d.registry))
+	for node, addrs := range d.registry {
+		addrsCopy := make([]string, len(addrs))
+		copy(addrsCopy, addrs)
+		nodes[node] = addrsCopy
+	}
+	d.registryLock.RUnlock()
+	return nodes
+}
+
 func (d *Discoverer) announcementPkt() []byte {
 	var addrs []Address
 	for _, astr := range d.listenAddrs {
 		addr, err := net.ResolveTCPAddr("tcp", astr)
 		if err != nil {
-			log.Printf("discover/announcement: %v: not announcing %s", err, astr)
+			l.Warnln("%v: not announcing %s", err, astr)
 			continue
 		} else if debug {
-			dlog.Printf("announcing %s: %#v", astr, addr)
+			l.Debugf("discover: announcing %s: %#v", astr, addr)
 		}
 		if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
 			addrs = append(addrs, Address{Port: uint16(addr.Port)})
@@ -95,18 +132,34 @@ func (d *Discoverer) announcementPkt() []byte {
 		}
 	}
 	var pkt = AnnounceV2{
-		Magic:     AnnouncementMagicV2,
-		NodeID:    d.myID,
-		Addresses: addrs,
+		Magic: AnnouncementMagicV2,
+		This:  Node{d.myID, addrs},
 	}
 	return pkt.MarshalXDR()
 }
 
 func (d *Discoverer) sendLocalAnnouncements() {
-	var buf = d.announcementPkt()
+	var addrs = resolveAddrs(d.listenAddrs)
+
+	var pkt = AnnounceV2{
+		Magic: AnnouncementMagicV2,
+		This:  Node{d.myID, addrs},
+	}
 
 	for {
-		d.beacon.Send(buf)
+		pkt.Extra = nil
+		d.registryLock.RLock()
+		for node, addrs := range d.registry {
+			if len(pkt.Extra) == 16 {
+				break
+			}
+
+			anode := Node{node, resolveAddrs(addrs)}
+			pkt.Extra = append(pkt.Extra, anode)
+		}
+		d.registryLock.RUnlock()
+
+		d.beacon.Send(pkt.MarshalXDR())
 
 		select {
 		case <-d.localBcastTick:
@@ -118,22 +171,21 @@ func (d *Discoverer) sendLocalAnnouncements() {
 func (d *Discoverer) sendExternalAnnouncements() {
 	remote, err := net.ResolveUDPAddr("udp", d.extServer)
 	if err != nil {
-		log.Printf("discover/external: %v; no external announcements", err)
+		l.Warnf("Global discovery: %v; no external announcements", err)
 		return
 	}
 
 	conn, err := net.ListenUDP("udp", nil)
 	if err != nil {
-		log.Printf("discover/external: %v; no external announcements", err)
+		l.Warnf("Global discovery: %v; no external announcements", err)
 		return
 	}
 
 	var buf []byte
 	if d.extPort != 0 {
 		var pkt = AnnounceV2{
-			Magic:     AnnouncementMagicV2,
-			NodeID:    d.myID,
-			Addresses: []Address{{Port: d.extPort}},
+			Magic: AnnouncementMagicV2,
+			This:  Node{d.myID, []Address{{Port: d.extPort}}},
 		}
 		buf = pkt.MarshalXDR()
 	} else {
@@ -145,12 +197,14 @@ func (d *Discoverer) sendExternalAnnouncements() {
 		var ok bool
 
 		if debug {
-			dlog.Printf("send announcement -> %v\n%s", remote, hex.Dump(buf))
+			l.Debugf("discover: send announcement -> %v\n%s", remote, hex.Dump(buf))
 		}
 
 		_, err = conn.WriteTo(buf, remote)
 		if err != nil {
-			log.Println("discover/write: warning:", err)
+			if debug {
+				l.Debugln("discover: warning:", err)
+			}
 			errCounter++
 			ok = false
 		} else {
@@ -161,7 +215,7 @@ func (d *Discoverer) sendExternalAnnouncements() {
 			time.Sleep(1 * time.Second)
 			res := d.externalLookup(d.myID)
 			if debug {
-				dlog.Println("external lookup check:", res)
+				l.Debugln("discover: external lookup check:", res)
 			}
 			ok = len(res) > 0
 
@@ -177,7 +231,7 @@ func (d *Discoverer) sendExternalAnnouncements() {
 			time.Sleep(60 * time.Second)
 		}
 	}
-	log.Printf("discover/write: %v: stopping due to too many errors: %v", remote, err)
+	l.Warnf("Global discovery: %v: stopping due to too many errors: %v", remote, err)
 }
 
 func (d *Discoverer) recvAnnouncements() {
@@ -185,77 +239,105 @@ func (d *Discoverer) recvAnnouncements() {
 		buf, addr := d.beacon.Recv()
 
 		if debug {
-			dlog.Printf("read announcement:\n%s", hex.Dump(buf))
+			l.Debugf("discover: read announcement:\n%s", hex.Dump(buf))
 		}
 
 		var pkt AnnounceV2
 		err := pkt.UnmarshalXDR(buf)
-		if err != nil {
+		if err != nil && err != io.EOF {
 			continue
 		}
 
 		if debug {
-			dlog.Printf("parsed announcement: %#v", pkt)
+			l.Debugf("discover: parsed announcement: %#v", pkt)
 		}
 
-		if pkt.NodeID != d.myID {
-			var addrs []string
-			for _, a := range pkt.Addresses {
-				var nodeAddr string
-				if len(a.IP) > 0 {
-					nodeAddr = fmt.Sprintf("%s:%d", ipStr(a.IP), a.Port)
-				} else {
-					ua := addr.(*net.UDPAddr)
-					ua.Port = int(a.Port)
-					nodeAddr = ua.String()
-				}
-				addrs = append(addrs, nodeAddr)
-			}
-			if debug {
-				dlog.Printf("register: %#v", addrs)
-			}
-			d.registryLock.Lock()
-			_, seen := d.registry[pkt.NodeID]
-			if !seen {
-				select {
-				case d.forcedBcastTick <- time.Now():
+		var newNode bool
+		if pkt.This.ID != d.myID {
+			n := d.registerNode(addr, pkt.This)
+			newNode = newNode || n
+			for _, node := range pkt.Extra {
+				if node.ID != d.myID {
+					n := d.registerNode(nil, node)
+					newNode = newNode || n
 				}
 			}
-			d.registry[pkt.NodeID] = addrs
-			d.registryLock.Unlock()
+		}
+
+		if newNode {
+			select {
+			case d.forcedBcastTick <- time.Now():
+			}
 		}
 	}
 }
 
+func (d *Discoverer) registerNode(addr net.Addr, node Node) bool {
+	var addrs []string
+	for _, a := range node.Addresses {
+		var nodeAddr string
+		if len(a.IP) > 0 {
+			nodeAddr = fmt.Sprintf("%s:%d", net.IP(a.IP), a.Port)
+			addrs = append(addrs, nodeAddr)
+		} else if addr != nil {
+			ua := addr.(*net.UDPAddr)
+			ua.Port = int(a.Port)
+			nodeAddr = ua.String()
+			addrs = append(addrs, nodeAddr)
+		}
+	}
+	if len(addrs) == 0 {
+		if debug {
+			l.Debugln("discover: no valid address for", node.ID)
+		}
+	}
+	if debug {
+		l.Debugf("discover: register: %s -> %#v", node.ID, addrs)
+	}
+	d.registryLock.Lock()
+	_, seen := d.registry[node.ID]
+	d.registry[node.ID] = addrs
+	d.registryLock.Unlock()
+	return !seen
+}
+
 func (d *Discoverer) externalLookup(node string) []string {
 	extIP, err := net.ResolveUDPAddr("udp", d.extServer)
 	if err != nil {
-		log.Printf("discover/external: %v; no external lookup", err)
+		if debug {
+			l.Debugf("discover: %v; no external lookup", err)
+		}
 		return nil
 	}
 
 	conn, err := net.DialUDP("udp", nil, extIP)
 	if err != nil {
-		log.Printf("discover/external: %v; no external lookup", err)
+		if debug {
+			l.Debugf("discover: %v; no external lookup", err)
+		}
 		return nil
 	}
 	defer conn.Close()
 
 	err = conn.SetDeadline(time.Now().Add(5 * time.Second))
 	if err != nil {
-		log.Printf("discover/external: %v; no external lookup", err)
+		if debug {
+			l.Debugf("discover: %v; no external lookup", err)
+		}
 		return nil
 	}
 
 	buf := QueryV2{QueryMagicV2, node}.MarshalXDR()
 	_, err = conn.Write(buf)
 	if err != nil {
-		log.Printf("discover/external: %v; no external lookup", err)
+		if debug {
+			l.Debugf("discover: %v; no external lookup", err)
+		}
 		return nil
 	}
 	buffers.Put(buf)
 
-	buf = buffers.Get(256)
+	buf = buffers.Get(2048)
 	defer buffers.Put(buf)
 
 	n, err := conn.Read(buf)
@@ -264,60 +346,59 @@ func (d *Discoverer) externalLookup(node string) []string {
 			// Expected if the server doesn't know about requested node ID
 			return nil
 		}
-		log.Printf("discover/external/read: %v; no external lookup", err)
+		if debug {
+			l.Debugf("discover: %v; no external lookup", err)
+		}
 		return nil
 	}
 
 	if debug {
-		dlog.Printf("read external:\n%s", hex.Dump(buf[:n]))
+		l.Debugf("discover: read external:\n%s", hex.Dump(buf[:n]))
 	}
 
 	var pkt AnnounceV2
 	err = pkt.UnmarshalXDR(buf[:n])
-	if err != nil {
-		log.Println("discover/external/decode:", err)
+	if err != nil && err != io.EOF {
+		if debug {
+			l.Debugln("discover:", err)
+		}
 		return nil
 	}
 
 	if debug {
-		dlog.Printf("parsed external: %#v", pkt)
+		l.Debugf("discover: parsed external: %#v", pkt)
 	}
 
 	var addrs []string
-	for _, a := range pkt.Addresses {
-		var nodeAddr string
-		if len(a.IP) > 0 {
-			nodeAddr = fmt.Sprintf("%s:%d", ipStr(a.IP), a.Port)
-		}
+	for _, a := range pkt.This.Addresses {
+		nodeAddr := fmt.Sprintf("%s:%d", net.IP(a.IP), a.Port)
 		addrs = append(addrs, nodeAddr)
 	}
 	return addrs
 }
 
-func (d *Discoverer) Lookup(node string) []string {
-	d.registryLock.Lock()
-	addr, ok := d.registry[node]
-	d.registryLock.Unlock()
-
-	if ok {
-		return addr
-	} else if len(d.extServer) != 0 {
-		// We might want to cache this, but not permanently so it needs some intelligence
-		return d.externalLookup(node)
+func addrToAddr(addr *net.TCPAddr) Address {
+	if len(addr.IP) == 0 || addr.IP.IsUnspecified() {
+		return Address{Port: uint16(addr.Port)}
+	} else if bs := addr.IP.To4(); bs != nil {
+		return Address{IP: bs, Port: uint16(addr.Port)}
+	} else if bs := addr.IP.To16(); bs != nil {
+		return Address{IP: bs, Port: uint16(addr.Port)}
 	}
-	return nil
+	return Address{}
 }
 
-func ipStr(ip []byte) string {
-	var f = "%d"
-	var s = "."
-	if len(ip) > 4 {
-		f = "%x"
-		s = ":"
+func resolveAddrs(addrs []string) []Address {
+	var raddrs []Address
+	for _, addrStr := range addrs {
+		addrRes, err := net.ResolveTCPAddr("tcp", addrStr)
+		if err != nil {
+			continue
+		}
+		addr := addrToAddr(addrRes)
+		if len(addr.IP) > 0 {
+			raddrs = append(raddrs, addr)
+		}
 	}
-	var ss = make([]string, len(ip))
-	for i := range ip {
-		ss[i] = fmt.Sprintf(f, ip[i])
-	}
-	return strings.Join(ss, s)
+	return raddrs
 }

discover/packets.go

@@ -11,8 +11,13 @@ type QueryV2 struct {
 }
 
 type AnnounceV2 struct {
-	Magic     uint32
-	NodeID    string    // max:64
+	Magic uint32
+	This  Node
+	Extra []Node // max:16
+}
+
+type Node struct {
+	ID        string    // max:64
 	Addresses []Address // max:16
 }
 

discover/packets_xdr.go

@@ -59,16 +59,13 @@ func (o AnnounceV2) MarshalXDR() []byte {
 
 func (o AnnounceV2) encodeXDR(xw *xdr.Writer) (int, error) {
 	xw.WriteUint32(o.Magic)
-	if len(o.NodeID) > 64 {
+	o.This.encodeXDR(xw)
+	if len(o.Extra) > 16 {
 		return xw.Tot(), xdr.ErrElementSizeExceeded
 	}
-	xw.WriteString(o.NodeID)
-	if len(o.Addresses) > 16 {
-		return xw.Tot(), xdr.ErrElementSizeExceeded
-	}
-	xw.WriteUint32(uint32(len(o.Addresses)))
-	for i := range o.Addresses {
-		o.Addresses[i].encodeXDR(xw)
+	xw.WriteUint32(uint32(len(o.Extra)))
+	for i := range o.Extra {
+		o.Extra[i].encodeXDR(xw)
 	}
 	return xw.Tot(), xw.Error()
 }
@@ -86,7 +83,58 @@ func (o *AnnounceV2) UnmarshalXDR(bs []byte) error {
 
 func (o *AnnounceV2) decodeXDR(xr *xdr.Reader) error {
 	o.Magic = xr.ReadUint32()
-	o.NodeID = xr.ReadStringMax(64)
+	(&o.This).decodeXDR(xr)
+	_ExtraSize := int(xr.ReadUint32())
+	if _ExtraSize > 16 {
+		return xdr.ErrElementSizeExceeded
+	}
+	o.Extra = make([]Node, _ExtraSize)
+	for i := range o.Extra {
+		(&o.Extra[i]).decodeXDR(xr)
+	}
+	return xr.Error()
+}
+
+func (o Node) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o Node) MarshalXDR() []byte {
+	var buf bytes.Buffer
+	var xw = xdr.NewWriter(&buf)
+	o.encodeXDR(xw)
+	return buf.Bytes()
+}
+
+func (o Node) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.ID) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.ID)
+	if len(o.Addresses) > 16 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteUint32(uint32(len(o.Addresses)))
+	for i := range o.Addresses {
+		o.Addresses[i].encodeXDR(xw)
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Node) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *Node) UnmarshalXDR(bs []byte) error {
+	var buf = bytes.NewBuffer(bs)
+	var xr = xdr.NewReader(buf)
+	return o.decodeXDR(xr)
+}
+
+func (o *Node) decodeXDR(xr *xdr.Reader) error {
+	o.ID = xr.ReadStringMax(64)
 	_AddressesSize := int(xr.ReadUint32())
 	if _AddressesSize > 16 {
 		return xdr.ErrElementSizeExceeded

files/debug.go

@@ -1,12 +1,13 @@
 package files
 
 import (
-	"log"
 	"os"
 	"strings"
+
+	"github.com/calmh/syncthing/logger"
 )
 
 var (
-	dlog  = log.New(os.Stderr, "files: ", log.Lmicroseconds|log.Lshortfile)
-	debug = strings.Contains(os.Getenv("STTRACE"), "files")
+	debug = strings.Contains(os.Getenv("STTRACE"), "files") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
 )

files/set.go

@@ -38,7 +38,7 @@ func NewSet() *Set {
 
 func (m *Set) Replace(id uint, fs []scanner.File) {
 	if debug {
-		dlog.Printf("Replace(%d, [%d])", id, len(fs))
+		l.Debugf("Replace(%d, [%d])", id, len(fs))
 	}
 	if id > 63 {
 		panic("Connection ID must be in the range 0 - 63 inclusive")
@@ -54,7 +54,7 @@ func (m *Set) Replace(id uint, fs []scanner.File) {
 
 func (m *Set) ReplaceWithDelete(id uint, fs []scanner.File) {
 	if debug {
-		dlog.Printf("ReplaceWithDelete(%d, [%d])", id, len(fs))
+		l.Debugf("ReplaceWithDelete(%d, [%d])", id, len(fs))
 	}
 	if id > 63 {
 		panic("Connection ID must be in the range 0 - 63 inclusive")
@@ -84,7 +84,7 @@ func (m *Set) ReplaceWithDelete(id uint, fs []scanner.File) {
 				}
 				fs = append(fs, cf)
 				if debug {
-					dlog.Println("deleted:", ck.Name)
+					l.Debugln("deleted:", ck.Name)
 				}
 			}
 		}
@@ -96,7 +96,7 @@ func (m *Set) ReplaceWithDelete(id uint, fs []scanner.File) {
 
 func (m *Set) Update(id uint, fs []scanner.File) {
 	if debug {
-		dlog.Printf("Update(%d, [%d])", id, len(fs))
+		l.Debugf("Update(%d, [%d])", id, len(fs))
 	}
 	m.Lock()
 	m.update(id, fs)
@@ -106,10 +106,10 @@ func (m *Set) Update(id uint, fs []scanner.File) {
 
 func (m *Set) Need(id uint) []scanner.File {
 	if debug {
-		dlog.Printf("Need(%d)", id)
+		l.Debugf("Need(%d)", id)
 	}
-	var fs = make([]scanner.File, 0, len(m.globalKey)/2) // Just a guess, but avoids too many reallocations
 	m.Lock()
+	var fs = make([]scanner.File, 0, len(m.globalKey)/2) // Just a guess, but avoids too many reallocations
 	rkID := m.remoteKey[id]
 	for gk, gf := range m.files {
 		if !gf.Global {
@@ -130,7 +130,7 @@ func (m *Set) Need(id uint) []scanner.File {
 
 func (m *Set) Have(id uint) []scanner.File {
 	if debug {
-		dlog.Printf("Have(%d)", id)
+		l.Debugf("Have(%d)", id)
 	}
 	var fs = make([]scanner.File, 0, len(m.remoteKey[id]))
 	m.Lock()
@@ -143,10 +143,10 @@ func (m *Set) Have(id uint) []scanner.File {
 
 func (m *Set) Global() []scanner.File {
 	if debug {
-		dlog.Printf("Global()")
+		l.Debugf("Global()")
 	}
-	var fs = make([]scanner.File, 0, len(m.globalKey))
 	m.Lock()
+	var fs = make([]scanner.File, 0, len(m.globalKey))
 	for _, file := range m.files {
 		if file.Global {
 			fs = append(fs, file.File)
@@ -160,7 +160,7 @@ func (m *Set) Get(id uint, file string) scanner.File {
 	m.Lock()
 	defer m.Unlock()
 	if debug {
-		dlog.Printf("Get(%d, %q)", id, file)
+		l.Debugf("Get(%d, %q)", id, file)
 	}
 	return m.files[m.remoteKey[id][file]].File
 }
@@ -169,7 +169,7 @@ func (m *Set) GetGlobal(file string) scanner.File {
 	m.Lock()
 	defer m.Unlock()
 	if debug {
-		dlog.Printf("GetGlobal(%q)", file)
+		l.Debugf("GetGlobal(%q)", file)
 	}
 	return m.files[m.globalKey[file]].File
 }
@@ -179,7 +179,7 @@ func (m *Set) Availability(name string) bitset {
 	defer m.Unlock()
 	av := m.globalAvailability[name]
 	if debug {
-		dlog.Printf("Availability(%q) = %0x", name, av)
+		l.Debugf("Availability(%q) = %0x", name, av)
 	}
 	return av
 }
@@ -188,7 +188,7 @@ func (m *Set) Changes(id uint) uint64 {
 	m.Lock()
 	defer m.Unlock()
 	if debug {
-		dlog.Printf("Changes(%d)", id)
+		l.Debugf("Changes(%d)", id)
 	}
 	return m.changes[id]
 }

gui/angular.min.js

@@ -200,4 +200,3 @@ isolateScope:Ea.isolateScope,controller:Ea.controller,injector:Ea.injector,inher
 $$csp:Tb});Ta=Uc(Z);try{Ta("ngLocale")}catch(c){Ta("ngLocale",[]).provider("$locale",rd)}Ta("ng",["ngLocale"],["$provide",function(a){a.provider({$$sanitizeUri:Bd});a.provider("$compile",ic).directive({a:Wd,input:Lc,textarea:Lc,form:Xd,script:De,select:Ge,style:Ie,option:He,ngBind:he,ngBindHtml:je,ngBindTemplate:ie,ngClass:ke,ngClassEven:me,ngClassOdd:le,ngCloak:ne,ngController:oe,ngForm:Yd,ngHide:xe,ngIf:pe,ngInclude:qe,ngInit:se,ngNonBindable:te,ngPluralize:ue,ngRepeat:ve,ngShow:we,ngStyle:ye,ngSwitch:ze,
 ngSwitchWhen:Ae,ngSwitchDefault:Be,ngOptions:Fe,ngTransclude:Ce,ngModel:ce,ngList:ee,ngChange:de,required:Mc,ngRequired:Mc,ngValue:ge}).directive({ngInclude:re}).directive(Nb).directive(Nc);a.provider({$anchorScroll:cd,$animate:Td,$browser:ed,$cacheFactory:fd,$controller:id,$document:jd,$exceptionHandler:kd,$filter:Ac,$interpolate:pd,$interval:qd,$http:ld,$httpBackend:nd,$location:td,$log:ud,$parse:xd,$rootScope:Ad,$q:yd,$sce:Ed,$sceDelegate:Dd,$sniffer:Fd,$templateCache:gd,$timeout:Gd,$window:Hd})}])})(Na);
 A(Q).ready(function(){Sc(Q,Yb)})})(window,document);!angular.$$csp()&&angular.element(document).find("head").prepend('<style type="text/css">@charset "UTF-8";[ng\\:cloak],[ng-cloak],[data-ng-cloak],[x-ng-cloak],.ng-cloak,.x-ng-cloak,.ng-hide{display:none !important;}ng\\:form{display:block;}</style>');
-//# sourceMappingURL=angular.min.js.map

gui/app.js

@@ -4,6 +4,7 @@
 'use strict';
 
 var syncthing = angular.module('syncthing', []);
+var urlbase = 'rest';
 
 syncthing.controller('SyncthingCtrl', function ($scope, $http) {
     var prevDate = 0;
@@ -18,7 +19,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
     $scope.errors = [];
     $scope.seenError = '';
     $scope.model = {};
-    $scope.repos = [];
+    $scope.repos = {};
 
     // Strings before bools look better
     $scope.settings = [
@@ -43,10 +44,12 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
 
     function getSucceeded() {
         if (!getOK) {
+            $scope.init();
             $('#networkError').modal('hide');
             getOK = true;
         }
         if (restarting) {
+            $scope.init();
             $('#restarting').modal('hide');
             restarting = false;
         }
@@ -62,31 +65,19 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         }
     }
 
-    function nodeCompare(a, b) {
-        if (typeof a.Name !== 'undefined' && typeof b.Name !== 'undefined') {
-            if (a.Name < b.Name)
-                return -1;
-            return a.Name > b.Name;
-        }
-        if (a.NodeID < b.NodeID) {
-            return -1;
-        }
-        return a.NodeID > b.NodeID;
-    }
-
     $scope.refresh = function () {
-        $http.get('/rest/system').success(function (data) {
+        $http.get(urlbase + '/system').success(function (data) {
             getSucceeded();
             $scope.system = data;
         }).error(function () {
             getFailed();
         });
-        $scope.repos.forEach(function (repo) {
-            $http.get('/rest/model?repo=' + encodeURIComponent(repo.ID)).success(function (data) {
-                $scope.model[repo.ID] = data;
+        Object.keys($scope.repos).forEach(function (id) {
+            $http.get(urlbase + '/model?repo=' + encodeURIComponent(id)).success(function (data) {
+                $scope.model[id] = data;
             });
         });
-        $http.get('/rest/connections').success(function (data) {
+        $http.get(urlbase + '/connections').success(function (data) {
             var now = Date.now(),
             td = (now - prevDate) / 1000,
             id;
@@ -111,7 +102,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
             }
             $scope.connections = data;
         });
-        $http.get('/rest/errors').success(function (data) {
+        $http.get(urlbase + '/errors').success(function (data) {
             $scope.errors = data;
         });
     };
@@ -121,6 +112,10 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
             return 'Unknown';
         }
 
+        if ($scope.model[repo].invalid !== '') {
+            return 'Stopped';
+        }
+
         var state = '' + $scope.model[repo].state;
         state = state[0].toUpperCase() + state.substr(1);
 
@@ -136,6 +131,10 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
             return 'text-info';
         }
 
+        if ($scope.model[repo].invalid !== '') {
+            return 'text-danger';
+        }
+
         var state = '' + $scope.model[repo].state;
         if (state == 'idle') {
             return 'text-success';
@@ -231,6 +230,18 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         return nodeCfg.NodeID.substr(0, 6);
     };
 
+    $scope.thisNodeName = function () {
+        var nodes = $scope.thisNode();
+        if (typeof nodes === 'undefined' || nodes.length != 1) {
+            return "(unknown node)";
+        }
+        var nodeCfg = nodes[0];
+        if (nodeCfg.Name) {
+            return nodeCfg.Name;
+        }
+        return nodeCfg.NodeID.substr(0, 6);
+    };
+
     $scope.editSettings = function () {
         $('#settings').modal({backdrop: 'static', keyboard: true});
     }
@@ -238,14 +249,14 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
     $scope.saveSettings = function () {
         $scope.configInSync = false;
         $scope.config.Options.ListenAddress = $scope.config.Options.ListenStr.split(',').map(function (x) { return x.trim(); });
-        $http.post('/rest/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
+        $http.post(urlbase + '/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
         $('#settings').modal("hide");
     };
 
     $scope.restart = function () {
         restarting = true;
         $('#restarting').modal('show');
-        $http.post('/rest/restart');
+        $http.post(urlbase + '/restart');
         $scope.configInSync = true;
     };
 
@@ -254,6 +265,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         $scope.editingExisting = true;
         $scope.editingSelf = (nodeCfg.NodeID == $scope.myID);
         $scope.currentNode.AddressesStr = nodeCfg.Addresses.join(', ');
+        $scope.nodeEditor.$setPristine();
         $('#editNode').modal({backdrop: 'static', keyboard: true});
     };
 
@@ -261,6 +273,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         $scope.currentNode = {AddressesStr: 'dynamic'};
         $scope.editingExisting = false;
         $scope.editingSelf = false;
+        $scope.nodeEditor.$setPristine();
         $('#editNode').modal({backdrop: 'static', keyboard: true});
     };
 
@@ -275,14 +288,14 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         });
         $scope.config.Nodes = $scope.nodes;
 
-        for (var i = 0; i < $scope.repos.length; i++) {
-            $scope.repos[i].Nodes = $scope.repos[i].Nodes.filter(function (n) {
+        for (var id in repos) {
+            $scope.repos[id].Nodes = $scope.repos[id].Nodes.filter(function (n) {
                 return n.NodeID !== $scope.currentNode.NodeID;
             });
         }
 
         $scope.configInSync = false;
-        $http.post('/rest/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
+        $http.post(urlbase + '/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
     };
 
     $scope.saveNode = function () {
@@ -291,6 +304,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         $scope.configInSync = false;
         $('#editNode').modal('hide');
         nodeCfg = $scope.currentNode;
+        nodeCfg.NodeID = nodeCfg.NodeID.replace(/ /g, '').trim();
         nodeCfg.Addresses = nodeCfg.AddressesStr.split(',').map(function (x) { return x.trim(); });
 
         done = false;
@@ -309,7 +323,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         $scope.nodes.sort(nodeCompare);
         $scope.config.Nodes = $scope.nodes;
 
-        $http.post('/rest/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
+        $http.post(urlbase + '/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
     };
 
     $scope.otherNodes = function () {
@@ -337,7 +351,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
 
     $scope.clearErrors = function () {
         $scope.seenError = $scope.errors[$scope.errors.length - 1].Time;
-        $http.post('/rest/error/clear');
+        $http.post(urlbase + '/error/clear');
     };
 
     $scope.friendlyNodes = function (str) {
@@ -348,18 +362,24 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         return str;
     };
 
+    $scope.repoList = function () {
+        return repoList($scope.repos);
+    }
+
     $scope.editRepo = function (nodeCfg) {
         $scope.currentRepo = $.extend({selectedNodes: {}}, nodeCfg);
         $scope.currentRepo.Nodes.forEach(function (n) {
             $scope.currentRepo.selectedNodes[n.NodeID] = true;
         });
         $scope.editingExisting = true;
+        $scope.repoEditor.$setPristine();
         $('#editRepo').modal({backdrop: 'static', keyboard: true});
     };
 
     $scope.addRepo = function () {
         $scope.currentRepo = {selectedNodes: {}};
         $scope.editingExisting = false;
+        $scope.repoEditor.$setPristine();
         $('#editRepo').modal({backdrop: 'static', keyboard: true});
     };
 
@@ -378,22 +398,10 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         }
         delete repoCfg.selectedNodes;
 
-        done = false;
-        for (i = 0; i < $scope.repos.length; i++) {
-            if ($scope.repos[i].ID === repoCfg.ID) {
-                $scope.repos[i] = repoCfg;
-                done = true;
-                break;
-            }
-        }
+        $scope.repos[repoCfg.ID] = repoCfg;
+        $scope.config.Repositories = repoList($scope.repos);
 
-        if (!done) {
-            $scope.repos.push(repoCfg);
-        }
-
-        $scope.config.Repositories = $scope.repos;
-
-        $http.post('/rest/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
+        $http.post(urlbase + '/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
     };
 
     $scope.deleteRepo = function () {
@@ -402,45 +410,80 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
             return;
         }
 
-        $scope.repos = $scope.repos.filter(function (r) {
-            return r.ID !== $scope.currentRepo.ID;
-        });
-
-        $scope.config.Repositories = $scope.repos;
+        delete $scope.repos[$scope.currentRepo.ID];
+        $scope.config.Repositories = repoList($scope.repos);
 
         $scope.configInSync = false;
-        $http.post('/rest/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
+        $http.post(urlbase + '/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
     };
 
-    $http.get('/rest/version').success(function (data) {
-        $scope.version = data;
-    });
+    $scope.init = function() {
+        $http.get(urlbase + '/version').success(function (data) {
+            $scope.version = data;
+        });
 
-    $http.get('/rest/system').success(function (data) {
-        $scope.system = data;
-        $scope.myID = data.myID;
-    });
+        $http.get(urlbase + '/system').success(function (data) {
+            $scope.system = data;
+            $scope.myID = data.myID;
+        });
 
-    $http.get('/rest/config').success(function (data) {
-        $scope.config = data;
-        $scope.config.Options.ListenStr = $scope.config.Options.ListenAddress.join(', ');
+        $http.get(urlbase + '/config').success(function (data) {
+            $scope.config = data;
+            $scope.config.Options.ListenStr = $scope.config.Options.ListenAddress.join(', ');
 
-        var nodes = $scope.config.Nodes;
-        nodes.sort(nodeCompare);
-        $scope.nodes = nodes;
+            $scope.nodes = $scope.config.Nodes;
+            $scope.nodes.sort(nodeCompare);
 
-        $scope.repos = $scope.config.Repositories;
+            $scope.repos = repoMap($scope.config.Repositories);
 
-        $scope.refresh();
-    });
+            $scope.refresh();
+        });
 
-    $http.get('/rest/config/sync').success(function (data) {
-        $scope.configInSync = data.configInSync;
-    });
+        $http.get(urlbase + '/config/sync').success(function (data) {
+            $scope.configInSync = data.configInSync;
+        });
+    };
 
+    $scope.init();
     setInterval($scope.refresh, 10000);
 });
 
+function nodeCompare(a, b) {
+    if (typeof a.Name !== 'undefined' && typeof b.Name !== 'undefined') {
+        if (a.Name < b.Name)
+            return -1;
+        return a.Name > b.Name;
+    }
+    if (a.NodeID < b.NodeID) {
+        return -1;
+    }
+    return a.NodeID > b.NodeID;
+}
+
+function repoCompare(a, b) {
+    if (a.Directory < b.Directory) {
+        return -1;
+    }
+    return a.Directory > b.Directory;
+}
+
+function repoMap(l) {
+    var m = {};
+    l.forEach(function (r) {
+        m[r.ID] = r;
+    });
+    return m;
+}
+
+function repoList(m) {
+    var l = [];
+    for (var id in m) {
+        l.push(m[id])
+    }
+    l.sort(repoCompare);
+    return l;
+}
+
 function decimals(val, num) {
     var digits, decs;
 
@@ -516,6 +559,17 @@ syncthing.filter('alwaysNumber', function () {
     };
 });
 
+syncthing.filter('chunkID', function () {
+    return function (input) {
+        if (input === undefined)
+            return "";
+        var parts = input.match(/.{1,6}/g);
+        if (!parts)
+            return "";
+        return parts.join(' ');
+    }
+});
+
 syncthing.directive('optionEditor', function () {
     return {
         restrict: 'C',
@@ -527,3 +581,24 @@ syncthing.directive('optionEditor', function () {
         template: '<input type="text" ng-model="config.Options[setting.id]"></input>',
     };
 });
+
+syncthing.directive('uniqueRepo', function() {
+    return {
+        require: 'ngModel',
+        link: function(scope, elm, attrs, ctrl) {
+            ctrl.$parsers.unshift(function(viewValue) {
+                if (scope.editingExisting) {
+                    // we shouldn't validate
+                    ctrl.$setValidity('uniqueRepo', true);
+                } else if (scope.repos[viewValue]) {
+                    // the repo exists already
+                    ctrl.$setValidity('uniqueRepo', false);
+                } else {
+                    // the repo is unique
+                    ctrl.$setValidity('uniqueRepo', true);
+                }
+                return viewValue;
+            });
+        }
+    };
+});

gui/index.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html lang="en" ng-app="syncthing">
+<html lang="en" ng-app="syncthing" ng-controller="SyncthingCtrl" class="ng-cloak">
 <head>
   <meta charset="utf-8">
   <meta http-equiv="X-UA-Compatible" content="IE=edge">
@@ -8,7 +8,7 @@
   <meta name="author" content="">
   <link rel="shortcut icon" href="favicon.png">
 
-  <title>syncthing</title>
+  <title>Syncthing | {{thisNodeName()}}</title>
   <link href="bootstrap/css/bootstrap.min.css" rel="stylesheet">
   <style type="text/css">
     body {
@@ -91,13 +91,13 @@
   </style>
 </head>
 
-<body ng-controller="SyncthingCtrl" class="ng-cloak">
+<body>
 
   <!-- Top bar -->
 
   <nav class="navbar navbar-top navbar-default" role="navigation">
     <div class="container">
-      <span class="navbar-brand"><img class="logo" src="st-logo-128.png" width="32" height="32"> Syncthing</span>
+      <span class="navbar-brand"><img class="logo" src="st-logo-128.png" width="32" height="32" /> Syncthing<small> | {{thisNodeName()}}</small></span>
       <button type="button" class="btn btn-default btn-sm pull-right navbar-btn" ng-click="editSettings()"><span class="glyphicon glyphicon-cog"></span> Settings</button>
     </div>
   </nav>
@@ -131,9 +131,10 @@
         <div class="panel panel-default">
           <div class="panel-heading"><h3 class="panel-title">Repositories</h3></div>
           <div class="panel-body">
-            <ul class="list-unstyled" ng-repeat="repo in repos">
+            <ul class="list-unstyled" ng-repeat="repo in repoList()">
               <li>
                 <span class="text-monospace">{{repo.Directory}}</span>
+                <span ng-if="model[repo.ID].invalid" class="label label-danger">{{model[repo.ID].invalid}}</span>
                 <ul class="list-no-bullet">
                   <li>
                     <div class="li-column" title="Repository ID">
@@ -298,7 +299,7 @@
       <ul class="nav navbar-nav navbar-right">
         <li><a class="navbar-link" href="http://discourse.syncthing.net/">Support / Forum</a></li>
         <li><a class="navbar-link hidden-sm" href="https://github.com/calmh/syncthing/releases">Latest Release</a></li>
-        <li><a class="navbar-link" href="https://github.com/calmh/syncthing/wiki">Documentation</a></li>
+        <li><a class="navbar-link" href="http://discourse.syncthing.net/category/documentation">Documentation</a></li>
         <li><a class="navbar-link hidden-sm" href="https://github.com/calmh/syncthing/issues">Bugs</a></li>
         <li><a class="navbar-link hidden-sm" href="https://github.com/calmh/syncthing">Source Code</a></li>
       </ul>
@@ -357,11 +358,15 @@
           <h4 ng-show="editingExisting" class="modal-title">Edit Node</h4>
         </div>
         <div class="modal-body">
-          <form role="form">
-            <div class="form-group">
+          <form role="form" name="nodeEditor">
+            <div class="form-group" ng-class="{'has-error': nodeEditor.nodeID.$invalid && nodeEditor.nodeID.$dirty}">
               <label for="nodeID">Node ID</label>
-              <input ng-disabled="editingExisting" id="nodeID" class="form-control" type="text" ng-model="currentNode.NodeID"></input>
-              <p class="help-block">The node ID can be found in the logs or in the "Add Node" dialog on the other node.</p>
+              <input ng-if="!editingExisting" name="nodeID" id="nodeID" class="form-control" type="text" ng-model="currentNode.NodeID" required></input>
+              <div ng-if="editingExisting" class="well well-sm">{{currentNode.NodeID | chunkID}}</div>
+              <p class="help-block">
+                <span ng-if="nodeEditor.nodeID.$valid || nodeEditor.nodeID.$pristine">The node ID to enter here can be found in the "Add Node" dialog on the other node. Spaces are ignored.</span>
+                <span ng-if="nodeEditor.nodeID.$error.required && nodeEditor.nodeID.$dirty">The node ID cannot be blank.</span>
+              </p>
             </div>
             <div class="form-group">
               <label for="name">Name</label>
@@ -376,11 +381,11 @@
           </form>
           <div ng-show="!editingExisting">
             When adding a new node, keep in mind that <em>this node</em> must be added on the other side too. The Node ID of this node is:
-            <pre>{{myID}}</pre>
+            <pre>{{myID | chunkID}}</pre>
           </div>
         </div>
         <div class="modal-footer">
-          <button type="button" class="btn btn-primary" ng-click="saveNode()">Save</button>
+          <button type="button" class="btn btn-primary" ng-click="saveNode()" ng-disabled="nodeEditor.$invalid">Save</button>
           <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
           <button ng-if="editingExisting && !editingSelf" type="button" class="btn btn-danger pull-left" ng-click="deleteNode()">Delete</button>
         </div>
@@ -399,23 +404,31 @@
           <h4 ng-show="editingExisting" class="modal-title">Edit Repository</h4>
         </div>
         <div class="modal-body">
-          <form role="form">
-            <div class="form-group">
+          <form role="form" name="repoEditor">
+            <div class="form-group" ng-class="{'has-error': repoEditor.repoID.$invalid && repoEditor.repoID.$dirty}">
               <label for="repoID">Repository ID</label>
-              <input placeholder="documents" ng-disabled="editingExisting" id="repoID" class="form-control" type="text" ng-model="currentRepo.ID"></input>
-              <p class="help-block">Short, unique identifier for the repository. Must be the same on all cluster nodes.</p>
+              <input name="repoID" placeholder="documents" ng-disabled="editingExisting" id="repoID" class="form-control" type="text" ng-model="currentRepo.ID" required unique-repo></input>
+              <p class="help-block">
+              <span ng-if="repoEditor.repoID.$valid || repoEditor.repoID.$pristine">Short identifier for the repository. Must be the same on all cluster nodes.</span>
+              <span ng-if="repoEditor.repoID.$error.uniqueRepo">The repository ID must be unique.</span>
+              <span ng-if="repoEditor.repoID.$error.required && repoEditor.repoID.$dirty">The repository ID cannot be blank.</span>
+              </p>
             </div>
-            <div class="form-group">
+            <div class="form-group" ng-class="{'has-error': repoEditor.repoPath.$invalid && repoEditor.repoPath.$dirty}">
               <label for="repoPath">Repository Path</label>
-              <input placeholder="~/Documents" id="repoPath" class="form-control" type="text" ng-model="currentRepo.Directory"></input>
-              <p class="help-block">Path to the repository on the local computer. Will be created if it does not exist.</p>
+              <input name="repoPath" placeholder="~/Documents" id="repoPath" class="form-control" type="text" ng-model="currentRepo.Directory" required></input>
+              <p class="help-block">
+              <span ng-if="repoEditor.repoPath.$valid || repoEditor.repoPath.$pristine">Path to the repository on the local computer. Will be created if it does not exist.</span>
+              <span ng-if="repoEditor.repoPath.$error.required && repoEditor.repoPath.$dirty">The repository path cannot be blank.</span>
+              </p>
             </div>
             <div class="form-group">
               <div class="checkbox">
                 <label>
-                  <input type="checkbox" ng-model="currentRepo.ReadOnly"> Read Only
+                  <input type="checkbox" ng-model="currentRepo.ReadOnly"> Repository Master
                 </label>
               </div>
+              <p class="help-block">Files are protected from changes made on other nodes, but changes made on <em>this</em> node will be sent to the rest of the cluster.</p>
             </div>
             <div class="form-group">
               <label for="nodes">Nodes</label>
@@ -432,7 +445,7 @@
           </div>
         </div>
         <div class="modal-footer">
-          <button type="button" class="btn btn-primary" ng-click="saveRepo()">Save</button>
+          <button type="button" class="btn btn-primary" ng-click="saveRepo()" ng-disabled="repoEditor.$invalid">Save</button>
           <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
           <button ng-if="editingExisting" type="button" class="btn btn-danger pull-left" ng-click="deleteRepo()">Delete</button>
         </div>

integration/h1/config.xml

@@ -9,6 +9,9 @@
         <node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3">
             <address>127.0.0.1:22003</address>
         </node>
+        <node id="EJHMPAQOGCVORISB4IS3SYYVJXTKJGLTU66DIQPGJ5D2GXGQ3OWQ" name="s4">
+            <address>127.0.0.1:22004</address>
+        </node>
     </repository>
     <repository id="s12" directory="s12-1">
         <node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1">
@@ -32,7 +35,7 @@
         <maxSendKbps>0</maxSendKbps>
         <rescanIntervalS>10</rescanIntervalS>
         <reconnectionIntervalS>5</reconnectionIntervalS>
-        <maxChangeKbps>1000</maxChangeKbps>
+        <maxChangeKbps>10000</maxChangeKbps>
         <startBrowser>false</startBrowser>
     </options>
 </configuration>

integration/h2/config.xml

@@ -40,7 +40,7 @@
         <maxSendKbps>0</maxSendKbps>
         <rescanIntervalS>15</rescanIntervalS>
         <reconnectionIntervalS>5</reconnectionIntervalS>
-        <maxChangeKbps>1000</maxChangeKbps>
+        <maxChangeKbps>10000</maxChangeKbps>
         <startBrowser>false</startBrowser>
     </options>
 </configuration>

integration/h3/config.xml

@@ -32,7 +32,7 @@
         <maxSendKbps>0</maxSendKbps>
         <rescanIntervalS>20</rescanIntervalS>
         <reconnectionIntervalS>5</reconnectionIntervalS>
-        <maxChangeKbps>1000</maxChangeKbps>
+        <maxChangeKbps>10000</maxChangeKbps>
         <startBrowser>false</startBrowser>
     </options>
 </configuration>

integration/h4/cert.pem

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID3jCCAkigAwIBAgIBADALBgkqhkiG9w0BAQswFDESMBAGA1UEAxMJc3luY3Ro
+aW5nMB4XDTE0MDUxMDAwNTM0N1oXDTQ5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJ
+c3luY3RoaW5nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEA9MRyBtAr
+Sjt29azNoCWxx5xZF3RodBcQu+wv5sRR8lWozrr4brfUJLslcQHowqaAprOU1NP+
+BH12P5CSymsUrwAmCwSQ54CimXrNi5RiNMl7dtInJksk4Kp6nJgfyR7TqeQgqxtv
++skVWdJY7ptxqpVuDfkf1JnNr68dbANw8hEJpPaGm3qOt81YvSg37R75HiOCzv+h
+FcSjKpPyFMvPARMCOHuZS0fYRJtI5nwmR0mWtKfnH/2204YNiQUne/8h2fgtkpxy
+OjxKOs2KJxbmpV6Uur/YyGyinb5+Aa0df3KCBuZmE+i/AsZcTsk0fgefe+bshWG/
+hzrNfV0wsX3TYjYOSBJ04+f/uQW00G1GGSxPwTsShGqVuwfJkTqkjAXX5wcH+PgJ
+ewG/dyMzKklMg19Y65WkhpWa/19o2KSZNw6TO8YM1arwT0STcMc+4fdrVB09lX6q
+NJA8UL8hUX+jbKBzatDY64h1d9E8PE0ODHYgYFO2Ko7e2GnWCQeijGmnAgMBAAGj
+PzA9MA4GA1UdDwEB/wQEAwIAoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH
+AwIwDAYDVR0TAQH/BAIwADALBgkqhkiG9w0BAQsDggGBANFiHcATP5Lm11o65wbh
+sKk7yteTapRohMoLNdW44YNyM8ZkELnrdNY8pe3CWSGy3spBH01+4jbUT+gSltQr
+KTLVxSZ7f91696Og5ag4BQCeFY6ghKD/G9+PlBSj6yb3Y98NZsx8huLfylH+XuJw
+2gP5Nqov4uXaKgYylx2gdaeCb2M+wM/br1DO2HCPCmgbZE5g8RM5JxzojGn/41Le
+IbCd39zdI6NKj9c7T1Bxmt20uzca4nRgXVVzJymedEoF+//sBRk6PQzqgjgn/r3S
+h9vrqo5j8ly/+ojFjBaVY7gq2XHM6/q0LTjeKkv2MUQw+vEEZX65GpBOgBZ8U0Wb
+/NMUUhhDjGE/0G6TCJgq/HdkjmsNaWjO5sWjhnwXNImYXBdH4OenhXIrHcLhcnxN
+2n5sPkDc6n0LVVV7VAjBPXcTmu2uOSK02yqNZLLWJygp1Wl6lbiqLS3bJgYrUv2m
+YkRaR+IqVPw5EPs/QlH0qLBeCyIasaSWUVZeitVwRmqIUA==
+-----END CERTIFICATE-----

integration/h4/config.xml

@@ -0,0 +1,36 @@
+<configuration version="2">
+    <repository id="unique" directory="s4" ro="false">
+        <node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1"></node>
+        <node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ" name="s2"></node>
+        <node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3"></node>
+        <node id="EJHMPAQOGCVORISB4IS3SYYVJXTKJGLTU66DIQPGJ5D2GXGQ3OWQ" name="s4"></node>
+    </repository>
+    <node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1">
+        <address>127.0.0.1:22001</address>
+    </node>
+    <node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ" name="s2">
+        <address>127.0.0.1:22002</address>
+    </node>
+    <node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3">
+        <address>127.0.0.1:22003</address>
+    </node>
+    <node id="EJHMPAQOGCVORISB4IS3SYYVJXTKJGLTU66DIQPGJ5D2GXGQ3OWQ" name="s4">
+        <address>dynamic</address>
+    </node>
+    <gui enabled="true">
+        <address>127.0.0.1:8084</address>
+    </gui>
+    <options>
+        <listenAddress>:22004</listenAddress>
+        <globalAnnounceServer>announce.syncthing.net:22025</globalAnnounceServer>
+        <globalAnnounceEnabled>false</globalAnnounceEnabled>
+        <localAnnounceEnabled>true</localAnnounceEnabled>
+        <parallelRequests>16</parallelRequests>
+        <maxSendKbps>0</maxSendKbps>
+        <rescanIntervalS>60</rescanIntervalS>
+        <reconnectionIntervalS>10</reconnectionIntervalS>
+        <maxChangeKbps>10000</maxChangeKbps>
+        <startBrowser>false</startBrowser>
+        <upnpEnabled>false</upnpEnabled>
+    </options>
+</configuration>

integration/h4/key.pem

@@ -0,0 +1,39 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIG5AIBAAKCAYEA9MRyBtArSjt29azNoCWxx5xZF3RodBcQu+wv5sRR8lWozrr4
+brfUJLslcQHowqaAprOU1NP+BH12P5CSymsUrwAmCwSQ54CimXrNi5RiNMl7dtIn
+Jksk4Kp6nJgfyR7TqeQgqxtv+skVWdJY7ptxqpVuDfkf1JnNr68dbANw8hEJpPaG
+m3qOt81YvSg37R75HiOCzv+hFcSjKpPyFMvPARMCOHuZS0fYRJtI5nwmR0mWtKfn
+H/2204YNiQUne/8h2fgtkpxyOjxKOs2KJxbmpV6Uur/YyGyinb5+Aa0df3KCBuZm
+E+i/AsZcTsk0fgefe+bshWG/hzrNfV0wsX3TYjYOSBJ04+f/uQW00G1GGSxPwTsS
+hGqVuwfJkTqkjAXX5wcH+PgJewG/dyMzKklMg19Y65WkhpWa/19o2KSZNw6TO8YM
+1arwT0STcMc+4fdrVB09lX6qNJA8UL8hUX+jbKBzatDY64h1d9E8PE0ODHYgYFO2
+Ko7e2GnWCQeijGmnAgMBAAECggGBAIjKaLdqC2d3CCqQonJH3q0hsaCsC9wlL9L2
+UmbzfKCkQq0WTNUDo2nLtUcMvBpclzWS0zCGMUYtH7Kyh3bclTigKqKpsJnQiA6i
+VNEW4jOCDp//HqYGBNwSKmftlIX/1mbx+VfnA5PyYR5LsivXb5TX4iOpAKL+Obdf
+dF/zJGIEJ5GrvNqTicMq3dcI7Qh18N9pFSe+MTZLKK0Y9Yetx0hgaTNL0AYEZtcg
+uYMmCvZ4J+Namo6EanKYTmQvHzvq/tZVMvud9Gcr6uKKtVBcgex9S/R7IicaKg78
+oDTgH0nDrpI55pZCX8vuVGk8nVTXXLTsMR1XojOpiYjS6ucfTkPEw3fOW/YRhHg5
+93TrdDiWkqSWube5LNUF87q65t/aw/y2EH2aTNqcPD5OQ+EZRS8OGYPqOrJ4Ycbp
+j6CMSE+LX2IDMQyJ+9J0vPHtFsAviBKQkPoQ1L6mvhJuw6ksy34NQGykNDHz7nQK
+SeqvCJ6XCtaWNkq+00lC3UFaGsjuUQKBwQD8+y370co5G7G5GDLbLE3i+pguUN7L
+5YfDj5qqsM9hOJNqeKAHrKFP2ii0F9WxGw/ruY0k8k7zUt6LepgwkCI5BYfckRKJ
+g8YsNTizjqPLRGtiqL9Garjo+xPxFGj+TkTg9fYD4xTWFa1I15zzCu7Ye7xObeEH
+LRtcm3R4fU54JDrKtKDccoQmTEAzsxRdNXi9ifc7qgjGBH9W02guuGPY4ltT1aZR
+bcO5vpi44Fnl2h6d7N6iwCtFJ0CaT1pAZ4UCgcEA97Asf5DTDWKByZBhk+VvuT1b
+6nMYjqKxDNMmCaomCmk8Mif0w9SEJmAg0b/gbs/H6T78a+9WjbN5q9xHcDU91uax
+TdCenTq7H981AjgUG7OA7XwYn+AKy+hGSnsTJglMJzJm6TGt+Sq0oO9EahBRDlsP
+PiQRot2gyQfubwcl3rhdErRwaCM92BUyPkC2fy2OppAeZOOxxuzxrvHflDOuDGCZ
+KPCmy6U9HV0JOAO2FSNJeZdNLBixXa1Pk8TgbLY7AoHBAPG7lhn9Qg3Fz9H9NINH
+13jfWdFQB0SwJEWTEAiwgMj2ha6Eau5KX63s2V4VNGVSZakqmZtHSneppOuEjq5A
+2+K+zS7PFPaACzos9OxmjU7rJu2UL4m66sv9NvXzOcxev+RyQs0+DKfw+K8VEG0Q
+8l+8BJiw2AjCalXYWbfUjMmyXNdbOCbN6kaqL+L26KuUL7Z1gd/qPw3wODmgMvoJ
+yabxzLDUA2PlzdPMMyTdhCllfkILmEXN+MrQkiOhVa0a/QKBwGZjAhH9ePD4fnQm
+5d8wIb3uGlfRGh6kLBIEGp42IqF9HPASykBFUhdW91odOhY0eAv4CHpJpnrO7QXY
++gLtT1HNbQ+gpGCUTZQAPbZcHhvRWQNSoA8+mtftfVj+hUzc3Qj68cWFzsfIGoDI
+R3ycoBUSGTvzxwKPIQ7Y43wr9UCa74Zy5mB16POw12MadxYda/F4c8f6w5taiRFr
+VKO7tT/Skp101U4rURcZRV1NU3BrdMz5eWI4FuGFafbIlIj7zwKBwHCt3VQt+JmZ
+OhCJR+8Q+jT0JvnMu1zi4CcMRiT8FbNdZDY/3B0wG4ySTNrEikFzIjihF4zIp2nv
+nD3qKQs+THl51GA8AnP9bNk7hknD7rXUuScndccTW58+PGrjqfwJp/1MEeOJQpoX
+0JML1w+dIKHzsKN0X6UL7Gyq8m+0SJKmQQguan3d3M8CMpnW0srgqOfJ+q1+bz8b
+6FuJeijoaN8+zyKkN+9R91Erw5pk+7vJRzEpDtkhprEE5tLNDKrXJw==
+-----END RSA PRIVATE KEY-----

integration/test.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-export STNORESTART=1
-
 iterations=${1:-5}
 
 id1=I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA
@@ -14,7 +12,7 @@ go build json.go
 
 start() {
 	echo "Starting..."
-	for i in 1 2 3 ; do
+	for i in 1 2 3 4 ; do
 		STPROFILER=":909$i" syncthing -home "h$i" &
 	done
 }
@@ -137,4 +135,6 @@ for ((t = 1; t <= $iterations; t++)) ; do
 	testConvergence
 done
 
-pkill syncthing
+for i in 1 2 3 4 ; do
+	curl -X POST "http://localhost:808$i/rest/shutdown"
+done

lamport/clock.go

@@ -1,3 +1,4 @@
+// Package lamport implements a simple Lamport Clock for versioning
 package lamport
 
 import "sync"

logger/logger.go

@@ -0,0 +1,143 @@
+// Package logger implements a standardized logger with callback functionality
+package logger
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"sync"
+)
+
+type LogLevel int
+
+const (
+	LevelDebug LogLevel = iota
+	LevelInfo
+	LevelOK
+	LevelWarn
+	LevelFatal
+	NumLevels
+)
+
+type MessageHandler func(l LogLevel, msg string)
+
+type Logger struct {
+	logger   *log.Logger
+	handlers [NumLevels][]MessageHandler
+	mut      sync.Mutex
+}
+
+var DefaultLogger = New()
+
+func New() *Logger {
+	return &Logger{
+		logger: log.New(os.Stderr, "", log.Ltime),
+	}
+}
+
+func (l *Logger) AddHandler(level LogLevel, h MessageHandler) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	l.handlers[level] = append(l.handlers[level], h)
+}
+
+func (l *Logger) SetFlags(flag int) {
+	l.logger.SetFlags(flag)
+}
+
+func (l *Logger) SetPrefix(prefix string) {
+	l.logger.SetPrefix(prefix)
+}
+
+func (l *Logger) callHandlers(level LogLevel, s string) {
+	for _, h := range l.handlers[level] {
+		h(level, s)
+	}
+}
+
+func (l *Logger) Debugln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "DEBUG: "+s)
+	l.callHandlers(LevelDebug, s)
+}
+
+func (l *Logger) Debugf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "DEBUG: "+s)
+	l.callHandlers(LevelDebug, s)
+}
+func (l *Logger) Infoln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "INFO: "+s)
+	l.callHandlers(LevelInfo, s)
+}
+
+func (l *Logger) Infof(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "INFO: "+s)
+	l.callHandlers(LevelInfo, s)
+}
+
+func (l *Logger) Okln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "OK: "+s)
+	l.callHandlers(LevelOK, s)
+}
+
+func (l *Logger) Okf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "OK: "+s)
+	l.callHandlers(LevelOK, s)
+}
+
+func (l *Logger) Warnln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "WARNING: "+s)
+	l.callHandlers(LevelWarn, s)
+}
+
+func (l *Logger) Warnf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "WARNING: "+s)
+	l.callHandlers(LevelWarn, s)
+}
+
+func (l *Logger) Fatalln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "FATAL: "+s)
+	l.callHandlers(LevelFatal, s)
+	os.Exit(3)
+}
+
+func (l *Logger) Fatalf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "FATAL: "+s)
+	l.callHandlers(LevelFatal, s)
+	os.Exit(3)
+}
+
+func (l *Logger) FatalErr(err error) {
+	if err != nil {
+		l.Fatalf(err.Error())
+	}
+}

mc/beacon.go

@@ -1,91 +0,0 @@
-package mc
-
-import (
-	"log"
-	"net"
-)
-
-type recv struct {
-	data []byte
-	src  net.Addr
-}
-
-type Beacon struct {
-	group  string
-	port   int
-	conns  []*net.UDPConn
-	inbox  chan []byte
-	outbox chan recv
-}
-
-func NewBeacon(group string, port int) *Beacon {
-	b := &Beacon{
-		group:  group,
-		port:   port,
-		inbox:  make(chan []byte),
-		outbox: make(chan recv),
-	}
-	go b.run()
-	return b
-}
-
-func (b *Beacon) Send(data []byte) {
-	b.inbox <- data
-}
-
-func (b *Beacon) Recv() ([]byte, net.Addr) {
-	recv := <-b.outbox
-	return recv.data, recv.src
-}
-
-func (b *Beacon) run() {
-	group := &net.UDPAddr{IP: net.ParseIP(b.group), Port: b.port}
-
-	intfs, err := net.Interfaces()
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	for _, intf := range intfs {
-		intf := intf
-
-		if debug {
-			dlog.Printf("trying interface %q", intf.Name)
-		}
-		conn, err := net.ListenMulticastUDP("udp4", &intf, group)
-		if err != nil {
-			if debug {
-				dlog.Printf("listen for multicast group on %q: %v", intf.Name, err)
-			}
-		} else {
-			b.conns = append(b.conns, conn)
-		}
-	}
-
-	for _, conn := range b.conns {
-		conn := conn
-		go func() {
-			for {
-				var bs = make([]byte, 1500)
-				n, addr, err := conn.ReadFrom(bs)
-				if err != nil {
-					dlog.Println(err)
-					return
-				}
-				b.outbox <- recv{bs[:n], addr}
-			}
-		}()
-	}
-
-	go func() {
-		for bs := range b.inbox {
-			for _, conn := range b.conns {
-				_, err := conn.WriteTo(bs, group)
-				if err != nil {
-					dlog.Println(err)
-					return
-				}
-			}
-		}
-	}()
-}

mc/debug.go

@@ -1,12 +0,0 @@
-package mc
-
-import (
-	"log"
-	"os"
-	"strings"
-)
-
-var (
-	dlog  = log.New(os.Stderr, "mc: ", log.Lmicroseconds|log.Lshortfile)
-	debug = strings.Contains(os.Getenv("STTRACE"), "mc")
-)

model/blockqueue.go

@@ -1,6 +1,10 @@
-package main
+package model
 
-import "github.com/calmh/syncthing/scanner"
+import (
+	"sync/atomic"
+
+	"github.com/calmh/syncthing/scanner"
+)
 
 type bqAdd struct {
 	file scanner.File
@@ -20,6 +24,7 @@ type blockQueue struct {
 	outbox chan bqBlock
 
 	queued []bqBlock
+	qlen   uint32
 }
 
 func newBlockQueue() *blockQueue {
@@ -77,6 +82,7 @@ func (q *blockQueue) run() {
 				q.queued = q.queued[1:]
 			}
 		}
+		atomic.StoreUint32(&q.qlen, uint32(len(q.queued)))
 	}
 }
 
@@ -89,6 +95,7 @@ func (q *blockQueue) get() bqBlock {
 }
 
 func (q *blockQueue) empty() bool {
-	// There is a race condition here. We're only mostly sure the queue is empty if the expression below is true.
-	return len(q.queued) == 0 && len(q.inbox) == 0 && len(q.outbox) == 0
+	var l uint32
+	atomic.LoadUint32(&l)
+	return l == 0
 }

model/debug.go

@@ -0,0 +1,13 @@
+package model
+
+import (
+	"os"
+	"strings"
+
+	"github.com/calmh/syncthing/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "model") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

model/doc.go

@@ -0,0 +1,2 @@
+// Package model implements repository abstraction and file pulling mechanisms
+package model

model/model.go

@@ -1,4 +1,4 @@
-package main
+package model
 
 import (
 	"compress/gzip"
@@ -14,6 +14,7 @@ import (
 
 	"github.com/calmh/syncthing/buffers"
 	"github.com/calmh/syncthing/cid"
+	"github.com/calmh/syncthing/config"
 	"github.com/calmh/syncthing/files"
 	"github.com/calmh/syncthing/lamport"
 	"github.com/calmh/syncthing/protocol"
@@ -30,12 +31,19 @@ const (
 )
 
 type Model struct {
-	repoDirs  map[string]string     // repo -> dir
-	repoFiles map[string]*files.Set // repo -> files
-	repoNodes map[string][]string   // repo -> nodeIDs
-	nodeRepos map[string][]string   // nodeID -> repos
-	repoState map[string]repoState  // repo -> state
-	rmut      sync.RWMutex          // protects the above
+	indexDir string
+	cfg      *config.Configuration
+
+	clientName    string
+	clientVersion string
+
+	repoDirs   map[string]string      // repo -> dir
+	repoFiles  map[string]*files.Set  // repo -> files
+	repoNodes  map[string][]string    // repo -> nodeIDs
+	nodeRepos  map[string][]string    // nodeID -> repos
+	repoState  map[string]repoState   // repo -> state
+	suppressor map[string]*suppressor // repo -> suppressor
+	rmut       sync.RWMutex           // protects the above
 
 	cm *cid.Map
 
@@ -58,18 +66,23 @@ var (
 // NewModel creates and starts a new model. The model starts in read-only mode,
 // where it sends index information to connected peers and responds to requests
 // for file data without altering the local repository in any way.
-func NewModel(maxChangeBw int) *Model {
+func NewModel(indexDir string, cfg *config.Configuration, clientName, clientVersion string) *Model {
 	m := &Model{
-		repoDirs:  make(map[string]string),
-		repoFiles: make(map[string]*files.Set),
-		repoNodes: make(map[string][]string),
-		nodeRepos: make(map[string][]string),
-		repoState: make(map[string]repoState),
-		cm:        cid.NewMap(),
-		protoConn: make(map[string]protocol.Connection),
-		rawConn:   make(map[string]io.Closer),
-		nodeVer:   make(map[string]string),
-		sup:       suppressor{threshold: int64(maxChangeBw)},
+		indexDir:      indexDir,
+		cfg:           cfg,
+		clientName:    clientName,
+		clientVersion: clientVersion,
+		repoDirs:      make(map[string]string),
+		repoFiles:     make(map[string]*files.Set),
+		repoNodes:     make(map[string][]string),
+		nodeRepos:     make(map[string][]string),
+		repoState:     make(map[string]repoState),
+		suppressor:    make(map[string]*suppressor),
+		cm:            cid.NewMap(),
+		protoConn:     make(map[string]protocol.Connection),
+		rawConn:       make(map[string]io.Closer),
+		nodeVer:       make(map[string]string),
+		sup:           suppressor{threshold: int64(cfg.Options.MaxChangeKbps)},
 	}
 
 	go m.broadcastIndexLoop()
@@ -80,13 +93,13 @@ func NewModel(maxChangeBw int) *Model {
 // read/write mode the model will attempt to keep in sync with the cluster by
 // pulling needed files from peer nodes.
 func (m *Model) StartRepoRW(repo string, threads int) {
-	m.rmut.Lock()
-	defer m.rmut.Unlock()
+	m.rmut.RLock()
+	defer m.rmut.RUnlock()
 
 	if dir, ok := m.repoDirs[repo]; !ok {
 		panic("cannot start without repo")
 	} else {
-		newPuller(repo, dir, m, threads)
+		newPuller(repo, dir, m, threads, m.cfg)
 	}
 }
 
@@ -213,8 +226,8 @@ func (m *Model) NeedFilesRepo(repo string) []scanner.File {
 // Index is called when a new node is connected and we receive their full index.
 // Implements the protocol.Model interface.
 func (m *Model) Index(nodeID string, repo string, fs []protocol.FileInfo) {
-	if debugNet {
-		dlog.Printf("IDX(in): %s / %q: %d files", nodeID, repo, len(fs))
+	if debug {
+		l.Debugf("IDX(in): %s / %q: %d files", nodeID, repo, len(fs))
 	}
 
 	var files = make([]scanner.File, len(fs))
@@ -228,7 +241,7 @@ func (m *Model) Index(nodeID string, repo string, fs []protocol.FileInfo) {
 	if r, ok := m.repoFiles[repo]; ok {
 		r.Replace(id, files)
 	} else {
-		warnf("Index from %s for nonexistant repo %q; dropping", nodeID, repo)
+		l.Warnf("Index from %s for nonexistant repo %q; dropping", nodeID, repo)
 	}
 	m.rmut.RUnlock()
 }
@@ -236,8 +249,8 @@ func (m *Model) Index(nodeID string, repo string, fs []protocol.FileInfo) {
 // IndexUpdate is called for incremental updates to connected nodes' indexes.
 // Implements the protocol.Model interface.
 func (m *Model) IndexUpdate(nodeID string, repo string, fs []protocol.FileInfo) {
-	if debugNet {
-		dlog.Printf("IDXUP(in): %s / %q: %d files", nodeID, repo, len(fs))
+	if debug {
+		l.Debugf("IDXUP(in): %s / %q: %d files", nodeID, repo, len(fs))
 	}
 
 	var files = make([]scanner.File, len(fs))
@@ -251,20 +264,20 @@ func (m *Model) IndexUpdate(nodeID string, repo string, fs []protocol.FileInfo)
 	if r, ok := m.repoFiles[repo]; ok {
 		r.Update(id, files)
 	} else {
-		warnf("Index update from %s for nonexistant repo %q; dropping", nodeID, repo)
+		l.Warnf("Index update from %s for nonexistant repo %q; dropping", nodeID, repo)
 	}
 	m.rmut.RUnlock()
 }
 
 func (m *Model) ClusterConfig(nodeID string, config protocol.ClusterConfigMessage) {
 	compErr := compareClusterConfig(m.clusterConfig(nodeID), config)
-	if debugNet {
-		dlog.Printf("ClusterConfig: %s: %#v", nodeID, config)
-		dlog.Printf("  ... compare: %s: %v", nodeID, compErr)
+	if debug {
+		l.Debugf("ClusterConfig: %s: %#v", nodeID, config)
+		l.Debugf("  ... compare: %s: %v", nodeID, compErr)
 	}
 
 	if compErr != nil {
-		warnf("%s: %v", nodeID, compErr)
+		l.Warnf("%s: %v", nodeID, compErr)
 		m.Close(nodeID, compErr)
 	}
 
@@ -280,14 +293,14 @@ func (m *Model) ClusterConfig(nodeID string, config protocol.ClusterConfigMessag
 // Close removes the peer from the model and closes the underlying connection if possible.
 // Implements the protocol.Model interface.
 func (m *Model) Close(node string, err error) {
-	if debugNet {
-		dlog.Printf("%s: %v", node, err)
+	if debug {
+		l.Debugf("%s: %v", node, err)
 	}
 
 	if err != io.EOF {
-		warnf("Connection to %s closed: %v", node, err)
+		l.Warnf("Connection to %s closed: %v", node, err)
 	} else if _, ok := err.(ClusterConfigMismatch); ok {
-		warnf("Connection to %s closed: %v", node, err)
+		l.Warnf("Connection to %s closed: %v", node, err)
 	}
 
 	cid := m.cm.Get(node)
@@ -318,22 +331,24 @@ func (m *Model) Request(nodeID, repo, name string, offset int64, size int) ([]by
 	m.rmut.RUnlock()
 
 	if !ok {
-		warnf("Request from %s for file %s in nonexistent repo %q", nodeID, name, repo)
+		l.Warnf("Request from %s for file %s in nonexistent repo %q", nodeID, name, repo)
 		return nil, ErrNoSuchFile
 	}
 
 	lf := r.Get(cid.LocalID, name)
-	if offset > lf.Size {
-		warnf("SECURITY (nonexistent file) REQ(in): %s: %q o=%d s=%d", nodeID, name, offset, size)
-		return nil, ErrNoSuchFile
-	}
-
-	if lf.Suppressed {
+	if lf.Suppressed || lf.Flags&protocol.FlagDeleted != 0 {
 		return nil, ErrInvalid
 	}
 
-	if debugNet && nodeID != "<local>" {
-		dlog.Printf("REQ(in): %s: %q / %q o=%d s=%d", nodeID, repo, name, offset, size)
+	if offset > lf.Size {
+		if debug {
+			l.Debugf("REQ(in; nonexistent): %s: %q o=%d s=%d", nodeID, name, offset, size)
+		}
+		return nil, ErrNoSuchFile
+	}
+
+	if debug && nodeID != "<local>" {
+		l.Debugf("REQ(in): %s: %q / %q o=%d s=%d", nodeID, repo, name, offset, size)
 	}
 	m.rmut.RLock()
 	fn := filepath.Join(m.repoDirs[repo], name)
@@ -423,16 +438,18 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection)
 	cm := m.clusterConfig(nodeID)
 	protoConn.ClusterConfig(cm)
 
+	var idxToSend = make(map[string][]protocol.FileInfo)
+
+	m.rmut.RLock()
+	for _, repo := range m.nodeRepos[nodeID] {
+		idxToSend[repo] = m.protocolIndex(repo)
+	}
+	m.rmut.RUnlock()
+
 	go func() {
-		m.rmut.RLock()
-		repos := m.nodeRepos[nodeID]
-		m.rmut.RUnlock()
-		for _, repo := range repos {
-			m.rmut.RLock()
-			idx := m.protocolIndex(repo)
-			m.rmut.RUnlock()
-			if debugNet {
-				dlog.Printf("IDX(out/initial): %s: %q: %d files", nodeID, repo, len(idx))
+		for repo, idx := range idxToSend {
+			if debug {
+				l.Debugf("IDX(out/initial): %s: %q: %d files", nodeID, repo, len(idx))
 			}
 			protoConn.Index(repo, idx)
 		}
@@ -447,12 +464,12 @@ func (m *Model) protocolIndex(repo string) []protocol.FileInfo {
 
 	for _, f := range fs {
 		mf := fileInfoFromFile(f)
-		if debugIdx {
+		if debug {
 			var flagComment string
 			if mf.Flags&protocol.FlagDeleted != 0 {
 				flagComment = " (deleted)"
 			}
-			dlog.Printf("IDX(out): %q/%q m=%d f=%o%s v=%d (%d blocks)", repo, mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks))
+			l.Debugf("IDX(out): %q/%q m=%d f=%o%s v=%d (%d blocks)", repo, mf.Name, mf.Modified, mf.Flags, flagComment, mf.Version, len(mf.Blocks))
 		}
 		index = append(index, mf)
 	}
@@ -475,8 +492,8 @@ func (m *Model) requestGlobal(nodeID, repo, name string, offset int64, size int,
 		return nil, fmt.Errorf("requestGlobal: no such node: %s", nodeID)
 	}
 
-	if debugNet {
-		dlog.Printf("REQ(out): %s: %q / %q o=%d s=%d h=%x", nodeID, repo, name, offset, size, hash)
+	if debug {
+		l.Debugf("REQ(out): %s: %q / %q o=%d s=%d h=%x", nodeID, repo, name, offset, size, hash)
 	}
 
 	return nc.Request(repo, name, offset, size)
@@ -498,14 +515,14 @@ func (m *Model) broadcastIndexLoop() {
 			lastChange[repo] = c
 
 			idx := m.protocolIndex(repo)
-			m.saveIndex(repo, confDir, idx)
+			m.saveIndex(repo, m.indexDir, idx)
 
 			var indexWg sync.WaitGroup
 			for _, nodeID := range m.repoNodes[repo] {
 				if conn, ok := m.protoConn[nodeID]; ok {
 					indexWg.Add(1)
-					if debugNet {
-						dlog.Printf("IDX(out/loop): %s: %d files", nodeID, len(idx))
+					if debug {
+						l.Debugf("IDX(out/loop): %s: %d files", nodeID, len(idx))
 					}
 					go func() {
 						conn.Index(repo, idx)
@@ -522,7 +539,7 @@ func (m *Model) broadcastIndexLoop() {
 	}
 }
 
-func (m *Model) AddRepo(id, dir string, nodes []NodeConfiguration) {
+func (m *Model) AddRepo(id, dir string, nodes []config.NodeConfiguration) {
 	if m.started {
 		panic("cannot add repo to started model")
 	}
@@ -533,6 +550,7 @@ func (m *Model) AddRepo(id, dir string, nodes []NodeConfiguration) {
 	m.rmut.Lock()
 	m.repoDirs[id] = dir
 	m.repoFiles[id] = files.NewSet()
+	m.suppressor[id] = &suppressor{threshold: int64(m.cfg.Options.MaxChangeKbps)}
 
 	m.repoNodes[id] = make([]string, len(nodes))
 	for i, node := range nodes {
@@ -552,27 +570,60 @@ func (m *Model) ScanRepos() {
 	}
 	m.rmut.RUnlock()
 
+	var wg sync.WaitGroup
+	wg.Add(len(repos))
 	for _, repo := range repos {
-		m.ScanRepo(repo)
+		repo := repo
+		go func() {
+			m.ScanRepo(repo)
+			wg.Done()
+		}()
 	}
+	wg.Wait()
 }
 
-func (m *Model) ScanRepo(repo string) {
-	sup := &suppressor{threshold: int64(cfg.Options.MaxChangeKbps)}
-	m.rmut.Lock()
+func (m *Model) CleanRepos() {
+	m.rmut.RLock()
+	var dirs = make([]string, 0, len(m.repoDirs))
+	for _, dir := range m.repoDirs {
+		dirs = append(dirs, dir)
+	}
+	m.rmut.RUnlock()
+
+	var wg sync.WaitGroup
+	wg.Add(len(dirs))
+	for _, dir := range dirs {
+		w := &scanner.Walker{
+			Dir:       dir,
+			TempNamer: defTempNamer,
+		}
+		go func() {
+			w.CleanTempFiles()
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+}
+
+func (m *Model) ScanRepo(repo string) error {
+	m.rmut.RLock()
 	w := &scanner.Walker{
 		Dir:          m.repoDirs[repo],
 		IgnoreFile:   ".stignore",
-		BlockSize:    BlockSize,
+		BlockSize:    scanner.StandardBlockSize,
 		TempNamer:    defTempNamer,
-		Suppressor:   sup,
+		Suppressor:   m.suppressor[repo],
 		CurrentFiler: cFiler{m, repo},
 	}
-	m.rmut.Unlock()
+	m.rmut.RUnlock()
 	m.setState(repo, RepoScanning)
-	fs, _ := w.Walk()
+	fs, _, err := w.Walk()
+	if err != nil {
+		return err
+	}
 	m.ReplaceLocal(repo, fs)
 	m.setState(repo, RepoIdle)
+	return nil
 }
 
 func (m *Model) SaveIndexes(dir string) {
@@ -644,11 +695,11 @@ func (m *Model) loadIndex(repo string, dir string) []protocol.FileInfo {
 // clusterConfig returns a ClusterConfigMessage that is correct for the given peer node
 func (m *Model) clusterConfig(node string) protocol.ClusterConfigMessage {
 	cm := protocol.ClusterConfigMessage{
-		ClientName:    "syncthing",
-		ClientVersion: Version,
+		ClientName:    m.clientName,
+		ClientVersion: m.clientVersion,
 	}
 
-	m.rmut.Lock()
+	m.rmut.RLock()
 	for _, repo := range m.nodeRepos[node] {
 		cr := protocol.Repository{
 			ID: repo,
@@ -662,7 +713,7 @@ func (m *Model) clusterConfig(node string) protocol.ClusterConfigMessage {
 		}
 		cm.Repositories = append(cm.Repositories, cr)
 	}
-	m.rmut.Unlock()
+	m.rmut.RUnlock()
 
 	return cm
 }
@@ -674,9 +725,9 @@ func (m *Model) setState(repo string, state repoState) {
 }
 
 func (m *Model) State(repo string) string {
-	m.rmut.Lock()
+	m.rmut.RLock()
 	state := m.repoState[repo]
-	m.rmut.Unlock()
+	m.rmut.RUnlock()
 	switch state {
 	case RepoIdle:
 		return "idle"

model/model_test.go

@@ -1,4 +1,4 @@
-package main
+package model
 
 import (
 	"bytes"
@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/calmh/syncthing/cid"
+	"github.com/calmh/syncthing/config"
 	"github.com/calmh/syncthing/protocol"
 	"github.com/calmh/syncthing/scanner"
 )
@@ -47,7 +48,7 @@ func init() {
 }
 
 func TestRequest(t *testing.T) {
-	m := NewModel(1e6)
+	m := NewModel("/tmp", &config.Configuration{}, "syncthing", "dev")
 	m.AddRepo("default", "testdata", nil)
 	m.ScanRepo("default")
 
@@ -83,7 +84,7 @@ func genFiles(n int) []protocol.FileInfo {
 }
 
 func BenchmarkIndex10000(b *testing.B) {
-	m := NewModel(1e6)
+	m := NewModel("/tmp", nil, "syncthing", "dev")
 	m.AddRepo("default", "testdata", nil)
 	m.ScanRepo("default")
 	files := genFiles(10000)
@@ -95,7 +96,7 @@ func BenchmarkIndex10000(b *testing.B) {
 }
 
 func BenchmarkIndex00100(b *testing.B) {
-	m := NewModel(1e6)
+	m := NewModel("/tmp", nil, "syncthing", "dev")
 	m.AddRepo("default", "testdata", nil)
 	m.ScanRepo("default")
 	files := genFiles(100)
@@ -107,7 +108,7 @@ func BenchmarkIndex00100(b *testing.B) {
 }
 
 func BenchmarkIndexUpdate10000f10000(b *testing.B) {
-	m := NewModel(1e6)
+	m := NewModel("/tmp", nil, "syncthing", "dev")
 	m.AddRepo("default", "testdata", nil)
 	m.ScanRepo("default")
 	files := genFiles(10000)
@@ -120,7 +121,7 @@ func BenchmarkIndexUpdate10000f10000(b *testing.B) {
 }
 
 func BenchmarkIndexUpdate10000f00100(b *testing.B) {
-	m := NewModel(1e6)
+	m := NewModel("/tmp", nil, "syncthing", "dev")
 	m.AddRepo("default", "testdata", nil)
 	m.ScanRepo("default")
 	files := genFiles(10000)
@@ -134,7 +135,7 @@ func BenchmarkIndexUpdate10000f00100(b *testing.B) {
 }
 
 func BenchmarkIndexUpdate10000f00001(b *testing.B) {
-	m := NewModel(1e6)
+	m := NewModel("/tmp", nil, "syncthing", "dev")
 	m.AddRepo("default", "testdata", nil)
 	m.ScanRepo("default")
 	files := genFiles(10000)
@@ -181,7 +182,7 @@ func (FakeConnection) Statistics() protocol.Statistics {
 }
 
 func BenchmarkRequest(b *testing.B) {
-	m := NewModel(1e6)
+	m := NewModel("/tmp", nil, "syncthing", "dev")
 	m.AddRepo("default", "testdata", nil)
 	m.ScanRepo("default")
 

model/puller.go

@@ -1,4 +1,4 @@
-package main
+package model
 
 import (
 	"bytes"
@@ -9,6 +9,7 @@ import (
 
 	"github.com/calmh/syncthing/buffers"
 	"github.com/calmh/syncthing/cid"
+	"github.com/calmh/syncthing/config"
 	"github.com/calmh/syncthing/protocol"
 	"github.com/calmh/syncthing/scanner"
 )
@@ -61,6 +62,7 @@ func (m activityMap) decrease(node string) {
 var errNoNode = errors.New("no available source node")
 
 type puller struct {
+	cfg               *config.Configuration
 	repo              string
 	dir               string
 	bq                *blockQueue
@@ -72,8 +74,9 @@ type puller struct {
 	requestResults    chan requestResult
 }
 
-func newPuller(repo, dir string, model *Model, slots int) *puller {
+func newPuller(repo, dir string, model *Model, slots int, cfg *config.Configuration) *puller {
 	p := &puller{
+		cfg:               cfg,
 		repo:              repo,
 		dir:               dir,
 		bq:                newBlockQueue(),
@@ -90,14 +93,14 @@ func newPuller(repo, dir string, model *Model, slots int) *puller {
 		for i := 0; i < slots; i++ {
 			p.requestSlots <- true
 		}
-		if debugPull {
-			dlog.Printf("starting puller; repo %q dir %q slots %d", repo, dir, slots)
+		if debug {
+			l.Debugf("starting puller; repo %q dir %q slots %d", repo, dir, slots)
 		}
 		go p.run()
 	} else {
 		// Read only
-		if debugPull {
-			dlog.Printf("starting puller; repo %q dir %q (read only)", repo, dir)
+		if debug {
+			l.Debugf("starting puller; repo %q dir %q (read only)", repo, dir)
 		}
 		go p.runRO()
 	}
@@ -110,14 +113,14 @@ func (p *puller) run() {
 		for {
 			<-p.requestSlots
 			b := p.bq.get()
-			if debugPull {
-				dlog.Printf("filler: queueing %q / %q offset %d copy %d", p.repo, b.file.Name, b.block.Offset, len(b.copy))
+			if debug {
+				l.Debugf("filler: queueing %q / %q offset %d copy %d", p.repo, b.file.Name, b.block.Offset, len(b.copy))
 			}
 			p.blocks <- b
 		}
 	}()
 
-	walkTicker := time.Tick(time.Duration(cfg.Options.RescanIntervalS) * time.Second)
+	walkTicker := time.Tick(time.Duration(p.cfg.Options.RescanIntervalS) * time.Second)
 	timeout := time.Tick(5 * time.Second)
 	changed := true
 
@@ -135,18 +138,21 @@ func (p *puller) run() {
 			case b := <-p.blocks:
 				p.model.setState(p.repo, RepoSyncing)
 				changed = true
-				p.handleBlock(b)
+				if p.handleBlock(b) {
+					// Block was fully handled, free up the slot
+					p.requestSlots <- true
+				}
 
 			case <-timeout:
 				if len(p.openFiles) == 0 && p.bq.empty() {
 					// Nothing more to do for the moment
 					break pull
 				}
-				if debugPull {
-					dlog.Printf("%q: idle but have %d open files", p.repo, len(p.openFiles))
+				if debug {
+					l.Debugf("%q: idle but have %d open files", p.repo, len(p.openFiles))
 					i := 5
 					for _, f := range p.openFiles {
-						dlog.Printf("  %v", f)
+						l.Debugf("  %v", f)
 						i--
 						if i == 0 {
 							break
@@ -167,10 +173,14 @@ func (p *puller) run() {
 		// Do a rescan if it's time for it
 		select {
 		case <-walkTicker:
-			if debugPull {
-				dlog.Printf("%q: time for rescan", p.repo)
+			if debug {
+				l.Debugf("%q: time for rescan", p.repo)
+			}
+			err := p.model.ScanRepo(p.repo)
+			if err != nil {
+				invalidateRepo(p.cfg, p.repo, err)
+				return
 			}
-			p.model.ScanRepo(p.repo)
 
 		default:
 		}
@@ -181,19 +191,23 @@ func (p *puller) run() {
 }
 
 func (p *puller) runRO() {
-	walkTicker := time.Tick(time.Duration(cfg.Options.RescanIntervalS) * time.Second)
+	walkTicker := time.Tick(time.Duration(p.cfg.Options.RescanIntervalS) * time.Second)
 
 	for _ = range walkTicker {
-		if debugPull {
-			dlog.Printf("%q: time for rescan", p.repo)
+		if debug {
+			l.Debugf("%q: time for rescan", p.repo)
+		}
+		err := p.model.ScanRepo(p.repo)
+		if err != nil {
+			invalidateRepo(p.cfg, p.repo, err)
+			return
 		}
-		p.model.ScanRepo(p.repo)
 	}
 }
 
 func (p *puller) fixupDirectories() {
 	var deleteDirs []string
-	fn := func(path string, info os.FileInfo, err error) error {
+	filepath.Walk(p.dir, func(path string, info os.FileInfo, err error) error {
 		if !info.IsDir() {
 			return nil
 		}
@@ -214,8 +228,8 @@ func (p *puller) fixupDirectories() {
 		}
 
 		if cur.Flags&protocol.FlagDeleted != 0 {
-			if debugPull {
-				dlog.Printf("queue delete dir: %v", cur)
+			if debug {
+				l.Debugf("queue delete dir: %v", cur)
 			}
 
 			// We queue the directories to delete since we walk the
@@ -228,31 +242,30 @@ func (p *puller) fixupDirectories() {
 
 		if cur.Flags&uint32(os.ModePerm) != uint32(info.Mode()&os.ModePerm) {
 			os.Chmod(path, os.FileMode(cur.Flags)&os.ModePerm)
-			if debugPull {
-				dlog.Printf("restored dir flags: %o -> %v", info.Mode()&os.ModePerm, cur)
+			if debug {
+				l.Debugf("restored dir flags: %o -> %v", info.Mode()&os.ModePerm, cur)
 			}
 		}
 
 		if cur.Modified != info.ModTime().Unix() {
 			t := time.Unix(cur.Modified, 0)
 			os.Chtimes(path, t, t)
-			if debugPull {
-				dlog.Printf("restored dir modtime: %d -> %v", info.ModTime().Unix(), cur)
+			if debug {
+				l.Debugf("restored dir modtime: %d -> %v", info.ModTime().Unix(), cur)
 			}
 		}
 
 		return nil
-	}
-	filepath.Walk(p.dir, fn)
+	})
 
 	// Delete any queued directories
 	for i := len(deleteDirs) - 1; i >= 0; i-- {
-		if debugPull {
-			dlog.Println("delete dir:", deleteDirs[i])
+		if debug {
+			l.Debugln("delete dir:", deleteDirs[i])
 		}
 		err := os.Remove(deleteDirs[i])
 		if err != nil {
-			warnln(err)
+			l.Warnln(err)
 		}
 	}
 }
@@ -273,59 +286,19 @@ func (p *puller) handleRequestResult(res requestResult) {
 	of.outstanding--
 	p.openFiles[f.Name] = of
 
-	if debugPull {
-		dlog.Printf("pull: wrote %q / %q offset %d outstanding %d done %v", p.repo, f.Name, res.offset, of.outstanding, of.done)
+	if debug {
+		l.Debugf("pull: wrote %q / %q offset %d outstanding %d done %v", p.repo, f.Name, res.offset, of.outstanding, of.done)
 	}
 
 	if of.done && of.outstanding == 0 {
-		if debugPull {
-			dlog.Printf("pull: closing %q / %q", p.repo, f.Name)
-		}
-		of.file.Close()
-		defer os.Remove(of.temp)
-
-		delete(p.openFiles, f.Name)
-
-		fd, err := os.Open(of.temp)
-		if err != nil {
-			if debugPull {
-				dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, err)
-			}
-			return
-		}
-		hb, _ := scanner.Blocks(fd, BlockSize)
-		fd.Close()
-
-		if l0, l1 := len(hb), len(f.Blocks); l0 != l1 {
-			if debugPull {
-				dlog.Printf("pull: %q / %q: nblocks %d != %d", p.repo, f.Name, l0, l1)
-			}
-			return
-		}
-
-		for i := range hb {
-			if bytes.Compare(hb[i].Hash, f.Blocks[i].Hash) != 0 {
-				dlog.Printf("pull: %q / %q: block %d hash mismatch", p.repo, f.Name, i)
-				return
-			}
-		}
-
-		t := time.Unix(f.Modified, 0)
-		os.Chtimes(of.temp, t, t)
-		os.Chmod(of.temp, os.FileMode(f.Flags&0777))
-		defTempNamer.Show(of.temp)
-		if debugPull {
-			dlog.Printf("pull: rename %q / %q: %q", p.repo, f.Name, of.filepath)
-		}
-		if err := Rename(of.temp, of.filepath); err == nil {
-			p.model.updateLocal(p.repo, f)
-		} else {
-			dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, err)
-		}
+		p.closeFile(f)
 	}
 }
 
-func (p *puller) handleBlock(b bqBlock) {
+// handleBlock fulfills the block request by copying, ignoring or fetching
+// from the network. Returns true if the block was fully handled
+// synchronously, i.e. if the slot can be reused.
+func (p *puller) handleBlock(b bqBlock) bool {
 	f := b.file
 
 	// For directories, simply making sure they exist is enough
@@ -336,16 +309,15 @@ func (p *puller) handleBlock(b bqBlock) {
 			os.MkdirAll(path, 0777)
 		}
 		p.model.updateLocal(p.repo, f)
-		p.requestSlots <- true
-		return
+		return true
 	}
 
 	of, ok := p.openFiles[f.Name]
 	of.done = b.last
 
 	if !ok {
-		if debugPull {
-			dlog.Printf("pull: %q: opening file %q", p.repo, f.Name)
+		if debug {
+			l.Debugf("pull: %q: opening file %q", p.repo, f.Name)
 		}
 
 		of.availability = uint64(p.model.repoFiles[p.repo].Availability(f.Name))
@@ -358,35 +330,32 @@ func (p *puller) handleBlock(b bqBlock) {
 			err = os.MkdirAll(dirName, 0777)
 		}
 		if err != nil {
-			dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, err)
+			l.Debugf("pull: error: %q / %q: %v", p.repo, f.Name, err)
 		}
 
 		of.file, of.err = os.Create(of.temp)
 		if of.err != nil {
-			if debugPull {
-				dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, of.err)
+			if debug {
+				l.Debugf("pull: error: %q / %q: %v", p.repo, f.Name, of.err)
 			}
 			if !b.last {
 				p.openFiles[f.Name] = of
 			}
-			p.requestSlots <- true
-			return
+			return true
 		}
 		defTempNamer.Hide(of.temp)
 	}
 
 	if of.err != nil {
 		// We have already failed this file.
-		if debugPull {
-			dlog.Printf("pull: error: %q / %q has already failed: %v", p.repo, f.Name, of.err)
+		if debug {
+			l.Debugf("pull: error: %q / %q has already failed: %v", p.repo, f.Name, of.err)
 		}
 		if b.last {
-			dlog.Printf("pull: removing failed file %q / %q", p.repo, f.Name)
 			delete(p.openFiles, f.Name)
 		}
 
-		p.requestSlots <- true
-		return
+		return true
 	}
 
 	p.openFiles[f.Name] = of
@@ -394,15 +363,14 @@ func (p *puller) handleBlock(b bqBlock) {
 	switch {
 	case len(b.copy) > 0:
 		p.handleCopyBlock(b)
-		p.requestSlots <- true
+		return true
 
 	case b.block.Size > 0:
-		p.handleRequestBlock(b)
-		// Request slot gets freed in <-p.blocks case
+		return p.handleRequestBlock(b)
 
 	default:
 		p.handleEmptyBlock(b)
-		p.requestSlots <- true
+		return true
 	}
 }
 
@@ -411,15 +379,15 @@ func (p *puller) handleCopyBlock(b bqBlock) {
 	f := b.file
 	of := p.openFiles[f.Name]
 
-	if debugPull {
-		dlog.Printf("pull: copying %d blocks for %q / %q", len(b.copy), p.repo, f.Name)
+	if debug {
+		l.Debugf("pull: copying %d blocks for %q / %q", len(b.copy), p.repo, f.Name)
 	}
 
 	var exfd *os.File
 	exfd, of.err = os.Open(of.filepath)
 	if of.err != nil {
-		if debugPull {
-			dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, of.err)
+		if debug {
+			l.Debugf("pull: error: %q / %q: %v", p.repo, f.Name, of.err)
 		}
 		of.file.Close()
 		of.file = nil
@@ -437,8 +405,8 @@ func (p *puller) handleCopyBlock(b bqBlock) {
 		}
 		buffers.Put(bs)
 		if of.err != nil {
-			if debugPull {
-				dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, of.err)
+			if debug {
+				l.Debugf("pull: error: %q / %q: %v", p.repo, f.Name, of.err)
 			}
 			exfd.Close()
 			of.file.Close()
@@ -450,11 +418,15 @@ func (p *puller) handleCopyBlock(b bqBlock) {
 	}
 }
 
-func (p *puller) handleRequestBlock(b bqBlock) {
-	// We have a block to get from the network
-
+// handleRequestBlock tries to pull a block from the network. Returns true if
+// the block could _not_ be fetched (i.e. it was fully handled, matching the
+// return criteria of handleBlock)
+func (p *puller) handleRequestBlock(b bqBlock) bool {
 	f := b.file
-	of := p.openFiles[f.Name]
+	of, ok := p.openFiles[f.Name]
+	if !ok {
+		panic("bug: request for non-open file")
+	}
 
 	node := p.oustandingPerNode.leastBusyNode(of.availability, p.model.cm)
 	if len(node) == 0 {
@@ -469,16 +441,15 @@ func (p *puller) handleRequestBlock(b bqBlock) {
 		} else {
 			p.openFiles[f.Name] = of
 		}
-		p.requestSlots <- true
-		return
+		return true
 	}
 
 	of.outstanding++
 	p.openFiles[f.Name] = of
 
 	go func(node string, b bqBlock) {
-		if debugPull {
-			dlog.Printf("pull: requesting %q / %q offset %d size %d from %q outstanding %d", p.repo, f.Name, b.block.Offset, b.block.Size, node, of.outstanding)
+		if debug {
+			l.Debugf("pull: requesting %q / %q offset %d size %d from %q outstanding %d", p.repo, f.Name, b.block.Offset, b.block.Size, node, of.outstanding)
 		}
 
 		bs, err := p.model.requestGlobal(node, p.repo, f.Name, b.block.Offset, int(b.block.Size), nil)
@@ -491,6 +462,8 @@ func (p *puller) handleRequestBlock(b bqBlock) {
 			err:      err,
 		}
 	}(node, b)
+
+	return false
 }
 
 func (p *puller) handleEmptyBlock(b bqBlock) {
@@ -504,14 +477,14 @@ func (p *puller) handleEmptyBlock(b bqBlock) {
 	}
 
 	if f.Flags&protocol.FlagDeleted != 0 {
-		if debugPull {
-			dlog.Printf("pull: delete %q", f.Name)
+		if debug {
+			l.Debugf("pull: delete %q", f.Name)
 		}
 		os.Remove(of.temp)
 		os.Remove(of.filepath)
 	} else {
-		if debugPull {
-			dlog.Printf("pull: no blocks to fetch and nothing to copy for %q / %q", p.repo, f.Name)
+		if debug {
+			l.Debugf("pull: no blocks to fetch and nothing to copy for %q / %q", p.repo, f.Name)
 		}
 		t := time.Unix(f.Modified, 0)
 		os.Chtimes(of.temp, t, t)
@@ -528,8 +501,8 @@ func (p *puller) queueNeededBlocks() {
 	for _, f := range p.model.NeedFilesRepo(p.repo) {
 		lf := p.model.CurrentRepoFile(p.repo, f.Name)
 		have, need := scanner.BlockDiff(lf.Blocks, f.Blocks)
-		if debugNeed {
-			dlog.Printf("need:\n  local: %v\n  global: %v\n  haveBlocks: %v\n  needBlocks: %v", lf, f, have, need)
+		if debug {
+			l.Debugf("need:\n  local: %v\n  global: %v\n  haveBlocks: %v\n  needBlocks: %v", lf, f, have, need)
 		}
 		queued++
 		p.bq.put(bqAdd{
@@ -538,7 +511,66 @@ func (p *puller) queueNeededBlocks() {
 			need: need,
 		})
 	}
-	if debugPull && queued > 0 {
-		dlog.Printf("%q: queued %d blocks", p.repo, queued)
+	if debug && queued > 0 {
+		l.Debugf("%q: queued %d blocks", p.repo, queued)
+	}
+}
+
+func (p *puller) closeFile(f scanner.File) {
+	if debug {
+		l.Debugf("pull: closing %q / %q", p.repo, f.Name)
+	}
+
+	of := p.openFiles[f.Name]
+	of.file.Close()
+	defer os.Remove(of.temp)
+
+	delete(p.openFiles, f.Name)
+
+	fd, err := os.Open(of.temp)
+	if err != nil {
+		if debug {
+			l.Debugf("pull: error: %q / %q: %v", p.repo, f.Name, err)
+		}
+		return
+	}
+	hb, _ := scanner.Blocks(fd, scanner.StandardBlockSize)
+	fd.Close()
+
+	if l0, l1 := len(hb), len(f.Blocks); l0 != l1 {
+		if debug {
+			l.Debugf("pull: %q / %q: nblocks %d != %d", p.repo, f.Name, l0, l1)
+		}
+		return
+	}
+
+	for i := range hb {
+		if bytes.Compare(hb[i].Hash, f.Blocks[i].Hash) != 0 {
+			l.Debugf("pull: %q / %q: block %d hash mismatch", p.repo, f.Name, i)
+			return
+		}
+	}
+
+	t := time.Unix(f.Modified, 0)
+	os.Chtimes(of.temp, t, t)
+	os.Chmod(of.temp, os.FileMode(f.Flags&0777))
+	defTempNamer.Show(of.temp)
+	if debug {
+		l.Debugf("pull: rename %q / %q: %q", p.repo, f.Name, of.filepath)
+	}
+	if err := Rename(of.temp, of.filepath); err == nil {
+		p.model.updateLocal(p.repo, f)
+	} else {
+		l.Debugf("pull: error: %q / %q: %v", p.repo, f.Name, err)
+	}
+}
+
+func invalidateRepo(cfg *config.Configuration, repoID string, err error) {
+	for i := range cfg.Repositories {
+		repo := &cfg.Repositories[i]
+		if repo.ID == repoID {
+			repo.Invalid = err.Error()
+			return
+		}
 	}
 }

model/suppressor.go

@@ -1,4 +1,4 @@
-package main
+package model
 
 import (
 	"os"
@@ -52,9 +52,8 @@ func (h *changeHistory) append(size int64, t time.Time) {
 	h.changes = append(h.changes, c)
 }
 
-func (s *suppressor) Suppress(name string, fi os.FileInfo) bool {
-	sup, _ := s.suppress(name, fi.Size(), time.Now())
-	return sup
+func (s *suppressor) Suppress(name string, fi os.FileInfo) (cur, prev bool) {
+	return s.suppress(name, fi.Size(), time.Now())
 }
 
 func (s *suppressor) suppress(name string, size int64, t time.Time) (bool, bool) {

model/suppressor_test.go

@@ -1,4 +1,4 @@
-package main
+package model
 
 import (
 	"testing"

model/tempname.go

@@ -1,6 +1,6 @@
 // +build !windows
 
-package main
+package model
 
 import (
 	"fmt"

model/tempname_windows.go

@@ -1,6 +1,6 @@
 // +build windows
 
-package main
+package model
 
 import (
 	"fmt"

model/util.go

@@ -1,4 +1,4 @@
-package main
+package model
 
 import (
 	"fmt"
@@ -10,37 +10,11 @@ import (
 	"github.com/calmh/syncthing/scanner"
 )
 
-func MetricPrefix(n int64) string {
-	if n > 1e9 {
-		return fmt.Sprintf("%.02f G", float64(n)/1e9)
-	}
-	if n > 1e6 {
-		return fmt.Sprintf("%.02f M", float64(n)/1e6)
-	}
-	if n > 1e3 {
-		return fmt.Sprintf("%.01f k", float64(n)/1e3)
-	}
-	return fmt.Sprintf("%d ", n)
-}
-
-func BinaryPrefix(n int64) string {
-	if n > 1<<30 {
-		return fmt.Sprintf("%.02f Gi", float64(n)/(1<<30))
-	}
-	if n > 1<<20 {
-		return fmt.Sprintf("%.02f Mi", float64(n)/(1<<20))
-	}
-	if n > 1<<10 {
-		return fmt.Sprintf("%.01f Ki", float64(n)/(1<<10))
-	}
-	return fmt.Sprintf("%d ", n)
-}
-
 func Rename(from, to string) error {
 	if runtime.GOOS == "windows" {
 		err := os.Remove(to)
 		if err != nil && !os.IsNotExist(err) {
-			warnln(err)
+			l.Warnln(err)
 		}
 	}
 	return os.Rename(from, to)

model/util_test.go

@@ -1,4 +1,4 @@
-package main
+package model
 
 import (
 	"testing"

protocol/PROTOCOL.md

@@ -38,13 +38,13 @@ level protocols providing compression, encryption and authentication.
     |-----------------------------|
     v             ...             v
 
-Compression is started directly after a successfull TLS handshake,
+Compression is started directly after a successful TLS handshake,
 before the first message is sent. The compression is flushed at each
 message boundary. Compression SHALL use the DEFLATE format as specified
 in RFC 1951.
 
 The encryption and authentication layer SHALL use TLS 1.2 or a higher
-revision. A strong cipher suite SHALL be used, with "string cipher
+revision. A strong cipher suite SHALL be used, with "strong cipher
 suite" being defined as being without known weaknesses and providing
 Perfect Forward Secrecy (PFS). Examples of strong cipher suites are
 given at the end of this document. This is not to be taken as an
@@ -82,7 +82,7 @@ For BEP v1 the Version field is set to zero. Future versions with
 incompatible message formats will increment the Version field. A message
 with an unknown version is a protocol error and MUST result in the
 connection being terminated. A client supporting multiple versions MAY
-retry with a different protcol version upon disconnection.
+retry with a different protocol version upon disconnection.
 
 The Type field indicates the type of data following the message header
 and is one of the integers defined below. A message of an unknown type
@@ -262,7 +262,7 @@ pairs, both of string type. Key ID:s are implementation specific. An
 implementation MUST ignore unknown keys. An implementation MAY impose
 limits on the length keys and values. The options list may be used to
 inform nodes of relevant local configuration options such as rate
-limiting or make recommendations about request parallellism, node
+limiting or make recommendations about request parallelism, node
 priorities, etc. An empty options list is valid for nodes not having any
 such information to share. Nodes MAY NOT make any assumptions about
 peers acting in a specific manner as a result of sent options.
@@ -392,7 +392,7 @@ The Flags field is made up of the following single bit flags:
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
  - The lower 12 bits hold the common Unix permission and mode bits. An
-   implemention MAY ignore or interpret these as is suitable on the host
+   implementation MAY ignore or interpret these as is suitable on the host
    operating system.
 
  - Bit 19 ("D") is set when the file has been deleted. The block list
@@ -575,7 +575,7 @@ restrictive than the following:
 ### Index and Index Update Messages
 
  - Repository: 64 bytes
- - Number of Files: 100.000
+ - Number of Files: 1.000.000
  - Name: 1024 bytes
  - Number of Blocks: 100.000
  - Hash: 64 bytes

protocol/message_types.go

@@ -2,7 +2,7 @@ package protocol
 
 type IndexMessage struct {
 	Repository string     // max:64
-	Files      []FileInfo // max:100000
+	Files      []FileInfo // max:1000000
 }
 
 type FileInfo struct {

protocol/message_xdr.go

@@ -24,7 +24,7 @@ func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 		return xw.Tot(), xdr.ErrElementSizeExceeded
 	}
 	xw.WriteString(o.Repository)
-	if len(o.Files) > 100000 {
+	if len(o.Files) > 1000000 {
 		return xw.Tot(), xdr.ErrElementSizeExceeded
 	}
 	xw.WriteUint32(uint32(len(o.Files)))
@@ -48,7 +48,7 @@ func (o *IndexMessage) UnmarshalXDR(bs []byte) error {
 func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error {
 	o.Repository = xr.ReadStringMax(64)
 	_FilesSize := int(xr.ReadUint32())
-	if _FilesSize > 100000 {
+	if _FilesSize > 1000000 {
 		return xdr.ErrElementSizeExceeded
 	}
 	o.Files = make([]FileInfo, _FilesSize)

protocol/protocol.go

@@ -9,7 +9,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/calmh/syncthing/buffers"
 	"github.com/calmh/syncthing/xdr"
 )
 
@@ -64,24 +63,26 @@ type Connection interface {
 }
 
 type rawConnection struct {
-	sync.RWMutex
+	id       string
+	receiver Model
+
+	reader io.ReadCloser
+	cr     *countingReader
+	xr     *xdr.Reader
+	writer io.WriteCloser
+
+	cw   *countingWriter
+	wb   *bufio.Writer
+	xw   *xdr.Writer
+	wmut sync.Mutex
 
-	id        string
-	receiver  Model
-	reader    io.ReadCloser
-	cr        *countingReader
-	xr        *xdr.Reader
-	writer    io.WriteCloser
-	cw        *countingWriter
-	wb        *bufio.Writer
-	xw        *xdr.Writer
-	closed    chan struct{}
-	awaiting  map[int]chan asyncResult
-	nextID    int
 	indexSent map[string]map[string][2]int64
+	awaiting  []chan asyncResult
+	imut      sync.Mutex
 
-	hasSentIndex  bool
-	hasRecvdIndex bool
+	nextID chan int
+	outbox chan []encodable
+	closed chan struct{}
 }
 
 type asyncResult struct {
@@ -90,7 +91,7 @@ type asyncResult struct {
 }
 
 const (
-	pingTimeout  = 2 * time.Minute
+	pingTimeout  = 4 * time.Minute
 	pingIdleTime = 5 * time.Minute
 )
 
@@ -115,13 +116,17 @@ func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver M
 		cw:        cw,
 		wb:        wb,
 		xw:        xdr.NewWriter(wb),
-		closed:    make(chan struct{}),
-		awaiting:  make(map[int]chan asyncResult),
+		awaiting:  make([]chan asyncResult, 0x1000),
 		indexSent: make(map[string]map[string][2]int64),
+		outbox:    make(chan []encodable),
+		nextID:    make(chan int),
+		closed:    make(chan struct{}),
 	}
 
 	go c.readerLoop()
+	go c.writerLoop()
 	go c.pingerLoop()
+	go c.idGenerator()
 
 	return wireFormatConnection{&c}
 }
@@ -132,11 +137,7 @@ func (c *rawConnection) ID() string {
 
 // Index writes the list of file information to the connected peer node
 func (c *rawConnection) Index(repo string, idx []FileInfo) {
-	c.Lock()
-	if c.isClosed() {
-		c.Unlock()
-		return
-	}
+	c.imut.Lock()
 	var msgType int
 	if c.indexSent[repo] == nil {
 		// This is the first time we send an index.
@@ -158,46 +159,33 @@ func (c *rawConnection) Index(repo string, idx []FileInfo) {
 		}
 		idx = diff
 	}
+	c.imut.Unlock()
 
-	header{0, c.nextID, msgType}.encodeXDR(c.xw)
-	_, err := IndexMessage{repo, idx}.encodeXDR(c.xw)
-	if err == nil {
-		err = c.flush()
-	}
-	c.nextID = (c.nextID + 1) & 0xfff
-	c.hasSentIndex = true
-	c.Unlock()
-
-	if err != nil {
-		c.close(err)
-		return
-	}
+	c.send(header{0, -1, msgType}, IndexMessage{repo, idx})
 }
 
 // Request returns the bytes for the specified block after fetching them from the connected peer.
 func (c *rawConnection) Request(repo string, name string, offset int64, size int) ([]byte, error) {
-	c.Lock()
-	if c.isClosed() {
-		c.Unlock()
+	var id int
+	select {
+	case id = <-c.nextID:
+	case <-c.closed:
 		return nil, ErrClosed
 	}
-	rc := make(chan asyncResult)
-	if _, ok := c.awaiting[c.nextID]; ok {
+
+	c.imut.Lock()
+	if ch := c.awaiting[id]; ch != nil {
 		panic("id taken")
 	}
-	c.awaiting[c.nextID] = rc
-	header{0, c.nextID, messageTypeRequest}.encodeXDR(c.xw)
-	_, err := RequestMessage{repo, name, uint64(offset), uint32(size)}.encodeXDR(c.xw)
-	if err == nil {
-		err = c.flush()
+	rc := make(chan asyncResult)
+	c.awaiting[id] = rc
+	c.imut.Unlock()
+
+	ok := c.send(header{0, id, messageTypeRequest},
+		RequestMessage{repo, name, uint64(offset), uint32(size)})
+	if !ok {
+		return nil, ErrClosed
 	}
-	if err != nil {
-		c.Unlock()
-		c.close(err)
-		return nil, err
-	}
-	c.nextID = (c.nextID + 1) & 0xfff
-	c.Unlock()
 
 	res, ok := <-rc
 	if !ok {
@@ -208,225 +196,276 @@ func (c *rawConnection) Request(repo string, name string, offset int64, size int
 
 // ClusterConfig send the cluster configuration message to the peer and returns any error
 func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) {
-	c.Lock()
-	defer c.Unlock()
-
-	if c.isClosed() {
-		return
-	}
-
-	header{0, c.nextID, messageTypeClusterConfig}.encodeXDR(c.xw)
-	c.nextID = (c.nextID + 1) & 0xfff
-
-	_, err := config.encodeXDR(c.xw)
-	if err == nil {
-		err = c.flush()
-	}
-	if err != nil {
-		c.close(err)
-	}
+	c.send(header{0, -1, messageTypeClusterConfig}, config)
 }
 
 func (c *rawConnection) ping() bool {
-	c.Lock()
-	if c.isClosed() {
-		c.Unlock()
+	var id int
+	select {
+	case id = <-c.nextID:
+	case <-c.closed:
 		return false
 	}
+
 	rc := make(chan asyncResult, 1)
-	c.awaiting[c.nextID] = rc
-	header{0, c.nextID, messageTypePing}.encodeXDR(c.xw)
-	err := c.flush()
-	if err != nil {
-		c.Unlock()
-		c.close(err)
-		return false
-	} else if c.xw.Error() != nil {
-		c.Unlock()
-		c.close(c.xw.Error())
+	c.imut.Lock()
+	c.awaiting[id] = rc
+	c.imut.Unlock()
+
+	ok := c.send(header{0, id, messageTypePing})
+	if !ok {
 		return false
 	}
-	c.nextID = (c.nextID + 1) & 0xfff
-	c.Unlock()
 
 	res, ok := <-rc
 	return ok && res.err == nil
 }
 
+func (c *rawConnection) readerLoop() (err error) {
+	defer func() {
+		c.close(err)
+	}()
+
+	for {
+		select {
+		case <-c.closed:
+			return ErrClosed
+		default:
+		}
+
+		var hdr header
+		hdr.decodeXDR(c.xr)
+		if err := c.xr.Error(); err != nil {
+			return err
+		}
+		if hdr.version != 0 {
+			return fmt.Errorf("protocol error: %s: unknown message version %#x", c.id, hdr.version)
+		}
+
+		switch hdr.msgType {
+		case messageTypeIndex:
+			if err := c.handleIndex(); err != nil {
+				return err
+			}
+
+		case messageTypeIndexUpdate:
+			if err := c.handleIndexUpdate(); err != nil {
+				return err
+			}
+
+		case messageTypeRequest:
+			if err := c.handleRequest(hdr); err != nil {
+				return err
+			}
+
+		case messageTypeResponse:
+			if err := c.handleResponse(hdr); err != nil {
+				return err
+			}
+
+		case messageTypePing:
+			c.send(header{0, hdr.msgID, messageTypePong})
+
+		case messageTypePong:
+			c.handlePong(hdr)
+
+		case messageTypeClusterConfig:
+			if err := c.handleClusterConfig(); err != nil {
+				return err
+			}
+
+		default:
+			return fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType)
+		}
+	}
+}
+
+func (c *rawConnection) handleIndex() error {
+	var im IndexMessage
+	im.decodeXDR(c.xr)
+	if err := c.xr.Error(); err != nil {
+		return err
+	} else {
+
+		// We run this (and the corresponding one for update, below)
+		// in a separate goroutine to avoid blocking the read loop.
+		// There is otherwise a potential deadlock where both sides
+		// has the model locked because it's sending a large index
+		// update and can't receive the large index update from the
+		// other side.
+
+		go c.receiver.Index(c.id, im.Repository, im.Files)
+	}
+	return nil
+}
+
+func (c *rawConnection) handleIndexUpdate() error {
+	var im IndexMessage
+	im.decodeXDR(c.xr)
+	if err := c.xr.Error(); err != nil {
+		return err
+	} else {
+		go c.receiver.IndexUpdate(c.id, im.Repository, im.Files)
+	}
+	return nil
+}
+
+func (c *rawConnection) handleRequest(hdr header) error {
+	var req RequestMessage
+	req.decodeXDR(c.xr)
+	if err := c.xr.Error(); err != nil {
+		return err
+	}
+	go c.processRequest(hdr.msgID, req)
+	return nil
+}
+
+func (c *rawConnection) handleResponse(hdr header) error {
+	data := c.xr.ReadBytesMax(256 * 1024) // Sufficiently larger than max expected block size
+
+	if err := c.xr.Error(); err != nil {
+		return err
+	}
+
+	go func(hdr header, err error) {
+		c.imut.Lock()
+		rc := c.awaiting[hdr.msgID]
+		c.awaiting[hdr.msgID] = nil
+		c.imut.Unlock()
+
+		if rc != nil {
+			rc <- asyncResult{data, err}
+			close(rc)
+		}
+	}(hdr, c.xr.Error())
+
+	return nil
+}
+
+func (c *rawConnection) handlePong(hdr header) {
+	c.imut.Lock()
+	if rc := c.awaiting[hdr.msgID]; rc != nil {
+		go func() {
+			rc <- asyncResult{}
+			close(rc)
+		}()
+
+		c.awaiting[hdr.msgID] = nil
+	}
+	c.imut.Unlock()
+}
+
+func (c *rawConnection) handleClusterConfig() error {
+	var cm ClusterConfigMessage
+	cm.decodeXDR(c.xr)
+	if err := c.xr.Error(); err != nil {
+		return err
+	} else {
+		go c.receiver.ClusterConfig(c.id, cm)
+	}
+	return nil
+}
+
+type encodable interface {
+	encodeXDR(*xdr.Writer) (int, error)
+}
+type encodableBytes []byte
+
+func (e encodableBytes) encodeXDR(xw *xdr.Writer) (int, error) {
+	return xw.WriteBytes(e)
+}
+
+func (c *rawConnection) send(h header, es ...encodable) bool {
+	if h.msgID < 0 {
+		select {
+		case id := <-c.nextID:
+			h.msgID = id
+		case <-c.closed:
+			return false
+		}
+	}
+	msg := append([]encodable{h}, es...)
+
+	select {
+	case c.outbox <- msg:
+		return true
+	case <-c.closed:
+		return false
+	}
+}
+
+func (c *rawConnection) writerLoop() {
+	var err error
+	for es := range c.outbox {
+		c.wmut.Lock()
+		for _, e := range es {
+			e.encodeXDR(c.xw)
+		}
+
+		if err = c.flush(); err != nil {
+			c.wmut.Unlock()
+			c.close(err)
+			return
+		}
+		c.wmut.Unlock()
+	}
+}
+
 type flusher interface {
 	Flush() error
 }
 
 func (c *rawConnection) flush() error {
-	c.wb.Flush()
+	if err := c.xw.Error(); err != nil {
+		return err
+	}
+
+	if err := c.wb.Flush(); err != nil {
+		return err
+	}
+
 	if f, ok := c.writer.(flusher); ok {
 		return f.Flush()
 	}
+
 	return nil
 }
 
 func (c *rawConnection) close(err error) {
-	c.Lock()
+	c.imut.Lock()
+	c.wmut.Lock()
+	defer c.imut.Unlock()
+	defer c.wmut.Unlock()
+
 	select {
 	case <-c.closed:
-		c.Unlock()
 		return
 	default:
-	}
-	close(c.closed)
-	for _, ch := range c.awaiting {
-		close(ch)
-	}
-	c.awaiting = nil
-	c.writer.Close()
-	c.reader.Close()
-	c.Unlock()
+		close(c.closed)
 
-	c.receiver.Close(c.id, err)
-}
+		for i, ch := range c.awaiting {
+			if ch != nil {
+				close(ch)
+				c.awaiting[i] = nil
+			}
+		}
 
-func (c *rawConnection) isClosed() bool {
-	select {
-	case <-c.closed:
-		return true
-	default:
-		return false
+		c.writer.Close()
+		c.reader.Close()
+
+		go c.receiver.Close(c.id, err)
 	}
 }
 
-func (c *rawConnection) readerLoop() {
-loop:
-	for !c.isClosed() {
-		var hdr header
-		hdr.decodeXDR(c.xr)
-		if c.xr.Error() != nil {
-			c.close(c.xr.Error())
-			break loop
+func (c *rawConnection) idGenerator() {
+	nextID := 0
+	for {
+		nextID = (nextID + 1) & 0xfff
+		select {
+		case c.nextID <- nextID:
+		case <-c.closed:
+			return
 		}
-		if hdr.version != 0 {
-			c.close(fmt.Errorf("protocol error: %s: unknown message version %#x", c.id, hdr.version))
-			break loop
-		}
-
-		switch hdr.msgType {
-		case messageTypeIndex:
-			var im IndexMessage
-			im.decodeXDR(c.xr)
-			if c.xr.Error() != nil {
-				c.close(c.xr.Error())
-				break loop
-			} else {
-
-				// We run this (and the corresponding one for update, below)
-				// in a separate goroutine to avoid blocking the read loop.
-				// There is otherwise a potential deadlock where both sides
-				// has the model locked because it's sending a large index
-				// update and can't receive the large index update from the
-				// other side.
-
-				go c.receiver.Index(c.id, im.Repository, im.Files)
-			}
-			c.Lock()
-			c.hasRecvdIndex = true
-			c.Unlock()
-
-		case messageTypeIndexUpdate:
-			var im IndexMessage
-			im.decodeXDR(c.xr)
-			if c.xr.Error() != nil {
-				c.close(c.xr.Error())
-				break loop
-			} else {
-				go c.receiver.IndexUpdate(c.id, im.Repository, im.Files)
-			}
-
-		case messageTypeRequest:
-			var req RequestMessage
-			req.decodeXDR(c.xr)
-			if c.xr.Error() != nil {
-				c.close(c.xr.Error())
-				break loop
-			}
-			go c.processRequest(hdr.msgID, req)
-
-		case messageTypeResponse:
-			data := c.xr.ReadBytesMax(256 * 1024) // Sufficiently larger than max expected block size
-
-			if c.xr.Error() != nil {
-				c.close(c.xr.Error())
-				break loop
-			}
-
-			go func(hdr header, err error) {
-				c.Lock()
-				rc, ok := c.awaiting[hdr.msgID]
-				delete(c.awaiting, hdr.msgID)
-				c.Unlock()
-
-				if ok {
-					rc <- asyncResult{data, err}
-					close(rc)
-				}
-			}(hdr, c.xr.Error())
-
-		case messageTypePing:
-			c.Lock()
-			header{0, hdr.msgID, messageTypePong}.encodeXDR(c.xw)
-			err := c.flush()
-			c.Unlock()
-			if err != nil {
-				c.close(err)
-				break loop
-			} else if c.xw.Error() != nil {
-				c.close(c.xw.Error())
-				break loop
-			}
-
-		case messageTypePong:
-			c.RLock()
-			rc, ok := c.awaiting[hdr.msgID]
-			c.RUnlock()
-
-			if ok {
-				rc <- asyncResult{}
-				close(rc)
-
-				c.Lock()
-				delete(c.awaiting, hdr.msgID)
-				c.Unlock()
-			}
-
-		case messageTypeClusterConfig:
-			var cm ClusterConfigMessage
-			cm.decodeXDR(c.xr)
-			if c.xr.Error() != nil {
-				c.close(c.xr.Error())
-				break loop
-			} else {
-				go c.receiver.ClusterConfig(c.id, cm)
-			}
-
-		default:
-			c.close(fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType))
-			break loop
-		}
-	}
-}
-
-func (c *rawConnection) processRequest(msgID int, req RequestMessage) {
-	data, _ := c.receiver.Request(c.id, req.Repository, req.Name, int64(req.Offset), int(req.Size))
-
-	c.Lock()
-	header{0, msgID, messageTypeResponse}.encodeXDR(c.xw)
-	_, err := c.xw.WriteBytes(data)
-	if err == nil {
-		err = c.flush()
-	}
-	c.Unlock()
-
-	buffers.Put(data)
-	if err != nil {
-		c.close(err)
 	}
 }
 
@@ -436,29 +475,33 @@ func (c *rawConnection) pingerLoop() {
 	for {
 		select {
 		case <-ticker:
-			c.RLock()
-			ready := c.hasRecvdIndex && c.hasSentIndex
-			c.RUnlock()
-
-			if ready {
-				go func() {
-					rc <- c.ping()
-				}()
-				select {
-				case ok := <-rc:
-					if !ok {
-						c.close(fmt.Errorf("ping failure"))
-					}
-				case <-time.After(pingTimeout):
-					c.close(fmt.Errorf("ping timeout"))
+			go func() {
+				rc <- c.ping()
+			}()
+			select {
+			case ok := <-rc:
+				if !ok {
+					c.close(fmt.Errorf("ping failure"))
 				}
+			case <-time.After(pingTimeout):
+				c.close(fmt.Errorf("ping timeout"))
+			case <-c.closed:
+				return
 			}
+
 		case <-c.closed:
 			return
 		}
 	}
 }
 
+func (c *rawConnection) processRequest(msgID int, req RequestMessage) {
+	data, _ := c.receiver.Request(c.id, req.Repository, req.Name, int64(req.Offset), int(req.Size))
+
+	c.send(header{0, msgID, messageTypeResponse},
+		encodableBytes(data))
+}
+
 type Statistics struct {
 	At            time.Time
 	InBytesTotal  int

protocol/protocol_test.go

@@ -174,9 +174,7 @@ func TestClose(t *testing.T) {
 
 	c0.close(nil)
 
-	if !c0.isClosed() {
-		t.Fatal("Connection should be closed")
-	}
+	<-c0.closed
 	if !m0.isClosed() {
 		t.Fatal("Connection should be closed")
 	}

scanner/blocks.go

@@ -6,6 +6,8 @@ import (
 	"io"
 )
 
+const StandardBlockSize = 128 * 1024
+
 type Block struct {
 	Offset int64
 	Size   uint32

scanner/debug.go

@@ -1,12 +1,13 @@
 package scanner
 
 import (
-	"log"
 	"os"
 	"strings"
+
+	"github.com/calmh/syncthing/logger"
 )
 
 var (
-	dlog  = log.New(os.Stderr, "scanner: ", log.Lmicroseconds|log.Lshortfile)
-	debug = strings.Contains(os.Getenv("STTRACE"), "scanner")
+	debug = strings.Contains(os.Getenv("STTRACE"), "scanner") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
 )

scanner/walk.go

@@ -2,8 +2,8 @@ package scanner
 
 import (
 	"bytes"
+	"errors"
 	"io/ioutil"
-	"log"
 	"os"
 	"path/filepath"
 	"strings"
@@ -28,8 +28,6 @@ type Walker struct {
 	// Suppressed files will be returned with empty metadata and the Suppressed flag set.
 	// Requires CurrentFiler to be set.
 	Suppressor Suppressor
-
-	suppressed map[string]bool // file name -> suppression status
 }
 
 type TempNamer interface {
@@ -41,7 +39,7 @@ type TempNamer interface {
 
 type Suppressor interface {
 	// Supress returns true if the update to the named file should be ignored.
-	Suppress(name string, fi os.FileInfo) bool
+	Suppress(name string, fi os.FileInfo) (bool, bool)
 }
 
 type CurrentFiler interface {
@@ -51,12 +49,16 @@ type CurrentFiler interface {
 
 // Walk returns the list of files found in the local repository by scanning the
 // file system. Files are blockwise hashed.
-func (w *Walker) Walk() (files []File, ignore map[string][]string) {
-	w.lazyInit()
-
+func (w *Walker) Walk() (files []File, ignore map[string][]string, err error) {
 	if debug {
-		dlog.Println("Walk", w.Dir, w.BlockSize, w.IgnoreFile)
+		l.Debugln("Walk", w.Dir, w.BlockSize, w.IgnoreFile)
 	}
+
+	err = checkDir(w.Dir)
+	if err != nil {
+		return
+	}
+
 	t0 := time.Now()
 
 	ignore = make(map[string][]string)
@@ -68,8 +70,10 @@ func (w *Walker) Walk() (files []File, ignore map[string][]string) {
 	if debug {
 		t1 := time.Now()
 		d := t1.Sub(t0).Seconds()
-		dlog.Printf("Walk in %.02f ms, %.0f files/s", d*1000, float64(len(files))/d)
+		l.Debugf("Walk in %.02f ms, %.0f files/s", d*1000, float64(len(files))/d)
 	}
+
+	err = checkDir(w.Dir)
 	return
 }
 
@@ -78,12 +82,6 @@ func (w *Walker) CleanTempFiles() {
 	filepath.Walk(w.Dir, w.cleanTempFile)
 }
 
-func (w *Walker) lazyInit() {
-	if w.suppressed == nil {
-		w.suppressed = make(map[string]bool)
-	}
-}
-
 func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.WalkFunc {
 	return func(p string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -116,7 +114,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 	return func(p string, info os.FileInfo, err error) error {
 		if err != nil {
 			if debug {
-				dlog.Println("error:", p, info, err)
+				l.Debugln("error:", p, info, err)
 			}
 			return nil
 		}
@@ -124,7 +122,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 		rn, err := filepath.Rel(w.Dir, p)
 		if err != nil {
 			if debug {
-				dlog.Println("rel error:", p, err)
+				l.Debugln("rel error:", p, err)
 			}
 			return nil
 		}
@@ -136,7 +134,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 		if w.TempNamer != nil && w.TempNamer.IsTemporary(rn) {
 			// A temporary file
 			if debug {
-				dlog.Println("temporary:", rn)
+				l.Debugln("temporary:", rn)
 			}
 			return nil
 		}
@@ -144,7 +142,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 		if _, sn := filepath.Split(rn); sn == w.IgnoreFile {
 			// An ignore-file; these are ignored themselves
 			if debug {
-				dlog.Println("ignorefile:", rn)
+				l.Debugln("ignorefile:", rn)
 			}
 			return nil
 		}
@@ -152,7 +150,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 		if w.ignoreFile(ign, rn) {
 			// An ignored file
 			if debug {
-				dlog.Println("ignored:", rn)
+				l.Debugln("ignored:", rn)
 			}
 			if info.IsDir() {
 				return filepath.SkipDir
@@ -165,7 +163,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 				cf := w.CurrentFiler.CurrentFile(rn)
 				if cf.Modified == info.ModTime().Unix() && cf.Flags == uint32(info.Mode()&os.ModePerm|protocol.FlagDirectory) {
 					if debug {
-						dlog.Println("unchanged:", cf)
+						l.Debugln("unchanged:", cf)
 					}
 					*res = append(*res, cf)
 				} else {
@@ -176,7 +174,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 						Modified: info.ModTime().Unix(),
 					}
 					if debug {
-						dlog.Println("dir:", cf, f)
+						l.Debugln("dir:", cf, f)
 					}
 					*res = append(*res, f)
 				}
@@ -189,34 +187,32 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 				cf := w.CurrentFiler.CurrentFile(rn)
 				if cf.Flags&protocol.FlagDeleted == 0 && cf.Modified == info.ModTime().Unix() {
 					if debug {
-						dlog.Println("unchanged:", cf)
+						l.Debugln("unchanged:", cf)
 					}
 					*res = append(*res, cf)
 					return nil
 				}
 
-				if w.Suppressor != nil && w.Suppressor.Suppress(rn, info) {
-					if !w.suppressed[rn] {
-						w.suppressed[rn] = true
-						log.Printf("INFO: Changes to %q are being temporarily suppressed because it changes too frequently.", p)
+				if w.Suppressor != nil {
+					if cur, prev := w.Suppressor.Suppress(rn, info); cur && !prev {
+						l.Infof("Changes to %q are being temporarily suppressed because it changes too frequently.", p)
 						cf.Suppressed = true
 						cf.Version++
+						if debug {
+							l.Debugln("suppressed:", cf)
+						}
+						*res = append(*res, cf)
+						return nil
+					} else if prev && !cur {
+						l.Infof("Changes to %q are no longer suppressed.", p)
 					}
-					if debug {
-						dlog.Println("suppressed:", cf)
-					}
-					*res = append(*res, cf)
-					return nil
-				} else if w.suppressed[rn] {
-					log.Printf("INFO: Changes to %q are no longer suppressed.", p)
-					delete(w.suppressed, rn)
 				}
 			}
 
 			fd, err := os.Open(p)
 			if err != nil {
 				if debug {
-					dlog.Println("open:", p, err)
+					l.Debugln("open:", p, err)
 				}
 				return nil
 			}
@@ -226,13 +222,13 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 			blocks, err := Blocks(fd, w.BlockSize)
 			if err != nil {
 				if debug {
-					dlog.Println("hash error:", rn, err)
+					l.Debugln("hash error:", rn, err)
 				}
 				return nil
 			}
 			if debug {
 				t1 := time.Now()
-				dlog.Println("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s")
+				l.Debugln("hashed:", rn, ";", len(blocks), "blocks;", info.Size(), "bytes;", int(float64(info.Size())/1024/t1.Sub(t0).Seconds()), "KB/s")
 			}
 			f := File{
 				Name:     rn,
@@ -272,3 +268,12 @@ func (w *Walker) ignoreFile(patterns map[string][]string, file string) bool {
 	}
 	return false
 }
+
+func checkDir(dir string) error {
+	if info, err := os.Stat(dir); err != nil {
+		return err
+	} else if !info.IsDir() {
+		return errors.New(dir + ": not a directory")
+	}
+	return nil
+}

scanner/walk_test.go

@@ -27,7 +27,11 @@ func TestWalk(t *testing.T) {
 		BlockSize:  128 * 1024,
 		IgnoreFile: ".stignore",
 	}
-	files, ignores := w.Walk()
+	files, ignores, err := w.Walk()
+
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	if l1, l2 := len(files), len(testdata); l1 != l2 {
 		t.Fatalf("Incorrect number of walked files %d != %d", l1, l2)
@@ -54,6 +58,30 @@ func TestWalk(t *testing.T) {
 	}
 }
 
+func TestWalkError(t *testing.T) {
+	w := Walker{
+		Dir:        "testdata-missing",
+		BlockSize:  128 * 1024,
+		IgnoreFile: ".stignore",
+	}
+	_, _, err := w.Walk()
+
+	if err == nil {
+		t.Error("no error from missing directory")
+	}
+
+	w = Walker{
+		Dir:        "testdata/bar",
+		BlockSize:  128 * 1024,
+		IgnoreFile: ".stignore",
+	}
+	_, _, err = w.Walk()
+
+	if err == nil {
+		t.Error("no error from non-directory")
+	}
+}
+
 func TestIgnore(t *testing.T) {
 	var patterns = map[string][]string{
 		"":        {"t2"},

upnp/debug.go

@@ -1,12 +1,13 @@
 package upnp
 
 import (
-	"log"
 	"os"
 	"strings"
+
+	"github.com/calmh/syncthing/logger"
 )
 
 var (
-	dlog  = log.New(os.Stderr, "upnp: ", log.Lmicroseconds|log.Lshortfile)
-	debug = strings.Contains(os.Getenv("STTRACE"), "upnp")
+	debug = strings.Contains(os.Getenv("STTRACE"), "upnp") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
 )

upnp/upnp.go

@@ -81,7 +81,7 @@ Mx: 3
 	}
 
 	if debug {
-		dlog.Println(string(resp[:n]))
+		l.Debugln(string(resp[:n]))
 	}
 
 	reader := bufio.NewReader(bytes.NewBuffer(resp[:n]))
@@ -225,8 +225,8 @@ func soapRequest(url, function, message string) error {
 	req.Header.Set("Pragma", "no-cache")
 
 	if debug {
-		dlog.Println(req.Header.Get("SOAPAction"))
-		dlog.Println(body)
+		l.Debugln(req.Header.Get("SOAPAction"))
+		l.Debugln(body)
 	}
 
 	r, err := http.DefaultClient.Do(req)
@@ -236,7 +236,7 @@ func soapRequest(url, function, message string) error {
 
 	if debug {
 		resp, _ := ioutil.ReadAll(r.Body)
-		dlog.Println(string(resp))
+		l.Debugln(string(resp))
 	}
 
 	r.Body.Close()

xdr/reader.go

@@ -72,20 +72,25 @@ func (r *Reader) ReadUint16() uint16 {
 }
 
 func (r *Reader) ReadUint32() uint32 {
+	var n int
 	if r.err != nil {
 		return 0
 	}
-	_, r.err = io.ReadFull(r.r, r.b[:4])
-	r.tot += 4
+	n, r.err = io.ReadFull(r.r, r.b[:4])
+	if n < 4 {
+		return 0
+	}
+	r.tot += n
 	return uint32(r.b[3]) | uint32(r.b[2])<<8 | uint32(r.b[1])<<16 | uint32(r.b[0])<<24
 }
 
 func (r *Reader) ReadUint64() uint64 {
+	var n int
 	if r.err != nil {
 		return 0
 	}
-	_, r.err = io.ReadFull(r.r, r.b[:8])
-	r.tot += 8
+	n, r.err = io.ReadFull(r.r, r.b[:8])
+	r.tot += n
 	return uint64(r.b[7]) | uint64(r.b[6])<<8 | uint64(r.b[5])<<16 | uint64(r.b[4])<<24 |
 		uint64(r.b[3])<<32 | uint64(r.b[2])<<40 | uint64(r.b[1])<<48 | uint64(r.b[0])<<56
 }