Increase reconnect delay towards max

Change Name -> Node Name to match Add Repo dialog.

.gitignore

@@ -8,4 +8,5 @@ stcli.exe
 *.sublime*
 discosrv
 stpidx
-.jshintrc
+.jshintrc
+coverage.out

.travis.yml

@@ -1,12 +1,20 @@
 language: go
 
 go:
-   - 1.2
    - tip
 
 install:
    - export PATH=$PATH:$HOME/gopath/bin
    - ./build.sh setup
+   - go get code.google.com/p/go.tools/cmd/cover
+   - go get github.com/mattn/goveralls
 
 script:
-   - ./build.sh test
+   - ./build.sh test-cov
+
+after_success:
+   - goveralls -coverprofile=coverage.out -service=travis-ci -package=calmh/syncthing -repotoken="$COVERALS_TOKEN"
+
+env:
+   global:
+      secure: "zEV2h2XtKHNLVdXJjM4LA/VjMfLVydm6goF+ARit+nOSGxGoH7f7jIdzJzhxgh7shKG93q61eLO1Tug+WBMYB2EpBuYnTB5AIMYhCDwNI8C4uBV6c3brHfcrie7MASNao8TID2QScASKNFFWvjv/i1Ccn5ztxdcQuhSsNjGZp8A="

README.md

@@ -1,4 +1,4 @@
-syncthing [![Build Status](https://travis-ci.org/calmh/syncthing.svg?branch=master)](https://travis-ci.org/calmh/syncthing)
+syncthing [![Build Status](https://travis-ci.org/calmh/syncthing.svg?branch=master)](https://travis-ci.org/calmh/syncthing) [![Coverage Status](https://img.shields.io/coveralls/calmh/syncthing.svg)](https://coveralls.io/r/calmh/syncthing?branch=master)
 =========
 
 This is the `syncthing` project. The following are the project goals:

beacon/beacon.go

@@ -83,7 +83,7 @@ func (b *Beacon) writer() {
 
 		var dsts []net.IP
 		for _, addr := range addrs {
-			if iaddr, ok := addr.(*net.IPNet); ok && iaddr.IP.IsGlobalUnicast() {
+			if iaddr, ok := addr.(*net.IPNet); ok && iaddr.IP.IsGlobalUnicast() && iaddr.IP.To4() != nil {
 				baddr := bcast(iaddr)
 				dsts = append(dsts, baddr.IP)
 			}

build.sh

@@ -9,7 +9,8 @@ date=$(git show -s --format=%ct)
 user=$(whoami)
 host=$(hostname)
 host=${host%%.*}
-ldflags="-w -X main.Version $version -X main.BuildStamp $date -X main.BuildUser $user -X main.BuildHost $host"
+bldenv=${ENVIRONMENT:-default}
+ldflags="-w -X main.Version $version -X main.BuildStamp $date -X main.BuildUser $user -X main.BuildHost $host -X main.BuildEnv $bldenv"
 
 check() {
 	if ! command -v godep >/dev/null ; then
@@ -31,6 +32,21 @@ assets() {
 	godep go run cmd/assets/assets.go gui > auto/gui.files.go
 }
 
+test-cov() {
+	echo "mode: set" > coverage.out
+	fail=0
+
+	for dir in $(go list ./...) ; do
+		godep go test -coverprofile=profile.out $dir || fail=1
+		if [ -f profile.out ] ; then
+			grep -v "mode: set" profile.out >> coverage.out
+		rm profile.out
+        fi
+    done
+
+   exit $fail
+}
+
 test() {
 	check
 	godep go test -cpu=1,2,4 ./...
@@ -61,7 +77,8 @@ zipDist() {
 	rm -rf "$name"
 	mkdir -p "$name"
 	for f in "${distFiles[@]}" ; do
-		sed 's/$/
+		sed 's/$/
+/' < "$f" > "$name/$f.txt"
 	done
 	cp syncthing.exe "$name"
 	sign "$name/syncthing.exe"
@@ -100,6 +117,10 @@ case "$1" in
 		test
 		;;
 
+	test-cov)
+		test-cov
+		;;
+
 	tar)
 		rm -f *.tar.gz *.zip
 		test || exit 1

cmd/syncthing/gui.go

@@ -58,6 +58,8 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
 	if cfg.UseTLS {
 		cert, err := loadCert(confDir, "https-")
 		if err != nil {
+			l.Infoln("Loading HTTPS certificate:", err)
+			l.Infoln("Creating new HTTPS certificate")
 			newCertificate(confDir, "https-")
 			cert, err = loadCert(confDir, "https-")
 		}
@@ -96,6 +98,7 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
 	router.Get("/rest/system", restGetSystem)
 	router.Get("/rest/errors", restGetErrors)
 	router.Get("/rest/discovery", restGetDiscovery)
+	router.Get("/rest/report", restGetReport)
 	router.Get("/qr/:text", getQR)
 
 	router.Post("/rest/config", restPostConfig)
@@ -193,7 +196,7 @@ func restGetConfig(w http.ResponseWriter) {
 	json.NewEncoder(w).Encode(encCfg)
 }
 
-func restPostConfig(req *http.Request) {
+func restPostConfig(req *http.Request, m *model.Model) {
 	var newCfg config.Configuration
 	err := json.NewDecoder(req.Body).Decode(&newCfg)
 	if err != nil {
@@ -240,7 +243,33 @@ func restPostConfig(req *http.Request) {
 			}
 		}
 
-		if !reflect.DeepEqual(cfg.Options, newCfg.Options) {
+		if newCfg.Options.UREnabled && !cfg.Options.UREnabled {
+			// UR was enabled
+			cfg.Options.UREnabled = true
+			cfg.Options.URDeclined = false
+			cfg.Options.URAccepted = usageReportVersion
+			// Set the corresponding options in newCfg so we don't trigger the restart check if this was the only option change
+			newCfg.Options.URDeclined = false
+			newCfg.Options.URAccepted = usageReportVersion
+			err := sendUsageReport(m)
+			if err != nil {
+				l.Infoln("Usage report:", err)
+			}
+			go usageReportingLoop(m)
+		} else if !newCfg.Options.UREnabled && cfg.Options.UREnabled {
+			// UR was disabled
+			cfg.Options.UREnabled = false
+			cfg.Options.URDeclined = true
+			cfg.Options.URAccepted = 0
+			// Set the corresponding options in newCfg so we don't trigger the restart check if this was the only option change
+			newCfg.Options.URDeclined = true
+			newCfg.Options.URAccepted = 0
+			stopUsageReporting()
+		} else {
+			cfg.Options.URDeclined = newCfg.Options.URDeclined
+		}
+
+		if !reflect.DeepEqual(cfg.Options, newCfg.Options) || !reflect.DeepEqual(cfg.GUI, newCfg.GUI) {
 			configInSync = false
 		}
 
@@ -345,6 +374,10 @@ func restGetDiscovery(w http.ResponseWriter) {
 	json.NewEncoder(w).Encode(discoverer.All())
 }
 
+func restGetReport(w http.ResponseWriter, m *model.Model) {
+	json.NewEncoder(w).Encode(reportData(m))
+}
+
 func getQR(w http.ResponseWriter, params martini.Params) {
 	code, err := qr.Encode(params["text"], qr.M)
 	if err != nil {

cmd/syncthing/main.go

@@ -5,6 +5,7 @@
 package main
 
 import (
+	"crypto/sha1"
 	"crypto/tls"
 	"flag"
 	"fmt"
@@ -36,6 +37,7 @@ import (
 
 var (
 	Version     = "unknown-dev"
+	BuildEnv    = "default"
 	BuildStamp  = "0"
 	BuildDate   time.Time
 	BuildHost   = "unknown"
@@ -50,7 +52,7 @@ func init() {
 	BuildDate = time.Unix(int64(stamp), 0)
 
 	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf("syncthing %s (%s %s-%s) %s@%s %s", Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
+	LongVersion = fmt.Sprintf("syncthing %s (%s %s-%s %s) %s@%s %s", Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildEnv, BuildUser, BuildHost, date)
 
 	if os.Getenv("STTRACE") != "" {
 		logFlags = log.Ltime | log.Ldate | log.Lmicroseconds | log.Lshortfile
@@ -107,6 +109,10 @@ The following enviroment variables are interpreted by syncthing:
  STGUIASSETS   Directory to load GUI assets from. Overrides compiled in assets.`
 )
 
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
 func main() {
 	var reset bool
 	var showVersion bool
@@ -348,14 +354,36 @@ func main() {
 	m.ScanRepos()
 	m.SaveIndexes(confDir)
 
+	// Remove all .idx* files that don't belong to an active repo.
+
+	validIndexes := make(map[string]bool)
+	for _, repo := range cfg.Repositories {
+		dir := expandTilde(repo.Directory)
+		id := fmt.Sprintf("%x", sha1.Sum([]byte(dir)))
+		validIndexes[id] = true
+	}
+
+	allIndexes, err := filepath.Glob(filepath.Join(confDir, "*.idx*"))
+	if err == nil {
+		for _, idx := range allIndexes {
+			bn := filepath.Base(idx)
+			fs := strings.Split(bn, ".")
+			if len(fs) > 1 {
+				if _, ok := validIndexes[fs[0]]; !ok {
+					l.Infoln("Removing old index", bn)
+					os.Remove(idx)
+				}
+			}
+		}
+	}
+
 	// UPnP
 
 	var externalPort = 0
 	if cfg.Options.UPnPEnabled {
 		// We seed the random number generator with the node ID to get a
 		// repeatable sequence of random external ports.
-		rand.Seed(certSeed(cert.Certificate[0]))
-		externalPort = setupUPnP()
+		externalPort = setupUPnP(rand.NewSource(certSeed(cert.Certificate[0])))
 	}
 
 	// Routine to connect out to configured nodes
@@ -393,6 +421,21 @@ func main() {
 		}
 	}
 
+	if cfg.Options.UREnabled && cfg.Options.URAccepted < usageReportVersion {
+		l.Infoln("Anonymous usage report has changed; revoking acceptance")
+		cfg.Options.UREnabled = false
+	}
+	if cfg.Options.UREnabled {
+		go usageReportingLoop(m)
+		go func() {
+			time.Sleep(10 * time.Minute)
+			err := sendUsageReport(m)
+			if err != nil {
+				l.Infoln("Usage report:", err)
+			}
+		}()
+	}
+
 	<-stop
 	l.Okln("Exiting")
 }
@@ -411,7 +454,7 @@ func waitForParentExit() {
 	l.Okln("Continuing")
 }
 
-func setupUPnP() int {
+func setupUPnP(r rand.Source) int {
 	var externalPort = 0
 	if len(cfg.Options.ListenAddress) == 1 {
 		_, portStr, err := net.SplitHostPort(cfg.Options.ListenAddress[0])
@@ -423,7 +466,7 @@ func setupUPnP() int {
 			igd, err := upnp.Discover()
 			if err == nil {
 				for i := 0; i < 10; i++ {
-					r := 1024 + rand.Intn(65535-1024)
+					r := 1024 + int(r.Int63()%(65535-1024))
 					err := igd.AddPortMapping(upnp.TCP, r, port, "syncthing", 0)
 					if err == nil {
 						externalPort = r
@@ -569,6 +612,7 @@ func listenConnect(myID string, m *model.Model, tlsCfg *tls.Config) {
 
 	// Connect
 	go func() {
+		var delay time.Duration = 1 * time.Second
 		for {
 		nextNode:
 			for _, nodeCfg := range cfg.Nodes {
@@ -619,7 +663,11 @@ func listenConnect(myID string, m *model.Model, tlsCfg *tls.Config) {
 				}
 			}
 
-			time.Sleep(time.Duration(cfg.Options.ReconnectIntervalS) * time.Second)
+			time.Sleep(delay)
+			delay *= 2
+			if maxD := time.Duration(cfg.Options.ReconnectIntervalS) * time.Second; delay > maxD {
+				delay = maxD
+			}
 		}
 	}()
 

cmd/syncthing/memsize_darwin.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"errors"
+	"os/exec"
+	"strconv"
+	"strings"
+)
+
+func memorySize() (uint64, error) {
+	cmd := exec.Command("sysctl", "hw.memsize")
+	out, err := cmd.Output()
+	if err != nil {
+		return 0, err
+	}
+	fs := strings.Fields(string(out))
+	if len(fs) != 2 {
+		return 0, errors.New("sysctl parse error")
+	}
+	bytes, err := strconv.ParseUint(fs[1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	return bytes, nil
+}

cmd/syncthing/memsize_linux.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"bufio"
+	"errors"
+	"os"
+	"strconv"
+	"strings"
+)
+
+func memorySize() (uint64, error) {
+	f, err := os.Open("/proc/meminfo")
+	if err != nil {
+		return 0, err
+	}
+
+	s := bufio.NewScanner(f)
+	if !s.Scan() {
+		return 0, errors.New("/proc/meminfo parse error 1")
+	}
+
+	l := s.Text()
+	fs := strings.Fields(l)
+	if len(fs) != 3 || fs[2] != "kB" {
+		return 0, errors.New("/proc/meminfo parse error 2")
+	}
+
+	kb, err := strconv.ParseUint(fs[1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+	return kb * 1024, nil
+}

cmd/syncthing/memsize_unimpl.go

@@ -0,0 +1,9 @@
+// +build freebsd solaris
+
+package main
+
+import "errors"
+
+func memorySize() (uint64, error) {
+	return 0, errors.New("not implemented")
+}

cmd/syncthing/memsize_windows.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"encoding/binary"
+	"syscall"
+	"unsafe"
+)
+
+var (
+	kernel32, _             = syscall.LoadLibrary("kernel32.dll")
+	globalMemoryStatusEx, _ = syscall.GetProcAddress(kernel32, "GlobalMemoryStatusEx")
+)
+
+func memorySize() (uint64, error) {
+	var memoryStatusEx [64]byte
+	binary.LittleEndian.PutUint32(memoryStatusEx[:], 64)
+	p := uintptr(unsafe.Pointer(&memoryStatusEx[0]))
+
+	ret, _, callErr := syscall.Syscall(uintptr(globalMemoryStatusEx), 1, p, 0, 0)
+	if ret == 0 {
+		return 0, callErr
+	}
+
+	return binary.LittleEndian.Uint64(memoryStatusEx[8:]), nil
+}

cmd/syncthing/usage_report.go

@@ -0,0 +1,129 @@
+package main
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/json"
+	"net"
+	"net/http"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/calmh/syncthing/model"
+)
+
+// Current version number of the usage report, for acceptance purposes. If
+// fields are added or changed this integer must be incremented so that users
+// are prompted for acceptance of the new report.
+const usageReportVersion = 1
+
+var stopUsageReportingCh = make(chan struct{})
+
+func reportData(m *model.Model) map[string]interface{} {
+	res := make(map[string]interface{})
+	res["uniqueID"] = strings.ToLower(certID([]byte(myID)))[:6]
+	res["version"] = Version
+	res["platform"] = runtime.GOOS + "-" + runtime.GOARCH
+	res["numRepos"] = len(cfg.Repositories)
+	res["numNodes"] = len(cfg.Nodes)
+
+	var totFiles, maxFiles int
+	var totBytes, maxBytes int64
+	for _, repo := range cfg.Repositories {
+		files, _, bytes := m.GlobalSize(repo.ID)
+		totFiles += files
+		totBytes += bytes
+		if files > maxFiles {
+			maxFiles = files
+		}
+		if bytes > maxBytes {
+			maxBytes = bytes
+		}
+	}
+
+	res["totFiles"] = totFiles
+	res["repoMaxFiles"] = maxFiles
+	res["totMiB"] = totBytes / 1024 / 1024
+	res["repoMaxMiB"] = maxBytes / 1024 / 1024
+
+	var mem runtime.MemStats
+	runtime.ReadMemStats(&mem)
+	res["memoryUsageMiB"] = mem.Sys / 1024 / 1024
+
+	var perf float64
+	for i := 0; i < 5; i++ {
+		p := cpuBench()
+		if p > perf {
+			perf = p
+		}
+	}
+	res["sha256Perf"] = perf
+
+	bytes, err := memorySize()
+	if err == nil {
+		res["memorySize"] = bytes / 1024 / 1024
+	}
+
+	return res
+}
+
+func sendUsageReport(m *model.Model) error {
+	d := reportData(m)
+	var b bytes.Buffer
+	json.NewEncoder(&b).Encode(d)
+
+	var client = http.DefaultClient
+	if BuildEnv == "android" {
+		// This works around the lack of DNS resolution on Android... :(
+		tr := &http.Transport{
+			Dial: func(network, addr string) (net.Conn, error) {
+				return net.Dial(network, "194.126.249.13:443")
+			},
+		}
+		client = &http.Client{Transport: tr}
+	}
+	_, err := client.Post("https://data.syncthing.net/newdata", "application/json", &b)
+	return err
+}
+
+func usageReportingLoop(m *model.Model) {
+	l.Infoln("Starting usage reporting")
+	t := time.NewTicker(86400 * time.Second)
+loop:
+	for {
+		select {
+		case <-stopUsageReportingCh:
+			break loop
+		case <-t.C:
+			err := sendUsageReport(m)
+			if err != nil {
+				l.Infoln("Usage report:", err)
+			}
+		}
+	}
+	l.Infoln("Stopping usage reporting")
+}
+
+func stopUsageReporting() {
+	stopUsageReportingCh <- struct{}{}
+}
+
+// Returns CPU performance as a measure of single threaded SHA-256 MiB/s
+func cpuBench() float64 {
+	chunkSize := 100 * 1 << 10
+	h := sha256.New()
+	bs := make([]byte, chunkSize)
+	rand.Reader.Read(bs)
+
+	t0 := time.Now()
+	b := 0
+	for time.Since(t0) < 125*time.Millisecond {
+		h.Write(bs)
+		b += chunkSize
+	}
+	h.Sum(nil)
+	d := time.Since(t0)
+	return float64(int(float64(b)/d.Seconds()/(1<<20)*100)) / 100
+}

config/config.go

@@ -157,6 +157,10 @@ type OptionsConfiguration struct {
 	StartBrowser       bool     `xml:"startBrowser" default:"true"`
 	UPnPEnabled        bool     `xml:"upnpEnabled" default:"true"`
 
+	UREnabled  bool `xml:"urEnabled"`  // If true, send usage reporting data
+	URDeclined bool `xml:"urDeclined"` // If true, don't ask again
+	URAccepted int  `xml:"urAccepted"` // Accepted usage reporting version
+
 	Deprecated_ReadOnly   bool   `xml:"readOnly,omitempty" json:"-"`
 	Deprecated_GUIEnabled bool   `xml:"guiEnabled,omitempty" json:"-"`
 	Deprecated_GUIAddress string `xml:"guiAddress,omitempty" json:"-"`

files/set.go

@@ -120,7 +120,12 @@ func (m *Set) Need(id uint) []scanner.File {
 			continue
 		}
 
-		if gk.newerThan(rkID[gk.Name]) {
+		if rk, ok := rkID[gk.Name]; gk.newerThan(rk) {
+			if protocol.IsDeleted(gf.File.Flags) && (!ok || protocol.IsDeleted(m.files[rk].File.Flags)) {
+				// We don't need to delete files we don't have or that are already deleted
+				continue
+			}
+
 			fs = append(fs, gf.File)
 		}
 	}

gui/app.js

@@ -30,21 +30,37 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
     $scope.seenError = '';
     $scope.model = {};
     $scope.repos = {};
+    $scope.reportData = {};
+    $scope.reportPreview = false;
+
+    $scope.needActions = {
+        'rm': 'Del',
+        'rmdir': 'Del (dir)',
+        'sync': 'Sync',
+        'touch': 'Update',
+    }
+    $scope.needIcons = {
+        'rm': 'remove',
+        'rmdir': 'remove',
+        'sync': 'download',
+        'touch': 'asterisk',
+    }
 
     // Strings before bools look better
     $scope.settings = [
-    {id: 'ListenStr', descr: 'Sync Protocol Listen Addresses', type: 'text', restart: true},
-    {id: 'MaxSendKbps', descr: 'Outgoing Rate Limit (KiB/s)', type: 'number', restart: true},
-    {id: 'RescanIntervalS', descr: 'Rescan Interval (s)', type: 'number', restart: true},
-    {id: 'ReconnectIntervalS', descr: 'Reconnect Interval (s)', type: 'number', restart: true},
-    {id: 'ParallelRequests', descr: 'Max Outstanding Requests', type: 'number', restart: true},
-    {id: 'MaxChangeKbps', descr: 'Max File Change Rate (KiB/s)', type: 'number', restart: true},
+    {id: 'ListenStr', descr: 'Sync Protocol Listen Addresses', type: 'text'},
+    {id: 'MaxSendKbps', descr: 'Outgoing Rate Limit (KiB/s)', type: 'number'},
+    {id: 'RescanIntervalS', descr: 'Rescan Interval (s)', type: 'number'},
+    {id: 'ReconnectIntervalS', descr: 'Reconnect Interval (s)', type: 'number'},
+    {id: 'ParallelRequests', descr: 'Max Outstanding Requests', type: 'number'},
+    {id: 'MaxChangeKbps', descr: 'Max File Change Rate (KiB/s)', type: 'number'},
 
-    {id: 'GlobalAnnEnabled', descr: 'Global Discovery', type: 'bool', restart: true},
-    {id: 'LocalAnnEnabled', descr: 'Local Discovery', type: 'bool', restart: true},
-    {id: 'LocalAnnPort', descr: 'Local Discovery Port', type: 'number', restart: true},
+    {id: 'LocalAnnPort', descr: 'Local Discovery Port', type: 'number'},
+    {id: 'LocalAnnEnabled', descr: 'Local Discovery', type: 'bool'},
+    {id: 'GlobalAnnEnabled', descr: 'Global Discovery', type: 'bool'},
     {id: 'StartBrowser', descr: 'Start Browser', type: 'bool'},
     {id: 'UPnPEnabled', descr: 'Enable UPnP', type: 'bool'},
+    {id: 'UREnabled', descr: 'Anonymous Usage Reporting', type: 'bool'},
     ];
 
     $scope.guiSettings = [
@@ -544,11 +560,71 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
             $scope.repos = repoMap($scope.config.Repositories);
 
             $scope.refresh();
+
+            if (!$scope.config.Options.UREnabled && !$scope.config.Options.URDeclined) {
+                // If usage reporting has been neither accepted nor declined,
+                // we want to ask the user to make a choice. But we don't want
+                // to bug them during initial setup, so we set a cookie with
+                // the time of the first visit. When that cookie is present
+                // and the time is more than four hours ago, we ask the
+                // question.
+
+                var firstVisit = document.cookie.replace(/(?:(?:^|.*;\s*)firstVisit\s*\=\s*([^;]*).*$)|^.*$/, "$1");
+                if (!firstVisit) {
+                    document.cookie = "firstVisit=" + Date.now() + ";max-age=" + 30*24*3600;
+                } else {
+                    if (+firstVisit < Date.now() - 4*3600*1000){
+                        $('#ur').modal({backdrop: 'static', keyboard: false});
+                    }
+                }
+            }
         });
 
         $http.get(urlbase + '/config/sync').success(function (data) {
             $scope.configInSync = data.configInSync;
         });
+
+        $http.get(urlbase + '/report').success(function (data) {
+            $scope.reportData = data;
+        });
+    };
+
+    $scope.acceptUR = function () {
+        $scope.config.Options.UREnabled = true;
+        $scope.config.Options.URDeclined = false;
+        $scope.saveConfig();
+        $('#ur').modal('hide');
+    };
+
+    $scope.declineUR = function () {
+        $scope.config.Options.UREnabled = false;
+        $scope.config.Options.URDeclined = true;
+        $scope.saveConfig();
+        $('#ur').modal('hide');
+    };
+
+    $scope.showNeed = function (repo) {
+        $scope.neededLoaded = false;
+        $('#needed').modal({backdrop: 'static', keyboard: true});
+        $http.get(urlbase + "/need?repo=" + encodeURIComponent(repo)).success(function (data) {
+            $scope.needed = data;
+            $scope.neededLoaded = true;
+        });
+    };
+
+    $scope.needAction = function (file) {
+        var fDelete = 4096;
+        var fDirectory = 16384;
+
+        if ((file.Flags & (fDelete+fDirectory)) === fDelete+fDirectory) {
+            return 'rmdir';
+        } else if ((file.Flags & fDelete) === fDelete) {
+            return 'rm';
+        } else if ((file.Flags & fDirectory) === fDirectory) {
+            return 'touch';
+        } else {
+            return 'sync';
+        }
     };
 
     $scope.init();
@@ -701,6 +777,18 @@ syncthing.filter('shortPath', function () {
     };
 });
 
+syncthing.filter('basename', function () {
+    return function (input) {
+        if (input === undefined)
+            return "";
+        var parts = input.split(/[\/\\]/);
+        if (!parts || parts.length < 1) {
+            return input;
+        }
+        return parts[parts.length-1];
+    };
+});
+
 syncthing.filter('clean', function () {
     return function (input) {
         return encodeURIComponent(input).replace(/%/g, '');

gui/index.html

@@ -65,7 +65,7 @@ found in the LICENSE file.
     }
 
     .table th {
-      white-space:nowrap;
+      white-space: nowrap;
       font-weight: 400;
     }
 
@@ -73,6 +73,10 @@ found in the LICENSE file.
       padding-left: 20px !important;
     }
 
+    .table td.small-data {
+      white-space: nowrap;
+    }
+
     @media (max-width:767px) {
       .table-responsive>.table>tbody>tr>td {
         /* revert a bootstrap setting e.g.:
@@ -168,15 +172,18 @@ found in the LICENSE file.
                       </tr>
                       <tr>
                         <th><span class="glyphicon glyphicon-globe"></span>&emsp;Global Repository</th>
-                        <td class="text-right">{{model[repo.ID].globalFiles | alwaysNumber}} files, {{model[repo.ID].globalBytes | binary}}B</td>
+                        <td class="text-right">{{model[repo.ID].globalFiles | alwaysNumber}} items, {{model[repo.ID].globalBytes | binary}}B</td>
                       </tr>
                       <tr>
                         <th><span class="glyphicon glyphicon-home"></span>&emsp;Local Repository</th>
-                        <td class="text-right">{{model[repo.ID].localFiles | alwaysNumber}} files, {{model[repo.ID].localBytes | binary}}B</td>
+                        <td class="text-right">{{model[repo.ID].localFiles | alwaysNumber}} items, {{model[repo.ID].localBytes | binary}}B</td>
                       </tr>
                       <tr>
                         <th><span class="glyphicon glyphicon-cloud-download"></span>&emsp;Out of Sync</th>
-                        <td class="text-right">{{model[repo.ID].needFiles | alwaysNumber}} files, {{model[repo.ID].needBytes | binary}}B</td>
+                        <td class="text-right">
+                        <a ng-if="model[repo.ID].needFiles > 0" ng-click="showNeed(repo.ID)" href="">{{model[repo.ID].needFiles | alwaysNumber}} items, {{model[repo.ID].needBytes | binary}}B</a>
+                        <span ng-if="model[repo.ID].needFiles == 0">0 items, 0 B</span>
+                        </td>
                       </tr>
                       <tr>
                         <th><span class="glyphicon glyphicon-lock"></span>&emsp;Master Repository</th>
@@ -442,7 +449,7 @@ found in the LICENSE file.
               </p>
             </div>
             <div class="form-group">
-              <label for="name">Name</label>
+              <label for="name">Node Name</label>
               <input placeholder="Home Server" id="name" class="form-control" type="text" ng-model="currentNode.Name"></input>
               <p class="help-block">Shown instead of Node ID in the cluster status.</p>
             </div>
@@ -564,7 +571,7 @@ found in the LICENSE file.
     <div class="modal-dialog modal-lg">
       <div class="modal-content">
         <div class="modal-header">
-          <h4 class="modal-title"> Settings</h4>
+          <h4 class="modal-title">Settings</h4>
         </div>
         <div class="modal-body">
           <form role="form">
@@ -611,6 +618,56 @@ found in the LICENSE file.
     </div>
   </div>
 
+  <!-- Usage report modal -->
+
+  <div id="ur" class="modal fade">
+    <div class="modal-dialog modal-lg">
+      <div class="modal-content">
+        <div class="modal-header alert alert-success">
+          <h4 class="modal-title">Allow Anonymous Usage Reporting?</h4>
+        </div>
+        <div class="modal-body">
+        <p>
+        The encrypted usage report is sent daily. It is used to track common platforms, repo sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.
+        </p>
+        <p>
+        The aggregated statistics are publicly available at <a href="https://data.syncthing.net/">https://data.syncthing.net/</a>.
+        </p>
+        <button type="button" class="btn btn-default" ng-show="!reportPreview" ng-click="reportPreview = true">Preview Usage Report</button>
+        <pre ng-if="reportPreview"><small>{{reportData | json}}</small></pre>
+        </div>
+        <div class="modal-footer">
+          <button type="button" class="btn btn-success" ng-click="acceptUR()"><span class="glyphicon glyphicon-ok"></span>&emsp;Yes</button>
+          <button type="button" class="btn btn-danger" ng-click="declineUR()"><span class="glyphicon glyphicon-remove"></span>&emsp;No</button>
+        </div>
+      </div>
+    </div>
+  </div>
+
+  <!-- Needed files modal -->
+
+  <div id="needed" class="modal fade">
+    <div class="modal-dialog modal-lg">
+      <div class="modal-content">
+        <div class="modal-header alert alert-info">
+          <h4 class="modal-title">Out of Sync Items</h4>
+        </div>
+        <div class="modal-body">
+          <table class="table table-striped table-condensed">
+          <tr ng-repeat="f in needed" ng-init="a = needAction(f)">
+            <td class="small-data"><span class="glyphicon glyphicon-{{needIcons[a]}}"></span> {{needActions[a]}}</td>
+            <td title="{{f.Name}}">{{f.Name | basename}}</td>
+            <td class="text-right small-data"><span ng-if="f.Size > 0">{{f.Size | binary}}B</span></td>
+          </tr>
+          </table>
+        </div>
+        <div class="modal-footer">
+          <button type="button" class="btn btn-default" data-dismiss="modal"><span class="glyphicon glyphicon-remove"></span>&emsp;Close</button>
+        </div>
+      </div>
+    </div>
+  </div>
+
 
   <script src="angular.min.js"></script>
   <script src="jquery-2.0.3.min.js"></script>

integration/test.sh

@@ -21,6 +21,15 @@ start() {
 	done
 }
 
+clean() {
+	if [[ $(uname -s) == "Linux" ]] ; then
+		grep -v utf8-nfd
+	else
+		cat
+	fi
+}
+
+
 testConvergence() {
 	while true ; do
 		sleep 5
@@ -38,13 +47,13 @@ testConvergence() {
 	done
 
 	echo "Verifying..."
-	cat md5-? | sort | uniq > md5-tot
-	cat md5-12-? | sort | uniq > md5-12-tot
-	cat md5-23-? | sort | uniq > md5-23-tot
+	cat md5-? | sort | clean | uniq > md5-tot
+	cat md5-12-? | sort | clean | uniq > md5-12-tot
+	cat md5-23-? | sort | clean | uniq > md5-23-tot
 
 	for i in 1 2 3 12-1 12-2 23-2 23-3; do
 		pushd "s$i" >/dev/null
-		../md5r -l | sort > ../md5-$i
+		../md5r -l | sort | clean > ../md5-$i
 		popd >/dev/null
 	done
 

protocol/PROTOCOL.md

@@ -70,12 +70,13 @@ Messages
 --------
 
 Every message starts with one 32 bit word indicating the message
-version, type and ID.
+version, type and ID. The header is in network byte order, i.e. big
+endian.
 
      0                   1                   2                   3
      0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |  Ver  |  Type |       Message ID      |        Reply To       |
+    |  Ver  |       Message ID      |      Type     |    Reserved   |
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 For BEP v1 the Version field is set to zero. Future versions with
@@ -84,19 +85,19 @@ with an unknown version is a protocol error and MUST result in the
 connection being terminated. A client supporting multiple versions MAY
 retry with a different protocol version upon disconnection.
 
+The Message ID is set to a unique value for each transmitted request
+message. In response messages it is set to the Message ID of the
+corresponding request message. The uniqueness requirement implies that
+no more than 4096 messages may be outstanding at any given moment. The
+ordering requirement implies that a response to a given message ID also
+means that all preceding messages have been received, specifically those
+which do not otherwise demand a response. Hence their message ID:s may
+be reused.
+
 The Type field indicates the type of data following the message header
 and is one of the integers defined below. A message of an unknown type
 is a protocol error and MUST result in the connection being terminated.
 
-The Message ID is set to a unique value for each transmitted message. In
-request messages the Reply To is set to zero. In response messages it is
-set to the message ID of the corresponding request. The uniqueness
-requirement implies that no more than 4096 messages may be outstanding
-at any given moment. The ordering requirement implies that a response to
-a given message ID also means that all preceding messages have been
-received, specifically those which do not otherwise demand a response.
-Hence their message ID:s may be reused.
-
 All data following the message header MUST be in XDR (RFC 1014)
 encoding. All fields shorter than 32 bits and all variable length data
 MUST be padded to a multiple of 32 bits. The actual data types in use by

protocol/protocol.go

@@ -96,8 +96,8 @@ type asyncResult struct {
 }
 
 const (
-	pingTimeout  = 300 * time.Second
-	pingIdleTime = 600 * time.Second
+	pingTimeout  = 30 * time.Second
+	pingIdleTime = 60 * time.Second
 )
 
 func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver Model) Connection {
@@ -128,6 +128,7 @@ func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver M
 		closed:    make(chan struct{}),
 	}
 
+	go c.indexSerializerLoop()
 	go c.readerLoop()
 	go c.writerLoop()
 	go c.pingerLoop()
@@ -164,9 +165,11 @@ func (c *rawConnection) Index(repo string, idx []FileInfo) {
 		}
 		idx = diff
 	}
-	c.imut.Unlock()
 
-	c.send(header{0, -1, msgType}, IndexMessage{repo, idx})
+	if len(idx) > 0 {
+		c.send(header{0, -1, msgType}, IndexMessage{repo, idx})
+	}
+	c.imut.Unlock()
 }
 
 // Request returns the bytes for the specified block after fetching them from the connected peer.
@@ -285,6 +288,31 @@ func (c *rawConnection) readerLoop() (err error) {
 	}
 }
 
+type incomingIndex struct {
+	update bool
+	id     string
+	repo   string
+	files  []FileInfo
+}
+
+var incomingIndexes = make(chan incomingIndex, 100) // should be enough for anyone, right?
+
+func (c *rawConnection) indexSerializerLoop() {
+	// We must avoid blocking the reader loop when processing large indexes.
+	// There is otherwise a potential deadlock where both sides has the model
+	// locked because it's sending a large index update and can't receive the
+	// large index update from the other side. But we must also ensure to
+	// process the indexes in the order they are received, hence the separate
+	// routine and buffered channel.
+	for ii := range incomingIndexes {
+		if ii.update {
+			c.receiver.IndexUpdate(ii.id, ii.repo, ii.files)
+		} else {
+			c.receiver.Index(ii.id, ii.repo, ii.files)
+		}
+	}
+}
+
 func (c *rawConnection) handleIndex() error {
 	var im IndexMessage
 	im.decodeXDR(c.xr)
@@ -299,7 +327,7 @@ func (c *rawConnection) handleIndex() error {
 		// update and can't receive the large index update from the
 		// other side.
 
-		go c.receiver.Index(c.id, im.Repository, im.Files)
+		incomingIndexes <- incomingIndex{false, c.id, im.Repository, im.Files}
 	}
 	return nil
 }
@@ -310,7 +338,7 @@ func (c *rawConnection) handleIndexUpdate() error {
 	if err := c.xr.Error(); err != nil {
 		return err
 	} else {
-		go c.receiver.IndexUpdate(c.id, im.Repository, im.Files)
+		incomingIndexes <- incomingIndex{true, c.id, im.Repository, im.Files}
 	}
 	return nil
 }

protocol/protocol_test.go

@@ -25,6 +25,31 @@ func TestHeaderFunctions(t *testing.T) {
 	}
 }
 
+func TestHeaderLayout(t *testing.T) {
+	var e, a uint32
+
+	// Version are the first four bits
+	e = 0xf0000000
+	a = encodeHeader(header{0xf, 0, 0})
+	if a != e {
+		t.Errorf("Header layout incorrect; %08x != %08x", a, e)
+	}
+
+	// Message ID are the following 12 bits
+	e = 0x0fff0000
+	a = encodeHeader(header{0, 0xfff, 0})
+	if a != e {
+		t.Errorf("Header layout incorrect; %08x != %08x", a, e)
+	}
+
+	// Type are the last 8 bits before reserved
+	e = 0x0000ff00
+	a = encodeHeader(header{0, 0, 0xff})
+	if a != e {
+		t.Errorf("Header layout incorrect; %08x != %08x", a, e)
+	}
+}
+
 func TestPing(t *testing.T) {
 	ar, aw := io.Pipe()
 	br, bw := io.Pipe()

scanner/walk.go

@@ -13,6 +13,7 @@ import (
 	"runtime"
 	"strings"
 	"time"
+	"code.google.com/p/go.text/unicode/norm"
 
 	"github.com/calmh/syncthing/lamport"
 	"github.com/calmh/syncthing/protocol"
@@ -159,6 +160,11 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 			return nil
 		}
 
+		if (runtime.GOOS == "linux" || runtime.GOOS == "windows") && !norm.NFC.IsNormalString(rn) {
+			l.Warnf("File %q contains non-NFC UTF-8 sequences and cannot be synced. Consider renaming.", rn)
+			return nil
+		}
+
 		if info.Mode().IsDir() {
 			if w.CurrentFiler != nil {
 				cf := w.CurrentFiler.CurrentFile(rn)