Revert "lib/model: Add folders on start in model (#6135)"

This reverts commit bee7cce081.

.golangci.yml

@@ -12,9 +12,13 @@ linters:
     - gochecknoglobals
     - gofmt
     - scopelint
+    - gocyclo
+    - funlen
+    - wsl
 
 service:
-  golangci-lint-version: 1.16.x
+  golangci-lint-version: 1.21.x
   prepare:
+    - rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
     - GO111MODULE=on go mod vendor
     - go run build.go assets

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.12 AS builder
+FROM golang:1.13 AS builder
 
 WORKDIR /src
 COPY . .
@@ -19,9 +19,10 @@ RUN apk add --no-cache ca-certificates su-exec
 COPY --from=builder /src/syncthing /bin/syncthing
 COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
 
-ENV PUID=1000 PGID=1000
+ENV PUID=1000 PGID=1000 HOME=/var/syncthing
 
 HEALTHCHECK --interval=1m --timeout=10s \
   CMD nc -z localhost 8384 || exit 1
 
-ENTRYPOINT ["/bin/entrypoint.sh", "-home", "/var/syncthing/config", "-gui-address", "0.0.0.0:8384"]
+ENV STGUIADDRESS=0.0.0.0:8384
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]

Dockerfile.stdiscosrv

@@ -0,0 +1,28 @@
+FROM golang:1.13 AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f stdiscosrv && go run build.go -no-upgrade build stdiscosrv
+
+FROM alpine
+
+EXPOSE 19200 8443
+
+VOLUME ["/var/stdiscosrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/stdiscosrv /bin/stdiscosrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/stdiscosrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 8443 || exit 1
+
+WORKDIR /var/stdiscosrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/stdiscosrv"]

Dockerfile.strelaysrv

@@ -0,0 +1,28 @@
+FROM golang:1.13 AS builder
+
+WORKDIR /src
+COPY . .
+
+ENV CGO_ENABLED=0
+ENV BUILD_HOST=syncthing.net
+ENV BUILD_USER=docker
+RUN rm -f strelaysrv && go run build.go -no-upgrade build strelaysrv
+
+FROM alpine
+
+EXPOSE 22067 22070
+
+VOLUME ["/var/strelaysrv"]
+
+RUN apk add --no-cache ca-certificates su-exec
+
+COPY --from=builder /src/strelaysrv /bin/strelaysrv
+COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
+
+ENV PUID=1000 PGID=1000 HOME=/var/strelaysrv
+
+HEALTHCHECK --interval=1m --timeout=10s \
+  CMD nc -z localhost 22067 || exit 1
+
+WORKDIR /var/strelaysrv
+ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaysrv"]

README-Docker.md

@@ -18,7 +18,11 @@ $ docker run -p 8384:8384 -p 22000:22000 \
     syncthing/syncthing:latest
 ```
 
-Note that local device discovery will not work with the above command, resulting in poor local transfer rates if local device addresses are not manually configured.
+## Discovery
+
+Note that local device discovery will not work with the above command,
+resulting in poor local transfer rates if local device addresses are not
+manually configured.
 
 To allow local discovery, the docker host network can be used instead:
 
@@ -32,3 +36,24 @@ $ docker run --network=host \
 Be aware that syncthing alone is now in control of what interfaces and ports it
 listens on. You can edit the syncthing configuration to change the defaults if
 there are conflicts.
+
+## GUI Security
+
+By default Syncthing inside the Docker image listens on 0.0.0.0:8384 to
+allow GUI connections via the Docker proxy. This is set by the
+`STGUIADDRESS` environment variable in the Dockerfile, as it differs from
+what Syncthing would otherwise use by default. This means you should set up
+authentication in the GUI, like for any other externally reachable Syncthing
+instance. If you do not require the GUI, or you use host networking, you can
+unset the `STGUIADDRESS` variable to have Syncthing fall back to listening
+on 127.0.0.1:
+
+```
+$ docker pull syncthing/syncthing
+$ docker run -e STGUIADDRESS= \
+    -v /wherever/st-sync:/var/syncthing \
+    syncthing/syncthing:latest
+```
+
+With the environment variable unset Syncthing will follow what is set in the
+configuration file / GUI settings dialog.

README.md

@@ -62,6 +62,10 @@ There are a few examples for keeping Syncthing running in the background
 on your system in [the etc directory][3]. There are also several [GUI
 implementations][11] for Windows, Mac and Linux.
 
+## Docker
+
+To run Syncthing in Docker, see [the Docker README][16].
+
 ## Vote on features/bugs
 
 We'd like to encourage you to [vote][12] on issues that matter to you.
@@ -110,4 +114,5 @@ All code is licensed under the [MPLv2 License][7].
 [13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
 [14]: assets/logo-text-128.png
 [15]: https://syncthing.net/
+[16]: https://github.com/syncthing/syncthing/blob/master/README-Docker.md
 

build.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"os/exec"
 	"os/user"
+	"path"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -34,23 +35,22 @@ import (
 )
 
 var (
-	versionRe        = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
-	goarch           string
-	goos             string
-	noupgrade        bool
-	version          string
-	goCmd            string
-	goVersion        float64
-	race             bool
-	debug            = os.Getenv("BUILDDEBUG") != ""
-	extraTags        string
-	installSuffix    string
-	pkgdir           string
-	cc               string
-	debugBinary      bool
-	coverage         bool
-	timeout          = "120s"
-	gogoProtoVersion = "v1.2.0"
+	versionRe     = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
+	goarch        string
+	goos          string
+	noupgrade     bool
+	version       string
+	goCmd         string
+	goVersion     float64
+	race          bool
+	debug         = os.Getenv("BUILDDEBUG") != ""
+	extraTags     string
+	installSuffix string
+	pkgdir        string
+	cc            string
+	debugBinary   bool
+	coverage      bool
+	timeout       = "120s"
 )
 
 type target struct {
@@ -60,7 +60,7 @@ type target struct {
 	debpre            string
 	debpost           string
 	description       string
-	buildPkg          string
+	buildPkgs         []string
 	binaryName        string
 	archiveFiles      []archiveFile
 	systemdServices   []string
@@ -77,9 +77,8 @@ type archiveFile struct {
 var targets = map[string]target{
 	"all": {
 		// Only valid for the "build" and "install" commands as it lacks all
-		// the archive creation stuff.
-		buildPkg: "github.com/syncthing/syncthing/cmd/...",
-		tags:     []string{"purego"},
+		// the archive creation stuff. buildPkgs gets filled out in init()
+		tags: []string{"purego"},
 	},
 	"syncthing": {
 		// The default target for "build", "install", "tar", "zip", "deb", etc.
@@ -88,7 +87,7 @@ var targets = map[string]target{
 		debdeps:     []string{"libc6", "procps"},
 		debpost:     "script/post-upgrade",
 		description: "Open Source Continuous File Synchronization",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/syncthing",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/syncthing"},
 		binaryName:  "syncthing", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -132,7 +131,7 @@ var targets = map[string]target{
 		debdeps:     []string{"libc6"},
 		debpre:      "cmd/stdiscosrv/scripts/preinst",
 		description: "Syncthing Discovery Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/stdiscosrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/stdiscosrv"},
 		binaryName:  "stdiscosrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -160,7 +159,7 @@ var targets = map[string]target{
 		debdeps:     []string{"libc6"},
 		debpre:      "cmd/strelaysrv/scripts/preinst",
 		description: "Syncthing Relay Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/strelaysrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaysrv"},
 		binaryName:  "strelaysrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -188,7 +187,7 @@ var targets = map[string]target{
 		debname:     "syncthing-relaypoolsrv",
 		debdeps:     []string{"libc6"},
 		description: "Syncthing Relay Pool Server",
-		buildPkg:    "github.com/syncthing/syncthing/cmd/strelaypoolsrv",
+		buildPkgs:   []string{"github.com/syncthing/syncthing/cmd/strelaypoolsrv"},
 		binaryName:  "strelaypoolsrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
@@ -214,11 +213,22 @@ type dependencyRepo struct {
 }
 
 var dependencyRepos = []dependencyRepo{
-	{path: "protobuf", repo: "https://github.com/gogo/protobuf.git", commit: gogoProtoVersion},
 	{path: "xdr", repo: "https://github.com/calmh/xdr.git", commit: "08e072f9cb16"},
 }
 
 func init() {
+	all := targets["all"]
+	pkgs, _ := filepath.Glob("cmd/*")
+	for _, pkg := range pkgs {
+		pkg = filepath.Base(pkg)
+		if strings.HasPrefix(pkg, ".") {
+			// ignore dotfiles
+			continue
+		}
+		all.buildPkgs = append(all.buildPkgs, fmt.Sprintf("github.com/syncthing/syncthing/cmd/%s", pkg))
+	}
+	targets["all"] = all
+
 	// The "syncthing" target includes a few more files found in the "etc"
 	// and "extra" dirs.
 	syncthingPkg := targets["syncthing"]
@@ -384,9 +394,6 @@ func install(target target, tags []string) {
 	}
 	os.Setenv("GOBIN", filepath.Join(cwd, "bin"))
 
-	args := []string{"install", "-v"}
-	args = appendParameters(args, tags, target)
-
 	os.Setenv("GOOS", goos)
 	os.Setenv("GOARCH", goarch)
 	os.Setenv("CC", cc)
@@ -402,19 +409,20 @@ func install(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	runPrint(goCmd, args...)
+	for _, pkg := range target.buildPkgs {
+		args := []string{"install", "-v"}
+		args = appendParameters(args, tags, pkg)
+
+		runPrint(goCmd, args...)
+	}
 }
 
 func build(target target, tags []string) {
 	lazyRebuildAssets()
-
 	tags = append(target.tags, tags...)
 
 	rmr(target.BinaryName())
 
-	args := []string{"build", "-v"}
-	args = appendParameters(args, tags, target)
-
 	os.Setenv("GOOS", goos)
 	os.Setenv("GOARCH", goarch)
 	os.Setenv("CC", cc)
@@ -434,10 +442,15 @@ func build(target target, tags []string) {
 		defer shouldCleanupSyso(sysoPath)
 	}
 
-	runPrint(goCmd, args...)
+	for _, pkg := range target.buildPkgs {
+		args := []string{"build", "-v"}
+		args = appendParameters(args, tags, pkg)
+
+		runPrint(goCmd, args...)
+	}
 }
 
-func appendParameters(args []string, tags []string, target target) []string {
+func appendParameters(args []string, tags []string, pkg string) []string {
 	if pkgdir != "" {
 		args = append(args, "-pkgdir", pkgdir)
 	}
@@ -453,7 +466,7 @@ func appendParameters(args []string, tags []string, target target) []string {
 
 	if !debugBinary {
 		// Regular binaries get version tagged and skip some debug symbols
-		args = append(args, "-ldflags", ldflags())
+		args = append(args, "-ldflags", ldflags(path.Base(pkg)))
 	} else {
 		// -gcflags to disable optimizations and inlining. Skip -ldflags
 		// because `Could not launch program: decoding dwarf section info at
@@ -462,7 +475,7 @@ func appendParameters(args []string, tags []string, target target) []string {
 		args = append(args, "-gcflags", "-N -l")
 	}
 
-	return append(args, target.buildPkg)
+	return append(args, pkg)
 }
 
 func buildTar(target target) {
@@ -710,6 +723,7 @@ func listFiles(dir string) []string {
 		if err != nil {
 			return err
 		}
+
 		if fi.Mode().IsRegular() {
 			res = append(res, path)
 		}
@@ -756,14 +770,21 @@ func shouldRebuildAssets(target, srcdir string) bool {
 }
 
 func proto() {
-	runPrint(goCmd, "get", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", gogoProtoVersion))
+	pv := protobufVersion()
+	dependencyRepos = append(dependencyRepos,
+		dependencyRepo{path: "protobuf", repo: "https://github.com/gogo/protobuf.git", commit: pv},
+	)
+
+	runPrint(goCmd, "get", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", pv))
 	os.MkdirAll("repos", 0755)
 	for _, dep := range dependencyRepos {
 		path := filepath.Join("repos", dep.path)
 		if _, err := os.Stat(path); err != nil {
 			runPrintInDir("repos", "git", "clone", dep.repo, dep.path)
-			runPrintInDir(path, "git", "checkout", dep.commit)
+		} else {
+			runPrintInDir(path, "git", "fetch")
 		}
+		runPrintInDir(path, "git", "checkout", dep.commit)
 	}
 	runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/stdiscosrv")
 }
@@ -784,7 +805,7 @@ func transifex() {
 	runPrint(goCmd, "run", "../../../../script/transifexdl.go")
 }
 
-func ldflags() string {
+func ldflags(program string) string {
 	sep := '='
 	if goVersion > 0 && goVersion < 1.5 {
 		sep = ' '
@@ -796,6 +817,10 @@ func ldflags() string {
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Stamp%c%d", sep, buildStamp())
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.User%c%s", sep, buildUser())
 	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Host%c%s", sep, buildHost())
+	fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Program%c%s", sep, program)
+	if v := os.Getenv("EXTRA_LDFLAGS"); v != "" {
+		fmt.Fprintf(b, " %s", v)
+	}
 	return b.String()
 }
 
@@ -1252,3 +1277,11 @@ func (t target) BinaryName() string {
 	}
 	return t.binaryName
 }
+
+func protobufVersion() string {
+	bs, err := runError(goCmd, "list", "-f", "{{.Version}}", "-m", "github.com/gogo/protobuf")
+	if err != nil {
+		log.Fatal("Getting protobuf version:", err)
+	}
+	return string(bs)
+}

cmd/stbench/main.go

@@ -1,143 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// This doesn't build on Windows due to the Rusage stuff.
-
-// +build !windows
-
-package main
-
-import (
-	"flag"
-	"fmt"
-	"log"
-	"runtime"
-	"syscall"
-	"time"
-
-	"github.com/syncthing/syncthing/lib/rc"
-)
-
-var homeDir = "h1"
-var syncthingBin = "./bin/syncthing"
-var test = "scan"
-
-func main() {
-	flag.StringVar(&homeDir, "home", homeDir, "Home directory location")
-	flag.StringVar(&syncthingBin, "bin", syncthingBin, "Binary location")
-	flag.StringVar(&test, "test", test, "Test to run")
-	flag.Parse()
-
-	switch test {
-	case "scan":
-		// scan measures the resource usage required to perform the initial
-		// scan, without cleaning away the database first.
-		testScan()
-	}
-}
-
-// testScan starts a process and reports on the resource usage required to
-// perform the initial scan.
-func testScan() {
-	log.Println("Starting...")
-	p := rc.NewProcess("127.0.0.1:8081")
-	if err := p.Start(syncthingBin, "-home", homeDir, "-no-browser"); err != nil {
-		log.Println(err)
-		return
-	}
-	defer p.Stop()
-
-	wallTime := awaitScanComplete(p)
-
-	report(p, wallTime)
-}
-
-// awaitScanComplete waits for a folder to transition idle->scanning and
-// then scanning->idle and returns the time taken for the scan.
-func awaitScanComplete(p *rc.Process) time.Duration {
-	log.Println("Awaiting scan completion...")
-	var t0, t1 time.Time
-	lastEvent := 0
-loop:
-	for {
-		evs, err := p.Events(lastEvent)
-		if err != nil {
-			continue
-		}
-
-		for _, ev := range evs {
-			if ev.Type == "StateChanged" {
-				data := ev.Data.(map[string]interface{})
-				log.Println(ev)
-
-				if data["to"].(string) == "scanning" {
-					t0 = ev.Time
-					continue
-				}
-
-				if !t0.IsZero() && data["to"].(string) == "idle" {
-					t1 = ev.Time
-					break loop
-				}
-			}
-			lastEvent = ev.ID
-		}
-
-		time.Sleep(250 * time.Millisecond)
-	}
-
-	return t1.Sub(t0)
-}
-
-// report stops the given process and reports on its resource usage in two
-// ways: human readable to stderr, and CSV to stdout.
-func report(p *rc.Process, wallTime time.Duration) {
-	sv, err := p.SystemVersion()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	ss, err := p.SystemStatus()
-	if err != nil {
-		log.Println(err)
-		return
-	}
-
-	proc, err := p.Stop()
-	if err != nil {
-		return
-	}
-
-	rusage, ok := proc.SysUsage().(*syscall.Rusage)
-	if !ok {
-		return
-	}
-
-	log.Println("Version:", sv.Version)
-	log.Println("Alloc:", ss.Alloc/1024, "KiB")
-	log.Println("Sys:", ss.Sys/1024, "KiB")
-	log.Println("Goroutines:", ss.Goroutines)
-	log.Println("Wall time:", wallTime)
-	log.Println("Utime:", time.Duration(rusage.Utime.Nano()))
-	log.Println("Stime:", time.Duration(rusage.Stime.Nano()))
-	if runtime.GOOS == "darwin" {
-		// Darwin reports in bytes, Linux seems to report in KiB even
-		// though the manpage says otherwise.
-		rusage.Maxrss /= 1024
-	}
-	log.Println("MaxRSS:", rusage.Maxrss, "KiB")
-
-	fmt.Printf("%s,%d,%d,%d,%.02f,%.02f,%.02f,%d\n",
-		sv.Version,
-		ss.Alloc/1024,
-		ss.Sys/1024,
-		ss.Goroutines,
-		wallTime.Seconds(),
-		time.Duration(rusage.Utime.Nano()).Seconds(),
-		time.Duration(rusage.Stime.Nano()).Seconds(),
-		rusage.Maxrss)
-}

cmd/stcli/main.go

@@ -14,7 +14,6 @@ import (
 	"log"
 	"os"
 	"reflect"
-	"strings"
 
 	"github.com/AudriusButkevicius/recli"
 	"github.com/flynn-archive/go-shlex"
@@ -22,6 +21,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/urfave/cli"
@@ -85,7 +85,7 @@ func main() {
 		myID := protocol.NewDeviceID(cert.Certificate[0])
 
 		// Load the config
-		cfg, err := config.Load(locations.Get(locations.ConfigFile), myID)
+		cfg, err := config.Load(locations.Get(locations.ConfigFile), myID, events.NoopLogger)
 		if err != nil {
 			log.Fatalln(errors.Wrap(err, "loading config"))
 		}
@@ -127,7 +127,7 @@ func main() {
 	app.HelpName = app.Name
 	app.Author = "The Syncthing Authors"
 	app.Usage = "Syncthing command line interface"
-	app.Version = strings.Replace(build.LongVersion, "syncthing", app.Name, 1)
+	app.Version = build.Version
 	app.Flags = fakeFlags
 	app.Metadata = map[string]interface{}{
 		"client": client,

cmd/stcli/utils.go

@@ -12,7 +12,6 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
-	"text/tabwriter"
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/urfave/cli"
@@ -45,12 +44,6 @@ func dumpOutput(url string) cli.ActionFunc {
 	}
 }
 
-func newTableWriter() *tabwriter.Writer {
-	writer := new(tabwriter.Writer)
-	writer.Init(os.Stdout, 0, 8, 0, '\t', 0)
-	return writer
-}
-
 func getConfig(c *APIClient) (config.Configuration, error) {
 	cfg := config.Configuration{}
 	response, err := c.Get("system/config")

cmd/stdiscosrv/apisrv.go

@@ -18,6 +18,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -279,6 +280,10 @@ func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addre
 		dbAddrs[i].Expires = expire
 	}
 
+	// The address slice must always be sorted for database merges to work
+	// properly.
+	sort.Sort(databaseAddressOrder(dbAddrs))
+
 	seen := now.UnixNano()
 	if s.repl != nil {
 		s.repl.send(key, dbAddrs, seen)
@@ -295,28 +300,66 @@ func certificateBytes(req *http.Request) []byte {
 		return req.TLS.PeerCertificates[0].Raw
 	}
 
+	var bs []byte
+
 	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
-		bs := []byte(hdr)
-		// The certificate is in PEM format but with spaces for newlines. We
-		// need to reinstate the newlines for the PEM decoder. But we need to
-		// leave the spaces in the BEGIN and END lines - the first and last
-		// space - alone.
-		firstSpace := bytes.Index(bs, []byte(" "))
-		lastSpace := bytes.LastIndex(bs, []byte(" "))
-		for i := firstSpace + 1; i < lastSpace; i++ {
-			if bs[i] == ' ' {
-				bs[i] = '\n'
+		if strings.Contains(hdr, "%") {
+			// Nginx using $ssl_client_escaped_cert
+			// The certificate is in PEM format with url encoding.
+			// We need to decode for the PEM decoder
+			hdr, err := url.QueryUnescape(hdr)
+			if err != nil {
+				// Decoding failed
+				return nil
+			}
+
+			bs = []byte(hdr)
+		} else {
+			// Nginx using $ssl_client_cert
+			// The certificate is in PEM format but with spaces for newlines. We
+			// need to reinstate the newlines for the PEM decoder. But we need to
+			// leave the spaces in the BEGIN and END lines - the first and last
+			// space - alone.
+			bs = []byte(hdr)
+			firstSpace := bytes.Index(bs, []byte(" "))
+			lastSpace := bytes.LastIndex(bs, []byte(" "))
+			for i := firstSpace + 1; i < lastSpace; i++ {
+				if bs[i] == ' ' {
+					bs[i] = '\n'
+				}
 			}
 		}
-		block, _ := pem.Decode(bs)
-		if block == nil {
+	} else if hdr := req.Header.Get("X-Forwarded-Tls-Client-Cert"); hdr != "" {
+		// Traefik 2 passtlsclientcert
+		// The certificate is in PEM format with url encoding but without newlines
+		// and start/end statements. We need to decode, reinstate the newlines every 64
+		// character and add statements for the PEM decoder
+		hdr, err := url.QueryUnescape(hdr)
+		if err != nil {
 			// Decoding failed
 			return nil
 		}
-		return block.Bytes
+
+		for i := 64; i < len(hdr); i += 65 {
+			hdr = hdr[:i] + "\n" + hdr[i:]
+		}
+
+		hdr = "-----BEGIN CERTIFICATE-----\n" + hdr
+		hdr = hdr + "\n-----END CERTIFICATE-----\n"
+		bs = []byte(hdr)
 	}
 
-	return nil
+	if bs == nil {
+		return nil
+	}
+
+	block, _ := pem.Decode(bs)
+	if block == nil {
+		// Decoding failed
+		return nil
+	}
+
+	return block.Bytes
 }
 
 // fixupAddresses checks the list of addresses, removing invalid ones and

cmd/stdiscosrv/database.go

@@ -10,6 +10,7 @@
 package main
 
 import (
+	"log"
 	"sort"
 	"time"
 
@@ -263,12 +264,15 @@ func (s *levelDBStore) Stop() {
 // chosen for any duplicates.
 func merge(a, b DatabaseRecord) DatabaseRecord {
 	// Both lists must be sorted for this to work.
-	sort.Slice(a.Addresses, func(i, j int) bool {
-		return a.Addresses[i].Address < a.Addresses[j].Address
-	})
-	sort.Slice(b.Addresses, func(i, j int) bool {
-		return b.Addresses[i].Address < b.Addresses[j].Address
-	})
+	if !sort.IsSorted(databaseAddressOrder(a.Addresses)) {
+		log.Println("Warning: bug: addresses not correctly sorted in merge")
+		a.Addresses = sortedAddressCopy(a.Addresses)
+	}
+	if !sort.IsSorted(databaseAddressOrder(b.Addresses)) {
+		// no warning because this is the side we read from disk and it may
+		// legitimately predate correct sorting.
+		b.Addresses = sortedAddressCopy(b.Addresses)
+	}
 
 	res := DatabaseRecord{
 		Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
@@ -352,3 +356,24 @@ func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
 	}
 	return addrs
 }
+
+func sortedAddressCopy(addrs []DatabaseAddress) []DatabaseAddress {
+	sorted := make([]DatabaseAddress, len(addrs))
+	copy(sorted, addrs)
+	sort.Sort(databaseAddressOrder(sorted))
+	return sorted
+}
+
+type databaseAddressOrder []DatabaseAddress
+
+func (s databaseAddressOrder) Less(a, b int) bool {
+	return s[a].Address < s[b].Address
+}
+
+func (s databaseAddressOrder) Swap(a, b int) {
+	s[a], s[b] = s[b], s[a]
+}
+
+func (s databaseAddressOrder) Len() int {
+	return len(s)
+}

cmd/stdiscosrv/database.pb.go

@@ -3,12 +3,14 @@
 
 package main
 
-import proto "github.com/gogo/protobuf/proto"
-import fmt "fmt"
-import math "math"
-import _ "github.com/gogo/protobuf/gogoproto"
-
-import io "io"
+import (
+	fmt "fmt"
+	_ "github.com/gogo/protobuf/gogoproto"
+	proto "github.com/gogo/protobuf/proto"
+	io "io"
+	math "math"
+	math_bits "math/bits"
+)
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
@@ -19,7 +21,7 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
 
 type DatabaseRecord struct {
 	Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
@@ -32,7 +34,7 @@ func (m *DatabaseRecord) Reset()         { *m = DatabaseRecord{} }
 func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
 func (*DatabaseRecord) ProtoMessage()    {}
 func (*DatabaseRecord) Descriptor() ([]byte, []int) {
-	return fileDescriptor_database_0f49e029703a04f5, []int{0}
+	return fileDescriptor_b90fe3356ea5df07, []int{0}
 }
 func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -42,15 +44,15 @@ func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, erro
 		return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalTo(b)
+		n, err := m.MarshalToSizedBuffer(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (dst *DatabaseRecord) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DatabaseRecord.Merge(dst, src)
+func (m *DatabaseRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseRecord.Merge(m, src)
 }
 func (m *DatabaseRecord) XXX_Size() int {
 	return m.Size()
@@ -71,7 +73,7 @@ func (m *ReplicationRecord) Reset()         { *m = ReplicationRecord{} }
 func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
 func (*ReplicationRecord) ProtoMessage()    {}
 func (*ReplicationRecord) Descriptor() ([]byte, []int) {
-	return fileDescriptor_database_0f49e029703a04f5, []int{1}
+	return fileDescriptor_b90fe3356ea5df07, []int{1}
 }
 func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -81,15 +83,15 @@ func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, e
 		return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalTo(b)
+		n, err := m.MarshalToSizedBuffer(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (dst *ReplicationRecord) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ReplicationRecord.Merge(dst, src)
+func (m *ReplicationRecord) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ReplicationRecord.Merge(m, src)
 }
 func (m *ReplicationRecord) XXX_Size() int {
 	return m.Size()
@@ -109,7 +111,7 @@ func (m *DatabaseAddress) Reset()         { *m = DatabaseAddress{} }
 func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
 func (*DatabaseAddress) ProtoMessage()    {}
 func (*DatabaseAddress) Descriptor() ([]byte, []int) {
-	return fileDescriptor_database_0f49e029703a04f5, []int{2}
+	return fileDescriptor_b90fe3356ea5df07, []int{2}
 }
 func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -119,15 +121,15 @@ func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, err
 		return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalTo(b)
+		n, err := m.MarshalToSizedBuffer(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (dst *DatabaseAddress) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_DatabaseAddress.Merge(dst, src)
+func (m *DatabaseAddress) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DatabaseAddress.Merge(m, src)
 }
 func (m *DatabaseAddress) XXX_Size() int {
 	return m.Size()
@@ -143,10 +145,34 @@ func init() {
 	proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
 	proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
 }
+
+func init() { proto.RegisterFile("database.proto", fileDescriptor_b90fe3356ea5df07) }
+
+var fileDescriptor_b90fe3356ea5df07 = []byte{
+	// 270 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
+	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
+	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
+	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
+	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
+	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
+	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
+	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
+	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
+	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
+	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
+	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
+	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
+	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
+	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
+	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
+	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
+}
+
 func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -154,44 +180,51 @@ func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Addresses) > 0 {
-		for _, msg := range m.Addresses {
-			dAtA[i] = 0xa
-			i++
-			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
-			n, err := msg.MarshalTo(dAtA[i:])
-			if err != nil {
-				return 0, err
-			}
-			i += n
-		}
-	}
-	if m.Misses != 0 {
-		dAtA[i] = 0x10
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+	if m.Missed != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+		i--
+		dAtA[i] = 0x20
 	}
 	if m.Seen != 0 {
-		dAtA[i] = 0x18
-		i++
 		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
 	}
-	if m.Missed != 0 {
-		dAtA[i] = 0x20
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
+	if m.Misses != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
+		i--
+		dAtA[i] = 0x10
 	}
-	return i, nil
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
 }
 
 func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -199,40 +232,48 @@ func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ReplicationRecord) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Key) > 0 {
-		dAtA[i] = 0xa
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
-		i += copy(dAtA[i:], m.Key)
+	if m.Seen != 0 {
+		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+		i--
+		dAtA[i] = 0x18
 	}
 	if len(m.Addresses) > 0 {
-		for _, msg := range m.Addresses {
-			dAtA[i] = 0x12
-			i++
-			i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
-			n, err := msg.MarshalTo(dAtA[i:])
-			if err != nil {
-				return 0, err
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintDatabase(dAtA, i, uint64(size))
 			}
-			i += n
+			i--
+			dAtA[i] = 0x12
 		}
 	}
-	if m.Seen != 0 {
-		dAtA[i] = 0x18
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
+	if len(m.Key) > 0 {
+		i -= len(m.Key)
+		copy(dAtA[i:], m.Key)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
+		i--
+		dAtA[i] = 0xa
 	}
-	return i, nil
+	return len(dAtA) - i, nil
 }
 
 func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -240,32 +281,40 @@ func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *DatabaseAddress) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	if len(m.Address) > 0 {
-		dAtA[i] = 0xa
-		i++
-		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
-		i += copy(dAtA[i:], m.Address)
-	}
 	if m.Expires != 0 {
-		dAtA[i] = 0x10
-		i++
 		i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
+		i--
+		dAtA[i] = 0x10
 	}
-	return i, nil
+	if len(m.Address) > 0 {
+		i -= len(m.Address)
+		copy(dAtA[i:], m.Address)
+		i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
 }
 
 func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
+	offset -= sovDatabase(v)
+	base := offset
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)
 		v >>= 7
 		offset++
 	}
 	dAtA[offset] = uint8(v)
-	return offset + 1
+	return base
 }
 func (m *DatabaseRecord) Size() (n int) {
 	if m == nil {
@@ -330,14 +379,7 @@ func (m *DatabaseAddress) Size() (n int) {
 }
 
 func sovDatabase(x uint64) (n int) {
-	for {
-		n++
-		x >>= 7
-		if x == 0 {
-			break
-		}
-	}
-	return n
+	return (math_bits.Len64(x|1) + 6) / 7
 }
 func sozDatabase(x uint64) (n int) {
 	return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
@@ -357,7 +399,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -385,7 +427,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= (int(b) & 0x7F) << shift
+				msglen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -394,6 +436,9 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -416,7 +461,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Misses |= (int32(b) & 0x7F) << shift
+				m.Misses |= int32(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -435,7 +480,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= (int64(b) & 0x7F) << shift
+				m.Seen |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -454,7 +499,7 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Missed |= (int64(b) & 0x7F) << shift
+				m.Missed |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -468,6 +513,9 @@ func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -495,7 +543,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -523,7 +571,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
+				stringLen |= uint64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -533,6 +581,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -552,7 +603,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				msglen |= (int(b) & 0x7F) << shift
+				msglen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -561,6 +612,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -583,7 +637,7 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Seen |= (int64(b) & 0x7F) << shift
+				m.Seen |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -597,6 +651,9 @@ func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -624,7 +681,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -652,7 +709,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
+				stringLen |= uint64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -662,6 +719,9 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthDatabase
 			}
 			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -681,7 +741,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.Expires |= (int64(b) & 0x7F) << shift
+				m.Expires |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -695,6 +755,9 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthDatabase
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthDatabase
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -710,6 +773,7 @@ func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
 func skipDatabase(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0
+	depth := 0
 	for iNdEx < l {
 		var wire uint64
 		for shift := uint(0); ; shift += 7 {
@@ -741,10 +805,8 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			return iNdEx, nil
 		case 1:
 			iNdEx += 8
-			return iNdEx, nil
 		case 2:
 			var length int
 			for shift := uint(0); ; shift += 7 {
@@ -761,76 +823,34 @@ func skipDatabase(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			iNdEx += length
 			if length < 0 {
 				return 0, ErrInvalidLengthDatabase
 			}
-			return iNdEx, nil
+			iNdEx += length
 		case 3:
-			for {
-				var innerWire uint64
-				var start int = iNdEx
-				for shift := uint(0); ; shift += 7 {
-					if shift >= 64 {
-						return 0, ErrIntOverflowDatabase
-					}
-					if iNdEx >= l {
-						return 0, io.ErrUnexpectedEOF
-					}
-					b := dAtA[iNdEx]
-					iNdEx++
-					innerWire |= (uint64(b) & 0x7F) << shift
-					if b < 0x80 {
-						break
-					}
-				}
-				innerWireType := int(innerWire & 0x7)
-				if innerWireType == 4 {
-					break
-				}
-				next, err := skipDatabase(dAtA[start:])
-				if err != nil {
-					return 0, err
-				}
-				iNdEx = start + next
-			}
-			return iNdEx, nil
+			depth++
 		case 4:
-			return iNdEx, nil
+			if depth == 0 {
+				return 0, ErrUnexpectedEndOfGroupDatabase
+			}
+			depth--
 		case 5:
 			iNdEx += 4
-			return iNdEx, nil
 		default:
 			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
 		}
+		if iNdEx < 0 {
+			return 0, ErrInvalidLengthDatabase
+		}
+		if depth == 0 {
+			return iNdEx, nil
+		}
 	}
-	panic("unreachable")
+	return 0, io.ErrUnexpectedEOF
 }
 
 var (
-	ErrInvalidLengthDatabase = fmt.Errorf("proto: negative length found during unmarshaling")
-	ErrIntOverflowDatabase   = fmt.Errorf("proto: integer overflow")
+	ErrInvalidLengthDatabase        = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowDatabase          = fmt.Errorf("proto: integer overflow")
+	ErrUnexpectedEndOfGroupDatabase = fmt.Errorf("proto: unexpected end of group")
 )
-
-func init() { proto.RegisterFile("database.proto", fileDescriptor_database_0f49e029703a04f5) }
-
-var fileDescriptor_database_0f49e029703a04f5 = []byte{
-	// 270 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
-	0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
-	0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
-	0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
-	0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
-	0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
-	0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
-	0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
-	0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
-	0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
-	0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
-	0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
-	0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
-	0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
-	0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
-	0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
-	0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
-}

cmd/stdiscosrv/main.go

@@ -9,17 +9,15 @@ package main
 import (
 	"crypto/tls"
 	"flag"
-	"fmt"
 	"log"
 	"net"
 	"net/http"
 	"os"
-	"runtime"
-	"strconv"
 	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/syncthing/syncthing/lib/build"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -65,24 +63,6 @@ var levelDBOptions = &opt.Options{
 	WriteBuffer: 32 << 20, // default 4<<20
 }
 
-var (
-	Version    string
-	BuildStamp string
-	BuildUser  string
-	BuildHost  string
-
-	BuildDate   time.Time
-	LongVersion string
-)
-
-func init() {
-	stamp, _ := strconv.Atoi(BuildStamp)
-	BuildDate = time.Unix(int64(stamp), 0)
-
-	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`stdiscosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
-}
-
 var (
 	debug = false
 )
@@ -109,14 +89,18 @@ func main() {
 	flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
 	flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
 	flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
+	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
-	log.Println(LongVersion)
+	log.Println(build.LongVersion)
+	if *showVersion {
+		return
+	}
 
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv")
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 20*365)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}

cmd/stfinddevice/main.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
@@ -82,7 +83,7 @@ func checkServers(deviceID protocol.DeviceID, servers ...string) {
 }
 
 func checkServer(deviceID protocol.DeviceID, server string) checkResult {
-	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil)
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil, events.NoopLogger)
 	if err != nil {
 		return checkResult{error: err}
 	}

cmd/strelaypoolsrv/main.go

@@ -633,7 +633,7 @@ func createTestCertificate() tls.Certificate {
 	}
 
 	certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
-	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv")
+	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 20*365)
 	if err != nil {
 		log.Fatalln("Failed to create test X509 key pair:", err)
 	}

cmd/strelaysrv/main.go

@@ -14,12 +14,13 @@ import (
 	"os/signal"
 	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
 	"sync/atomic"
 	"syscall"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/build"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/relay/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
@@ -33,24 +34,6 @@ import (
 	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
 )
 
-var (
-	Version    string
-	BuildStamp string
-	BuildUser  string
-	BuildHost  string
-
-	BuildDate   time.Time
-	LongVersion string
-)
-
-func init() {
-	stamp, _ := strconv.Atoi(BuildStamp)
-	BuildDate = time.Unix(int64(stamp), 0)
-
-	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`strelaysrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
-}
-
 var (
 	listen string
 	debug  bool
@@ -116,8 +99,14 @@ func main() {
 	flag.IntVar(&natTimeout, "nat-timeout", 10, "NAT discovery timeout in seconds")
 	flag.BoolVar(&pprofEnabled, "pprof", false, "Enable the built in profiling on the status server")
 	flag.IntVar(&networkBufferSize, "network-buffer", 2048, "Network buffer size (two of these per proxied connection)")
+	showVersion := flag.Bool("version", false, "Show version")
 	flag.Parse()
 
+	if *showVersion {
+		fmt.Println(build.LongVersion)
+		return
+	}
+
 	if extAddress == "" {
 		extAddress = listen
 	}
@@ -146,7 +135,7 @@ func main() {
 		}
 	}
 
-	log.Println(LongVersion)
+	log.Println(build.LongVersion)
 
 	maxDescriptors, err := osutil.MaximizeOpenFileLimit()
 	if maxDescriptors > 0 {
@@ -166,7 +155,7 @@ func main() {
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv")
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 20*365)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}
@@ -194,7 +183,7 @@ func main() {
 		log.Println("ID:", id)
 	}
 
-	wrapper := config.Wrap("config", config.New(id))
+	wrapper := config.Wrap("config", config.New(id), events.NoopLogger)
 	wrapper.SetOptions(config.OptionsConfiguration{
 		NATLeaseM:   natLease,
 		NATRenewalM: natRenewal,

cmd/strelaysrv/status.go

@@ -10,6 +10,8 @@ import (
 	"runtime"
 	"sync/atomic"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/build"
 )
 
 var rc *rateCalculator
@@ -40,10 +42,10 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 
 	sessionMut.Lock()
 	// This can potentially be double the number of pending sessions, as each session has two keys, one for each side.
-	status["version"] = Version
-	status["buildHost"] = BuildHost
-	status["buildUser"] = BuildUser
-	status["buildDate"] = BuildDate
+	status["version"] = build.Version
+	status["buildHost"] = build.Host
+	status["buildUser"] = build.User
+	status["buildDate"] = build.Date
 	status["startTime"] = rc.startTime
 	status["uptimeSeconds"] = time.Since(rc.startTime) / time.Second
 	status["numPendingSessionKeys"] = len(pendingSessions)

cmd/stupgrades/main.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"os"
+	"sort"
+
+	"github.com/syncthing/syncthing/lib/upgrade"
+)
+
+const defaultURL = "https://api.github.com/repos/syncthing/syncthing/releases?per_page=25"
+
+func main() {
+	url := flag.String("u", defaultURL, "GitHub releases url")
+	flag.Parse()
+
+	rels := upgrade.FetchLatestReleases(*url, "")
+	if rels == nil {
+		// An error was already logged
+		os.Exit(1)
+	}
+
+	sort.Sort(upgrade.SortByRelease(rels))
+	rels = filterForLatest(rels)
+
+	if err := json.NewEncoder(os.Stdout).Encode(rels); err != nil {
+		os.Exit(1)
+	}
+}
+
+// filterForLatest returns the latest stable and prerelease only. If the
+// stable version is newer (comes first in the list) there is no need to go
+// looking for a prerelease at all.
+func filterForLatest(rels []upgrade.Release) []upgrade.Release {
+	var filtered []upgrade.Release
+	var havePre bool
+	for _, rel := range rels {
+		if !rel.Prerelease {
+			// We found a stable version, we're good now.
+			filtered = append(filtered, rel)
+			break
+		}
+		if rel.Prerelease && !havePre {
+			// We remember the first prerelease we find.
+			filtered = append(filtered, rel)
+			havePre = true
+		}
+	}
+	return filtered
+}

cmd/syncthing/debug.go

@@ -7,16 +7,9 @@
 package main
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
 	l = logger.DefaultLogger.NewFacility("main", "Main package")
 )
-
-func init() {
-	l.SetDebug("main", strings.Contains(os.Getenv("STTRACE"), "main") || os.Getenv("STTRACE") == "all")
-}

cmd/syncthing/main.go

@@ -45,19 +45,8 @@ import (
 )
 
 const (
-	exitSuccess            = 0
-	exitError              = 1
-	exitNoUpgradeAvailable = 2
-	exitRestarting         = 3
-	exitUpgrading          = 4
-)
-
-const (
-	bepProtocolName      = "bep/1.0"
-	tlsDefaultCommonName = "syncthing"
-	maxSystemErrors      = 5
-	initialSystemLog     = 10
-	maxSystemLog         = 250
+	tlsDefaultCommonName   = "syncthing"
+	deviceCertLifetimeDays = 20 * 365
 )
 
 const (
@@ -271,7 +260,7 @@ func main() {
 	// default location
 	if options.noRestart && (options.logFile != "" && options.logFile != "-") {
 		l.Warnln("-logfile may not be used with -no-restart or STNORESTART")
-		os.Exit(exitError)
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 
 	if options.hideConsole {
@@ -285,12 +274,12 @@ func main() {
 			options.confDir, err = filepath.Abs(options.confDir)
 			if err != nil {
 				l.Warnln("Failed to make options path absolute:", err)
-				os.Exit(exitError)
+				os.Exit(syncthing.ExitError.AsInt())
 			}
 		}
 		if err := locations.SetBaseDir(locations.ConfigBaseDir, options.confDir); err != nil {
 			l.Warnln(err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 	}
 
@@ -328,7 +317,7 @@ func main() {
 		)
 		if err != nil {
 			l.Warnln("Error reading device ID:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 
 		fmt.Println(protocol.NewDeviceID(cert.Certificate[0]))
@@ -338,7 +327,7 @@ func main() {
 	if options.browserOnly {
 		if err := openGUI(protocol.EmptyDeviceID); err != nil {
 			l.Warnln("Failed to open web UI:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		return
 	}
@@ -346,7 +335,7 @@ func main() {
 	if options.generateDir != "" {
 		if err := generate(options.generateDir); err != nil {
 			l.Warnln("Failed to generate config and keys:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		return
 	}
@@ -354,14 +343,14 @@ func main() {
 	// Ensure that our home directory exists.
 	if err := ensureDir(locations.GetBaseDir(locations.ConfigBaseDir), 0700); err != nil {
 		l.Warnln("Failure on home directory:", err)
-		os.Exit(exitError)
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 
 	if options.upgradeTo != "" {
 		err := upgrade.ToURL(options.upgradeTo)
 		if err != nil {
 			l.Warnln("Error while Upgrading:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		l.Infoln("Upgraded from", options.upgradeTo)
 		return
@@ -381,7 +370,7 @@ func main() {
 	if options.resetDatabase {
 		if err := resetDB(); err != nil {
 			l.Warnln("Resetting database:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		return
 	}
@@ -394,7 +383,7 @@ func main() {
 }
 
 func openGUI(myID protocol.DeviceID) error {
-	cfg, err := loadOrDefaultConfig(myID)
+	cfg, err := loadOrDefaultConfig(myID, events.NoopLogger)
 	if err != nil {
 		return err
 	}
@@ -424,7 +413,7 @@ func generate(generateDir string) error {
 	if err == nil {
 		l.Warnln("Key exists; will not overwrite.")
 	} else {
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, tlsDefaultCommonName)
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, tlsDefaultCommonName, deviceCertLifetimeDays)
 		if err != nil {
 			return errors.Wrap(err, "create certificate")
 		}
@@ -437,7 +426,7 @@ func generate(generateDir string) error {
 		l.Warnln("Config exists; will not overwrite.")
 		return nil
 	}
-	cfg, err := syncthing.DefaultConfig(cfgFile, myID, noDefaultFolder)
+	cfg, err := syncthing.DefaultConfig(cfgFile, myID, events.NoopLogger, noDefaultFolder)
 	if err != nil {
 		return err
 	}
@@ -471,18 +460,18 @@ func debugFacilities() string {
 }
 
 func checkUpgrade() upgrade.Release {
-	cfg, _ := loadOrDefaultConfig(protocol.EmptyDeviceID)
+	cfg, _ := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger)
 	opts := cfg.Options()
 	release, err := upgrade.LatestRelease(opts.ReleasesURL, build.Version, opts.UpgradeToPreReleases)
 	if err != nil {
 		l.Warnln("Upgrade:", err)
-		os.Exit(exitError)
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 
 	if upgrade.CompareVersions(release.Tag, build.Version) <= 0 {
 		noUpgradeMessage := "No upgrade available (current %q >= latest %q)."
 		l.Infof(noUpgradeMessage, build.Version, release.Tag)
-		os.Exit(exitNoUpgradeAvailable)
+		os.Exit(syncthing.ExitNoUpgradeAvailable.AsInt())
 	}
 
 	l.Infof("Upgrade available (current %q < latest %q)", build.Version, release.Tag)
@@ -491,12 +480,12 @@ func checkUpgrade() upgrade.Release {
 
 func performUpgrade(release upgrade.Release) {
 	// Use leveldb database locks to protect against concurrent upgrades
-	_, err := syncthing.OpenGoleveldb(locations.Get(locations.Database))
+	_, err := syncthing.OpenGoleveldb(locations.Get(locations.Database), config.TuningAuto)
 	if err == nil {
 		err = upgrade.To(release)
 		if err != nil {
 			l.Warnln("Upgrade:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		l.Infof("Upgraded to %q", release.Tag)
 	} else {
@@ -504,15 +493,15 @@ func performUpgrade(release upgrade.Release) {
 		err = upgradeViaRest()
 		if err != nil {
 			l.Warnln("Upgrade:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		l.Infoln("Syncthing upgrading")
-		os.Exit(exitUpgrading)
+		os.Exit(syncthing.ExitUpgrade.AsInt())
 	}
 }
 
 func upgradeViaRest() error {
-	cfg, _ := loadOrDefaultConfig(protocol.EmptyDeviceID)
+	cfg, _ := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger)
 	u, err := url.Parse(cfg.GUI().URL())
 	if err != nil {
 		return err
@@ -566,10 +555,14 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		os.Exit(1)
 	}
 
-	cfg, err := syncthing.LoadConfigAtStartup(locations.Get(locations.ConfigFile), cert, runtimeOptions.allowNewerConfig, noDefaultFolder)
+	evLogger := events.NewLogger()
+	go evLogger.Serve()
+	defer evLogger.Stop()
+
+	cfg, err := syncthing.LoadConfigAtStartup(locations.Get(locations.ConfigFile), cert, evLogger, runtimeOptions.allowNewerConfig, noDefaultFolder)
 	if err != nil {
 		l.Warnln("Failed to initialize config:", err)
-		os.Exit(exitError)
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 
 	if runtimeOptions.unpaused {
@@ -579,7 +572,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 	}
 
 	dbFile := locations.Get(locations.Database)
-	ldb, err := syncthing.OpenGoleveldb(dbFile)
+	ldb, err := syncthing.OpenGoleveldb(dbFile, cfg.Options().DatabaseTuning)
 	if err != nil {
 		l.Warnln("Error opening database:", err)
 		os.Exit(1)
@@ -594,7 +587,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		appOpts.DeadlockTimeoutS = secs
 	}
 
-	app := syncthing.New(cfg, ldb, cert, appOpts)
+	app := syncthing.New(cfg, ldb, evLogger, cert, appOpts)
 
 	setupSignalHandling(app)
 
@@ -606,11 +599,11 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		f, err := os.Create(fmt.Sprintf("cpu-%d.pprof", os.Getpid()))
 		if err != nil {
 			l.Warnln("Creating profile:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		if err := pprof.StartCPUProfile(f); err != nil {
 			l.Warnln("Starting profile:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 	}
 
@@ -639,11 +632,13 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		if runtimeOptions.NoUpgrade {
 			l.Infof("No automatic upgrades; STNOUPGRADE environment variable defined.")
 		} else {
-			go autoUpgrade(cfg, app)
+			go autoUpgrade(cfg, app, evLogger)
 		}
 	}
 
-	app.Start()
+	if err := app.Start(); err != nil {
+		os.Exit(syncthing.ExitError.AsInt())
+	}
 
 	cleanConfigDirectory()
 
@@ -684,12 +679,12 @@ func setupSignalHandling(app *syncthing.App) {
 	}()
 }
 
-func loadOrDefaultConfig(myID protocol.DeviceID) (config.Wrapper, error) {
+func loadOrDefaultConfig(myID protocol.DeviceID, evLogger events.Logger) (config.Wrapper, error) {
 	cfgFile := locations.Get(locations.ConfigFile)
-	cfg, err := config.Load(cfgFile, myID)
+	cfg, err := config.Load(cfgFile, myID, evLogger)
 
 	if err != nil {
-		cfg, err = syncthing.DefaultConfig(cfgFile, myID, noDefaultFolder)
+		cfg, err = syncthing.DefaultConfig(cfgFile, myID, evLogger, noDefaultFolder)
 	}
 
 	return cfg, err
@@ -717,7 +712,7 @@ func auditWriter(auditFile string) io.Writer {
 		fd, err = os.OpenFile(auditFile, auditFlags, 0600)
 		if err != nil {
 			l.Warnln("Audit:", err)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 		auditDest = auditFile
 	}
@@ -774,9 +769,9 @@ func standbyMonitor(app *syncthing.App) {
 	}
 }
 
-func autoUpgrade(cfg config.Wrapper, app *syncthing.App) {
+func autoUpgrade(cfg config.Wrapper, app *syncthing.App, evLogger events.Logger) {
 	timer := time.NewTimer(0)
-	sub := events.Default.Subscribe(events.DeviceConnected)
+	sub := evLogger.Subscribe(events.DeviceConnected)
 	for {
 		select {
 		case event := <-sub.C():
@@ -798,7 +793,7 @@ func autoUpgrade(cfg config.Wrapper, app *syncthing.App) {
 
 		rel, err := upgrade.LatestRelease(opts.ReleasesURL, build.Version, opts.UpgradeToPreReleases)
 		if err == upgrade.ErrUpgradeUnsupported {
-			events.Default.Unsubscribe(sub)
+			sub.Unsubscribe()
 			return
 		}
 		if err != nil {
@@ -822,7 +817,7 @@ func autoUpgrade(cfg config.Wrapper, app *syncthing.App) {
 			timer.Reset(checkInterval)
 			continue
 		}
-		events.Default.Unsubscribe(sub)
+		sub.Unsubscribe()
 		l.Warnf("Automatically upgraded to version %q. Restarting in 1 minute.", rel.Tag)
 		time.Sleep(time.Minute)
 		app.Stop(syncthing.ExitUpgrade)
@@ -893,6 +888,6 @@ func setPauseState(cfg config.Wrapper, paused bool) {
 	}
 	if _, err := cfg.Replace(raw); err != nil {
 		l.Warnln("Cannot adjust paused state:", err)
-		os.Exit(exitError)
+		os.Exit(syncthing.ExitError.AsInt())
 	}
 }

cmd/syncthing/monitor.go

@@ -18,10 +18,12 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/syncthing"
 )
 
 var (
@@ -80,7 +82,7 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 
 		if t := time.Since(restarts[0]); t < loopThreshold {
 			l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
-			os.Exit(exitError)
+			os.Exit(syncthing.ExitError.AsInt())
 		}
 
 		copy(restarts[0:], restarts[1:])
@@ -149,17 +151,14 @@ func monitorMain(runtimeOptions RuntimeOptions) {
 				// Successful exit indicates an intentional shutdown
 				return
 			} else if exiterr, ok := err.(*exec.ExitError); ok {
-				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
-					switch status.ExitStatus() {
-					case exitUpgrading:
-						// Restart the monitor process to release the .old
-						// binary as part of the upgrade process.
-						l.Infoln("Restarting monitor...")
-						if err = restartMonitor(args); err != nil {
-							l.Warnln("Restart:", err)
-						}
-						return
+				if exiterr.ExitCode() == syncthing.ExitUpgrade.AsInt() {
+					// Restart the monitor process to release the .old
+					// binary as part of the upgrade process.
+					l.Infoln("Restarting monitor...")
+					if err = restartMonitor(args); err != nil {
+						l.Warnln("Restart:", err)
 					}
+					return
 				}
 			}
 		}
@@ -450,7 +449,7 @@ func childEnv() []string {
 // panicUploadMaxWait uploading panics...
 func maybeReportPanics() {
 	// Try to get a config to see if/where panics should be reported.
-	cfg, err := loadOrDefaultConfig(protocol.EmptyDeviceID)
+	cfg, err := loadOrDefaultConfig(protocol.EmptyDeviceID, events.NoopLogger)
 	if err != nil {
 		l.Warnln("Couldn't load config; not reporting crash")
 		return

cmd/todos/main.go

@@ -1,39 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at https://mozilla.org/MPL/2.0/.
-
-// +build ignore
-
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"os"
-)
-
-func main() {
-	buf := make([]byte, 4096)
-	var err error
-	for err == nil {
-		n, err := io.ReadFull(os.Stdin, buf)
-		if n > 0 {
-			buf = buf[:n]
-			repl := bytes.Replace(buf, []byte("\n"), []byte("\r\n"), -1)
-			_, err = os.Stdout.Write(repl)
-			if err != nil {
-				fmt.Println(err)
-				os.Exit(1)
-			}
-		}
-		if err == io.EOF {
-			return
-		}
-		buf = buf[:cap(buf)]
-	}
-	fmt.Println(err)
-	os.Exit(1)
-}

cmd/ursrv/main.go

@@ -755,6 +755,8 @@ func main() {
 	http.HandleFunc("/blockstats.json", withDB(db, blockStatsHandler))
 	http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
 
+	go cacheRefresher(db)
+
 	err = srv.Serve(listener)
 	if err != nil {
 		log.Fatalln("https:", err)
@@ -767,7 +769,31 @@ var (
 	cacheMut  sync.Mutex
 )
 
-const maxCacheTime = 5 * 60 * time.Second
+const maxCacheTime = 15 * time.Minute
+
+func cacheRefresher(db *sql.DB) {
+	ticker := time.NewTicker(maxCacheTime - time.Minute)
+	defer ticker.Stop()
+	for range ticker.C {
+		cacheMut.Lock()
+		if err := refreshCacheLocked(db); err != nil {
+			log.Println(err)
+		}
+		cacheMut.Unlock()
+	}
+}
+
+func refreshCacheLocked(db *sql.DB) error {
+	rep := getReport(db)
+	buf := new(bytes.Buffer)
+	err := tpl.Execute(buf, rep)
+	if err != nil {
+		return err
+	}
+	cacheData = buf.Bytes()
+	cacheTime = time.Now()
+	return nil
+}
 
 func rootHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
 	if r.URL.Path == "/" || r.URL.Path == "/index.html" {
@@ -775,16 +801,11 @@ func rootHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
 		defer cacheMut.Unlock()
 
 		if time.Since(cacheTime) > maxCacheTime {
-			rep := getReport(db)
-			buf := new(bytes.Buffer)
-			err := tpl.Execute(buf, rep)
-			if err != nil {
+			if err := refreshCacheLocked(db); err != nil {
 				log.Println(err)
 				http.Error(w, "Template Error", http.StatusInternalServerError)
 				return
 			}
-			cacheData = buf.Bytes()
-			cacheTime = time.Now()
 		}
 
 		w.Header().Set("Content-Type", "text/html; charset=utf-8")

go.mod

@@ -4,46 +4,48 @@ require (
 	github.com/AudriusButkevicius/go-nat-pmp v0.0.0-20160522074932-452c97607362
 	github.com/AudriusButkevicius/pfilter v0.0.0-20190627213056-c55ef6137fc6
 	github.com/AudriusButkevicius/recli v0.0.5
+	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/bkaradzic/go-lz4 v0.0.0-20160924222819-7224d8d8f27e
 	github.com/calmh/xdr v1.1.0
 	github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d
-	github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713 // indirect
+	github.com/certifi/gocertifi v0.0.0-20190905060710-a5e0173ced67 // indirect
 	github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5
 	github.com/d4l3k/messagediff v1.2.1
 	github.com/flynn-archive/go-shlex v0.0.0-20150515145356-3f9db97f8568
 	github.com/getsentry/raven-go v0.2.0
-	github.com/gobwas/glob v0.0.0-20170212200151-51eb1ee00b6d
-	github.com/gogo/protobuf v1.2.1
-	github.com/golang/groupcache v0.0.0-20171101203131-84a468cf14b4
-	github.com/jackpal/gateway v0.0.0-20161225004348-5795ac81146e
-	github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657
+	github.com/go-ole/go-ole v1.2.4 // indirect
+	github.com/gobwas/glob v0.2.3
+	github.com/gogo/protobuf v1.3.1
+	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6
+	github.com/golang/mock v1.3.1 // indirect
+	github.com/jackpal/gateway v1.0.5
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/lib/pq v1.2.0
-	github.com/lucas-clemente/quic-go v0.11.2
+	github.com/lucas-clemente/quic-go v0.12.1
 	github.com/maruel/panicparse v1.3.0
-	github.com/mattn/go-isatty v0.0.7
-	github.com/minio/sha256-simd v0.0.0-20190117184323-cc1980cb0338
-	github.com/onsi/ginkgo v1.8.0 // indirect
-	github.com/onsi/gomega v1.5.0 // indirect
+	github.com/mattn/go-isatty v0.0.10
+	github.com/minio/sha256-simd v0.1.1
+	github.com/onsi/ginkgo v1.9.0 // indirect
+	github.com/onsi/gomega v1.6.0 // indirect
 	github.com/oschwald/geoip2-golang v1.3.0
-	github.com/oschwald/maxminddb-golang v0.0.0-20170901134056-26fe5ace1c70 // indirect
-	github.com/petermattis/goid v0.0.0-20170816195418-3db12ebb2a59 // indirect
+	github.com/oschwald/maxminddb-golang v1.4.0 // indirect
+	github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect
 	github.com/pkg/errors v0.8.1
-	github.com/prometheus/client_golang v0.9.4
-	github.com/rcrowley/go-metrics v0.0.0-20171128170426-e181e095bae9
+	github.com/prometheus/client_golang v1.2.1
+	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563
 	github.com/sasha-s/go-deadlock v0.2.0
 	github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf
 	github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2
-	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
+	github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d
 	github.com/thejerf/suture v3.0.2+incompatible
-	github.com/urfave/cli v1.21.0
+	github.com/urfave/cli v1.22.1
 	github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0
-	golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8
-	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
-	golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb // indirect
+	golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472
+	golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297
 	golang.org/x/text v0.3.2
-	golang.org/x/time v0.0.0-20170927054726-6dc17368e09b
-	gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225 // indirect
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
+	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/ldap.v2 v2.5.1
 )

go.sum

@@ -7,24 +7,34 @@ github.com/AudriusButkevicius/recli v0.0.5/go.mod h1:Q2E26yc6RvWWEz/TJ/goUp6yXvi
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
+github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bkaradzic/go-lz4 v0.0.0-20160924222819-7224d8d8f27e h1:2augTYh6E+XoNrrivZJBadpThP/dsvYKj0nzqfQ8tM4=
 github.com/bkaradzic/go-lz4 v0.0.0-20160924222819-7224d8d8f27e/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
 github.com/calmh/xdr v1.1.0 h1:U/Dd4CXNLoo8EiQ4ulJUXkgO1/EyQLgDKLgpY1SOoJE=
 github.com/calmh/xdr v1.1.0/go.mod h1:E8sz2ByAdXC8MbANf1LCRYzedSnnc+/sXXJs/PVqoeg=
 github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d h1:As4937T5NVbJ/DmZT9z33pyLEprMd6CUSfhbmMY57Io=
 github.com/ccding/go-stun v0.0.0-20180726100737-be486d185f3d/go.mod h1:3FK1bMar37f7jqVY7q/63k3OMX1c47pGCufzt3X0sYE=
-github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713 h1:UNOqI3EKhvbqV8f1Vm3NIwkrhq388sGCeAH2Op7w0rc=
-github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
+github.com/certifi/gocertifi v0.0.0-20190905060710-a5e0173ced67 h1:8k9FLYBLKT+9v2HQJ/a95ZemmTx+/ltJcAiRhVushG8=
+github.com/certifi/gocertifi v0.0.0-20190905060710-a5e0173ced67/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
+github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
+github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5 h1:Wg96Dh0MLTanEaPO0OkGtUIaa2jOnShAIOVUIzRHUxo=
 github.com/chmduquesne/rollinghash v0.0.0-20180912150627-a60f8e7142b5/go.mod h1:Uc2I36RRfTAf7Dge82bi3RU0OQUmXT9iweIcPqvr8A0=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U=
 github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -37,34 +47,49 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
 github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI=
+github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gobwas/glob v0.0.0-20170212200151-51eb1ee00b6d h1:IngNQgbqr5ZOU0exk395Szrvkzes9Ilk1fmJfkw7d+M=
-github.com/gobwas/glob v0.0.0-20170212200151-51eb1ee00b6d/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/golang/groupcache v0.0.0-20171101203131-84a468cf14b4 h1:6o8aP0LGMKzo3NzwhhX6EJsiJ3ejmj+9yA/3p8Fjjlw=
-github.com/golang/groupcache v0.0.0-20171101203131-84a468cf14b4/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE=
+github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/jackpal/gateway v0.0.0-20161225004348-5795ac81146e h1:lS8IitpqG4RkZbEDlZg5Z7FvBdWLVjSVfsPGOKafEkI=
-github.com/jackpal/gateway v0.0.0-20161225004348-5795ac81146e/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
+github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc=
+github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657 h1:vE7J1m7cCpiRVEIr1B5ccDxRpbPsWT5JU3if2Di5nE4=
-github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -73,16 +98,15 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
-github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lucas-clemente/quic-go v0.11.2 h1:Mop0ac3zALaBR3wGs6j8OYe/tcFvFsxTUFMkE/7yUOI=
-github.com/lucas-clemente/quic-go v0.11.2/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw=
-github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA=
-github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
-github.com/maruel/panicparse v1.2.1 h1:mNlHGiakrixj+AwF/qRpTwnj+zsWYPRLQ7wRqnJsfO0=
-github.com/maruel/panicparse v1.2.1/go.mod h1:vszMjr5QQ4F5FSRfraldcIA/BCw5xrdLL+zEcU2nRBs=
+github.com/lucas-clemente/quic-go v0.12.0 h1:TRbvZ6F++sofeGbh+Z2IIyIOhl8KyGnYuA06g2yrHdI=
+github.com/lucas-clemente/quic-go v0.12.0/go.mod h1:UXJJPE4RfFef/xPO5wQm0tITK8gNfqwTxjbE7s3Vb8s=
+github.com/lucas-clemente/quic-go v0.12.1 h1:BPITli+6KnKogtTxBk2aS4okr5dUHz2LtIDAP1b8UL4=
+github.com/lucas-clemente/quic-go v0.12.1/go.mod h1:UXJJPE4RfFef/xPO5wQm0tITK8gNfqwTxjbE7s3Vb8s=
+github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
+github.com/marten-seemann/qtls v0.3.2 h1:O7awy4bHEzSX/K3h+fZig3/Vo03s/RxlxgsAk9sYamI=
+github.com/marten-seemann/qtls v0.3.2/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
 github.com/maruel/panicparse v1.3.0 h1:1Ep/RaYoSL1r5rTILHQQbyzHG8T4UP5ZbQTYTo4bdDc=
 github.com/maruel/panicparse v1.3.0/go.mod h1:vszMjr5QQ4F5FSRfraldcIA/BCw5xrdLL+zEcU2nRBs=
 github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
@@ -90,57 +114,81 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
-github.com/minio/sha256-simd v0.0.0-20190117184323-cc1980cb0338 h1:USW1+zAUkUSvk097CAX/i8KR3r6f+DHNhk6Xe025Oyw=
-github.com/minio/sha256-simd v0.0.0-20190117184323-cc1980cb0338/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
+github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
+github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
-github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.9.0 h1:SZjF721BByVj8QH636/8S2DnX4n0Re3SteMmw3N+tzc=
+github.com/onsi/ginkgo v1.9.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
-github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.6.0 h1:8XTW0fcJZEq9q+Upcyws4JSGua2MFysCL5xkaSgHc+M=
+github.com/onsi/gomega v1.6.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/oschwald/geoip2-golang v1.3.0 h1:D+Hsdos1NARPbzZ2aInUHZL+dApIzo8E0ErJVsWcku8=
 github.com/oschwald/geoip2-golang v1.3.0/go.mod h1:0LTTzix/Ao1uMvOhAV4iLU0Lz7eCrP94qZWBTDKf0iE=
-github.com/oschwald/maxminddb-golang v0.0.0-20170901134056-26fe5ace1c70 h1:XGLYUmodtNzThosQ8GkMvj9TiIB/uWsP8NfxKSa3aDc=
-github.com/oschwald/maxminddb-golang v0.0.0-20170901134056-26fe5ace1c70/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
-github.com/petermattis/goid v0.0.0-20170816195418-3db12ebb2a59 h1:2pHcLyJYXivxVvpoCc29uo3GDU1qFfJ1ggXKGYMrM0E=
-github.com/petermattis/goid v0.0.0-20170816195418-3db12ebb2a59/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
+github.com/oschwald/maxminddb-golang v1.4.0 h1:5/rpmW41qrgSed4wK32rdznbkTSXHcraY2LOMJX4DMc=
+github.com/oschwald/maxminddb-golang v1.4.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY=
+github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ=
+github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.4 h1:Y8E/JaaPbmFSW2V81Ab/d8yZFYQQGbni1b1jPcG9Y6A=
-github.com/prometheus/client_golang v0.9.4/go.mod h1:oCXIBxdI62A4cR6aTRJCgetEjecSIYzOEaeAn4iYEpM=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8=
+github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI=
+github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo=
+github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/rcrowley/go-metrics v0.0.0-20171128170426-e181e095bae9 h1:jmLW6izPBVlIbk4d+XgK9+sChGbVKxxOPmd9eqRHCjw=
-github.com/rcrowley/go-metrics v0.0.0-20171128170426-e181e095bae9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/prometheus/procfs v0.0.3 h1:CTwfnzjQ+8dS6MhHHu4YswVAD99sL2wjPqP+VkURmKE=
+github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.0.4 h1:w8DjqFMJDjuVwdZBQoOozr4MVWOnwF7RcL/7uxBjY78=
+github.com/prometheus/procfs v0.0.4/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
+github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=
+github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y=
 github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
 github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf h1:c9SV5NzG4KOk448TUE7iqCmb4E4y79CZF4zDdc1Jx3Q=
 github.com/shirou/gopsutil v0.0.0-20190714054239-47ef3260b6bf/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
-github.com/shirou/gopsutil v2.19.6+incompatible h1:49/Gru26Lne9Cl3IoAVDZVM09hvkSrUodgIIsCVRwbs=
-github.com/shirou/gopsutil v2.19.6+incompatible/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
@@ -151,29 +199,35 @@ github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2 h1:6tuEEEpg+mxM82
 github.com/syncthing/notify v0.0.0-20190709140112-69c7a957d3e2/go.mod h1:Sn4ChoS7e4FxjCN1XHPVBT43AgnRLbuaB8pEc1Zcdjg=
 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 h1:1oFLiOyVl+W7bnBzGhf7BbIv9loSFQcieWWYIjLqcAw=
 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
+github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d h1:gZZadD8H+fF+n9CmNhYL1Y0dJB+kLOmKd7FbPJLeGHs=
+github.com/syndtr/goleveldb v1.0.1-0.20190923125748-758128399b1d/go.mod h1:9OrXJhf154huy1nPWmuSrkgjPUtUNhA+Zmy+6AESzuA=
 github.com/thejerf/suture v3.0.2+incompatible h1:GtMydYcnK4zBJ0KL6Lx9vLzl6Oozb65wh252FTBxrvM=
 github.com/thejerf/suture v3.0.2+incompatible/go.mod h1:ibKwrVj+Uzf3XZdAiNWUouPaAbSoemxOHLmJmwheEMc=
 github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/urfave/cli v1.21.0 h1:wYSSj06510qPIzGSua9ZqsncMmWE3Zr55KBERygyrxE=
-github.com/urfave/cli v1.21.0/go.mod h1:lxDj6qX9Q6lWQxIrbrT0nwecwUtRnhVZAJjJZrVUZZQ=
+github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
+github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0 h1:okhMind4q9H1OxF44gNegWkiP4H/gsTFLalHFa4OOUI=
 github.com/vitrun/qart v0.0.0-20160531060029-bf64b92db6b0/go.mod h1:TTbGUfE+cXXceWtbTHq6lqcTvYPBKLNejBEbnUsQJtU=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXTqk7U7zPQCb+T4rbU9ZEoU=
-golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 h1:Gv7RPwsi3eZ2Fgewe3CBsuOebPwO27PoXzRpJPsvSSM=
+golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190228165749-92fc7df08ae7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -185,19 +239,29 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e h1:ZytStCyV048ZqDsWHiYDdoI2Vd4msMcrDECFxS+tL9c=
 golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k=
-golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/time v0.0.0-20170927054726-6dc17368e09b h1:3X+R0qq1+64izd8es+EttB6qcY+JDlVmAhpRXl7gpzU=
-golang.org/x/time v0.0.0-20170927054726-6dc17368e09b/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
-gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225 h1:JBwmEvLfCqgPcIq8MjVMQxsF3LVL4XG/HH0qiG0+IFY=
-gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
+gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM=
+gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

gui/black/assets/css/theme.css

@@ -24,6 +24,27 @@ a:hover,a:focus,a.focus{
     border-width: 2px !important;
 }
 
+
+.nav-tabs > li.active > a,
+.nav-tabs > li.active > a:hover,
+.nav-tabs > li.active > a:focus {
+  color: #3498db !important;
+  background-color: #222222 !important;
+  border: 1px solid #222222 !important;
+  border-bottom-color: transparent !important;
+  cursor: default;
+}
+
+.nav-tabs{
+    border-bottom: 1px solid #333;
+}
+
+.nav-tabs > li > a:hover,
+.nav-tabs > li > a:focus {
+  background-color: #222222 !important;
+  border: none !important;
+}
+
 .navbar-text, .dropdown>a, .dropdown-menu>li>a, .hidden-xs>a, .navbar-link {
     color: #aaa !important;
 }

gui/dark/assets/css/theme.css

@@ -28,6 +28,27 @@ a:hover,a:focus,a.focus{
 
 }
 
+.nav-tabs > li.active > a,
+.nav-tabs > li.active > a:hover,
+.nav-tabs > li.active > a:focus {
+  color: #3498db !important;
+  background-color: #424242 !important;
+  border: 1px solid #424242 !important;
+  border-bottom-color: transparent !important;
+  cursor: default;
+}
+
+.nav-tabs{
+    border-bottom: 1px solid #333;
+}
+
+.nav-tabs > li > a:hover,
+.nav-tabs > li > a:focus {
+  background-color: #424242 !important;
+  border: none !important;
+}
+
+
 .navbar-text, .dropdown>a, .dropdown-menu>li>a, .hidden-xs>a, .navbar-link {
     color: #aaa !important;
 }

gui/default/assets/css/theme.css

@@ -7,29 +7,5 @@
 
 */
 
-.panel-progress {
-    background: #3498db;
-}
-
-.identicon rect {
-    fill: #333;
-}
-
-.panel-warning .identicon rect {
-    fill: #fff;
-}
-
-.li-column {
-    background-color: rgb(236, 240, 241);
-    border-radius: 3px;
-}
-
-.panel-heading:hover, .panel-heading:focus {
-    text-decoration: none;
-}
-
-.fancytree-ext-filter-hide tr.fancytree-submatch span.fancytree-title,
-.fancytree-ext-filter-hide span.fancytree-node.fancytree-submatch span.fancytree-title {
-    color: black !important;
-    font-weight: lighter !important;
-}
+@import "../../theme-assets/dark/assets/css/theme.css" screen and (prefers-color-scheme: dark);
+@import "../../theme-assets/light/assets/css/theme.css" (prefers-color-scheme: light), (prefers-color-scheme: no-preference);

gui/default/index.html

@@ -323,6 +323,11 @@
                     <span class="visible-xs" aria-label="{{'Scanning' | translate}}"><i class="fas fa-fw fa-search"></i></span>
                   </span>
                   <span ng-switch-when="idle"><span class="hidden-xs" translate>Up to Date</span><span class="visible-xs" aria-label="{{'Up to Date' | translate}}"><i class="fas fa-fw fa-check"></i></span></span>
+                  <span ng-switch-when="localadditions"><span class="hidden-xs" translate>Local Additions</span><span class="visible-xs" aria-label="{{'Local Additions' | translate}}"><i class="fas fa-fw fa-check"></i></span></span>
+                  <span ng-switch-when="sync-preparing">
+                    <span class="hidden-xs" translate>Preparing to Sync</span>
+                    <span class="visible-xs" aria-label="{{'Preparing to Sync' | translate}}"><i class="fas fa-fw fa-hourglass-half"></i></span>
+                  </span>
                   <span ng-switch-when="syncing">
                     <span class="hidden-xs" translate>Syncing</span>
                     <span>({{syncPercentage(folder.id) | percent}}, {{model[folder.id].needBytes | binary}}B)</span>

gui/default/syncthing/app.js

@@ -66,6 +66,14 @@ function folderCompare(a, b) {
     return labelA > labelB;
 }
 
+function deviceMap(l) {
+    var m = {};
+    l.forEach(function (r) {
+        m[r.deviceID] = r;
+    });
+    return m;
+}
+
 function folderMap(l) {
     var m = {};
     l.forEach(function (r) {

gui/default/syncthing/core/syncthingController.js

@@ -60,7 +60,9 @@ angular.module('syncthing.core')
         } catch (exception) { }
 
         $scope.folderDefaults = {
+            sharedDevices: {},
             selectedDevices: {},
+            unrelatedDevices: {},
             type: "sendreceive",
             rescanIntervalS: 3600,
             fsWatcherDelayS: 10,
@@ -386,15 +388,8 @@ angular.module('syncthing.core')
                 });
             });
 
-            // If we're not listening on localhost, and there is no
-            // authentication configured, and the magic setting to silence the
-            // warning isn't set, then yell at the user.
-            var guiCfg = $scope.config.gui;
-            $scope.openNoAuth = guiCfg.address.substr(0, 4) !== "127."
-                && guiCfg.address.substr(0, 6) !== "[::1]:"
-                && (!guiCfg.user || !guiCfg.password)
-                && guiCfg.authMode !== 'ldap'
-                && !guiCfg.insecureAdminAccess;
+            refreshNoAuthWarning();
+            setDefaultTheme();
 
             if (!hasConfig) {
                 $scope.$emit('ConfigLoaded');
@@ -427,10 +422,33 @@ angular.module('syncthing.core')
                     }
                 }
                 $scope.discoveryFailed = discoveryFailed;
+
+                refreshNoAuthWarning();
+
                 console.log("refreshSystem", data);
             }).error($scope.emitHTTPError);
         }
 
+        function refreshNoAuthWarning() {
+            if (!$scope.system || !$scope.config) {
+                // We need both to be able to determine the state.
+                return
+            }
+
+            // If we're not listening on localhost, and there is no
+            // authentication configured, and the magic setting to silence the
+            // warning isn't set, then yell at the user.
+            var addr = $scope.system.guiAddressUsed;
+            var guiCfg = $scope.config.gui;
+            $scope.openNoAuth = addr.substr(0, 4) !== "127."
+                && addr.substr(0, 6) !== "[::1]:"
+                && addr.substr(0, 1) !== "/"
+                && (!guiCfg.user || !guiCfg.password)
+                && guiCfg.authMode !== 'ldap'
+                && !guiCfg.insecureAdminAccess;
+        }
+
+
         function refreshDiscoveryCache() {
             $http.get(urlbase + '/system/discovery').success(function (data) {
                 for (var device in data) {
@@ -634,6 +652,23 @@ angular.module('syncthing.core')
             $scope.remoteNeedDevice = undefined;
         }
 
+
+        function setDefaultTheme() {
+            if (!document.getElementById("fallback-theme-css")){
+
+                // check if no support for prefers-color-scheme
+                var colorSchemeNotSupported = typeof window.matchMedia === "undefined" || window.matchMedia('(prefers-color-scheme: dark)').media === 'not all';
+
+                if ($scope.config.gui.theme === "default" && colorSchemeNotSupported) {
+                    document.documentElement.style.display = 'none';
+                    document.head.insertAdjacentHTML(
+                      'beforeend',
+                      '<link id="fallback-theme-css" rel="stylesheet" href="theme-assets/light/assets/css/theme.css" onload="document.documentElement.style.display = \'\'">'
+                    );
+                }
+            }
+        }
+
         function saveIgnores(ignores, cb) {
             $http.post(urlbase + '/db/ignores?folder=' + encodeURIComponent($scope.currentFolder.id), {
                 ignore: ignores
@@ -753,25 +788,31 @@ angular.module('syncthing.core')
                 return 'paused';
             }
 
+            var folderInfo = $scope.model[folderCfg.id];
+
             // after restart syncthing process state may be empty
-            if (!$scope.model[folderCfg.id].state) {
+            if (!folderInfo.state) {
                 return 'unknown';
             }
 
-            var state = '' + $scope.model[folderCfg.id].state;
+            var state = '' + folderInfo.state;
             if (state === 'error') {
                 return 'stopped'; // legacy, the state is called "stopped" in the GUI
             }
-            if (state === 'idle' && $scope.model[folderCfg.id].needTotalItems > 0) {
+
+            if (state !== 'idle') {
+                return state;
+            }
+
+            if (folderInfo.needTotalItems > 0) {
                 return 'outofsync';
             }
             if ($scope.hasFailedFiles(folderCfg.id)) {
                 return 'faileditems';
             }
-            if (state === 'scanning') {
-                return state;
+            if (folderInfo.receiveOnlyTotalItems) {
+                return 'localadditions';
             }
-
             if (folderCfg.devices.length <= 1) {
                 return 'unshared';
             }
@@ -782,13 +823,13 @@ angular.module('syncthing.core')
         $scope.folderClass = function (folderCfg) {
             var status = $scope.folderStatus(folderCfg);
 
-            if (status === 'idle') {
+            if (status === 'idle' || status === 'localadditions') {
                 return 'success';
             }
             if (status == 'paused') {
                 return 'default';
             }
-            if (status === 'syncing' || status === 'scanning') {
+            if (status === 'syncing' || status === 'sync-preparing' || status === 'scanning') {
                 return 'primary';
             }
             if (status === 'unknown') {
@@ -838,6 +879,9 @@ angular.module('syncthing.core')
             // 32m 40s
             // 2h 32m
             // 4d 2h
+            // In case remaining scan time appears to be >31d, omit the
+            // details, i.e.:
+            // > 1 month
 
             if (!$scope.scanProgress[folder]) {
                 return "";
@@ -857,6 +901,9 @@ angular.module('syncthing.core')
             var res = [];
             if (seconds >= 86400) {
                 days = Math.floor(seconds / 86400);
+                if (days > 31) {
+                    return '> 1 month';
+                }
                 res.push('' + days + 'd')
                 seconds = seconds % 86400;
             }
@@ -947,6 +994,7 @@ angular.module('syncthing.core')
             for (var i = 0; i < folderListCache.length; i++) {
                 var status = $scope.folderStatus(folderListCache[i]);
                 switch (status) {
+                    case 'sync-preparing':
                     case 'syncing':
                         syncCount++;
                         break;
@@ -1377,13 +1425,13 @@ angular.module('syncthing.core')
         };
 
         $scope.selectAllFolders = function () {
-            angular.forEach($scope.folders, function (id) {
+            angular.forEach($scope.folders, function (_, id) {
                 $scope.currentDevice.selectedFolders[id] = true;
             });
         };
 
         $scope.deSelectAllFolders = function () {
-            angular.forEach($scope.folders, function (id) {
+            angular.forEach($scope.folders, function (_, id) {
                 $scope.currentDevice.selectedFolders[id] = false;
             });
         };
@@ -1657,10 +1705,20 @@ angular.module('syncthing.core')
             if ($scope.currentFolder.path.length > 1 && $scope.currentFolder.path.slice(-1) === $scope.system.pathSeparator) {
                 $scope.currentFolder.path = $scope.currentFolder.path.slice(0, -1);
             }
+            // Cache complete device objects indexed by ID for lookups
+            var devMap = deviceMap($scope.devices)
+            $scope.currentFolder.sharedDevices = [];
             $scope.currentFolder.selectedDevices = {};
             $scope.currentFolder.devices.forEach(function (n) {
+                if (n.deviceID !== $scope.myID) {
+                    $scope.currentFolder.sharedDevices.push(devMap[n.deviceID]);
+                }
                 $scope.currentFolder.selectedDevices[n.deviceID] = true;
             });
+            $scope.currentFolder.unrelatedDevices = $scope.devices.filter(function (n) {
+                return n.deviceID !== $scope.myID
+                    && ! $scope.currentFolder.selectedDevices[n.deviceID]
+            });
             if ($scope.currentFolder.versioning && $scope.currentFolder.versioning.type === "trashcan") {
                 $scope.currentFolder.trashcanFileVersioning = true;
                 $scope.currentFolder.fileVersioningSelector = "trashcan";
@@ -1711,17 +1769,17 @@ angular.module('syncthing.core')
             $scope.editFolderModal();
         };
 
-        $scope.selectAllDevices = function () {
-            var devices = $scope.otherDevices();
+        $scope.selectAllSharedDevices = function (state) {
+            var devices = $scope.currentFolder.sharedDevices;
             for (var i = 0; i < devices.length; i++) {
-                $scope.currentFolder.selectedDevices[devices[i].deviceID] = true;
+                $scope.currentFolder.selectedDevices[devices[i].deviceID] = !!state;
             }
         };
 
-        $scope.deSelectAllDevices = function () {
-            var devices = $scope.otherDevices();
+        $scope.selectAllUnrelatedDevices = function (state) {
+            var devices = $scope.currentFolder.unrelatedDevices;
             for (var i = 0; i < devices.length; i++) {
-                $scope.currentFolder.selectedDevices[devices[i].deviceID] = false;
+                $scope.currentFolder.selectedDevices[devices[i].deviceID] = !!state;
             }
         };
 
@@ -1730,6 +1788,7 @@ angular.module('syncthing.core')
                 $scope.editingExisting = false;
                 $scope.currentFolder = angular.copy($scope.folderDefaults);
                 $scope.currentFolder.id = (data.random.substr(0, 5) + '-' + data.random.substr(5, 5)).toLowerCase();
+                $scope.currentFolder.unrelatedDevices = $scope.otherDevices();
                 $('#folder-ignores textarea').val("");
                 $('#folder-ignores textarea').removeAttr('disabled');
                 $scope.editFolderModal();
@@ -1745,6 +1804,7 @@ angular.module('syncthing.core')
                 importFromOtherDevice: true
             };
             $scope.currentFolder.selectedDevices[device] = true;
+            $scope.currentFolder.unrelatedDevices = $scope.otherDevices();
             $('#folder-ignores textarea').val("");
             $('#folder-ignores textarea').removeAttr('disabled');
             $scope.editFolderModal();
@@ -1770,7 +1830,9 @@ angular.module('syncthing.core')
                     });
                 }
             }
+            delete folderCfg.sharedDevices;
             delete folderCfg.selectedDevices;
+            delete folderCfg.unrelatedDevices;
 
             if (folderCfg.fileVersioningSelector === "trashcan") {
                 folderCfg.versioning = {

gui/default/syncthing/folder/editFolderModalView.html

@@ -44,15 +44,35 @@
           </div>
         </div>
         <div id="folder-sharing" class="tab-pane">
-          <div class="form-group">
-            <label translate for="devices">Share With Devices</label>
+          <div class="form-group" ng-if="currentFolder.sharedDevices.length">
+            <label translate>Currently Shared With Devices</label>
             <p class="help-block">
-              <span translate>Select the devices to share this folder with.</span>&emsp;
-              <small><a href="#" ng-click="selectAllDevices()" translate>Select All</a>&emsp;
-                <a href="#" ng-click="deSelectAllDevices()" translate>Deselect All</a></small>
+              <span translate>Deselect devices to stop sharing this folder with.</span>&emsp;
+              <small><a href="#" ng-click="selectAllSharedDevices(true)" translate>Select All</a>&emsp;
+                <a href="#" ng-click="selectAllSharedDevices(false)" translate>Deselect All</a></small>
             </p>
             <div class="row">
-              <div class="col-md-4" ng-repeat="device in otherDevices()">
+              <div class="col-md-4" ng-repeat="device in currentFolder.sharedDevices">
+                <div class="checkbox">
+                  <label>
+                    <input type="checkbox" ng-model="currentFolder.selectedDevices[device.deviceID]" /> {{deviceName(device)}}
+                  </label>
+                </div>
+              </div>
+            </div>
+          </div>
+          <div class="form-group" ng-if="currentFolder.unrelatedDevices.length || otherDevices().length <= 0">
+            <label translate>Unshared Devices</label>
+            <p class="help-block" ng-if="otherDevices().length > 0">
+              <span translate>Select additional devices to share this folder with.</span>&emsp;
+              <small><a href="#" ng-click="selectAllUnrelatedDevices(true)" translate>Select All</a>&emsp;
+                <a href="#" ng-click="selectAllUnrelatedDevices(false)" translate>Deselect All</a></small>
+            </p>
+            <p class="help-block" ng-if="otherDevices().length <= 0">
+              <span translate>There are no devices to share this folder with.</span>
+            </p>
+            <div class="row">
+              <div class="col-md-4" ng-repeat="device in currentFolder.unrelatedDevices">
                 <div class="checkbox">
                   <label>
                     <input type="checkbox" ng-model="currentFolder.selectedDevices[device.deviceID]" /> {{deviceName(device)}}

gui/light/assets/css/theme.css

@@ -0,0 +1,35 @@
+/*
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+*/
+
+.panel-progress {
+    background: #3498db;
+}
+
+.identicon rect {
+    fill: #333;
+}
+
+.panel-warning .identicon rect {
+    fill: #fff;
+}
+
+.li-column {
+    background-color: rgb(236, 240, 241);
+    border-radius: 3px;
+}
+
+.panel-heading:hover, .panel-heading:focus {
+    text-decoration: none;
+}
+
+.fancytree-ext-filter-hide tr.fancytree-submatch span.fancytree-title,
+.fancytree-ext-filter-hide span.fancytree-node.fancytree-submatch span.fancytree-title {
+    color: black !important;
+    font-weight: lighter !important;
+}

lib/api/api.go

@@ -9,7 +9,9 @@ package api
 import (
 	"bytes"
 	"crypto/tls"
+	"crypto/x509"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -56,10 +58,11 @@ import (
 var bcryptExpr = regexp.MustCompile(`^\$2[aby]\$\d+\$.{50,}`)
 
 const (
-	DefaultEventMask    = events.AllEvents &^ events.LocalChangeDetected &^ events.RemoteChangeDetected
-	DiskEventMask       = events.LocalChangeDetected | events.RemoteChangeDetected
-	EventSubBufferSize  = 1000
-	defaultEventTimeout = time.Minute
+	DefaultEventMask      = events.AllEvents &^ events.LocalChangeDetected &^ events.RemoteChangeDetected
+	DiskEventMask         = events.LocalChangeDetected | events.RemoteChangeDetected
+	EventSubBufferSize    = 1000
+	defaultEventTimeout   = time.Minute
+	httpsCertLifetimeDays = 820
 )
 
 type service struct {
@@ -71,6 +74,7 @@ type service struct {
 	model                model.Model
 	eventSubs            map[events.EventType]events.BufferedSubscription
 	eventSubsMut         sync.Mutex
+	evLogger             events.Logger
 	discoverer           discover.CachingMux
 	connectionsService   connections.Service
 	fss                  model.FolderSummaryService
@@ -84,6 +88,7 @@ type service struct {
 	started              chan string   // signals startup complete by sending the listener address, for testing only
 	startedOnce          chan struct{} // the service has started successfully at least once
 	startupErr           error
+	listenerAddr         net.Addr
 
 	guiErrors logger.Recorder
 	systemLog logger.Recorder
@@ -105,7 +110,7 @@ type Service interface {
 	WaitForStart() error
 }
 
-func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonName string, m model.Model, defaultSub, diskSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService connections.Service, urService *ur.Service, fss model.FolderSummaryService, errors, systemLog logger.Recorder, cpu Rater, contr Controller, noUpgrade bool) Service {
+func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonName string, m model.Model, defaultSub, diskSub events.BufferedSubscription, evLogger events.Logger, discoverer discover.CachingMux, connectionsService connections.Service, urService *ur.Service, fss model.FolderSummaryService, errors, systemLog logger.Recorder, cpu Rater, contr Controller, noUpgrade bool) Service {
 	s := &service{
 		id:      id,
 		cfg:     cfg,
@@ -116,6 +121,7 @@ func New(id protocol.DeviceID, cfg config.Wrapper, assetDir, tlsDefaultCommonNam
 			DiskEventMask:    diskSub,
 		},
 		eventSubsMut:         sync.NewMutex(),
+		evLogger:             evLogger,
 		discoverer:           discoverer,
 		connectionsService:   connectionsService,
 		fss:                  fss,
@@ -143,6 +149,12 @@ func (s *service) getListener(guiCfg config.GUIConfiguration) (net.Listener, err
 	httpsCertFile := locations.Get(locations.HTTPSCertFile)
 	httpsKeyFile := locations.Get(locations.HTTPSKeyFile)
 	cert, err := tls.LoadX509KeyPair(httpsCertFile, httpsKeyFile)
+
+	// If the certificate has expired or will expire in the next month, fail
+	// it and generate a new one.
+	if err == nil {
+		err = checkExpiry(cert)
+	}
 	if err != nil {
 		l.Infoln("Loading HTTPS certificate:", err)
 		l.Infoln("Creating new HTTPS certificate")
@@ -155,7 +167,7 @@ func (s *service) getListener(guiCfg config.GUIConfiguration) (net.Listener, err
 			name = s.tlsDefaultCommonName
 		}
 
-		cert, err = tlsutil.NewCertificate(httpsCertFile, httpsKeyFile, name)
+		cert, err = tlsutil.NewCertificate(httpsCertFile, httpsKeyFile, name, httpsCertLifetimeDays)
 	}
 	if err != nil {
 		return nil, err
@@ -220,6 +232,7 @@ func (s *service) serve(stop chan struct{}) {
 		return
 	}
 
+	s.listenerAddr = listener.Addr()
 	defer listener.Close()
 
 	s.cfg.Subscribe(s)
@@ -308,14 +321,14 @@ func (s *service) serve(stop chan struct{}) {
 
 	// Wrap everything in CSRF protection. The /rest prefix should be
 	// protected, other requests will grant cookies.
-	handler := csrfMiddleware(s.id.String()[:5], "/rest", guiCfg, mux)
+	var handler http.Handler = newCsrfManager(s.id.String()[:5], "/rest", guiCfg, mux, locations.Get(locations.CsrfTokens))
 
 	// Add our version and ID as a header to responses
 	handler = withDetailsMiddleware(s.id, handler)
 
 	// Wrap everything in basic auth, if user/password is set.
 	if guiCfg.IsAuthEnabled() {
-		handler = basicAuthAndSessionMiddleware("sessionid-"+s.id.String()[:5], guiCfg, s.cfg.LDAP(), handler)
+		handler = basicAuthAndSessionMiddleware("sessionid-"+s.id.String()[:5], guiCfg, s.cfg.LDAP(), handler, s.evLogger)
 	}
 
 	// Redirect to HTTPS if we are supposed to
@@ -911,6 +924,7 @@ func (s *service) getSystemStatus(w http.ResponseWriter, r *http.Request) {
 	res["uptime"] = s.urService.UptimeS()
 	res["startTime"] = ur.StartTime
 	res["guiAddressOverridden"] = s.cfg.GUI().IsOverridden()
+	res["guiAddressUsed"] = s.listenerAddr.String()
 
 	sendJSON(w, res)
 }
@@ -1215,7 +1229,7 @@ func (s *service) getEventSub(mask events.EventType) events.BufferedSubscription
 	s.eventSubsMut.Lock()
 	bufsub, ok := s.eventSubs[mask]
 	if !ok {
-		evsub := events.Default.Subscribe(mask)
+		evsub := s.evLogger.Subscribe(mask)
 		bufsub = events.NewBufferedSubscription(evsub, EventSubBufferSize)
 		s.eventSubs[mask] = bufsub
 	}
@@ -1568,10 +1582,10 @@ func (s *service) getHeapProf(w http.ResponseWriter, r *http.Request) {
 	pprof.WriteHeapProfile(w)
 }
 
-func toJsonFileInfoSlice(fs []db.FileInfoTruncated) []jsonDBFileInfo {
-	res := make([]jsonDBFileInfo, len(fs))
+func toJsonFileInfoSlice(fs []db.FileInfoTruncated) []jsonFileInfoTrunc {
+	res := make([]jsonFileInfoTrunc, len(fs))
 	for i, f := range fs {
-		res[i] = jsonDBFileInfo(f)
+		res[i] = jsonFileInfoTrunc(f)
 	}
 	return res
 }
@@ -1581,45 +1595,39 @@ func toJsonFileInfoSlice(fs []db.FileInfoTruncated) []jsonDBFileInfo {
 type jsonFileInfo protocol.FileInfo
 
 func (f jsonFileInfo) MarshalJSON() ([]byte, error) {
-	return json.Marshal(map[string]interface{}{
-		"name":          f.Name,
-		"type":          f.Type,
-		"size":          f.Size,
-		"permissions":   fmt.Sprintf("%#o", f.Permissions),
-		"deleted":       f.Deleted,
-		"invalid":       protocol.FileInfo(f).IsInvalid(),
-		"ignored":       protocol.FileInfo(f).IsIgnored(),
-		"mustRescan":    protocol.FileInfo(f).MustRescan(),
-		"noPermissions": f.NoPermissions,
-		"modified":      protocol.FileInfo(f).ModTime(),
-		"modifiedBy":    f.ModifiedBy.String(),
-		"sequence":      f.Sequence,
-		"numBlocks":     len(f.Blocks),
-		"version":       jsonVersionVector(f.Version),
-		"localFlags":    f.LocalFlags,
-	})
+	m := fileIntfJSONMap(protocol.FileInfo(f))
+	m["numBlocks"] = len(f.Blocks)
+	return json.Marshal(m)
 }
 
-type jsonDBFileInfo db.FileInfoTruncated
+type jsonFileInfoTrunc db.FileInfoTruncated
 
-func (f jsonDBFileInfo) MarshalJSON() ([]byte, error) {
-	return json.Marshal(map[string]interface{}{
-		"name":          f.Name,
-		"type":          f.Type.String(),
-		"size":          f.Size,
-		"permissions":   fmt.Sprintf("%#o", f.Permissions),
-		"deleted":       f.Deleted,
-		"invalid":       db.FileInfoTruncated(f).IsInvalid(),
-		"ignored":       db.FileInfoTruncated(f).IsIgnored(),
-		"mustRescan":    db.FileInfoTruncated(f).MustRescan(),
-		"noPermissions": f.NoPermissions,
-		"modified":      db.FileInfoTruncated(f).ModTime(),
-		"modifiedBy":    f.ModifiedBy.String(),
-		"sequence":      f.Sequence,
-		"numBlocks":     nil, // explicitly unknown
-		"version":       jsonVersionVector(f.Version),
-		"localFlags":    f.LocalFlags,
-	})
+func (f jsonFileInfoTrunc) MarshalJSON() ([]byte, error) {
+	m := fileIntfJSONMap(db.FileInfoTruncated(f))
+	m["numBlocks"] = nil // explicitly unknown
+	return json.Marshal(m)
+}
+
+func fileIntfJSONMap(f db.FileIntf) map[string]interface{} {
+	out := map[string]interface{}{
+		"name":          f.FileName(),
+		"type":          f.FileType().String(),
+		"size":          f.FileSize(),
+		"deleted":       f.IsDeleted(),
+		"invalid":       f.IsInvalid(),
+		"ignored":       f.IsIgnored(),
+		"mustRescan":    f.MustRescan(),
+		"noPermissions": !f.HasPermissionBits(),
+		"modified":      f.ModTime(),
+		"modifiedBy":    f.FileModifiedBy().String(),
+		"sequence":      f.SequenceNo(),
+		"version":       jsonVersionVector(f.FileVersion()),
+		"localFlags":    f.FileLocalFlags(),
+	}
+	if f.HasPermissionBits() {
+		out["permissions"] = fmt.Sprintf("%#o", f.FilePermissions())
+	}
+	return out
 }
 
 type jsonVersionVector protocol.Vector
@@ -1673,3 +1681,45 @@ func addressIsLocalhost(addr string) bool {
 		return ip.IsLoopback()
 	}
 }
+
+func checkExpiry(cert tls.Certificate) error {
+	leaf := cert.Leaf
+	if leaf == nil {
+		// Leaf can be nil or not, depending on how parsed the certificate
+		// was when we got it.
+		if len(cert.Certificate) < 1 {
+			// can't happen
+			return errors.New("no certificate in certificate")
+		}
+		var err error
+		leaf, err = x509.ParseCertificate(cert.Certificate[0])
+		if err != nil {
+			return err
+		}
+	}
+
+	if leaf.Subject.String() != leaf.Issuer.String() ||
+		len(leaf.DNSNames) != 0 || len(leaf.IPAddresses) != 0 {
+		// The certificate is not self signed, or has DNS/IP attributes we don't
+		// add, so we leave it alone.
+		return nil
+	}
+
+	if leaf.NotAfter.Before(time.Now()) {
+		return errors.New("certificate has expired")
+	}
+	if leaf.NotAfter.Before(time.Now().Add(30 * 24 * time.Hour)) {
+		return errors.New("certificate will soon expire")
+	}
+
+	// On macOS, check for certificates issued on or after July 1st, 2019,
+	// with a longer validity time than 825 days.
+	cutoff := time.Date(2019, 7, 1, 0, 0, 0, 0, time.UTC)
+	if runtime.GOOS == "darwin" &&
+		leaf.NotBefore.After(cutoff) &&
+		leaf.NotAfter.Sub(leaf.NotBefore) > 825*24*time.Hour {
+		return errors.New("certificate incompatible with macOS 10.15 (Catalina)")
+	}
+
+	return nil
+}

lib/api/api_auth.go

@@ -28,14 +28,14 @@ var (
 	sessionsMut = sync.NewMutex()
 )
 
-func emitLoginAttempt(success bool, username string) {
-	events.Default.Log(events.LoginAttempt, map[string]interface{}{
+func emitLoginAttempt(success bool, username string, evLogger events.Logger) {
+	evLogger.Log(events.LoginAttempt, map[string]interface{}{
 		"success":  success,
 		"username": username,
 	})
 }
 
-func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfiguration, ldapCfg config.LDAPConfiguration, next http.Handler) http.Handler {
+func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfiguration, ldapCfg config.LDAPConfiguration, next http.Handler, evLogger events.Logger) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if guiCfg.IsValidAPIKey(r.Header.Get("X-API-Key")) {
 			next.ServeHTTP(w, r)
@@ -94,7 +94,7 @@ func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfigura
 		}
 
 		if !authOk {
-			emitLoginAttempt(false, username)
+			emitLoginAttempt(false, username, evLogger)
 			error()
 			return
 		}
@@ -109,7 +109,7 @@ func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfigura
 			MaxAge: 0,
 		})
 
-		emitLoginAttempt(true, username)
+		emitLoginAttempt(true, username, evLogger)
 		next.ServeHTTP(w, r)
 	})
 }

lib/api/api_auth_test.go

@@ -19,6 +19,8 @@ func init() {
 }
 
 func TestStaticAuthOK(t *testing.T) {
+	t.Parallel()
+
 	ok := authStatic("user", "pass", "user", string(passwordHashBytes))
 	if !ok {
 		t.Fatalf("should pass auth")
@@ -26,6 +28,8 @@ func TestStaticAuthOK(t *testing.T) {
 }
 
 func TestSimpleAuthUsernameFail(t *testing.T) {
+	t.Parallel()
+
 	ok := authStatic("userWRONG", "pass", "user", string(passwordHashBytes))
 	if ok {
 		t.Fatalf("should fail auth")
@@ -33,6 +37,8 @@ func TestSimpleAuthUsernameFail(t *testing.T) {
 }
 
 func TestStaticAuthPasswordFail(t *testing.T) {
+	t.Parallel()
+
 	ok := authStatic("user", "passWRONG", "user", string(passwordHashBytes))
 	if ok {
 		t.Fatalf("should fail auth")

lib/api/api_csrf.go

@@ -13,83 +13,103 @@ import (
 	"os"
 	"strings"
 
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/locations"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
-// csrfTokens is a list of valid tokens. It is sorted so that the most
-// recently used token is first in the list. New tokens are added to the front
-// of the list (as it is the most recently used at that time). The list is
-// pruned to a maximum of maxCsrfTokens, throwing away the least recently used
-// tokens.
-var csrfTokens []string
-var csrfMut = sync.NewMutex()
-
 const maxCsrfTokens = 25
 
+type csrfManager struct {
+	// tokens is a list of valid tokens. It is sorted so that the most
+	// recently used token is first in the list. New tokens are added to the front
+	// of the list (as it is the most recently used at that time). The list is
+	// pruned to a maximum of maxCsrfTokens, throwing away the least recently used
+	// tokens.
+	tokens    []string
+	tokensMut sync.Mutex
+
+	unique          string
+	prefix          string
+	apiKeyValidator apiKeyValidator
+	next            http.Handler
+	saveLocation    string
+}
+
+type apiKeyValidator interface {
+	IsValidAPIKey(key string) bool
+}
+
 // Check for CSRF token on /rest/ URLs. If a correct one is not given, reject
 // the request with 403. For / and /index.html, set a new CSRF cookie if none
 // is currently set.
-func csrfMiddleware(unique string, prefix string, cfg config.GUIConfiguration, next http.Handler) http.Handler {
-	loadCsrfTokens()
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// Allow requests carrying a valid API key
-		if cfg.IsValidAPIKey(r.Header.Get("X-API-Key")) {
-			// Set the access-control-allow-origin header for CORS requests
-			// since a valid API key has been provided
-			w.Header().Add("Access-Control-Allow-Origin", "*")
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		if strings.HasPrefix(r.URL.Path, "/rest/debug") {
-			// Debugging functions are only available when explicitly
-			// enabled, and can be accessed without a CSRF token
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		// Allow requests for anything not under the protected path prefix,
-		// and set a CSRF cookie if there isn't already a valid one.
-		if !strings.HasPrefix(r.URL.Path, prefix) {
-			cookie, err := r.Cookie("CSRF-Token-" + unique)
-			if err != nil || !validCsrfToken(cookie.Value) {
-				l.Debugln("new CSRF cookie in response to request for", r.URL)
-				cookie = &http.Cookie{
-					Name:  "CSRF-Token-" + unique,
-					Value: newCsrfToken(),
-				}
-				http.SetCookie(w, cookie)
-			}
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		// Verify the CSRF token
-		token := r.Header.Get("X-CSRF-Token-" + unique)
-		if !validCsrfToken(token) {
-			http.Error(w, "CSRF Error", 403)
-			return
-		}
-
-		next.ServeHTTP(w, r)
-	})
+func newCsrfManager(unique string, prefix string, apiKeyValidator apiKeyValidator, next http.Handler, saveLocation string) *csrfManager {
+	m := &csrfManager{
+		tokensMut:       sync.NewMutex(),
+		unique:          unique,
+		prefix:          prefix,
+		apiKeyValidator: apiKeyValidator,
+		next:            next,
+		saveLocation:    saveLocation,
+	}
+	m.load()
+	return m
 }
 
-func validCsrfToken(token string) bool {
-	csrfMut.Lock()
-	defer csrfMut.Unlock()
-	for i, t := range csrfTokens {
+func (m *csrfManager) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Allow requests carrying a valid API key
+	if m.apiKeyValidator.IsValidAPIKey(r.Header.Get("X-API-Key")) {
+		// Set the access-control-allow-origin header for CORS requests
+		// since a valid API key has been provided
+		w.Header().Add("Access-Control-Allow-Origin", "*")
+		m.next.ServeHTTP(w, r)
+		return
+	}
+
+	if strings.HasPrefix(r.URL.Path, "/rest/debug") {
+		// Debugging functions are only available when explicitly
+		// enabled, and can be accessed without a CSRF token
+		m.next.ServeHTTP(w, r)
+		return
+	}
+
+	// Allow requests for anything not under the protected path prefix,
+	// and set a CSRF cookie if there isn't already a valid one.
+	if !strings.HasPrefix(r.URL.Path, m.prefix) {
+		cookie, err := r.Cookie("CSRF-Token-" + m.unique)
+		if err != nil || !m.validToken(cookie.Value) {
+			l.Debugln("new CSRF cookie in response to request for", r.URL)
+			cookie = &http.Cookie{
+				Name:  "CSRF-Token-" + m.unique,
+				Value: m.newToken(),
+			}
+			http.SetCookie(w, cookie)
+		}
+		m.next.ServeHTTP(w, r)
+		return
+	}
+
+	// Verify the CSRF token
+	token := r.Header.Get("X-CSRF-Token-" + m.unique)
+	if !m.validToken(token) {
+		http.Error(w, "CSRF Error", http.StatusForbidden)
+		return
+	}
+
+	m.next.ServeHTTP(w, r)
+}
+
+func (m *csrfManager) validToken(token string) bool {
+	m.tokensMut.Lock()
+	defer m.tokensMut.Unlock()
+	for i, t := range m.tokens {
 		if t == token {
 			if i > 0 {
 				// Move this token to the head of the list. Copy the tokens at
 				// the front one step to the right and then replace the token
 				// at the head.
-				copy(csrfTokens[1:], csrfTokens[:i+1])
-				csrfTokens[0] = token
+				copy(m.tokens[1:], m.tokens[:i+1])
+				m.tokens[0] = token
 			}
 			return true
 		}
@@ -97,40 +117,47 @@ func validCsrfToken(token string) bool {
 	return false
 }
 
-func newCsrfToken() string {
+func (m *csrfManager) newToken() string {
 	token := rand.String(32)
 
-	csrfMut.Lock()
-	csrfTokens = append([]string{token}, csrfTokens...)
-	if len(csrfTokens) > maxCsrfTokens {
-		csrfTokens = csrfTokens[:maxCsrfTokens]
+	m.tokensMut.Lock()
+	m.tokens = append([]string{token}, m.tokens...)
+	if len(m.tokens) > maxCsrfTokens {
+		m.tokens = m.tokens[:maxCsrfTokens]
 	}
-	defer csrfMut.Unlock()
+	defer m.tokensMut.Unlock()
 
-	saveCsrfTokens()
+	m.save()
 
 	return token
 }
 
-func saveCsrfTokens() {
+func (m *csrfManager) save() {
 	// We're ignoring errors in here. It's not super critical and there's
 	// nothing relevant we can do about them anyway...
 
-	name := locations.Get(locations.CsrfTokens)
-	f, err := osutil.CreateAtomic(name)
+	if m.saveLocation == "" {
+		return
+	}
+
+	f, err := osutil.CreateAtomic(m.saveLocation)
 	if err != nil {
 		return
 	}
 
-	for _, t := range csrfTokens {
+	for _, t := range m.tokens {
 		fmt.Fprintln(f, t)
 	}
 
 	f.Close()
 }
 
-func loadCsrfTokens() {
-	f, err := os.Open(locations.Get(locations.CsrfTokens))
+func (m *csrfManager) load() {
+	if m.saveLocation == "" {
+		return
+	}
+
+	f, err := os.Open(m.saveLocation)
 	if err != nil {
 		return
 	}
@@ -138,6 +165,6 @@ func loadCsrfTokens() {
 
 	s := bufio.NewScanner(f)
 	for s.Scan() {
-		csrfTokens = append(csrfTokens, s.Text())
+		m.tokens = append(m.tokens, s.Text())
 	}
 }

lib/api/api_statics.go

@@ -23,21 +23,25 @@ import (
 	"github.com/syncthing/syncthing/lib/sync"
 )
 
+const themePrefix = "theme-assets/"
+
 type staticsServer struct {
 	assetDir        string
 	assets          map[string][]byte
 	availableThemes []string
 
-	mut   sync.RWMutex
-	theme string
+	mut             sync.RWMutex
+	theme           string
+	lastThemeChange time.Time
 }
 
 func newStaticsServer(theme, assetDir string) *staticsServer {
 	s := &staticsServer{
-		assetDir: assetDir,
-		assets:   auto.Assets(),
-		mut:      sync.NewRWMutex(),
-		theme:    theme,
+		assetDir:        assetDir,
+		assets:          auto.Assets(),
+		mut:             sync.NewRWMutex(),
+		theme:           theme,
+		lastThemeChange: time.Now().UTC(),
 	}
 
 	seen := make(map[string]struct{})
@@ -86,8 +90,23 @@ func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) {
 
 	s.mut.RLock()
 	theme := s.theme
+	modificationTime := s.lastThemeChange
 	s.mut.RUnlock()
 
+	// If path starts with special prefix, get theme and file from path
+	if strings.HasPrefix(file, themePrefix) {
+		path := file[len(themePrefix):]
+		i := strings.IndexRune(path, '/')
+
+		if i == -1 {
+			http.NotFound(w, r)
+			return
+		}
+
+		theme = path[:i]
+		file = path[i+1:]
+	}
+
 	// Check for an override for the current theme.
 	if s.assetDir != "" {
 		p := filepath.Join(s.assetDir, theme, filepath.FromSlash(file))
@@ -125,14 +144,12 @@ func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	etag := fmt.Sprintf("%d", auto.Generated)
-	modified := time.Unix(auto.Generated, 0).UTC()
-
-	w.Header().Set("Last-Modified", modified.Format(http.TimeFormat))
+	etag := fmt.Sprintf("%d", modificationTime.Unix())
+	w.Header().Set("Last-Modified", modificationTime.Format(http.TimeFormat))
 	w.Header().Set("Etag", etag)
 
 	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil {
-		if modified.Equal(t) || modified.Before(t) {
+		if modificationTime.Equal(t) || modificationTime.Before(t) {
 			w.WriteHeader(http.StatusNotModified)
 			return
 		}
@@ -199,6 +216,7 @@ func (s *staticsServer) mimeTypeForFile(file string) string {
 func (s *staticsServer) setTheme(theme string) {
 	s.mut.Lock()
 	s.theme = theme
+	s.lastThemeChange = time.Now().UTC()
 	s.mut.Unlock()
 }
 

lib/api/api_test.go

@@ -18,6 +18,7 @@ import (
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"testing"
@@ -31,6 +32,7 @@ import (
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/ur"
 	"github.com/thejerf/suture"
 )
@@ -52,7 +54,7 @@ func TestMain(m *testing.M) {
 }
 
 func TestCSRFToken(t *testing.T) {
-	defer os.Remove(token)
+	t.Parallel()
 
 	max := 250
 	int := 5
@@ -61,11 +63,13 @@ func TestCSRFToken(t *testing.T) {
 		int = 2
 	}
 
-	t1 := newCsrfToken()
-	t2 := newCsrfToken()
+	m := newCsrfManager("unique", "prefix", config.GUIConfiguration{}, nil, "")
 
-	t3 := newCsrfToken()
-	if !validCsrfToken(t3) {
+	t1 := m.newToken()
+	t2 := m.newToken()
+
+	t3 := m.newToken()
+	if !m.validToken(t3) {
 		t.Fatal("t3 should be valid")
 	}
 
@@ -73,36 +77,38 @@ func TestCSRFToken(t *testing.T) {
 		if i%int == 0 {
 			// t1 and t2 should remain valid by virtue of us checking them now
 			// and then.
-			if !validCsrfToken(t1) {
+			if !m.validToken(t1) {
 				t.Fatal("t1 should be valid at iteration", i)
 			}
-			if !validCsrfToken(t2) {
+			if !m.validToken(t2) {
 				t.Fatal("t2 should be valid at iteration", i)
 			}
 		}
 
 		// The newly generated token is always valid
-		t4 := newCsrfToken()
-		if !validCsrfToken(t4) {
+		t4 := m.newToken()
+		if !m.validToken(t4) {
 			t.Fatal("t4 should be valid at iteration", i)
 		}
 	}
 
-	if validCsrfToken(t3) {
+	if m.validToken(t3) {
 		t.Fatal("t3 should have expired by now")
 	}
 }
 
 func TestStopAfterBrokenConfig(t *testing.T) {
+	t.Parallel()
+
 	cfg := config.Configuration{
 		GUI: config.GUIConfiguration{
 			RawAddress: "127.0.0.1:0",
 			RawUseTLS:  false,
 		},
 	}
-	w := config.Wrap("/dev/null", cfg)
+	w := config.Wrap("/dev/null", cfg, events.NoopLogger)
 
-	srv := New(protocol.LocalDeviceID, w, "", "syncthing", nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, false).(*service)
+	srv := New(protocol.LocalDeviceID, w, "", "syncthing", nil, nil, nil, events.NoopLogger, nil, nil, nil, nil, nil, nil, nil, nil, false).(*service)
 	defer os.Remove(token)
 	srv.started = make(chan string)
 
@@ -134,6 +140,8 @@ func TestStopAfterBrokenConfig(t *testing.T) {
 }
 
 func TestAssetsDir(t *testing.T) {
+	t.Parallel()
+
 	// For any given request to $FILE, we should return the first found of
 	//  - assetsdir/$THEME/$FILE
 	//  - compiled in asset $THEME/$FILE
@@ -208,6 +216,8 @@ func expectURLToContain(t *testing.T, url, exp string) {
 }
 
 func TestDirNames(t *testing.T) {
+	t.Parallel()
+
 	names := dirNames("testdata")
 	expected := []string{"config", "default", "foo", "testfolder"}
 	if diff, equal := messagediff.PrettyDiff(expected, names); !equal {
@@ -224,6 +234,8 @@ type httpTestCase struct {
 }
 
 func TestAPIServiceRequests(t *testing.T) {
+	t.Parallel()
+
 	const testAPIKey = "foobarbaz"
 	cfg := new(mockedConfig)
 	cfg.gui.APIKey = testAPIKey
@@ -434,6 +446,8 @@ func testHTTPRequest(t *testing.T, baseURL string, tc httpTestCase, apikey strin
 }
 
 func TestHTTPLogin(t *testing.T) {
+	t.Parallel()
+
 	cfg := new(mockedConfig)
 	cfg.gui.User = "üser"
 	cfg.gui.Password = "$2a$10$IdIZTxTg/dCNuNEGlmLynOjqg4B1FvDKuIV5e0BB3pnWVHNb8.GSq" // bcrypt of "räksmörgås" in UTF-8
@@ -512,8 +526,8 @@ func startHTTP(cfg *mockedConfig) (string, error) {
 
 	// Instantiate the API service
 	urService := ur.New(cfg, m, connections, false)
-	summaryService := model.NewFolderSummaryService(cfg, m, protocol.LocalDeviceID)
-	svc := New(protocol.LocalDeviceID, cfg, assetDir, "syncthing", m, eventSub, diskEventSub, discoverer, connections, urService, summaryService, errorLog, systemLog, cpu, nil, false).(*service)
+	summaryService := model.NewFolderSummaryService(cfg, m, protocol.LocalDeviceID, events.NoopLogger)
+	svc := New(protocol.LocalDeviceID, cfg, assetDir, "syncthing", m, eventSub, diskEventSub, events.NoopLogger, discoverer, connections, urService, summaryService, errorLog, systemLog, cpu, nil, false).(*service)
 	defer os.Remove(token)
 	svc.started = addrChan
 
@@ -541,6 +555,8 @@ func startHTTP(cfg *mockedConfig) (string, error) {
 }
 
 func TestCSRFRequired(t *testing.T) {
+	t.Parallel()
+
 	const testAPIKey = "foobarbaz"
 	cfg := new(mockedConfig)
 	cfg.gui.APIKey = testAPIKey
@@ -550,7 +566,7 @@ func TestCSRFRequired(t *testing.T) {
 	}
 
 	cli := &http.Client{
-		Timeout: time.Second,
+		Timeout: time.Minute,
 	}
 
 	// Getting the base URL (i.e. "/") should succeed.
@@ -614,6 +630,8 @@ func TestCSRFRequired(t *testing.T) {
 }
 
 func TestRandomString(t *testing.T) {
+	t.Parallel()
+
 	const testAPIKey = "foobarbaz"
 	cfg := new(mockedConfig)
 	cfg.gui.APIKey = testAPIKey
@@ -663,6 +681,8 @@ func TestRandomString(t *testing.T) {
 }
 
 func TestConfigPostOK(t *testing.T) {
+	t.Parallel()
+
 	cfg := bytes.NewBuffer([]byte(`{
 		"version": 15,
 		"folders": [
@@ -684,6 +704,8 @@ func TestConfigPostOK(t *testing.T) {
 }
 
 func TestConfigPostDupFolder(t *testing.T) {
+	t.Parallel()
+
 	cfg := bytes.NewBuffer([]byte(`{
 		"version": 15,
 		"folders": [
@@ -719,6 +741,8 @@ func testConfigPost(data io.Reader) (*http.Response, error) {
 }
 
 func TestHostCheck(t *testing.T) {
+	t.Parallel()
+
 	// An API service bound to localhost should reject non-localhost host Headers
 
 	cfg := new(mockedConfig)
@@ -826,6 +850,11 @@ func TestHostCheck(t *testing.T) {
 
 	// This should all work over IPv6 as well
 
+	if runningInContainer() {
+		// Working IPv6 in Docker can't be taken for granted.
+		return
+	}
+
 	cfg = new(mockedConfig)
 	cfg.gui.RawAddress = "[::1]:0"
 	baseURL, err = startHTTP(cfg)
@@ -872,6 +901,8 @@ func TestHostCheck(t *testing.T) {
 }
 
 func TestAddressIsLocalhost(t *testing.T) {
+	t.Parallel()
+
 	testcases := []struct {
 		address string
 		result  bool
@@ -915,6 +946,8 @@ func TestAddressIsLocalhost(t *testing.T) {
 }
 
 func TestAccessControlAllowOriginHeader(t *testing.T) {
+	t.Parallel()
+
 	const testAPIKey = "foobarbaz"
 	cfg := new(mockedConfig)
 	cfg.gui.APIKey = testAPIKey
@@ -943,6 +976,8 @@ func TestAccessControlAllowOriginHeader(t *testing.T) {
 }
 
 func TestOptionsRequest(t *testing.T) {
+	t.Parallel()
+
 	const testAPIKey = "foobarbaz"
 	cfg := new(mockedConfig)
 	cfg.gui.APIKey = testAPIKey
@@ -976,10 +1011,12 @@ func TestOptionsRequest(t *testing.T) {
 }
 
 func TestEventMasks(t *testing.T) {
+	t.Parallel()
+
 	cfg := new(mockedConfig)
 	defSub := new(mockedEventSub)
 	diskSub := new(mockedEventSub)
-	svc := New(protocol.LocalDeviceID, cfg, "", "syncthing", nil, defSub, diskSub, nil, nil, nil, nil, nil, nil, nil, nil, false).(*service)
+	svc := New(protocol.LocalDeviceID, cfg, "", "syncthing", nil, defSub, diskSub, events.NoopLogger, nil, nil, nil, nil, nil, nil, nil, nil, false).(*service)
 	defer os.Remove(token)
 
 	if mask := svc.getEventMask(""); mask != DefaultEventMask {
@@ -1008,6 +1045,8 @@ func TestEventMasks(t *testing.T) {
 }
 
 func TestBrowse(t *testing.T) {
+	t.Parallel()
+
 	pathSep := string(os.PathSeparator)
 
 	tmpDir, err := ioutil.TempDir("", "syncthing")
@@ -1060,6 +1099,8 @@ func TestBrowse(t *testing.T) {
 }
 
 func TestPrefixMatch(t *testing.T) {
+	t.Parallel()
+
 	cases := []struct {
 		s        string
 		prefix   string
@@ -1079,6 +1120,44 @@ func TestPrefixMatch(t *testing.T) {
 	}
 }
 
+func TestCheckExpiry(t *testing.T) {
+	dir, err := ioutil.TempDir("", "syncthing-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	// Self signed certificates expiring in less than a month are errored so we
+	// can regenerate in time.
+	crt, err := tlsutil.NewCertificate(filepath.Join(dir, "crt"), filepath.Join(dir, "key"), "foo.example.com", 29)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := checkExpiry(crt); err == nil {
+		t.Error("expected expiry error")
+	}
+
+	// Certificates with at least 31 days of life left are fine.
+	crt, err = tlsutil.NewCertificate(filepath.Join(dir, "crt"), filepath.Join(dir, "key"), "foo.example.com", 31)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := checkExpiry(crt); err != nil {
+		t.Error("expected no error:", err)
+	}
+
+	if runtime.GOOS == "darwin" {
+		// Certificates with too long an expiry time are not allowed on macOS
+		crt, err = tlsutil.NewCertificate(filepath.Join(dir, "crt"), filepath.Join(dir, "key"), "foo.example.com", 1000)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if err := checkExpiry(crt); err == nil {
+			t.Error("expected expiry error")
+		}
+	}
+}
+
 func equalStrings(a, b []string) bool {
 	if len(a) != len(b) {
 		return false
@@ -1090,3 +1169,24 @@ func equalStrings(a, b []string) bool {
 	}
 	return true
 }
+
+// runningInContainer returns true if we are inside Docker or LXC. It might
+// be prone to false negatives if things change in the future, but likely
+// not false positives.
+func runningInContainer() bool {
+	if runtime.GOOS != "linux" {
+		return false
+	}
+
+	bs, err := ioutil.ReadFile("/proc/1/cgroup")
+	if err != nil {
+		return false
+	}
+	if bytes.Contains(bs, []byte("/docker/")) {
+		return true
+	}
+	if bytes.Contains(bs, []byte("/lxc/")) {
+		return true
+	}
+	return false
+}

lib/api/debug.go

@@ -7,9 +7,6 @@
 package api
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
@@ -24,5 +21,7 @@ func shouldDebugHTTP() bool {
 func init() {
 	// The debug facility was originally named "http", changed in:
 	// https://github.com/syncthing/syncthing/pull/5548
-	l.SetDebug("api", strings.Contains(os.Getenv("STTRACE"), "api") || strings.Contains(os.Getenv("STTRACE"), "http") || os.Getenv("STTRACE") == "all")
+	if l.IsTraced("http") {
+		l.SetDebug("api", true)
+	}
 }

lib/beacon/beacon.go

@@ -7,9 +7,13 @@
 package beacon
 
 import (
+	"fmt"
 	"net"
+	"time"
 
 	"github.com/thejerf/suture"
+
+	"github.com/syncthing/syncthing/lib/util"
 )
 
 type recv struct {
@@ -19,7 +23,93 @@ type recv struct {
 
 type Interface interface {
 	suture.Service
+	fmt.Stringer
 	Send(data []byte)
 	Recv() ([]byte, net.Addr)
 	Error() error
 }
+
+type cast struct {
+	*suture.Supervisor
+	name    string
+	reader  util.ServiceWithError
+	writer  util.ServiceWithError
+	outbox  chan recv
+	inbox   chan []byte
+	stopped chan struct{}
+}
+
+// newCast creates a base object for multi- or broadcasting. Afterwards the
+// caller needs to set reader and writer with the addReader and addWriter
+// methods to get a functional implementation of Interface.
+func newCast(name string) *cast {
+	return &cast{
+		Supervisor: suture.New(name, suture.Spec{
+			// Don't retry too frenetically: an error to open a socket or
+			// whatever is usually something that is either permanent or takes
+			// a while to get solved...
+			FailureThreshold: 2,
+			FailureBackoff:   60 * time.Second,
+			// Only log restarts in debug mode.
+			Log: func(line string) {
+				l.Debugln(line)
+			},
+			PassThroughPanics: true,
+		}),
+		name:    name,
+		inbox:   make(chan []byte),
+		outbox:  make(chan recv, 16),
+		stopped: make(chan struct{}),
+	}
+}
+
+func (c *cast) addReader(svc func(chan struct{}) error) {
+	c.reader = c.createService(svc, "reader")
+	c.Add(c.reader)
+}
+
+func (c *cast) addWriter(svc func(stop chan struct{}) error) {
+	c.writer = c.createService(svc, "writer")
+	c.Add(c.writer)
+}
+
+func (c *cast) createService(svc func(chan struct{}) error, suffix string) util.ServiceWithError {
+	return util.AsServiceWithError(func(stop chan struct{}) error {
+		l.Debugln("Starting", c.name, suffix)
+		err := svc(stop)
+		l.Debugf("Stopped %v %v: %v", c.name, suffix, err)
+		return err
+	})
+}
+
+func (c *cast) Stop() {
+	c.Supervisor.Stop()
+	close(c.stopped)
+}
+
+func (c *cast) String() string {
+	return fmt.Sprintf("%s@%p", c.name, c)
+}
+
+func (c *cast) Send(data []byte) {
+	select {
+	case c.inbox <- data:
+	case <-c.stopped:
+	}
+}
+
+func (c *cast) Recv() ([]byte, net.Addr) {
+	select {
+	case recv := <-c.outbox:
+		return recv.data, recv.src
+	case <-c.stopped:
+	}
+	return nil, nil
+}
+
+func (c *cast) Error() error {
+	if err := c.reader.Error(); err != nil {
+		return err
+	}
+	return c.writer.Error()
+}

lib/beacon/broadcast.go

@@ -7,85 +7,22 @@
 package beacon
 
 import (
-	"fmt"
 	"net"
 	"time"
-
-	"github.com/thejerf/suture"
-
-	"github.com/syncthing/syncthing/lib/util"
 )
 
-type Broadcast struct {
-	*suture.Supervisor
-	port   int
-	inbox  chan []byte
-	outbox chan recv
-	br     *broadcastReader
-	bw     *broadcastWriter
+func NewBroadcast(port int) Interface {
+	c := newCast("broadcastBeacon")
+	c.addReader(func(stop chan struct{}) error {
+		return readBroadcasts(c.outbox, port, stop)
+	})
+	c.addWriter(func(stop chan struct{}) error {
+		return writeBroadcasts(c.inbox, port, stop)
+	})
+	return c
 }
 
-func NewBroadcast(port int) *Broadcast {
-	b := &Broadcast{
-		Supervisor: suture.New("broadcastBeacon", suture.Spec{
-			// Don't retry too frenetically: an error to open a socket or
-			// whatever is usually something that is either permanent or takes
-			// a while to get solved...
-			FailureThreshold: 2,
-			FailureBackoff:   60 * time.Second,
-			// Only log restarts in debug mode.
-			Log: func(line string) {
-				l.Debugln(line)
-			},
-			PassThroughPanics: true,
-		}),
-		port:   port,
-		inbox:  make(chan []byte),
-		outbox: make(chan recv, 16),
-	}
-
-	b.br = &broadcastReader{
-		port:   port,
-		outbox: b.outbox,
-	}
-	b.br.ServiceWithError = util.AsServiceWithError(b.br.serve)
-	b.Add(b.br)
-	b.bw = &broadcastWriter{
-		port:  port,
-		inbox: b.inbox,
-	}
-	b.bw.ServiceWithError = util.AsServiceWithError(b.bw.serve)
-	b.Add(b.bw)
-
-	return b
-}
-
-func (b *Broadcast) Send(data []byte) {
-	b.inbox <- data
-}
-
-func (b *Broadcast) Recv() ([]byte, net.Addr) {
-	recv := <-b.outbox
-	return recv.data, recv.src
-}
-
-func (b *Broadcast) Error() error {
-	if err := b.br.Error(); err != nil {
-		return err
-	}
-	return b.bw.Error()
-}
-
-type broadcastWriter struct {
-	util.ServiceWithError
-	port  int
-	inbox chan []byte
-}
-
-func (w *broadcastWriter) serve(stop chan struct{}) error {
-	l.Debugln(w, "starting")
-	defer l.Debugln(w, "stopping")
-
+func writeBroadcasts(inbox <-chan []byte, port int, stop chan struct{}) error {
 	conn, err := net.ListenUDP("udp4", nil)
 	if err != nil {
 		l.Debugln(err)
@@ -104,7 +41,7 @@ func (w *broadcastWriter) serve(stop chan struct{}) error {
 	for {
 		var bs []byte
 		select {
-		case bs = <-w.inbox:
+		case bs = <-inbox:
 		case <-stop:
 			return nil
 		}
@@ -112,8 +49,7 @@ func (w *broadcastWriter) serve(stop chan struct{}) error {
 		addrs, err := net.InterfaceAddrs()
 		if err != nil {
 			l.Debugln(err)
-			w.SetError(err)
-			continue
+			return err
 		}
 
 		var dsts []net.IP
@@ -133,13 +69,13 @@ func (w *broadcastWriter) serve(stop chan struct{}) error {
 
 		success := 0
 		for _, ip := range dsts {
-			dst := &net.UDPAddr{IP: ip, Port: w.port}
+			dst := &net.UDPAddr{IP: ip, Port: port}
 
 			conn.SetWriteDeadline(time.Now().Add(time.Second))
-			_, err := conn.WriteTo(bs, dst)
+			_, err = conn.WriteTo(bs, dst)
 			conn.SetWriteDeadline(time.Time{})
 
-			if err, ok := err.(net.Error); ok && err.Timeout() {
+			if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
 				// Write timeouts should not happen. We treat it as a fatal
 				// error on the socket.
 				l.Debugln(err)
@@ -149,7 +85,6 @@ func (w *broadcastWriter) serve(stop chan struct{}) error {
 			if err != nil {
 				// Some other error that we don't expect. Debug and continue.
 				l.Debugln(err)
-				w.SetError(err)
 				continue
 			}
 
@@ -157,27 +92,15 @@ func (w *broadcastWriter) serve(stop chan struct{}) error {
 			success++
 		}
 
-		if success > 0 {
-			w.SetError(nil)
+		if success == 0 {
+			l.Debugln("couldn't send any braodcasts")
+			return err
 		}
 	}
 }
 
-func (w *broadcastWriter) String() string {
-	return fmt.Sprintf("broadcastWriter@%p", w)
-}
-
-type broadcastReader struct {
-	util.ServiceWithError
-	port   int
-	outbox chan recv
-}
-
-func (r *broadcastReader) serve(stop chan struct{}) error {
-	l.Debugln(r, "starting")
-	defer l.Debugln(r, "stopping")
-
-	conn, err := net.ListenUDP("udp4", &net.UDPAddr{Port: r.port})
+func readBroadcasts(outbox chan<- recv, port int, stop chan struct{}) error {
+	conn, err := net.ListenUDP("udp4", &net.UDPAddr{Port: port})
 	if err != nil {
 		l.Debugln(err)
 		return err
@@ -200,14 +123,12 @@ func (r *broadcastReader) serve(stop chan struct{}) error {
 			return err
 		}
 
-		r.SetError(nil)
-
 		l.Debugf("recv %d bytes from %s", n, addr)
 
 		c := make([]byte, n)
 		copy(c, bs)
 		select {
-		case r.outbox <- recv{c, addr}:
+		case outbox <- recv{c, addr}:
 		case <-stop:
 			return nil
 		default:
@@ -216,10 +137,6 @@ func (r *broadcastReader) serve(stop chan struct{}) error {
 	}
 }
 
-func (r *broadcastReader) String() string {
-	return fmt.Sprintf("broadcastReader@%p", r)
-}
-
 func bcast(ip *net.IPNet) *net.IPNet {
 	var bc = &net.IPNet{}
 	bc.IP = make([]byte, len(ip.IP))

lib/beacon/debug.go

@@ -7,16 +7,9 @@
 package beacon
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
 	l = logger.DefaultLogger.NewFacility("beacon", "Multicast and broadcast discovery")
 )
-
-func init() {
-	l.SetDebug("beacon", strings.Contains(os.Getenv("STTRACE"), "beacon") || os.Getenv("STTRACE") == "all")
-}

lib/beacon/multicast.go

@@ -8,86 +8,25 @@ package beacon
 
 import (
 	"errors"
-	"fmt"
 	"net"
 	"time"
 
-	"github.com/thejerf/suture"
 	"golang.org/x/net/ipv6"
-
-	"github.com/syncthing/syncthing/lib/util"
 )
 
-type Multicast struct {
-	*suture.Supervisor
-	inbox  chan []byte
-	outbox chan recv
-	mr     *multicastReader
-	mw     *multicastWriter
+func NewMulticast(addr string) Interface {
+	c := newCast("multicastBeacon")
+	c.addReader(func(stop chan struct{}) error {
+		return readMulticasts(c.outbox, addr, stop)
+	})
+	c.addWriter(func(stop chan struct{}) error {
+		return writeMulticasts(c.inbox, addr, stop)
+	})
+	return c
 }
 
-func NewMulticast(addr string) *Multicast {
-	m := &Multicast{
-		Supervisor: suture.New("multicastBeacon", suture.Spec{
-			// Don't retry too frenetically: an error to open a socket or
-			// whatever is usually something that is either permanent or takes
-			// a while to get solved...
-			FailureThreshold: 2,
-			FailureBackoff:   60 * time.Second,
-			// Only log restarts in debug mode.
-			Log: func(line string) {
-				l.Debugln(line)
-			},
-			PassThroughPanics: true,
-		}),
-		inbox:  make(chan []byte),
-		outbox: make(chan recv, 16),
-	}
-
-	m.mr = &multicastReader{
-		addr:   addr,
-		outbox: m.outbox,
-	}
-	m.mr.ServiceWithError = util.AsServiceWithError(m.mr.serve)
-	m.Add(m.mr)
-
-	m.mw = &multicastWriter{
-		addr:  addr,
-		inbox: m.inbox,
-	}
-	m.mw.ServiceWithError = util.AsServiceWithError(m.mw.serve)
-	m.Add(m.mw)
-
-	return m
-}
-
-func (m *Multicast) Send(data []byte) {
-	m.inbox <- data
-}
-
-func (m *Multicast) Recv() ([]byte, net.Addr) {
-	recv := <-m.outbox
-	return recv.data, recv.src
-}
-
-func (m *Multicast) Error() error {
-	if err := m.mr.Error(); err != nil {
-		return err
-	}
-	return m.mw.Error()
-}
-
-type multicastWriter struct {
-	util.ServiceWithError
-	addr  string
-	inbox <-chan []byte
-}
-
-func (w *multicastWriter) serve(stop chan struct{}) error {
-	l.Debugln(w, "starting")
-	defer l.Debugln(w, "stopping")
-
-	gaddr, err := net.ResolveUDPAddr("udp6", w.addr)
+func writeMulticasts(inbox <-chan []byte, addr string, stop chan struct{}) error {
+	gaddr, err := net.ResolveUDPAddr("udp6", addr)
 	if err != nil {
 		l.Debugln(err)
 		return err
@@ -117,7 +56,7 @@ func (w *multicastWriter) serve(stop chan struct{}) error {
 	for {
 		var bs []byte
 		select {
-		case bs = <-w.inbox:
+		case bs = <-inbox:
 		case <-stop:
 			return nil
 		}
@@ -137,7 +76,6 @@ func (w *multicastWriter) serve(stop chan struct{}) error {
 
 			if err != nil {
 				l.Debugln(err, "on write to", gaddr, intf.Name)
-				w.SetError(err)
 				continue
 			}
 
@@ -152,33 +90,20 @@ func (w *multicastWriter) serve(stop chan struct{}) error {
 			}
 		}
 
-		if success > 0 {
-			w.SetError(nil)
+		if success == 0 {
+			return err
 		}
 	}
 }
 
-func (w *multicastWriter) String() string {
-	return fmt.Sprintf("multicastWriter@%p", w)
-}
-
-type multicastReader struct {
-	util.ServiceWithError
-	addr   string
-	outbox chan<- recv
-}
-
-func (r *multicastReader) serve(stop chan struct{}) error {
-	l.Debugln(r, "starting")
-	defer l.Debugln(r, "stopping")
-
-	gaddr, err := net.ResolveUDPAddr("udp6", r.addr)
+func readMulticasts(outbox chan<- recv, addr string, stop chan struct{}) error {
+	gaddr, err := net.ResolveUDPAddr("udp6", addr)
 	if err != nil {
 		l.Debugln(err)
 		return err
 	}
 
-	conn, err := net.ListenPacket("udp6", r.addr)
+	conn, err := net.ListenPacket("udp6", addr)
 	if err != nil {
 		l.Debugln(err)
 		return err
@@ -226,21 +151,16 @@ func (r *multicastReader) serve(stop chan struct{}) error {
 		n, _, addr, err := pconn.ReadFrom(bs)
 		if err != nil {
 			l.Debugln(err)
-			r.SetError(err)
-			continue
+			return err
 		}
 		l.Debugf("recv %d bytes from %s", n, addr)
 
 		c := make([]byte, n)
 		copy(c, bs)
 		select {
-		case r.outbox <- recv{c, addr}:
+		case outbox <- recv{c, addr}:
 		default:
 			l.Debugln("dropping message")
 		}
 	}
 }
-
-func (r *multicastReader) String() string {
-	return fmt.Sprintf("multicastReader@%p", r)
-}

lib/build/build.go

@@ -18,10 +18,11 @@ import (
 
 var (
 	// Injected by build script
+	Program = "syncthing"
 	Version = "unknown-dev"
-	Host    = "unknown" // Set by build script
-	User    = "unknown" // Set by build script
-	Stamp   = "0"       // Set by build script
+	Host    = "unknown"
+	User    = "unknown"
+	Stamp   = "0"
 
 	// Static
 	Codename = "Fermium Flea"
@@ -73,7 +74,7 @@ func setBuildData() {
 	Date = time.Unix(int64(stamp), 0)
 
 	date := Date.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`syncthing %s "%s" (%s %s-%s) %s@%s %s`, Version, Codename, runtime.Version(), runtime.GOOS, runtime.GOARCH, User, Host, date)
+	LongVersion = fmt.Sprintf(`%s %s "%s" (%s %s-%s) %s@%s %s`, Program, Version, Codename, runtime.Version(), runtime.GOOS, runtime.GOARCH, User, Host, date)
 
 	if len(Tags) > 0 {
 		LongVersion = fmt.Sprintf("%s [%s]", LongVersion, strings.Join(Tags, ", "))

lib/config/commit_test.go

@@ -44,7 +44,7 @@ func (validationError) String() string {
 func TestReplaceCommit(t *testing.T) {
 	t.Skip("broken, fails randomly, #3834")
 
-	w := Wrap("/dev/null", Configuration{Version: 0})
+	w := wrap("/dev/null", Configuration{Version: 0})
 	if w.RawCopy().Version != 0 {
 		t.Fatal("Config incorrect")
 	}

lib/config/config.go

@@ -87,7 +87,6 @@ var (
 		"stun.voiparound.com:3478",
 		"stun.voipbuster.com:3478",
 		"stun.voipstunt.com:3478",
-		"stun.voxgratia.org:3478",
 		"stun.xten.com:3478",
 	}
 )

lib/config/config_test.go

@@ -20,6 +20,7 @@ import (
 	"testing"
 
 	"github.com/d4l3k/messagediff"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
@@ -86,7 +87,7 @@ func TestDefaultValues(t *testing.T) {
 func TestDeviceConfig(t *testing.T) {
 	for i := OldestHandledVersion; i <= CurrentVersion; i++ {
 		os.RemoveAll(filepath.Join("testdata", DefaultMarkerName))
-		wr, err := Load(fmt.Sprintf("testdata/v%d.xml", i), device1)
+		wr, err := load(fmt.Sprintf("testdata/v%d.xml", i), device1)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -168,7 +169,7 @@ func TestDeviceConfig(t *testing.T) {
 }
 
 func TestNoListenAddresses(t *testing.T) {
-	cfg, err := Load("testdata/nolistenaddress.xml", device1)
+	cfg, err := load("testdata/nolistenaddress.xml", device1)
 	if err != nil {
 		t.Error(err)
 	}
@@ -225,7 +226,7 @@ func TestOverriddenValues(t *testing.T) {
 	}
 
 	os.Unsetenv("STNOUPGRADE")
-	cfg, err := Load("testdata/overridenvalues.xml", device1)
+	cfg, err := load("testdata/overridenvalues.xml", device1)
 	if err != nil {
 		t.Error(err)
 	}
@@ -270,7 +271,7 @@ func TestDeviceAddressesDynamic(t *testing.T) {
 		},
 	}
 
-	cfg, err := Load("testdata/deviceaddressesdynamic.xml", device4)
+	cfg, err := load("testdata/deviceaddressesdynamic.xml", device4)
 	if err != nil {
 		t.Error(err)
 	}
@@ -319,7 +320,7 @@ func TestDeviceCompression(t *testing.T) {
 		},
 	}
 
-	cfg, err := Load("testdata/devicecompression.xml", device4)
+	cfg, err := load("testdata/devicecompression.xml", device4)
 	if err != nil {
 		t.Error(err)
 	}
@@ -365,7 +366,7 @@ func TestDeviceAddressesStatic(t *testing.T) {
 		},
 	}
 
-	cfg, err := Load("testdata/deviceaddressesstatic.xml", device4)
+	cfg, err := load("testdata/deviceaddressesstatic.xml", device4)
 	if err != nil {
 		t.Error(err)
 	}
@@ -377,7 +378,7 @@ func TestDeviceAddressesStatic(t *testing.T) {
 }
 
 func TestVersioningConfig(t *testing.T) {
-	cfg, err := Load("testdata/versioningconfig.xml", device4)
+	cfg, err := load("testdata/versioningconfig.xml", device4)
 	if err != nil {
 		t.Error(err)
 	}
@@ -404,7 +405,7 @@ func TestIssue1262(t *testing.T) {
 		t.Skipf("path gets converted to absolute as part of the filesystem initialization on linux")
 	}
 
-	cfg, err := Load("testdata/issue-1262.xml", device4)
+	cfg, err := load("testdata/issue-1262.xml", device4)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -418,7 +419,7 @@ func TestIssue1262(t *testing.T) {
 }
 
 func TestIssue1750(t *testing.T) {
-	cfg, err := Load("testdata/issue-1750.xml", device4)
+	cfg, err := load("testdata/issue-1750.xml", device4)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -520,7 +521,7 @@ func TestNewSaveLoad(t *testing.T) {
 	}
 
 	intCfg := New(device1)
-	cfg := Wrap(path, intCfg)
+	cfg := wrap(path, intCfg)
 
 	// To make the equality pass later
 	cfg.(*wrapper).cfg.XMLName.Local = "configuration"
@@ -537,7 +538,7 @@ func TestNewSaveLoad(t *testing.T) {
 		t.Error(path, "does not exist")
 	}
 
-	cfg2, err := Load(path, device1)
+	cfg2, err := load(path, device1)
 	if err != nil {
 		t.Error(err)
 	}
@@ -564,7 +565,7 @@ func TestPrepare(t *testing.T) {
 }
 
 func TestCopy(t *testing.T) {
-	wrapper, err := Load("testdata/example.xml", device1)
+	wrapper, err := load("testdata/example.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -603,7 +604,7 @@ func TestCopy(t *testing.T) {
 }
 
 func TestPullOrder(t *testing.T) {
-	wrapper, err := Load("testdata/pullorder.xml", device1)
+	wrapper, err := load("testdata/pullorder.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -643,7 +644,7 @@ func TestPullOrder(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	wrapper = Wrap("testdata/pullorder.xml", cfg)
+	wrapper = wrap("testdata/pullorder.xml", cfg)
 	folders = wrapper.Folders()
 
 	for _, tc := range expected {
@@ -654,7 +655,7 @@ func TestPullOrder(t *testing.T) {
 }
 
 func TestLargeRescanInterval(t *testing.T) {
-	wrapper, err := Load("testdata/largeinterval.xml", device1)
+	wrapper, err := load("testdata/largeinterval.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -692,7 +693,7 @@ func TestGUIConfigURL(t *testing.T) {
 func TestDuplicateDevices(t *testing.T) {
 	// Duplicate devices should be removed
 
-	wrapper, err := Load("testdata/dupdevices.xml", device1)
+	wrapper, err := load("testdata/dupdevices.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -710,7 +711,7 @@ func TestDuplicateDevices(t *testing.T) {
 func TestDuplicateFolders(t *testing.T) {
 	// Duplicate folders are a loading error
 
-	_, err := Load("testdata/dupfolders.xml", device1)
+	_, err := load("testdata/dupfolders.xml", device1)
 	if err == nil || !strings.Contains(err.Error(), errFolderIDDuplicate.Error()) {
 		t.Fatal(`Expected error to mention "duplicate folder ID":`, err)
 	}
@@ -721,7 +722,7 @@ func TestEmptyFolderPaths(t *testing.T) {
 	// get messed up by the prepare steps (e.g., become the current dir or
 	// get a slash added so that it becomes the root directory or similar).
 
-	_, err := Load("testdata/nopath.xml", device1)
+	_, err := load("testdata/nopath.xml", device1)
 	if err == nil || !strings.Contains(err.Error(), errFolderPathEmpty.Error()) {
 		t.Fatal("Expected error due to empty folder path, got", err)
 	}
@@ -790,7 +791,7 @@ func TestIgnoredDevices(t *testing.T) {
 	// Verify that ignored devices that are also present in the
 	// configuration are not in fact ignored.
 
-	wrapper, err := Load("testdata/ignoreddevices.xml", device1)
+	wrapper, err := load("testdata/ignoreddevices.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -808,7 +809,7 @@ func TestIgnoredFolders(t *testing.T) {
 	// configuration are not in fact ignored.
 	// Also, verify that folders that are shared with a device are not ignored.
 
-	wrapper, err := Load("testdata/ignoredfolders.xml", device1)
+	wrapper, err := load("testdata/ignoredfolders.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -844,7 +845,7 @@ func TestIgnoredFolders(t *testing.T) {
 func TestGetDevice(t *testing.T) {
 	// Verify that the Device() call does the right thing
 
-	wrapper, err := Load("testdata/ignoreddevices.xml", device1)
+	wrapper, err := load("testdata/ignoreddevices.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -871,7 +872,7 @@ func TestGetDevice(t *testing.T) {
 }
 
 func TestSharesRemovedOnDeviceRemoval(t *testing.T) {
-	wrapper, err := Load("testdata/example.xml", device1)
+	wrapper, err := load("testdata/example.xml", device1)
 	if err != nil {
 		t.Errorf("Failed: %s", err)
 	}
@@ -956,7 +957,7 @@ func TestIssue4219(t *testing.T) {
 		t.Errorf("There should be three ignored folders, not %d", ignoredFolders)
 	}
 
-	w := Wrap("/tmp/cfg", cfg)
+	w := wrap("/tmp/cfg", cfg)
 	if !w.IgnoredFolder(device2, "t1") {
 		t.Error("Folder device2 t1 should be ignored")
 	}
@@ -1145,3 +1146,11 @@ func defaultConfigAsMap() map[string]interface{} {
 	}
 	return tmp
 }
+
+func load(path string, myID protocol.DeviceID) (Wrapper, error) {
+	return Load(path, myID, events.NoopLogger)
+}
+
+func wrap(path string, cfg Configuration) Wrapper {
+	return Wrap(path, cfg, events.NoopLogger)
+}

lib/config/debug.go

@@ -7,16 +7,9 @@
 package config
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
 	l = logger.DefaultLogger.NewFacility("config", "Configuration loading and saving")
 )
-
-func init() {
-	l.SetDebug("config", strings.Contains(os.Getenv("STTRACE"), "config") || os.Getenv("STTRACE") == "all")
-}

lib/config/optionsconfiguration.go

@@ -57,6 +57,7 @@ type OptionsConfiguration struct {
 	StunKeepaliveStartS     int      `xml:"stunKeepaliveStartS" json:"stunKeepaliveStartS" default:"180"` // 0 for off
 	StunKeepaliveMinS       int      `xml:"stunKeepaliveMinS" json:"stunKeepaliveMinS" default:"20"`      // 0 for off
 	StunServers             []string `xml:"stunServer" json:"stunServers" default:"default"`
+	DatabaseTuning          Tuning   `xml:"databaseTuning" json:"databaseTuning" restart:"true"`
 
 	DeprecatedUPnPEnabled        bool     `xml:"upnpEnabled,omitempty" json:"-"`
 	DeprecatedUPnPLeaseM         int      `xml:"upnpLeaseMinutes,omitempty" json:"-"`

lib/config/tuning.go

@@ -0,0 +1,47 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package config
+
+type Tuning int
+
+const (
+	// N.b. these constants must match those in lib/db.Tuning!
+	TuningAuto Tuning = iota // default is auto
+	TuningSmall
+	TuningLarge
+)
+
+func (t Tuning) String() string {
+	switch t {
+	case TuningAuto:
+		return "auto"
+	case TuningSmall:
+		return "small"
+	case TuningLarge:
+		return "large"
+	default:
+		return "unknown"
+	}
+}
+
+func (t Tuning) MarshalText() ([]byte, error) {
+	return []byte(t.String()), nil
+}
+
+func (t *Tuning) UnmarshalText(bs []byte) error {
+	switch string(bs) {
+	case "auto":
+		*t = TuningAuto
+	case "small":
+		*t = TuningSmall
+	case "large":
+		*t = TuningLarge
+	default:
+		*t = TuningAuto
+	}
+	return nil
+}

lib/config/tuning_test.go

@@ -0,0 +1,26 @@
+// Copyright (C) 2019 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package config_test
+
+import (
+	"testing"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/db"
+)
+
+func TestTuningMatches(t *testing.T) {
+	if int(config.TuningAuto) != int(db.TuningAuto) {
+		t.Error("mismatch for TuningAuto")
+	}
+	if int(config.TuningSmall) != int(db.TuningSmall) {
+		t.Error("mismatch for TuningSmall")
+	}
+	if int(config.TuningLarge) != int(db.TuningLarge) {
+		t.Error("mismatch for TuningLarge")
+	}
+}

lib/config/wrapper.go

@@ -96,8 +96,9 @@ type Wrapper interface {
 }
 
 type wrapper struct {
-	cfg  Configuration
-	path string
+	cfg      Configuration
+	path     string
+	evLogger events.Logger
 
 	deviceMap map[protocol.DeviceID]DeviceConfiguration
 	folderMap map[string]FolderConfiguration
@@ -133,18 +134,19 @@ func (w *wrapper) StunServers() []string {
 
 // Wrap wraps an existing Configuration structure and ties it to a file on
 // disk.
-func Wrap(path string, cfg Configuration) Wrapper {
+func Wrap(path string, cfg Configuration, evLogger events.Logger) Wrapper {
 	w := &wrapper{
-		cfg:  cfg,
-		path: path,
-		mut:  sync.NewMutex(),
+		cfg:      cfg,
+		path:     path,
+		evLogger: evLogger,
+		mut:      sync.NewMutex(),
 	}
 	return w
 }
 
 // Load loads an existing file on disk and returns a new configuration
 // wrapper.
-func Load(path string, myID protocol.DeviceID) (Wrapper, error) {
+func Load(path string, myID protocol.DeviceID, evLogger events.Logger) (Wrapper, error) {
 	fd, err := os.Open(path)
 	if err != nil {
 		return nil, err
@@ -156,7 +158,7 @@ func Load(path string, myID protocol.DeviceID) (Wrapper, error) {
 		return nil, err
 	}
 
-	return Wrap(path, cfg), nil
+	return Wrap(path, cfg, evLogger), nil
 }
 
 func (w *wrapper) ConfigPath() string {
@@ -450,7 +452,7 @@ func (w *wrapper) Save() error {
 		return err
 	}
 
-	events.Default.Log(events.ConfigSaved, w.cfg)
+	w.evLogger.Log(events.ConfigSaved, w.cfg)
 	return nil
 }
 

lib/connections/debug.go

@@ -7,16 +7,9 @@
 package connections
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
 	l = logger.DefaultLogger.NewFacility("connections", "Connection handling")
 )
-
-func init() {
-	l.SetDebug("connections", strings.Contains(os.Getenv("STTRACE"), "connections") || os.Getenv("STTRACE") == "all")
-}

lib/connections/lan_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
 )
 
 func TestIsLANHost(t *testing.T) {
@@ -35,7 +36,7 @@ func TestIsLANHost(t *testing.T) {
 		Options: config.OptionsConfiguration{
 			AlwaysLocalNets: []string{"10.20.30.0/24"},
 		},
-	})
+	}, events.NoopLogger)
 	s := &service{cfg: cfg}
 
 	for _, tc := range cases {

lib/connections/limiter.go

@@ -32,9 +32,13 @@ type limiter struct {
 type waiter interface {
 	// This is the rate limiting operation
 	WaitN(ctx context.Context, n int) error
+	Limit() rate.Limit
 }
 
-const limiterBurstSize = 4 * 128 << 10
+const (
+	limiterBurstSize   = 4 * 128 << 10
+	maxSingleWriteSize = 8 << 10
+)
 
 func newLimiter(cfg config.Wrapper) *limiter {
 	l := &limiter{
@@ -186,19 +190,23 @@ func (lim *limiter) getLimiters(remoteID protocol.DeviceID, rw io.ReadWriter, is
 
 func (lim *limiter) newLimitedReaderLocked(remoteID protocol.DeviceID, r io.Reader, isLAN bool) io.Reader {
 	return &limitedReader{
-		reader:    r,
-		limitsLAN: &lim.limitsLAN,
-		waiter:    totalWaiter{lim.getReadLimiterLocked(remoteID), lim.read},
-		isLAN:     isLAN,
+		reader: r,
+		waiterHolder: waiterHolder{
+			waiter:    totalWaiter{lim.getReadLimiterLocked(remoteID), lim.read},
+			limitsLAN: &lim.limitsLAN,
+			isLAN:     isLAN,
+		},
 	}
 }
 
 func (lim *limiter) newLimitedWriterLocked(remoteID protocol.DeviceID, w io.Writer, isLAN bool) io.Writer {
 	return &limitedWriter{
-		writer:    w,
-		limitsLAN: &lim.limitsLAN,
-		waiter:    totalWaiter{lim.getWriteLimiterLocked(remoteID), lim.write},
-		isLAN:     isLAN,
+		writer: w,
+		waiterHolder: waiterHolder{
+			waiter:    totalWaiter{lim.getWriteLimiterLocked(remoteID), lim.write},
+			limitsLAN: &lim.limitsLAN,
+			isLAN:     isLAN,
+		},
 	}
 }
 
@@ -221,53 +229,87 @@ func getRateLimiter(m map[protocol.DeviceID]*rate.Limiter, deviceID protocol.Dev
 
 // limitedReader is a rate limited io.Reader
 type limitedReader struct {
-	reader    io.Reader
-	limitsLAN *atomicBool
-	waiter    waiter
-	isLAN     bool
+	reader io.Reader
+	waiterHolder
 }
 
 func (r *limitedReader) Read(buf []byte) (int, error) {
 	n, err := r.reader.Read(buf)
-	if !r.isLAN || r.limitsLAN.get() {
-		take(r.waiter, n)
+	if !r.unlimited() {
+		r.take(n)
 	}
 	return n, err
 }
 
 // limitedWriter is a rate limited io.Writer
 type limitedWriter struct {
-	writer    io.Writer
-	limitsLAN *atomicBool
-	waiter    waiter
-	isLAN     bool
+	writer io.Writer
+	waiterHolder
 }
 
 func (w *limitedWriter) Write(buf []byte) (int, error) {
-	if !w.isLAN || w.limitsLAN.get() {
-		take(w.waiter, len(buf))
+	if w.unlimited() {
+		return w.writer.Write(buf)
 	}
-	return w.writer.Write(buf)
+
+	// This does (potentially) multiple smaller writes in order to be less
+	// bursty with large writes and slow rates.
+	written := 0
+	for written < len(buf) {
+		toWrite := maxSingleWriteSize
+		if toWrite > len(buf)-written {
+			toWrite = len(buf) - written
+		}
+		w.take(toWrite)
+		n, err := w.writer.Write(buf[written : written+toWrite])
+		written += n
+		if err != nil {
+			return written, err
+		}
+	}
+
+	return written, nil
 }
 
-// take is a utility function to consume tokens from a overall rate.Limiter and deviceLimiter.
-// No call to WaitN can be larger than the limiter burst size so we split it up into
-// several calls when necessary.
-func take(waiter waiter, tokens int) {
+// waiterHolder is the common functionality around having and evaluating a
+// waiter, valid for both writers and readers
+type waiterHolder struct {
+	waiter    waiter
+	limitsLAN *atomicBool
+	isLAN     bool
+}
+
+// unlimited returns true if the waiter is not limiting the rate
+func (w waiterHolder) unlimited() bool {
+	if w.isLAN && !w.limitsLAN.get() {
+		return true
+	}
+	return w.waiter.Limit() == rate.Inf
+}
+
+// take is a utility function to consume tokens, because no call to WaitN
+// must be larger than the limiter burst size or it will hang.
+func (w waiterHolder) take(tokens int) {
+	// For writes we already split the buffer into smaller operations so those
+	// will always end up in the fast path below. For reads, however, we don't
+	// control the size of the incoming buffer and don't split the calls
+	// into the lower level reads so we might get a large amount of data and
+	// end up in the loop further down.
+
 	if tokens < limiterBurstSize {
-		// This is the by far more common case so we get it out of the way
-		// early.
-		waiter.WaitN(context.TODO(), tokens)
+		// Fast path. We won't get an error from WaitN as we don't pass a
+		// context with a deadline.
+		_ = w.waiter.WaitN(context.TODO(), tokens)
 		return
 	}
 
 	for tokens > 0 {
 		// Consume limiterBurstSize tokens at a time until we're done.
 		if tokens > limiterBurstSize {
-			waiter.WaitN(context.TODO(), limiterBurstSize)
+			_ = w.waiter.WaitN(context.TODO(), limiterBurstSize)
 			tokens -= limiterBurstSize
 		} else {
-			waiter.WaitN(context.TODO(), tokens)
+			_ = w.waiter.WaitN(context.TODO(), tokens)
 			tokens = 0
 		}
 	}
@@ -300,3 +342,13 @@ func (tw totalWaiter) WaitN(ctx context.Context, n int) error {
 	}
 	return nil
 }
+
+func (tw totalWaiter) Limit() rate.Limit {
+	min := rate.Inf
+	for _, w := range tw {
+		if l := w.Limit(); l < min {
+			min = l
+		}
+	}
+	return min
+}

lib/connections/limiter_test.go

@@ -7,11 +7,16 @@
 package connections
 
 import (
-	"github.com/syncthing/syncthing/lib/config"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"golang.org/x/time/rate"
+	"bytes"
+	crand "crypto/rand"
+	"io"
 	"math/rand"
 	"testing"
+
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"golang.org/x/time/rate"
 )
 
 var device1, device2, device3, device4 protocol.DeviceID
@@ -25,7 +30,7 @@ func init() {
 }
 
 func initConfig() config.Wrapper {
-	cfg := config.Wrap("/dev/null", config.New(device1))
+	cfg := config.Wrap("/dev/null", config.New(device1), events.NoopLogger)
 	dev1Conf = config.NewDeviceConfiguration(device1, "device1")
 	dev2Conf = config.NewDeviceConfiguration(device2, "device2")
 	dev3Conf = config.NewDeviceConfiguration(device3, "device3")
@@ -184,6 +189,151 @@ func TestAddAndRemove(t *testing.T) {
 	checkActualAndExpected(t, actualR, actualW, expectedR, expectedW)
 }
 
+func TestLimitedWriterWrite(t *testing.T) {
+	// Check that the limited writer writes the correct data in the correct manner.
+
+	// A buffer with random data that is larger than the write size and not
+	// a precise multiple either.
+	src := make([]byte, int(12.5*maxSingleWriteSize))
+	if _, err := crand.Reader.Read(src); err != nil {
+		t.Fatal(err)
+	}
+
+	// Write it to the destination using a limited writer, with a wrapper to
+	// count the write calls. The defaults on the limited writer should mean
+	// it is used (and doesn't take the fast path). In practice the limiter
+	// won't delay the test as the burst size is large enough to accommodate
+	// regardless of the rate.
+	dst := new(bytes.Buffer)
+	cw := &countingWriter{w: dst}
+	lw := &limitedWriter{
+		writer: cw,
+		waiterHolder: waiterHolder{
+			waiter:    rate.NewLimiter(rate.Limit(42), limiterBurstSize),
+			limitsLAN: new(atomicBool),
+			isLAN:     false, // enables limiting
+		},
+	}
+	if _, err := io.Copy(lw, bytes.NewReader(src)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify there were lots of writes and that the end result is identical.
+	if cw.writeCount != 13 {
+		t.Error("expected lots of smaller writes, but not too many")
+	}
+	if !bytes.Equal(src, dst.Bytes()) {
+		t.Error("results should be equal")
+	}
+
+	// Write it to the destination using a limited writer, with a wrapper to
+	// count the write calls. Now we make sure the fast path is used.
+	dst = new(bytes.Buffer)
+	cw = &countingWriter{w: dst}
+	lw = &limitedWriter{
+		writer: cw,
+		waiterHolder: waiterHolder{
+			waiter:    rate.NewLimiter(rate.Limit(42), limiterBurstSize),
+			limitsLAN: new(atomicBool),
+			isLAN:     true, // disables limiting
+		},
+	}
+	if _, err := io.Copy(lw, bytes.NewReader(src)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify there were a single write and that the end result is identical.
+	if cw.writeCount != 1 {
+		t.Error("expected just the one write")
+	}
+	if !bytes.Equal(src, dst.Bytes()) {
+		t.Error("results should be equal")
+	}
+
+	// Once more, but making sure the fast path is used for an unlimited
+	// rate, with multiple unlimited raters even (global and per-device).
+	dst = new(bytes.Buffer)
+	cw = &countingWriter{w: dst}
+	lw = &limitedWriter{
+		writer: cw,
+		waiterHolder: waiterHolder{
+			waiter:    totalWaiter{rate.NewLimiter(rate.Inf, limiterBurstSize), rate.NewLimiter(rate.Inf, limiterBurstSize)},
+			limitsLAN: new(atomicBool),
+			isLAN:     false, // enables limiting
+		},
+	}
+	if _, err := io.Copy(lw, bytes.NewReader(src)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify there were a single write and that the end result is identical.
+	if cw.writeCount != 1 {
+		t.Error("expected just the one write")
+	}
+	if !bytes.Equal(src, dst.Bytes()) {
+		t.Error("results should be equal")
+	}
+
+	// Once more, but making sure we *don't* take the fast path when there
+	// is a combo of limited and unlimited writers.
+	dst = new(bytes.Buffer)
+	cw = &countingWriter{w: dst}
+	lw = &limitedWriter{
+		writer: cw,
+		waiterHolder: waiterHolder{
+			waiter: totalWaiter{
+				rate.NewLimiter(rate.Inf, limiterBurstSize),
+				rate.NewLimiter(rate.Limit(42), limiterBurstSize),
+				rate.NewLimiter(rate.Inf, limiterBurstSize),
+			},
+			limitsLAN: new(atomicBool),
+			isLAN:     false, // enables limiting
+		},
+	}
+	if _, err := io.Copy(lw, bytes.NewReader(src)); err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify there were lots of writes and that the end result is identical.
+	if cw.writeCount != 13 {
+		t.Error("expected just the one write")
+	}
+	if !bytes.Equal(src, dst.Bytes()) {
+		t.Error("results should be equal")
+	}
+}
+
+func TestTotalWaiterLimit(t *testing.T) {
+	cases := []struct {
+		w waiter
+		r rate.Limit
+	}{
+		{
+			totalWaiter{},
+			rate.Inf,
+		},
+		{
+			totalWaiter{rate.NewLimiter(rate.Inf, 42)},
+			rate.Inf,
+		},
+		{
+			totalWaiter{rate.NewLimiter(rate.Inf, 42), rate.NewLimiter(rate.Inf, 42)},
+			rate.Inf,
+		},
+		{
+			totalWaiter{rate.NewLimiter(rate.Inf, 42), rate.NewLimiter(rate.Limit(12), 42), rate.NewLimiter(rate.Limit(15), 42)},
+			rate.Limit(12),
+		},
+	}
+
+	for _, tc := range cases {
+		l := tc.w.Limit()
+		if l != tc.r {
+			t.Error("incorrect limit returned")
+		}
+	}
+}
+
 func checkActualAndExpected(t *testing.T, actualR, actualW, expectedR, expectedW map[protocol.DeviceID]*rate.Limiter) {
 	t.Helper()
 	if len(expectedW) != len(actualW) || len(expectedR) != len(actualR) {
@@ -203,3 +353,13 @@ func checkActualAndExpected(t *testing.T, actualR, actualW, expectedR, expectedW
 		}
 	}
 }
+
+type countingWriter struct {
+	w          io.Writer
+	writeCount int
+}
+
+func (w *countingWriter) Write(data []byte) (int, error) {
+	w.writeCount++
+	return w.w.Write(data)
+}

lib/connections/quic_dial.go

@@ -11,6 +11,7 @@ package connections
 import (
 	"context"
 	"crypto/tls"
+	"fmt"
 	"net"
 	"net/url"
 	"time"
@@ -22,7 +23,13 @@ import (
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-const quicPriority = 100
+const (
+	quicPriority = 100
+
+	// The timeout for connecting, accepting and creating the various
+	// streams.
+	quicOperationTimeout = 10 * time.Second
+)
 
 func init() {
 	factory := &quicDialerFactory{}
@@ -60,38 +67,25 @@ func (d *quicDialer) Dial(_ protocol.DeviceID, uri *url.URL) (internalConn, erro
 		}
 	}
 
-	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), quicOperationTimeout)
+	defer cancel()
+
 	session, err := quic.DialContext(ctx, conn, addr, uri.Host, d.tlsCfg, quicConfig)
 	if err != nil {
 		if createdConn != nil {
 			_ = createdConn.Close()
 		}
-		return internalConn{}, err
+		return internalConn{}, fmt.Errorf("dial: %v", err)
 	}
 
-	// OpenStreamSync is blocks, but we want to make sure the connection is usable
-	// before we start killing off other connections, so do the dance.
-	ok := make(chan struct{})
-	go func() {
-		select {
-		case <-ok:
-			return
-		case <-time.After(10 * time.Second):
-			l.Debugln("timed out waiting for OpenStream on", session.RemoteAddr())
-			// This will unblock OpenStreamSync
-			_ = session.Close()
-		}
-	}()
-
-	stream, err := session.OpenStreamSync()
-	close(ok)
+	stream, err := session.OpenStreamSync(ctx)
 	if err != nil {
 		// It's ok to close these, this does not close the underlying packetConn.
 		_ = session.Close()
 		if createdConn != nil {
 			_ = createdConn.Close()
 		}
-		return internalConn{}, err
+		return internalConn{}, fmt.Errorf("open stream: %v", err)
 	}
 
 	return internalConn{&quicTlsConn{session, stream, createdConn}, connTypeQUICClient, quicPriority}, nil

lib/connections/quic_listen.go

@@ -9,6 +9,7 @@
 package connections
 
 import (
+	"context"
 	"crypto/tls"
 	"net"
 	"net/url"
@@ -80,6 +81,10 @@ func (t *quicListener) OnExternalAddressChanged(address *stun.Host, via string)
 func (t *quicListener) serve(stop chan struct{}) error {
 	network := strings.Replace(t.uri.Scheme, "quic", "udp", -1)
 
+	// Convert the stop channel into a context
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() { <-stop; cancel() }()
+
 	packetConn, err := net.ListenPacket(network, t.uri.Host)
 	if err != nil {
 		l.Infoln("Listen (BEP/quic):", err)
@@ -101,57 +106,49 @@ func (t *quicListener) serve(stop chan struct{}) error {
 		l.Infoln("Listen (BEP/quic):", err)
 		return err
 	}
+	defer listener.Close()
 
 	l.Infof("QUIC listener (%v) starting", packetConn.LocalAddr())
 	defer l.Infof("QUIC listener (%v) shutting down", packetConn.LocalAddr())
 
-	// Accept is forever, so handle stops externally.
-	go func() {
-		select {
-		case <-stop:
-			_ = listener.Close()
-		}
-	}()
+	acceptFailures := 0
+	const maxAcceptFailures = 10
 
 	for {
-		// Blocks forever, see https://github.com/lucas-clemente/quic-go/issues/1915
-		session, err := listener.Accept()
-
 		select {
-		case <-stop:
-			if err == nil {
-				_ = session.Close()
-			}
+		case <-ctx.Done():
 			return nil
 		default:
 		}
-		if err != nil {
-			if err, ok := err.(net.Error); !ok || !err.Timeout() {
-				l.Warnln("Listen (BEP/quic): Accepting connection:", err)
+
+		session, err := listener.Accept(ctx)
+		if err == context.Canceled {
+			return nil
+		} else if err != nil {
+			l.Infoln("Listen (BEP/quic): Accepting connection:", err)
+
+			acceptFailures++
+			if acceptFailures > maxAcceptFailures {
+				// Return to restart the listener, because something
+				// seems permanently damaged.
+				return err
 			}
+
+			// Slightly increased delay for each failure.
+			time.Sleep(time.Duration(acceptFailures) * time.Second)
+
 			continue
 		}
 
+		acceptFailures = 0
+
 		l.Debugln("connect from", session.RemoteAddr())
 
-		// Accept blocks forever, give it 10s to do it's thing.
-		ok := make(chan struct{})
-		go func() {
-			select {
-			case <-ok:
-				return
-			case <-stop:
-				_ = session.Close()
-			case <-time.After(10 * time.Second):
-				l.Debugln("timed out waiting for AcceptStream on", session.RemoteAddr())
-				_ = session.Close()
-			}
-		}()
-
-		stream, err := session.AcceptStream()
-		close(ok)
+		streamCtx, cancel := context.WithTimeout(ctx, quicOperationTimeout)
+		stream, err := session.AcceptStream(streamCtx)
+		cancel()
 		if err != nil {
-			l.Debugln("failed to accept stream from", session.RemoteAddr(), err.Error())
+			l.Debugf("failed to accept stream from %s: %v", session.RemoteAddr(), err)
 			_ = session.Close()
 			continue
 		}

lib/connections/service.go

@@ -83,7 +83,7 @@ var tlsCipherSuiteNames = map[uint16]string{
 
 var tlsVersionNames = map[uint16]string{
 	tls.VersionTLS12: "TLS1.2",
-	772:              "TLS1.3", // tls.VersionTLS13 constant available in Go 1.12+
+	tls.VersionTLS13: "TLS1.3",
 }
 
 // Service listens and dials all configured unconnected devices, via supported
@@ -120,6 +120,7 @@ type service struct {
 	limiter              *limiter
 	natService           *nat.Service
 	natServiceToken      *suture.ServiceToken
+	evLogger             events.Logger
 
 	listenersMut       sync.RWMutex
 	listeners          map[string]genericListener
@@ -130,7 +131,7 @@ type service struct {
 	connectionStatus    map[string]ConnectionStatusEntry // address -> latest error/status
 }
 
-func NewService(cfg config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *tls.Config, discoverer discover.Finder, bepProtocolName string, tlsDefaultCommonName string) Service {
+func NewService(cfg config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *tls.Config, discoverer discover.Finder, bepProtocolName string, tlsDefaultCommonName string, evLogger events.Logger) Service {
 	service := &service{
 		Supervisor: suture.New("connections.Service", suture.Spec{
 			Log: func(line string) {
@@ -148,6 +149,7 @@ func NewService(cfg config.Wrapper, myID protocol.DeviceID, mdl Model, tlsCfg *t
 		tlsDefaultCommonName: tlsDefaultCommonName,
 		limiter:              newLimiter(cfg),
 		natService:           nat.NewService(myID, cfg),
+		evLogger:             evLogger,
 
 		listenersMut:   sync.NewRWMutex(),
 		listeners:      make(map[string]genericListener),
@@ -552,7 +554,7 @@ func (s *service) createListener(factory listenerFactory, uri *url.URL) bool {
 }
 
 func (s *service) logListenAddressesChangedEvent(l genericListener) {
-	events.Default.Log(events.ListenAddressesChanged, map[string]interface{}{
+	s.evLogger.Log(events.ListenAddressesChanged, map[string]interface{}{
 		"address": l.URI(),
 		"lan":     l.LANAddresses(),
 		"wan":     l.WANAddresses(),
@@ -579,7 +581,7 @@ func (s *service) CommitConfiguration(from, to config.Configuration) bool {
 
 	s.listenersMut.Lock()
 	seen := make(map[string]struct{})
-	for _, addr := range config.Wrap("", to).ListenAddresses() {
+	for _, addr := range config.Wrap("", to, s.evLogger).ListenAddresses() {
 		if addr == "" {
 			// We can get an empty address if there is an empty listener
 			// element in the config, indicating no listeners should be

lib/db/debug.go

@@ -7,9 +7,6 @@
 package db
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
@@ -17,10 +14,6 @@ var (
 	l = logger.DefaultLogger.NewFacility("db", "The database layer")
 )
 
-func init() {
-	l.SetDebug("db", strings.Contains(os.Getenv("STTRACE"), "db") || os.Getenv("STTRACE") == "all")
-}
-
 func shouldDebug() bool {
 	return l.ShouldDebug("db")
 }

lib/db/instance.go

@@ -562,7 +562,7 @@ func (e errorSuggestion) Error() string {
 
 // unchanged checks if two files are the same and thus don't need to be updated.
 // Local flags or the invalid bit might change without the version
-// being bumped. The IsInvalid() method handles both.
+// being bumped.
 func unchanged(nf, ef FileIntf) bool {
-	return ef.FileVersion().Equal(nf.FileVersion()) && ef.IsInvalid() == nf.IsInvalid()
+	return ef.FileVersion().Equal(nf.FileVersion()) && ef.IsInvalid() == nf.IsInvalid() && ef.FileLocalFlags() == nf.FileLocalFlags()
 }

lib/db/lowlevel.go

@@ -23,11 +23,26 @@ import (
 
 const (
 	dbMaxOpenFiles = 100
-	dbWriteBuffer  = 16 << 20
+	dbFlushBatch   = 4 << MiB
+
+	// A large database is > 200 MiB. It's a mostly arbitrary value, but
+	// it's also the case that each file is 2 MiB by default and when we
+	// have dbMaxOpenFiles of them we will need to start thrashing fd:s.
+	// Switching to large database settings causes larger files to be used
+	// when compacting, reducing the number.
+	dbLargeThreshold = dbMaxOpenFiles * (2 << MiB)
+
+	KiB = 10
+	MiB = 20
 )
 
-var (
-	dbFlushBatch = debugEnvValue("WriteBuffer", dbWriteBuffer) / 4 // Some leeway for any leveldb in-memory optimizations
+type Tuning int
+
+const (
+	// N.b. these constants must match those in lib/config.Tuning!
+	TuningAuto Tuning = iota
+	TuningSmall
+	TuningLarge
 )
 
 // Lowlevel is the lowest level database interface. It has a very simple
@@ -49,18 +64,58 @@ type Lowlevel struct {
 // Open attempts to open the database at the given location, and runs
 // recovery on it if opening fails. Worst case, if recovery is not possible,
 // the database is erased and created from scratch.
-func Open(location string) (*Lowlevel, error) {
+func Open(location string, tuning Tuning) (*Lowlevel, error) {
+	opts := optsFor(location, tuning)
+	return open(location, opts)
+}
+
+// optsFor returns the database options to use when opening a database with
+// the given location and tuning. Settings can be overridden by debug
+// environment variables.
+func optsFor(location string, tuning Tuning) *opt.Options {
+	large := false
+	switch tuning {
+	case TuningLarge:
+		large = true
+	case TuningAuto:
+		large = dbIsLarge(location)
+	}
+
+	var (
+		// Set defaults used for small databases.
+		defaultBlockCacheCapacity            = 0 // 0 means let leveldb use default
+		defaultBlockSize                     = 0
+		defaultCompactionTableSize           = 0
+		defaultCompactionTableSizeMultiplier = 0
+		defaultWriteBuffer                   = 16 << MiB                      // increased from leveldb default of 4 MiB
+		defaultCompactionL0Trigger           = opt.DefaultCompactionL0Trigger // explicit because we use it as base for other stuff
+	)
+
+	if large {
+		// Change the parameters for better throughput at the price of some
+		// RAM and larger files. This results in larger batches of writes
+		// and compaction at a lower frequency.
+		l.Infoln("Using large-database tuning")
+
+		defaultBlockCacheCapacity = 64 << MiB
+		defaultBlockSize = 64 << KiB
+		defaultCompactionTableSize = 16 << MiB
+		defaultCompactionTableSizeMultiplier = 20 // 2.0 after division by ten
+		defaultWriteBuffer = 64 << MiB
+		defaultCompactionL0Trigger = 8 // number of l0 files
+	}
+
 	opts := &opt.Options{
-		BlockCacheCapacity:            debugEnvValue("BlockCacheCapacity", 0),
+		BlockCacheCapacity:            debugEnvValue("BlockCacheCapacity", defaultBlockCacheCapacity),
 		BlockCacheEvictRemoved:        debugEnvValue("BlockCacheEvictRemoved", 0) != 0,
 		BlockRestartInterval:          debugEnvValue("BlockRestartInterval", 0),
-		BlockSize:                     debugEnvValue("BlockSize", 0),
+		BlockSize:                     debugEnvValue("BlockSize", defaultBlockSize),
 		CompactionExpandLimitFactor:   debugEnvValue("CompactionExpandLimitFactor", 0),
 		CompactionGPOverlapsFactor:    debugEnvValue("CompactionGPOverlapsFactor", 0),
-		CompactionL0Trigger:           debugEnvValue("CompactionL0Trigger", 0),
+		CompactionL0Trigger:           debugEnvValue("CompactionL0Trigger", defaultCompactionL0Trigger),
 		CompactionSourceLimitFactor:   debugEnvValue("CompactionSourceLimitFactor", 0),
-		CompactionTableSize:           debugEnvValue("CompactionTableSize", 0),
-		CompactionTableSizeMultiplier: float64(debugEnvValue("CompactionTableSizeMultiplier", 0)) / 10.0,
+		CompactionTableSize:           debugEnvValue("CompactionTableSize", defaultCompactionTableSize),
+		CompactionTableSizeMultiplier: float64(debugEnvValue("CompactionTableSizeMultiplier", defaultCompactionTableSizeMultiplier)) / 10.0,
 		CompactionTotalSize:           debugEnvValue("CompactionTotalSize", 0),
 		CompactionTotalSizeMultiplier: float64(debugEnvValue("CompactionTotalSizeMultiplier", 0)) / 10.0,
 		DisableBufferPool:             debugEnvValue("DisableBufferPool", 0) != 0,
@@ -70,15 +125,16 @@ func Open(location string) (*Lowlevel, error) {
 		NoSync:                        debugEnvValue("NoSync", 0) != 0,
 		NoWriteMerge:                  debugEnvValue("NoWriteMerge", 0) != 0,
 		OpenFilesCacheCapacity:        debugEnvValue("OpenFilesCacheCapacity", dbMaxOpenFiles),
-		WriteBuffer:                   debugEnvValue("WriteBuffer", dbWriteBuffer),
+		WriteBuffer:                   debugEnvValue("WriteBuffer", defaultWriteBuffer),
 		// The write slowdown and pause can be overridden, but even if they
 		// are not and the compaction trigger is overridden we need to
 		// adjust so that we don't pause writes for L0 compaction before we
 		// even *start* L0 compaction...
-		WriteL0SlowdownTrigger: debugEnvValue("WriteL0SlowdownTrigger", 2*debugEnvValue("CompactionL0Trigger", opt.DefaultCompactionL0Trigger)),
-		WriteL0PauseTrigger:    debugEnvValue("WriteL0SlowdownTrigger", 3*debugEnvValue("CompactionL0Trigger", opt.DefaultCompactionL0Trigger)),
+		WriteL0SlowdownTrigger: debugEnvValue("WriteL0SlowdownTrigger", 2*debugEnvValue("CompactionL0Trigger", defaultCompactionL0Trigger)),
+		WriteL0PauseTrigger:    debugEnvValue("WriteL0SlowdownTrigger", 3*debugEnvValue("CompactionL0Trigger", defaultCompactionL0Trigger)),
 	}
-	return open(location, opts)
+
+	return opts
 }
 
 // OpenRO attempts to open the database at the given location, read only.
@@ -114,6 +170,7 @@ func open(location string, opts *opt.Options) (*Lowlevel, error) {
 			l.Warnln("Compacting database:", err)
 		}
 	}
+
 	return NewLowlevel(db, location), nil
 }
 
@@ -207,6 +264,38 @@ func (db *Lowlevel) Close() {
 	db.DB.Close()
 }
 
+// dbIsLarge returns whether the estimated size of the database at location
+// is large enough to warrant optimization for large databases.
+func dbIsLarge(location string) bool {
+	if ^uint(0)>>63 == 0 {
+		// We're compiled for a 32 bit architecture. We've seen trouble with
+		// large settings there.
+		// (https://forum.syncthing.net/t/many-small-ldb-files-with-database-tuning/13842)
+		return false
+	}
+
+	dir, err := os.Open(location)
+	if err != nil {
+		return false
+	}
+
+	fis, err := dir.Readdir(-1)
+	if err != nil {
+		return false
+	}
+
+	var size int64
+	for _, fi := range fis {
+		if fi.Name() == "LOG" {
+			// don't count the size
+			continue
+		}
+		size += fi.Size()
+	}
+
+	return size > dbLargeThreshold
+}
+
 // NewLowlevel wraps the given *leveldb.DB into a *lowlevel
 func NewLowlevel(db *leveldb.DB, location string) *Lowlevel {
 	return &Lowlevel{

lib/db/set.go

@@ -51,6 +51,10 @@ type FileIntf interface {
 	SequenceNo() int64
 	BlockSize() int
 	FileVersion() protocol.Vector
+	FileType() protocol.FileInfoType
+	FilePermissions() uint32
+	FileModifiedBy() protocol.ShortID
+	ModTime() time.Time
 }
 
 // The Iterator is called with either a protocol.FileInfo or a

lib/db/set_test.go

@@ -729,7 +729,7 @@ func BenchmarkUpdateOneFile(b *testing.B) {
 		protocol.FileInfo{Name: "zajksdhaskjdh/askjdhaskjdashkajshd/kasjdhaskjdhaskdjhaskdjash/dkjashdaksjdhaskdjahskdjh", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(8)},
 	}
 
-	ldb, err := db.Open("testdata/benchmarkupdate.db")
+	ldb, err := db.Open("testdata/benchmarkupdate.db", db.TuningAuto)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -1462,6 +1462,33 @@ func TestSequenceIndex(t *testing.T) {
 	}
 }
 
+func TestIgnoreAfterReceiveOnly(t *testing.T) {
+	ldb := db.OpenMemory()
+
+	file := "foo"
+	s := db.NewFileSet("test", fs.NewFilesystem(fs.FilesystemTypeBasic, "."), ldb)
+
+	fs := fileList{{
+		Name:       file,
+		Version:    protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1}}},
+		LocalFlags: protocol.FlagLocalReceiveOnly,
+	}}
+
+	s.Update(protocol.LocalDeviceID, fs)
+
+	fs[0].LocalFlags = protocol.FlagLocalIgnored
+
+	s.Update(protocol.LocalDeviceID, fs)
+
+	if f, ok := s.Get(protocol.LocalDeviceID, file); !ok {
+		t.Error("File missing in db")
+	} else if f.IsReceiveOnlyChanged() {
+		t.Error("File is still receive-only changed")
+	} else if !f.IsIgnored() {
+		t.Error("File is not ignored")
+	}
+}
+
 func replace(fs *db.FileSet, device protocol.DeviceID, files []protocol.FileInfo) {
 	fs.Drop(device)
 	fs.Update(device, files)

lib/db/structs.go

@@ -116,6 +116,17 @@ func (f FileInfoTruncated) FileVersion() protocol.Vector {
 	return f.Version
 }
 
+func (f FileInfoTruncated) FileType() protocol.FileInfoType {
+	return f.Type
+}
+
+func (f FileInfoTruncated) FilePermissions() uint32 {
+	return f.Permissions
+}
+
+func (f FileInfoTruncated) FileModifiedBy() protocol.ShortID {
+	return f.ModifiedBy
+}
 func (f FileInfoTruncated) ConvertToIgnoredFileInfo(by protocol.ShortID) protocol.FileInfo {
 	return protocol.FileInfo{
 		Name:         f.Name,

lib/dialer/internal.go

@@ -11,7 +11,6 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"strings"
 	"time"
 
 	"golang.org/x/net/proxy"
@@ -29,8 +28,6 @@ var (
 type dialFunc func(network, addr string) (net.Conn, error)
 
 func init() {
-	l.SetDebug("dialer", strings.Contains(os.Getenv("STTRACE"), "dialer") || os.Getenv("STTRACE") == "all")
-
 	proxy.RegisterDialerType("socks", socksDialerFunction)
 	proxyDialer = getDialer(proxy.Direct)
 	usingProxy = proxyDialer != proxy.Direct

lib/discover/debug.go

@@ -7,16 +7,9 @@
 package discover
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
 	l = logger.DefaultLogger.NewFacility("discover", "Remote device discovery")
 )
-
-func init() {
-	l.SetDebug("discover", strings.Contains(os.Getenv("STTRACE"), "discover") || os.Getenv("STTRACE") == "all")
-}

lib/discover/global.go

@@ -35,6 +35,7 @@ type globalClient struct {
 	queryClient    httpClient
 	noAnnounce     bool
 	noLookup       bool
+	evLogger       events.Logger
 	errorHolder
 }
 
@@ -70,7 +71,7 @@ func (e lookupError) CacheFor() time.Duration {
 	return e.cacheFor
 }
 
-func NewGlobal(server string, cert tls.Certificate, addrList AddressLister) (FinderService, error) {
+func NewGlobal(server string, cert tls.Certificate, addrList AddressLister, evLogger events.Logger) (FinderService, error) {
 	server, opts, err := parseOptions(server)
 	if err != nil {
 		return nil, err
@@ -125,6 +126,7 @@ func NewGlobal(server string, cert tls.Certificate, addrList AddressLister) (Fin
 		queryClient:    queryClient,
 		noAnnounce:     opts.noAnnounce,
 		noLookup:       opts.noLookup,
+		evLogger:       evLogger,
 	}
 	cl.Service = util.AsService(cl.serve)
 	if !opts.noAnnounce {
@@ -197,8 +199,8 @@ func (c *globalClient) serve(stop chan struct{}) {
 	timer := time.NewTimer(0)
 	defer timer.Stop()
 
-	eventSub := events.Default.Subscribe(events.ListenAddressesChanged)
-	defer events.Default.Unsubscribe(eventSub)
+	eventSub := c.evLogger.Subscribe(events.ListenAddressesChanged)
+	defer eventSub.Unsubscribe()
 
 	for {
 		select {

lib/discover/global_test.go

@@ -15,6 +15,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 )
@@ -54,15 +55,15 @@ func TestGlobalOverHTTP(t *testing.T) {
 	// is only allowed in combination with the "insecure" and "noannounce"
 	// parameters.
 
-	if _, err := NewGlobal("http://192.0.2.42/", tls.Certificate{}, nil); err == nil {
+	if _, err := NewGlobal("http://192.0.2.42/", tls.Certificate{}, nil, events.NoopLogger); err == nil {
 		t.Fatal("http is not allowed without insecure and noannounce")
 	}
 
-	if _, err := NewGlobal("http://192.0.2.42/?insecure", tls.Certificate{}, nil); err == nil {
+	if _, err := NewGlobal("http://192.0.2.42/?insecure", tls.Certificate{}, nil, events.NoopLogger); err == nil {
 		t.Fatal("http is not allowed without noannounce")
 	}
 
-	if _, err := NewGlobal("http://192.0.2.42/?noannounce", tls.Certificate{}, nil); err == nil {
+	if _, err := NewGlobal("http://192.0.2.42/?noannounce", tls.Certificate{}, nil, events.NoopLogger); err == nil {
 		t.Fatal("http is not allowed without insecure")
 	}
 
@@ -111,7 +112,7 @@ func TestGlobalOverHTTPS(t *testing.T) {
 	}
 
 	// Generate a server certificate.
-	cert, err := tlsutil.NewCertificate(dir+"/cert.pem", dir+"/key.pem", "syncthing")
+	cert, err := tlsutil.NewCertificate(dir+"/cert.pem", dir+"/key.pem", "syncthing", 30)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -176,7 +177,7 @@ func TestGlobalAnnounce(t *testing.T) {
 	}
 
 	// Generate a server certificate.
-	cert, err := tlsutil.NewCertificate(dir+"/cert.pem", dir+"/key.pem", "syncthing")
+	cert, err := tlsutil.NewCertificate(dir+"/cert.pem", dir+"/key.pem", "syncthing", 30)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -193,7 +194,7 @@ func TestGlobalAnnounce(t *testing.T) {
 	go func() { _ = http.Serve(list, mux) }()
 
 	url := "https://" + list.Addr().String() + "?insecure"
-	disco, err := NewGlobal(url, cert, new(fakeAddressLister))
+	disco, err := NewGlobal(url, cert, new(fakeAddressLister), events.NoopLogger)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -217,7 +218,7 @@ func TestGlobalAnnounce(t *testing.T) {
 }
 
 func testLookup(url string) ([]string, error) {
-	disco, err := NewGlobal(url, tls.Certificate{}, nil)
+	disco, err := NewGlobal(url, tls.Certificate{}, nil, events.NoopLogger)
 	if err != nil {
 		return nil, err
 	}

lib/discover/local.go

@@ -22,6 +22,7 @@ import (
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/rand"
+	"github.com/syncthing/syncthing/lib/util"
 	"github.com/thejerf/suture"
 )
 
@@ -30,6 +31,7 @@ type localClient struct {
 	myID     protocol.DeviceID
 	addrList AddressLister
 	name     string
+	evLogger events.Logger
 
 	beacon          beacon.Interface
 	localBcastStart time.Time
@@ -46,13 +48,14 @@ const (
 	v13Magic          = uint32(0x7D79BC40) // previous version
 )
 
-func NewLocal(id protocol.DeviceID, addr string, addrList AddressLister) (FinderService, error) {
+func NewLocal(id protocol.DeviceID, addr string, addrList AddressLister, evLogger events.Logger) (FinderService, error) {
 	c := &localClient{
 		Supervisor: suture.New("local", suture.Spec{
 			PassThroughPanics: true,
 		}),
 		myID:            id,
 		addrList:        addrList,
+		evLogger:        evLogger,
 		localBcastTick:  time.NewTicker(BroadcastInterval).C,
 		forcedBcastTick: make(chan time.Time),
 		localBcastStart: time.Now(),
@@ -71,30 +74,20 @@ func NewLocal(id protocol.DeviceID, addr string, addrList AddressLister) (Finder
 		if err != nil {
 			return nil, err
 		}
-		c.startLocalIPv4Broadcasts(bcPort)
+		c.beacon = beacon.NewBroadcast(bcPort)
 	} else {
 		// A multicast client
 		c.name = "IPv6 local"
-		c.startLocalIPv6Multicasts(addr)
+		c.beacon = beacon.NewMulticast(addr)
 	}
+	c.Add(c.beacon)
+	c.Add(util.AsService(c.recvAnnouncements))
 
-	go c.sendLocalAnnouncements()
+	c.Add(util.AsService(c.sendLocalAnnouncements))
 
 	return c, nil
 }
 
-func (c *localClient) startLocalIPv4Broadcasts(localPort int) {
-	c.beacon = beacon.NewBroadcast(localPort)
-	c.Add(c.beacon)
-	go c.recvAnnouncements(c.beacon)
-}
-
-func (c *localClient) startLocalIPv6Multicasts(localMCAddr string) {
-	c.beacon = beacon.NewMulticast(localMCAddr)
-	c.Add(c.beacon)
-	go c.recvAnnouncements(c.beacon)
-}
-
 // Lookup returns a list of addresses the device is available at.
 func (c *localClient) Lookup(device protocol.DeviceID) (addresses []string, err error) {
 	if cache, ok := c.Get(device); ok {
@@ -142,7 +135,7 @@ func (c *localClient) announcementPkt(instanceID int64, msg []byte) ([]byte, boo
 	return msg, true
 }
 
-func (c *localClient) sendLocalAnnouncements() {
+func (c *localClient) sendLocalAnnouncements(stop chan struct{}) {
 	var msg []byte
 	var ok bool
 	instanceID := rand.Int63()
@@ -154,13 +147,22 @@ func (c *localClient) sendLocalAnnouncements() {
 		select {
 		case <-c.localBcastTick:
 		case <-c.forcedBcastTick:
+		case <-stop:
+			return
 		}
 	}
 }
 
-func (c *localClient) recvAnnouncements(b beacon.Interface) {
+func (c *localClient) recvAnnouncements(stop chan struct{}) {
+	b := c.beacon
 	warnedAbout := make(map[string]bool)
 	for {
+		select {
+		case <-stop:
+			return
+		default:
+		}
+
 		buf, addr := b.Recv()
 		if len(buf) < 4 {
 			l.Debugf("discover: short packet from %s")
@@ -272,7 +274,7 @@ func (c *localClient) registerDevice(src net.Addr, device Announce) bool {
 	})
 
 	if isNewDevice {
-		events.Default.Log(events.DeviceDiscovered, map[string]interface{}{
+		c.evLogger.Log(events.DeviceDiscovered, map[string]interface{}{
 			"device": device.ID.String(),
 			"addrs":  validAddresses,
 		})

lib/discover/local.pb.go

@@ -3,14 +3,15 @@
 
 package discover
 
-import proto "github.com/gogo/protobuf/proto"
-import fmt "fmt"
-import math "math"
-import _ "github.com/gogo/protobuf/gogoproto"
-
-import github_com_syncthing_syncthing_lib_protocol "github.com/syncthing/syncthing/lib/protocol"
-
-import io "io"
+import (
+	fmt "fmt"
+	_ "github.com/gogo/protobuf/gogoproto"
+	proto "github.com/gogo/protobuf/proto"
+	github_com_syncthing_syncthing_lib_protocol "github.com/syncthing/syncthing/lib/protocol"
+	io "io"
+	math "math"
+	math_bits "math/bits"
+)
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
@@ -21,7 +22,7 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
 
 type Announce struct {
 	ID         github_com_syncthing_syncthing_lib_protocol.DeviceID `protobuf:"bytes,1,opt,name=id,proto3,customtype=github.com/syncthing/syncthing/lib/protocol.DeviceID" json:"id"`
@@ -33,7 +34,7 @@ func (m *Announce) Reset()         { *m = Announce{} }
 func (m *Announce) String() string { return proto.CompactTextString(m) }
 func (*Announce) ProtoMessage()    {}
 func (*Announce) Descriptor() ([]byte, []int) {
-	return fileDescriptor_local_652287d527eec38f, []int{0}
+	return fileDescriptor_aaf1a48d01603033, []int{0}
 }
 func (m *Announce) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -43,15 +44,15 @@ func (m *Announce) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 		return xxx_messageInfo_Announce.Marshal(b, m, deterministic)
 	} else {
 		b = b[:cap(b)]
-		n, err := m.MarshalTo(b)
+		n, err := m.MarshalToSizedBuffer(b)
 		if err != nil {
 			return nil, err
 		}
 		return b[:n], nil
 	}
 }
-func (dst *Announce) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Announce.Merge(dst, src)
+func (m *Announce) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Announce.Merge(m, src)
 }
 func (m *Announce) XXX_Size() int {
 	return m.ProtoSize()
@@ -65,10 +66,33 @@ var xxx_messageInfo_Announce proto.InternalMessageInfo
 func init() {
 	proto.RegisterType((*Announce)(nil), "discover.Announce")
 }
+
+func init() { proto.RegisterFile("local.proto", fileDescriptor_aaf1a48d01603033) }
+
+var fileDescriptor_aaf1a48d01603033 = []byte{
+	// 252 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x4c, 0x8e, 0x31, 0x4e, 0x84, 0x40,
+	0x14, 0x86, 0x67, 0x20, 0x31, 0xbb, 0xb3, 0xc6, 0x82, 0x8a, 0x18, 0x33, 0x10, 0x6d, 0xa8, 0xa0,
+	0xd0, 0x0b, 0x48, 0x68, 0xa6, 0xe5, 0x02, 0x06, 0x66, 0x46, 0xf6, 0x25, 0x38, 0x6f, 0xc3, 0xc0,
+	0x26, 0xde, 0xc2, 0x23, 0x78, 0x01, 0xef, 0x41, 0xb9, 0xa5, 0xb1, 0x20, 0x3a, 0x5c, 0xc4, 0x04,
+	0x34, 0xda, 0x7d, 0xef, 0xcb, 0x97, 0xbc, 0x9f, 0xed, 0x5a, 0x94, 0x55, 0x9b, 0x1e, 0x3a, 0xec,
+	0x31, 0xd8, 0x28, 0xb0, 0x12, 0x8f, 0xba, 0xbb, 0xbc, 0xe9, 0xf4, 0x01, 0x6d, 0xb6, 0xe8, 0x7a,
+	0x78, 0xcc, 0x1a, 0x6c, 0x70, 0x39, 0x16, 0x5a, 0xf3, 0xeb, 0x37, 0xca, 0x36, 0xf7, 0xc6, 0xe0,
+	0x60, 0xa4, 0x0e, 0x4a, 0xe6, 0x81, 0x0a, 0x69, 0x4c, 0x93, 0xf3, 0x3c, 0x1f, 0xa7, 0x88, 0x7c,
+	0x4c, 0xd1, 0x5d, 0x03, 0xfd, 0x7e, 0xa8, 0x53, 0x89, 0x4f, 0x99, 0x7d, 0x36, 0xb2, 0xdf, 0x83,
+	0x69, 0xfe, 0x51, 0x0b, 0xf5, 0xfa, 0x42, 0x62, 0x9b, 0x16, 0xfa, 0x08, 0x52, 0x8b, 0xc2, 0x4d,
+	0x91, 0x27, 0x8a, 0xd2, 0x03, 0x15, 0x5c, 0xb1, 0x6d, 0xa5, 0x54, 0xa7, 0xad, 0xd5, 0x36, 0xf4,
+	0x62, 0x3f, 0xd9, 0x96, 0x7f, 0x22, 0xc8, 0xd8, 0x0e, 0x8c, 0xed, 0x2b, 0x23, 0xf5, 0x03, 0xa8,
+	0xd0, 0x8f, 0x69, 0xe2, 0xe7, 0x17, 0x6e, 0x8a, 0x98, 0xf8, 0xd1, 0xa2, 0x28, 0xd9, 0x6f, 0x22,
+	0x54, 0x1e, 0x8f, 0x5f, 0x9c, 0x8c, 0x8e, 0xd3, 0x93, 0xe3, 0xf4, 0xd3, 0x71, 0xf2, 0x32, 0x73,
+	0xf2, 0x3a, 0x73, 0x7a, 0x9a, 0x39, 0x79, 0x9f, 0x39, 0xa9, 0xcf, 0x96, 0x35, 0xb7, 0xdf, 0x01,
+	0x00, 0x00, 0xff, 0xff, 0xbc, 0x46, 0xaf, 0x1d, 0x16, 0x01, 0x00, 0x00,
+}
+
 func (m *Announce) Marshal() (dAtA []byte, err error) {
 	size := m.ProtoSize()
 	dAtA = make([]byte, size)
-	n, err := m.MarshalTo(dAtA)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
 	if err != nil {
 		return nil, err
 	}
@@ -76,49 +100,52 @@ func (m *Announce) Marshal() (dAtA []byte, err error) {
 }
 
 func (m *Announce) MarshalTo(dAtA []byte) (int, error) {
-	var i int
+	size := m.ProtoSize()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *Announce) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
 	_ = i
 	var l int
 	_ = l
-	dAtA[i] = 0xa
-	i++
-	i = encodeVarintLocal(dAtA, i, uint64(m.ID.ProtoSize()))
-	n1, err := m.ID.MarshalTo(dAtA[i:])
-	if err != nil {
-		return 0, err
+	if m.InstanceID != 0 {
+		i = encodeVarintLocal(dAtA, i, uint64(m.InstanceID))
+		i--
+		dAtA[i] = 0x18
 	}
-	i += n1
 	if len(m.Addresses) > 0 {
-		for _, s := range m.Addresses {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			i -= len(m.Addresses[iNdEx])
+			copy(dAtA[i:], m.Addresses[iNdEx])
+			i = encodeVarintLocal(dAtA, i, uint64(len(m.Addresses[iNdEx])))
+			i--
 			dAtA[i] = 0x12
-			i++
-			l = len(s)
-			for l >= 1<<7 {
-				dAtA[i] = uint8(uint64(l)&0x7f | 0x80)
-				l >>= 7
-				i++
-			}
-			dAtA[i] = uint8(l)
-			i++
-			i += copy(dAtA[i:], s)
 		}
 	}
-	if m.InstanceID != 0 {
-		dAtA[i] = 0x18
-		i++
-		i = encodeVarintLocal(dAtA, i, uint64(m.InstanceID))
+	{
+		size := m.ID.ProtoSize()
+		i -= size
+		if _, err := m.ID.MarshalTo(dAtA[i:]); err != nil {
+			return 0, err
+		}
+		i = encodeVarintLocal(dAtA, i, uint64(size))
 	}
-	return i, nil
+	i--
+	dAtA[i] = 0xa
+	return len(dAtA) - i, nil
 }
 
 func encodeVarintLocal(dAtA []byte, offset int, v uint64) int {
+	offset -= sovLocal(v)
+	base := offset
 	for v >= 1<<7 {
 		dAtA[offset] = uint8(v&0x7f | 0x80)
 		v >>= 7
 		offset++
 	}
 	dAtA[offset] = uint8(v)
-	return offset + 1
+	return base
 }
 func (m *Announce) ProtoSize() (n int) {
 	if m == nil {
@@ -141,14 +168,7 @@ func (m *Announce) ProtoSize() (n int) {
 }
 
 func sovLocal(x uint64) (n int) {
-	for {
-		n++
-		x >>= 7
-		if x == 0 {
-			break
-		}
-	}
-	return n
+	return (math_bits.Len64(x|1) + 6) / 7
 }
 func sozLocal(x uint64) (n int) {
 	return sovLocal(uint64((x << 1) ^ uint64((int64(x) >> 63))))
@@ -168,7 +188,7 @@ func (m *Announce) Unmarshal(dAtA []byte) error {
 			}
 			b := dAtA[iNdEx]
 			iNdEx++
-			wire |= (uint64(b) & 0x7F) << shift
+			wire |= uint64(b&0x7F) << shift
 			if b < 0x80 {
 				break
 			}
@@ -196,7 +216,7 @@ func (m *Announce) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				byteLen |= (int(b) & 0x7F) << shift
+				byteLen |= int(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -205,6 +225,9 @@ func (m *Announce) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthLocal
 			}
 			postIndex := iNdEx + byteLen
+			if postIndex < 0 {
+				return ErrInvalidLengthLocal
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -226,7 +249,7 @@ func (m *Announce) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				stringLen |= (uint64(b) & 0x7F) << shift
+				stringLen |= uint64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -236,6 +259,9 @@ func (m *Announce) Unmarshal(dAtA []byte) error {
 				return ErrInvalidLengthLocal
 			}
 			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthLocal
+			}
 			if postIndex > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -255,7 +281,7 @@ func (m *Announce) Unmarshal(dAtA []byte) error {
 				}
 				b := dAtA[iNdEx]
 				iNdEx++
-				m.InstanceID |= (int64(b) & 0x7F) << shift
+				m.InstanceID |= int64(b&0x7F) << shift
 				if b < 0x80 {
 					break
 				}
@@ -269,6 +295,9 @@ func (m *Announce) Unmarshal(dAtA []byte) error {
 			if skippy < 0 {
 				return ErrInvalidLengthLocal
 			}
+			if (iNdEx + skippy) < 0 {
+				return ErrInvalidLengthLocal
+			}
 			if (iNdEx + skippy) > l {
 				return io.ErrUnexpectedEOF
 			}
@@ -284,6 +313,7 @@ func (m *Announce) Unmarshal(dAtA []byte) error {
 func skipLocal(dAtA []byte) (n int, err error) {
 	l := len(dAtA)
 	iNdEx := 0
+	depth := 0
 	for iNdEx < l {
 		var wire uint64
 		for shift := uint(0); ; shift += 7 {
@@ -315,10 +345,8 @@ func skipLocal(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			return iNdEx, nil
 		case 1:
 			iNdEx += 8
-			return iNdEx, nil
 		case 2:
 			var length int
 			for shift := uint(0); ; shift += 7 {
@@ -335,75 +363,34 @@ func skipLocal(dAtA []byte) (n int, err error) {
 					break
 				}
 			}
-			iNdEx += length
 			if length < 0 {
 				return 0, ErrInvalidLengthLocal
 			}
-			return iNdEx, nil
+			iNdEx += length
 		case 3:
-			for {
-				var innerWire uint64
-				var start int = iNdEx
-				for shift := uint(0); ; shift += 7 {
-					if shift >= 64 {
-						return 0, ErrIntOverflowLocal
-					}
-					if iNdEx >= l {
-						return 0, io.ErrUnexpectedEOF
-					}
-					b := dAtA[iNdEx]
-					iNdEx++
-					innerWire |= (uint64(b) & 0x7F) << shift
-					if b < 0x80 {
-						break
-					}
-				}
-				innerWireType := int(innerWire & 0x7)
-				if innerWireType == 4 {
-					break
-				}
-				next, err := skipLocal(dAtA[start:])
-				if err != nil {
-					return 0, err
-				}
-				iNdEx = start + next
-			}
-			return iNdEx, nil
+			depth++
 		case 4:
-			return iNdEx, nil
+			if depth == 0 {
+				return 0, ErrUnexpectedEndOfGroupLocal
+			}
+			depth--
 		case 5:
 			iNdEx += 4
-			return iNdEx, nil
 		default:
 			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
 		}
+		if iNdEx < 0 {
+			return 0, ErrInvalidLengthLocal
+		}
+		if depth == 0 {
+			return iNdEx, nil
+		}
 	}
-	panic("unreachable")
+	return 0, io.ErrUnexpectedEOF
 }
 
 var (
-	ErrInvalidLengthLocal = fmt.Errorf("proto: negative length found during unmarshaling")
-	ErrIntOverflowLocal   = fmt.Errorf("proto: integer overflow")
+	ErrInvalidLengthLocal        = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowLocal          = fmt.Errorf("proto: integer overflow")
+	ErrUnexpectedEndOfGroupLocal = fmt.Errorf("proto: unexpected end of group")
 )
-
-func init() { proto.RegisterFile("local.proto", fileDescriptor_local_652287d527eec38f) }
-
-var fileDescriptor_local_652287d527eec38f = []byte{
-	// 252 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x4c, 0x8e, 0x31, 0x4e, 0x84, 0x40,
-	0x14, 0x86, 0x67, 0x20, 0x31, 0xbb, 0xb3, 0xc6, 0x82, 0x8a, 0x18, 0x33, 0x10, 0x6d, 0xa8, 0xa0,
-	0xd0, 0x0b, 0x48, 0x68, 0xa6, 0xe5, 0x02, 0x06, 0x66, 0x46, 0xf6, 0x25, 0x38, 0x6f, 0xc3, 0xc0,
-	0x26, 0xde, 0xc2, 0x23, 0x78, 0x01, 0xef, 0x41, 0xb9, 0xa5, 0xb1, 0x20, 0x3a, 0x5c, 0xc4, 0x04,
-	0x34, 0xda, 0x7d, 0xef, 0xcb, 0x97, 0xbc, 0x9f, 0xed, 0x5a, 0x94, 0x55, 0x9b, 0x1e, 0x3a, 0xec,
-	0x31, 0xd8, 0x28, 0xb0, 0x12, 0x8f, 0xba, 0xbb, 0xbc, 0xe9, 0xf4, 0x01, 0x6d, 0xb6, 0xe8, 0x7a,
-	0x78, 0xcc, 0x1a, 0x6c, 0x70, 0x39, 0x16, 0x5a, 0xf3, 0xeb, 0x37, 0xca, 0x36, 0xf7, 0xc6, 0xe0,
-	0x60, 0xa4, 0x0e, 0x4a, 0xe6, 0x81, 0x0a, 0x69, 0x4c, 0x93, 0xf3, 0x3c, 0x1f, 0xa7, 0x88, 0x7c,
-	0x4c, 0xd1, 0x5d, 0x03, 0xfd, 0x7e, 0xa8, 0x53, 0x89, 0x4f, 0x99, 0x7d, 0x36, 0xb2, 0xdf, 0x83,
-	0x69, 0xfe, 0x51, 0x0b, 0xf5, 0xfa, 0x42, 0x62, 0x9b, 0x16, 0xfa, 0x08, 0x52, 0x8b, 0xc2, 0x4d,
-	0x91, 0x27, 0x8a, 0xd2, 0x03, 0x15, 0x5c, 0xb1, 0x6d, 0xa5, 0x54, 0xa7, 0xad, 0xd5, 0x36, 0xf4,
-	0x62, 0x3f, 0xd9, 0x96, 0x7f, 0x22, 0xc8, 0xd8, 0x0e, 0x8c, 0xed, 0x2b, 0x23, 0xf5, 0x03, 0xa8,
-	0xd0, 0x8f, 0x69, 0xe2, 0xe7, 0x17, 0x6e, 0x8a, 0x98, 0xf8, 0xd1, 0xa2, 0x28, 0xd9, 0x6f, 0x22,
-	0x54, 0x1e, 0x8f, 0x5f, 0x9c, 0x8c, 0x8e, 0xd3, 0x93, 0xe3, 0xf4, 0xd3, 0x71, 0xf2, 0x32, 0x73,
-	0xf2, 0x3a, 0x73, 0x7a, 0x9a, 0x39, 0x79, 0x9f, 0x39, 0xa9, 0xcf, 0x96, 0x35, 0xb7, 0xdf, 0x01,
-	0x00, 0x00, 0xff, 0xff, 0xbc, 0x46, 0xaf, 0x1d, 0x16, 0x01, 0x00, 0x00,
-}

lib/discover/local_test.go

@@ -11,11 +11,12 @@ import (
 	"net"
 	"testing"
 
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
 )
 
 func TestLocalInstanceID(t *testing.T) {
-	c, err := NewLocal(protocol.LocalDeviceID, ":0", &fakeAddressLister{})
+	c, err := NewLocal(protocol.LocalDeviceID, ":0", &fakeAddressLister{}, events.NoopLogger)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -38,7 +39,7 @@ func TestLocalInstanceID(t *testing.T) {
 }
 
 func TestLocalInstanceIDShouldTriggerNew(t *testing.T) {
-	c, err := NewLocal(protocol.LocalDeviceID, ":0", &fakeAddressLister{})
+	c, err := NewLocal(protocol.LocalDeviceID, ":0", &fakeAddressLister{}, events.NoopLogger)
 	if err != nil {
 		t.Fatal(err)
 	}

lib/events/debug.go

@@ -7,16 +7,9 @@
 package events
 
 import (
-	"os"
-	"strings"
-
-	"github.com/syncthing/syncthing/lib/logger"
+	liblogger "github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
-	dl = logger.DefaultLogger.NewFacility("events", "Event generation and logging")
+	dl = liblogger.DefaultLogger.NewFacility("events", "Event generation and logging")
 )
-
-func init() {
-	dl.SetDebug("events", strings.Contains(os.Getenv("STTRACE"), "events") || os.Getenv("STTRACE") == "all")
-}

lib/events/events.go

@@ -13,7 +13,10 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/thejerf/suture"
+
 	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/util"
 )
 
 type EventType int
@@ -51,7 +54,10 @@ const (
 	AllEvents = (1 << iota) - 1
 )
 
-var runningTests = false
+var (
+	runningTests = false
+	errNoop      = errors.New("method of a noop object called")
+)
 
 const eventLogTimeout = 15 * time.Millisecond
 
@@ -199,13 +205,21 @@ func UnmarshalEventType(s string) EventType {
 
 const BufferSize = 64
 
-type Logger struct {
-	subs                []*Subscription
+type Logger interface {
+	suture.Service
+	Log(t EventType, data interface{})
+	Subscribe(mask EventType) Subscription
+}
+
+type logger struct {
+	suture.Service
+	subs                []*subscription
 	nextSubscriptionIDs []int
 	nextGlobalID        int
 	timeout             *time.Timer
 	events              chan Event
 	funcs               chan func()
+	toUnsubscribe       chan *subscription
 	stop                chan struct{}
 }
 
@@ -219,19 +233,17 @@ type Event struct {
 	Data     interface{} `json:"data"`
 }
 
-type Subscription struct {
-	mask    EventType
-	events  chan Event
-	timeout *time.Timer
+type Subscription interface {
+	C() <-chan Event
+	Poll(timeout time.Duration) (Event, error)
+	Unsubscribe()
 }
 
-var Default = NewLogger()
-
-func init() {
-	// The default logger never stops. To ensure this we nil out the stop
-	// channel so any attempt to stop it will panic.
-	Default.stop = nil
-	go Default.Serve()
+type subscription struct {
+	mask          EventType
+	events        chan Event
+	toUnsubscribe chan *subscription
+	timeout       *time.Timer
 }
 
 var (
@@ -239,13 +251,14 @@ var (
 	ErrClosed  = errors.New("closed")
 )
 
-func NewLogger() *Logger {
-	l := &Logger{
-		timeout: time.NewTimer(time.Second),
-		events:  make(chan Event, BufferSize),
-		funcs:   make(chan func()),
-		stop:    make(chan struct{}),
+func NewLogger() Logger {
+	l := &logger{
+		timeout:       time.NewTimer(time.Second),
+		events:        make(chan Event, BufferSize),
+		funcs:         make(chan func()),
+		toUnsubscribe: make(chan *subscription),
 	}
+	l.Service = util.AsService(l.serve)
 	// Make sure the timer is in the stopped state and hasn't fired anything
 	// into the channel.
 	if !l.timeout.Stop() {
@@ -254,7 +267,7 @@ func NewLogger() *Logger {
 	return l
 }
 
-func (l *Logger) Serve() {
+func (l *logger) serve(stop chan struct{}) {
 loop:
 	for {
 		select {
@@ -263,10 +276,13 @@ loop:
 			l.sendEvent(e)
 
 		case fn := <-l.funcs:
-			// Subscriptions etc are handled here.
+			// Subscriptions are handled here.
 			fn()
 
-		case <-l.stop:
+		case s := <-l.toUnsubscribe:
+			l.unsubscribe(s)
+
+		case <-stop:
 			break loop
 		}
 	}
@@ -279,11 +295,7 @@ loop:
 	}
 }
 
-func (l *Logger) Stop() {
-	close(l.stop)
-}
-
-func (l *Logger) Log(t EventType, data interface{}) {
+func (l *logger) Log(t EventType, data interface{}) {
 	l.events <- Event{
 		Time: time.Now(),
 		Type: t,
@@ -292,7 +304,7 @@ func (l *Logger) Log(t EventType, data interface{}) {
 	}
 }
 
-func (l *Logger) sendEvent(e Event) {
+func (l *logger) sendEvent(e Event) {
 	l.nextGlobalID++
 	dl.Debugln("log", l.nextGlobalID, e.Type, e.Data)
 
@@ -323,15 +335,16 @@ func (l *Logger) sendEvent(e Event) {
 	}
 }
 
-func (l *Logger) Subscribe(mask EventType) *Subscription {
-	res := make(chan *Subscription)
+func (l *logger) Subscribe(mask EventType) Subscription {
+	res := make(chan Subscription)
 	l.funcs <- func() {
 		dl.Debugln("subscribe", mask)
 
-		s := &Subscription{
-			mask:    mask,
-			events:  make(chan Event, BufferSize),
-			timeout: time.NewTimer(0),
+		s := &subscription{
+			mask:          mask,
+			events:        make(chan Event, BufferSize),
+			toUnsubscribe: l.toUnsubscribe,
+			timeout:       time.NewTimer(0),
 		}
 
 		// We need to create the timeout timer in the stopped, non-fired state so
@@ -355,32 +368,30 @@ func (l *Logger) Subscribe(mask EventType) *Subscription {
 	return <-res
 }
 
-func (l *Logger) Unsubscribe(s *Subscription) {
-	l.funcs <- func() {
-		dl.Debugln("unsubscribe")
-		for i, ss := range l.subs {
-			if s == ss {
-				last := len(l.subs) - 1
+func (l *logger) unsubscribe(s *subscription) {
+	dl.Debugln("unsubscribe", s.mask)
+	for i, ss := range l.subs {
+		if s == ss {
+			last := len(l.subs) - 1
 
-				l.subs[i] = l.subs[last]
-				l.subs[last] = nil
-				l.subs = l.subs[:last]
+			l.subs[i] = l.subs[last]
+			l.subs[last] = nil
+			l.subs = l.subs[:last]
 
-				l.nextSubscriptionIDs[i] = l.nextSubscriptionIDs[last]
-				l.nextSubscriptionIDs[last] = 0
-				l.nextSubscriptionIDs = l.nextSubscriptionIDs[:last]
+			l.nextSubscriptionIDs[i] = l.nextSubscriptionIDs[last]
+			l.nextSubscriptionIDs[last] = 0
+			l.nextSubscriptionIDs = l.nextSubscriptionIDs[:last]
 
-				break
-			}
+			break
 		}
-		close(s.events)
 	}
+	close(s.events)
 }
 
 // Poll returns an event from the subscription or an error if the poll times
 // out of the event channel is closed. Poll should not be called concurrently
 // from multiple goroutines for a single subscription.
-func (s *Subscription) Poll(timeout time.Duration) (Event, error) {
+func (s *subscription) Poll(timeout time.Duration) (Event, error) {
 	dl.Debugln("poll", timeout)
 
 	s.timeout.Reset(timeout)
@@ -409,12 +420,16 @@ func (s *Subscription) Poll(timeout time.Duration) (Event, error) {
 	}
 }
 
-func (s *Subscription) C() <-chan Event {
+func (s *subscription) C() <-chan Event {
 	return s.events
 }
 
+func (s *subscription) Unsubscribe() {
+	s.toUnsubscribe <- s
+}
+
 type bufferedSubscription struct {
-	sub  *Subscription
+	sub  Subscription
 	buf  []Event
 	next int
 	cur  int // Current SubscriptionID
@@ -426,7 +441,7 @@ type BufferedSubscription interface {
 	Since(id int, into []Event, timeout time.Duration) []Event
 }
 
-func NewBufferedSubscription(s *Subscription, size int) BufferedSubscription {
+func NewBufferedSubscription(s Subscription, size int) BufferedSubscription {
 	bs := &bufferedSubscription{
 		sub: s,
 		buf: make([]Event, size),
@@ -489,3 +504,29 @@ func Error(err error) *string {
 	str := err.Error()
 	return &str
 }
+
+type noopLogger struct{}
+
+var NoopLogger Logger = &noopLogger{}
+
+func (*noopLogger) Serve() {}
+
+func (*noopLogger) Stop() {}
+
+func (*noopLogger) Log(t EventType, data interface{}) {}
+
+func (*noopLogger) Subscribe(mask EventType) Subscription {
+	return &noopSubscription{}
+}
+
+type noopSubscription struct{}
+
+func (*noopSubscription) C() <-chan Event {
+	return nil
+}
+
+func (*noopSubscription) Poll(timeout time.Duration) (Event, error) {
+	return Event{}, errNoop
+}
+
+func (*noopSubscription) Unsubscribe() {}

lib/events/events_test.go

@@ -33,7 +33,7 @@ func TestSubscriber(t *testing.T) {
 	go l.Serve()
 
 	s := l.Subscribe(0)
-	defer l.Unsubscribe(s)
+	defer s.Unsubscribe()
 	if s == nil {
 		t.Fatal("Unexpected nil Subscription")
 	}
@@ -45,7 +45,7 @@ func TestTimeout(t *testing.T) {
 	go l.Serve()
 
 	s := l.Subscribe(0)
-	defer l.Unsubscribe(s)
+	defer s.Unsubscribe()
 	_, err := s.Poll(timeout)
 	if err != ErrTimeout {
 		t.Fatal("Unexpected non-Timeout error:", err)
@@ -59,7 +59,7 @@ func TestEventBeforeSubscribe(t *testing.T) {
 
 	l.Log(DeviceConnected, "foo")
 	s := l.Subscribe(0)
-	defer l.Unsubscribe(s)
+	defer s.Unsubscribe()
 
 	_, err := s.Poll(timeout)
 	if err != ErrTimeout {
@@ -73,7 +73,7 @@ func TestEventAfterSubscribe(t *testing.T) {
 	go l.Serve()
 
 	s := l.Subscribe(AllEvents)
-	defer l.Unsubscribe(s)
+	defer s.Unsubscribe()
 	l.Log(DeviceConnected, "foo")
 
 	ev, err := s.Poll(timeout)
@@ -100,7 +100,7 @@ func TestEventAfterSubscribeIgnoreMask(t *testing.T) {
 	go l.Serve()
 
 	s := l.Subscribe(DeviceDisconnected)
-	defer l.Unsubscribe(s)
+	defer s.Unsubscribe()
 	l.Log(DeviceConnected, "foo")
 
 	_, err := s.Poll(timeout)
@@ -115,7 +115,7 @@ func TestBufferOverflow(t *testing.T) {
 	go l.Serve()
 
 	s := l.Subscribe(AllEvents)
-	defer l.Unsubscribe(s)
+	defer s.Unsubscribe()
 
 	// The first BufferSize events will be logged pretty much
 	// instantaneously. The next BufferSize events will each block for up to
@@ -147,7 +147,7 @@ func TestUnsubscribe(t *testing.T) {
 		t.Fatal("Unexpected error:", err)
 	}
 
-	l.Unsubscribe(s)
+	s.Unsubscribe()
 	l.Log(DeviceConnected, "foo")
 
 	_, err = s.Poll(timeout)
@@ -162,7 +162,7 @@ func TestGlobalIDs(t *testing.T) {
 	go l.Serve()
 
 	s := l.Subscribe(AllEvents)
-	defer l.Unsubscribe(s)
+	defer s.Unsubscribe()
 	l.Log(DeviceConnected, "foo")
 	l.Subscribe(AllEvents)
 	l.Log(DeviceConnected, "bar")
@@ -194,7 +194,7 @@ func TestSubscriptionIDs(t *testing.T) {
 	go l.Serve()
 
 	s := l.Subscribe(DeviceConnected)
-	defer l.Unsubscribe(s)
+	defer s.Unsubscribe()
 
 	l.Log(DeviceDisconnected, "a")
 	l.Log(DeviceConnected, "b")
@@ -236,7 +236,7 @@ func TestBufferedSub(t *testing.T) {
 	go l.Serve()
 
 	s := l.Subscribe(AllEvents)
-	defer l.Unsubscribe(s)
+	defer s.Unsubscribe()
 	bs := NewBufferedSubscription(s, 10*BufferSize)
 
 	go func() {
@@ -267,7 +267,7 @@ func BenchmarkBufferedSub(b *testing.B) {
 	go l.Serve()
 
 	s := l.Subscribe(AllEvents)
-	defer l.Unsubscribe(s)
+	defer s.Unsubscribe()
 	bufferSize := BufferSize
 	bs := NewBufferedSubscription(s, bufferSize)
 
@@ -323,7 +323,7 @@ func TestSinceUsesSubscriptionId(t *testing.T) {
 	go l.Serve()
 
 	s := l.Subscribe(DeviceConnected)
-	defer l.Unsubscribe(s)
+	defer s.Unsubscribe()
 	bs := NewBufferedSubscription(s, 10*BufferSize)
 
 	l.Log(DeviceConnected, "a") // SubscriptionID = 1
@@ -390,7 +390,7 @@ func TestUnsubscribeContention(t *testing.T) {
 			defer listenerWg.Done()
 
 			s := l.Subscribe(AllEvents)
-			defer l.Unsubscribe(s)
+			defer s.Unsubscribe()
 
 			for {
 				select {
@@ -449,7 +449,7 @@ func BenchmarkLogEvent(b *testing.B) {
 	go l.Serve()
 
 	s := l.Subscribe(AllEvents)
-	defer l.Unsubscribe(s)
+	defer s.Unsubscribe()
 	NewBufferedSubscription(s, 1) // runs in the background
 
 	for i := 0; i < b.N; i++ {

lib/fs/basicfs.go

@@ -345,6 +345,6 @@ func (e *ErrWatchEventOutsideRoot) Error() string {
 	return e.msg
 }
 
-func (f *BasicFilesystem) newErrWatchEventOutsideRoot(absPath, root string) *ErrWatchEventOutsideRoot {
-	return &ErrWatchEventOutsideRoot{fmt.Sprintf("Watching for changes encountered an event outside of the filesystem root: f.root==%v, root==%v, path==%v. This should never happen, please report this message to forum.syncthing.net.", f.root, root, absPath)}
+func (f *BasicFilesystem) newErrWatchEventOutsideRoot(absPath string, roots []string) *ErrWatchEventOutsideRoot {
+	return &ErrWatchEventOutsideRoot{fmt.Sprintf("Watching for changes encountered an event outside of the filesystem root: f.root==%v, roots==%v, path==%v. This should never happen, please report this message to forum.syncthing.net.", f.root, roots, absPath)}
 }

lib/fs/basicfs_unix.go

@@ -60,14 +60,20 @@ func (f *BasicFilesystem) Roots() ([]string, error) {
 // unrooted) or an error if the given path is not a subpath and handles the
 // special case when the given path is the folder root without a trailing
 // pathseparator.
-func (f *BasicFilesystem) unrootedChecked(absPath, root string) (string, *ErrWatchEventOutsideRoot) {
-	if absPath+string(PathSeparator) == root {
-		return ".", nil
+func (f *BasicFilesystem) unrootedChecked(absPath string, roots []string) (string, *ErrWatchEventOutsideRoot) {
+	for _, root := range roots {
+		// Make sure the root ends with precisely one path separator, to
+		// ease prefix comparisons.
+		root := strings.TrimRight(root, string(PathSeparator)) + string(PathSeparator)
+
+		if absPath+string(PathSeparator) == root {
+			return ".", nil
+		}
+		if strings.HasPrefix(absPath, root) {
+			return rel(absPath, root), nil
+		}
 	}
-	if !strings.HasPrefix(absPath, root) {
-		return "", f.newErrWatchEventOutsideRoot(absPath, root)
-	}
-	return rel(absPath, root), nil
+	return "", f.newErrWatchEventOutsideRoot(absPath, roots)
 }
 
 func rel(path, prefix string) string {
@@ -78,16 +84,16 @@ var evalSymlinks = filepath.EvalSymlinks
 
 // watchPaths adjust the folder root for use with the notify backend and the
 // corresponding absolute path to be passed to notify to watch name.
-func (f *BasicFilesystem) watchPaths(name string) (string, string, error) {
+func (f *BasicFilesystem) watchPaths(name string) (string, []string, error) {
 	root, err := evalSymlinks(f.root)
 	if err != nil {
-		return "", "", err
+		return "", nil, err
 	}
 
 	absName, err := rooted(name, root)
 	if err != nil {
-		return "", "", err
+		return "", nil, err
 	}
 
-	return filepath.Join(absName, "..."), root, nil
+	return filepath.Join(absName, "..."), []string{root}, nil
 }

lib/fs/basicfs_watch.go

@@ -21,7 +21,7 @@ import (
 var backendBuffer = 500
 
 func (f *BasicFilesystem) Watch(name string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, <-chan error, error) {
-	watchPath, root, err := f.watchPaths(name)
+	watchPath, roots, err := f.watchPaths(name)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -36,7 +36,7 @@ func (f *BasicFilesystem) Watch(name string, ignore Matcher, ctx context.Context
 
 	if ignore.SkipIgnoredDirs() {
 		absShouldIgnore := func(absPath string) bool {
-			rel, err := f.unrootedChecked(absPath, root)
+			rel, err := f.unrootedChecked(absPath, roots)
 			if err != nil {
 				return true
 			}
@@ -55,12 +55,12 @@ func (f *BasicFilesystem) Watch(name string, ignore Matcher, ctx context.Context
 	}
 
 	errChan := make(chan error)
-	go f.watchLoop(name, root, backendChan, outChan, errChan, ignore, ctx)
+	go f.watchLoop(name, roots, backendChan, outChan, errChan, ignore, ctx)
 
 	return outChan, errChan, nil
 }
 
-func (f *BasicFilesystem) watchLoop(name, evalRoot string, backendChan chan notify.EventInfo, outChan chan<- Event, errChan chan<- error, ignore Matcher, ctx context.Context) {
+func (f *BasicFilesystem) watchLoop(name string, roots []string, backendChan chan notify.EventInfo, outChan chan<- Event, errChan chan<- error, ignore Matcher, ctx context.Context) {
 	for {
 		// Detect channel overflow
 		if len(backendChan) == backendBuffer {
@@ -79,7 +79,7 @@ func (f *BasicFilesystem) watchLoop(name, evalRoot string, backendChan chan noti
 
 		select {
 		case ev := <-backendChan:
-			relPath, err := f.unrootedChecked(ev.Path(), evalRoot)
+			relPath, err := f.unrootedChecked(ev.Path(), roots)
 			if err != nil {
 				select {
 				case errChan <- err:

lib/fs/basicfs_watch_test.go

@@ -166,7 +166,7 @@ func TestWatchWinRoot(t *testing.T) {
 	// testFs is Filesystem, but we need BasicFilesystem here
 	root := `D:\`
 	fs := newBasicFilesystem(root)
-	watch, root, err := fs.watchPaths(".")
+	watch, roots, err := fs.watchPaths(".")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -178,7 +178,7 @@ func TestWatchWinRoot(t *testing.T) {
 			}
 			cancel()
 		}()
-		fs.watchLoop(".", root, backendChan, outChan, errChan, fakeMatcher{}, ctx)
+		fs.watchLoop(".", roots, backendChan, outChan, errChan, fakeMatcher{}, ctx)
 	}()
 
 	// filepath.Dir as watch has a /... suffix
@@ -201,23 +201,33 @@ func TestWatchWinRoot(t *testing.T) {
 
 // TestWatchOutside checks that no changes from outside the folder make it in
 func TestWatchOutside(t *testing.T) {
+	expectErrorForPath(t, filepath.Join(filepath.Dir(testDirAbs), "outside"))
+
+	rootWithoutSlash := strings.TrimRight(filepath.ToSlash(testDirAbs), "/")
+	expectErrorForPath(t, rootWithoutSlash+"outside")
+	expectErrorForPath(t, rootWithoutSlash+"outside/thing")
+}
+
+func expectErrorForPath(t *testing.T, path string) {
 	outChan := make(chan Event)
 	backendChan := make(chan notify.EventInfo, backendBuffer)
 	errChan := make(chan error)
 
 	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
 
 	// testFs is Filesystem, but we need BasicFilesystem here
 	fs := newBasicFilesystem(testDirAbs)
 
-	go fs.watchLoop(".", testDirAbs, backendChan, outChan, errChan, fakeMatcher{}, ctx)
+	go fs.watchLoop(".", []string{testDirAbs}, backendChan, outChan, errChan, fakeMatcher{}, ctx)
 
-	backendChan <- fakeEventInfo(filepath.Join(filepath.Dir(testDirAbs), "outside"))
+	backendChan <- fakeEventInfo(path)
 
 	select {
 	case <-time.After(10 * time.Second):
-		cancel()
 		t.Errorf("Timed out before receiving error")
+	case e := <-outChan:
+		t.Errorf("Unexpected passed through event %v", e)
 	case <-errChan:
 	case <-ctx.Done():
 	}
@@ -234,7 +244,7 @@ func TestWatchSubpath(t *testing.T) {
 	fs := newBasicFilesystem(testDirAbs)
 
 	abs, _ := fs.rooted("sub")
-	go fs.watchLoop("sub", testDirAbs, backendChan, outChan, errChan, fakeMatcher{}, ctx)
+	go fs.watchLoop("sub", []string{testDirAbs}, backendChan, outChan, errChan, fakeMatcher{}, ctx)
 
 	backendChan <- fakeEventInfo(filepath.Join(abs, "file"))
 
@@ -347,7 +357,7 @@ func TestWatchSymlinkedRoot(t *testing.T) {
 
 func TestUnrootedChecked(t *testing.T) {
 	fs := newBasicFilesystem(testDirAbs)
-	if unrooted, err := fs.unrootedChecked("/random/other/path", testDirAbs); err == nil {
+	if unrooted, err := fs.unrootedChecked("/random/other/path", []string{testDirAbs}); err == nil {
 		t.Error("unrootedChecked did not return an error on outside path, but returned", unrooted)
 	}
 }

lib/fs/basicfs_windows.go

@@ -156,17 +156,19 @@ func (f *BasicFilesystem) Roots() ([]string, error) {
 // unrooted) or an error if the given path is not a subpath and handles the
 // special case when the given path is the folder root without a trailing
 // pathseparator.
-func (f *BasicFilesystem) unrootedChecked(absPath, root string) (string, error) {
+func (f *BasicFilesystem) unrootedChecked(absPath string, roots []string) (string, error) {
 	absPath = f.resolveWin83(absPath)
 	lowerAbsPath := UnicodeLowercase(absPath)
-	lowerRoot := UnicodeLowercase(root)
-	if lowerAbsPath+string(PathSeparator) == lowerRoot {
-		return ".", nil
+	for _, root := range roots {
+		lowerRoot := UnicodeLowercase(root)
+		if lowerAbsPath+string(PathSeparator) == lowerRoot {
+			return ".", nil
+		}
+		if strings.HasPrefix(lowerAbsPath, lowerRoot) {
+			return rel(absPath, root), nil
+		}
 	}
-	if !strings.HasPrefix(lowerAbsPath, lowerRoot) {
-		return "", f.newErrWatchEventOutsideRoot(lowerAbsPath, lowerRoot)
-	}
-	return rel(absPath, root), nil
+	return "", f.newErrWatchEventOutsideRoot(lowerAbsPath, roots)
 }
 
 func rel(path, prefix string) string {
@@ -294,10 +296,10 @@ func evalSymlinks(in string) (string, error) {
 
 // watchPaths adjust the folder root for use with the notify backend and the
 // corresponding absolute path to be passed to notify to watch name.
-func (f *BasicFilesystem) watchPaths(name string) (string, string, error) {
+func (f *BasicFilesystem) watchPaths(name string) (string, []string, error) {
 	root, err := evalSymlinks(f.root)
 	if err != nil {
-		return "", "", err
+		return "", nil, err
 	}
 
 	// Remove `\\?\` prefix if the path is just a drive letter as a dirty
@@ -308,11 +310,17 @@ func (f *BasicFilesystem) watchPaths(name string) (string, string, error) {
 
 	absName, err := rooted(name, root)
 	if err != nil {
-		return "", "", err
+		return "", nil, err
 	}
 
-	root = f.resolveWin83(root)
+	roots := []string{f.resolveWin83(root)}
 	absName = f.resolveWin83(absName)
 
-	return filepath.Join(absName, "..."), root, nil
+	// Events returned from fs watching are all over the place, so allow
+	// both the user's input and the result of "canonicalizing" the path.
+	if roots[0] != f.root {
+		roots = append(roots, f.root)
+	}
+
+	return filepath.Join(absName, "..."), roots, nil
 }

lib/fs/basicfs_windows_test.go

@@ -131,7 +131,7 @@ func TestRelUnrootedCheckedWindows(t *testing.T) {
 		// on these test cases.
 		for _, root := range []string{tc.root, strings.ToLower(tc.root), strings.ToUpper(tc.root)} {
 			fs := BasicFilesystem{root: root}
-			if res, err := fs.unrootedChecked(tc.abs, tc.root); err != nil {
+			if res, err := fs.unrootedChecked(tc.abs, []string{tc.root}); err != nil {
 				t.Errorf(`Unexpected error from unrootedChecked("%v", "%v"): %v (fs.root: %v)`, tc.abs, tc.root, err, root)
 			} else if res != tc.expectedRel {
 				t.Errorf(`unrootedChecked("%v", "%v") == "%v", expected "%v" (fs.root: %v)`, tc.abs, tc.root, res, tc.expectedRel, root)
@@ -140,6 +140,21 @@ func TestRelUnrootedCheckedWindows(t *testing.T) {
 	}
 }
 
+// TestMultipleRoot checks that fs.unrootedChecked returns the correct path
+// when given more than one possible root path.
+func TestMultipleRoot(t *testing.T) {
+	root := `c:\foO`
+	roots := []string{root, `d:\`}
+	rel := `bar`
+	path := filepath.Join(root, rel)
+	fs := BasicFilesystem{root: root}
+	if res, err := fs.unrootedChecked(path, roots); err != nil {
+		t.Errorf(`Unexpected error from unrootedChecked("%v", "%v"): %v (fs.root: %v)`, path, roots, err, root)
+	} else if res != rel {
+		t.Errorf(`unrootedChecked("%v", "%v") == "%v", expected "%v" (fs.root: %v)`, path, roots, res, rel, root)
+	}
+}
+
 func TestGetFinalPath(t *testing.T) {
 	testCases := []struct {
 		input         string

lib/fs/debug.go

@@ -7,9 +7,6 @@
 package fs
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
@@ -19,11 +16,7 @@ var (
 
 func init() {
 	logger.DefaultLogger.NewFacility("walkfs", "Filesystem access while walking")
-	switch {
-	case strings.Contains(os.Getenv("STTRACE"), "walkfs") || os.Getenv("STTRACE") == "all":
-		l.SetDebug("walkfs", true)
-		fallthrough
-	case strings.Contains(os.Getenv("STTRACE"), "fs"):
+	if logger.DefaultLogger.ShouldDebug("walkfs") {
 		l.SetDebug("fs", true)
 	}
 }

lib/fs/util.go

@@ -35,23 +35,16 @@ func ExpandTilde(path string) (string, error) {
 }
 
 func getHomeDir() (string, error) {
-	var home string
-
-	switch runtime.GOOS {
-	case "windows":
-		home = filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath"))
-		if home == "" {
-			home = os.Getenv("UserProfile")
+	if runtime.GOOS == "windows" {
+		// Legacy -- we prioritize this for historical reasons, whereas
+		// os.UserHomeDir uses %USERPROFILE% always.
+		home := filepath.Join(os.Getenv("HomeDrive"), os.Getenv("HomePath"))
+		if home != "" {
+			return home, nil
 		}
-	default:
-		home = os.Getenv("HOME")
 	}
 
-	if home == "" {
-		return "", errNoHome
-	}
-
-	return home, nil
+	return os.UserHomeDir()
 }
 
 var windowsDisallowedCharacters = string([]rune{

lib/ignore/ignore.go

@@ -461,7 +461,18 @@ func parseIgnoreFile(fs fs.Filesystem, fd io.Reader, currentFile string, cd Chan
 		line = filepath.ToSlash(line)
 		switch {
 		case strings.HasPrefix(line, "#include"):
-			includeRel := strings.TrimSpace(line[len("#include "):])
+			fields := strings.SplitN(line, " ", 2)
+			if len(fields) != 2 {
+				err = fmt.Errorf("failed to parse #include line: no file?")
+				break
+			}
+
+			includeRel := strings.TrimSpace(fields[1])
+			if includeRel == "" {
+				err = fmt.Errorf("failed to parse #include line: no file?")
+				break
+			}
+
 			includeFile := filepath.Join(filepath.Dir(currentFile), includeRel)
 			var includePatterns []Pattern
 			if includePatterns, err = loadParseIncludeFile(fs, includeFile, cd, linesSeen); err == nil {

lib/ignore/ignore_test.go

@@ -1079,3 +1079,22 @@ func TestSpecialChars(t *testing.T) {
 		}
 	}
 }
+
+func TestPartialIncludeLine(t *testing.T) {
+	// Loading a partial #include line (no file mentioned) should error but not crash.
+
+	pats := New(fs.NewFilesystem(fs.FilesystemTypeBasic, "."), WithCache(true))
+	cases := []string{
+		"#include",
+		"#include\n",
+		"#include ",
+		"#include \n",
+		"#include   \n\n\n",
+	}
+
+	for _, tc := range cases {
+		if err := pats.Parse(bytes.NewBufferString(tc), ".stignore"); err == nil {
+			t.Fatal("should error out")
+		}
+	}
+}

lib/logger/logger.go

@@ -50,6 +50,7 @@ type Logger interface {
 	Warnf(format string, vals ...interface{})
 	ShouldDebug(facility string) bool
 	SetDebug(facility string, enabled bool)
+	IsTraced(facility string) bool
 	Facilities() map[string]string
 	FacilityDebugging() []string
 	NewFacility(facility, description string) Logger
@@ -60,6 +61,7 @@ type logger struct {
 	handlers   [NumLevels][]MessageHandler
 	facilities map[string]string   // facility name => description
 	debug      map[string]struct{} // only facility names with debugging enabled
+	traces     string
 	mut        sync.Mutex
 }
 
@@ -78,6 +80,7 @@ func New() Logger {
 func newLogger(w io.Writer) Logger {
 	return &logger{
 		logger:     log.New(w, "", DefaultFlags),
+		traces:     os.Getenv("STTRACE"),
 		facilities: make(map[string]string),
 		debug:      make(map[string]struct{}),
 	}
@@ -210,6 +213,11 @@ func (l *logger) SetDebug(facility string, enabled bool) {
 	}
 }
 
+// IsTraced returns whether the facility name is contained in STTRACE.
+func (l *logger) IsTraced(facility string) bool {
+	return strings.Contains(l.traces, facility) || l.traces == "all"
+}
+
 // FacilityDebugging returns the set of facilities that have debugging
 // enabled.
 func (l *logger) FacilityDebugging() []string {
@@ -236,6 +244,8 @@ func (l *logger) Facilities() map[string]string {
 
 // NewFacility returns a new logger bound to the named facility.
 func (l *logger) NewFacility(facility, description string) Logger {
+	l.SetDebug(facility, l.IsTraced(facility))
+
 	l.mut.Lock()
 	l.facilities[facility] = description
 	l.mut.Unlock()

lib/model/debug.go

@@ -7,9 +7,6 @@
 package model
 
 import (
-	"os"
-	"strings"
-
 	"github.com/syncthing/syncthing/lib/logger"
 )
 
@@ -17,10 +14,6 @@ var (
 	l = logger.DefaultLogger.NewFacility("model", "The root hub")
 )
 
-func init() {
-	l.SetDebug("model", strings.Contains(os.Getenv("STTRACE"), "model") || os.Getenv("STTRACE") == "all")
-}
-
 func shouldDebug() bool {
 	return l.ShouldDebug("model")
 }

lib/model/folder.go

@@ -77,11 +77,11 @@ type puller interface {
 	pull() bool // true when successfull and should not be retried
 }
 
-func newFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration) folder {
+func newFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, evLogger events.Logger) folder {
 	ctx, cancel := context.WithCancel(context.Background())
 
 	return folder{
-		stateTracker:              newStateTracker(cfg.ID),
+		stateTracker:              newStateTracker(cfg.ID, evLogger),
 		FolderConfiguration:       cfg,
 		FolderStatisticsReference: stats.NewFolderStatisticsReference(model.db, cfg.ID),
 
@@ -171,20 +171,23 @@ func (f *folder) serve(_ chan struct{}) {
 			}
 
 		case <-f.scanTimer.C:
-			l.Debugln(f, "Scanning subdirectories")
+			l.Debugln(f, "Scanning due to timer")
 			f.scanTimerFired()
 
 		case req := <-f.scanNow:
+			l.Debugln(f, "Scanning due to request")
 			req.err <- f.scanSubdirs(req.subdirs)
 
 		case next := <-f.scanDelay:
+			l.Debugln(f, "Delaying scan")
 			f.scanTimer.Reset(next)
 
 		case fsEvents := <-f.watchChan:
-			l.Debugln(f, "filesystem notification rescan")
+			l.Debugln(f, "Scan due to watcher")
 			f.scanSubdirs(fsEvents)
 
 		case <-f.restartWatchChan:
+			l.Debugln(f, "Restart watcher")
 			f.restartWatch()
 		}
 	}
@@ -283,13 +286,38 @@ func (f *folder) getHealthError() error {
 }
 
 func (f *folder) scanSubdirs(subDirs []string) error {
-	if err := f.CheckHealth(); err != nil {
+	if err := f.getHealthError(); err != nil {
+		// If there is a health error we set it as the folder error. We do not
+		// clear the folder error if there is no health error, as there might be
+		// an *other* folder error (failed to load ignores, for example). Hence
+		// we do not use the CheckHealth() convenience function here.
+		f.setError(err)
 		return err
 	}
 
-	mtimefs := f.fset.MtimeFS()
+	oldHash := f.ignores.Hash()
+	if err := f.ignores.Load(".stignore"); err != nil && !fs.IsNotExist(err) {
+		err = fmt.Errorf("loading ignores: %v", err)
+		f.setError(err)
+		return err
+	}
 
+	// Check on the way out if the ignore patterns changed as part of scanning
+	// this folder. If they did we should schedule a pull of the folder so that
+	// we request things we might have suddenly become unignored and so on.
+	defer func() {
+		if f.ignores.Hash() != oldHash {
+			l.Debugln("Folder", f.Description(), "ignore patterns change detected while scanning; triggering puller")
+			f.ignoresUpdated()
+			f.SchedulePull()
+		}
+	}()
+
+	// We've passed all health checks so now mark ourselves healthy and queued
+	// for scanning.
+	f.setError(nil)
 	f.setState(FolderScanWaiting)
+
 	scanLimiter.take(1)
 	defer scanLimiter.give(1)
 
@@ -306,24 +334,6 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 		subDirs[i] = sub
 	}
 
-	// Check if the ignore patterns changed as part of scanning this folder.
-	// If they did we should schedule a pull of the folder so that we
-	// request things we might have suddenly become unignored and so on.
-	oldHash := f.ignores.Hash()
-	defer func() {
-		if f.ignores.Hash() != oldHash {
-			l.Debugln("Folder", f.Description(), "ignore patterns change detected while scanning; triggering puller")
-			f.ignoresUpdated()
-			f.SchedulePull()
-		}
-	}()
-
-	if err := f.ignores.Load(".stignore"); err != nil && !fs.IsNotExist(err) {
-		err = fmt.Errorf("loading ignores: %v", err)
-		f.setError(err)
-		return err
-	}
-
 	// Clean the list of subitems to ensure that we start at a known
 	// directory, and don't scan subdirectories of things we've already
 	// scanned.
@@ -334,6 +344,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 
 	f.setState(FolderScanning)
 
+	mtimefs := f.fset.MtimeFS()
 	fchan := scanner.Walk(f.ctx, scanner.Config{
 		Folder:                f.ID,
 		Subs:                  subDirs,
@@ -348,6 +359,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 		ProgressTickIntervalS: f.ScanProgressIntervalS,
 		LocalFlags:            f.localFlags,
 		ModTimeWindow:         f.ModTimeWindow(),
+		EventLogger:           f.evLogger,
 	})
 
 	batchFn := func(fs []protocol.FileInfo) error {
@@ -466,7 +478,7 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 					return true
 				}
 
-				l.Debugln("marking file as ignored", f)
+				l.Debugln("marking file as ignored", file)
 				nf := file.ConvertToIgnoredFileInfo(f.shortID)
 				batch.append(nf)
 				changes++
@@ -500,12 +512,11 @@ func (f *folder) scanSubdirs(subDirs []string) error {
 					LocalFlags: f.localFlags,
 				}
 				// We do not want to override the global version
-				// with the deleted file. Keeping only our local
-				// counter makes sure we are in conflict with any
-				// other existing versions, which will be resolved
-				// by the normal pulling mechanisms.
+				// with the deleted file. Setting to an empty
+				// version makes sure the file gets in sync on
+				// the following pull.
 				if file.ShouldConflict() {
-					nf.Version = nf.Version.DropOthers(f.shortID)
+					nf.Version = protocol.Vector{}
 				}
 
 				batch.append(nf)
@@ -630,7 +641,7 @@ func (f *folder) monitorWatch(ctx context.Context) {
 				failTimer.Reset(time.Minute)
 				continue
 			}
-			watchaggregator.Aggregate(eventChan, f.watchChan, f.FolderConfiguration, f.model.cfg, aggrCtx)
+			watchaggregator.Aggregate(eventChan, f.watchChan, f.FolderConfiguration, f.model.cfg, f.evLogger, aggrCtx)
 			l.Debugln("Started filesystem watcher for folder", f.Description())
 		case err = <-errChan:
 			f.setWatchError(err)
@@ -669,7 +680,7 @@ func (f *folder) setWatchError(err error) {
 		if err != nil {
 			data["to"] = err.Error()
 		}
-		events.Default.Log(events.FolderWatchStateChanged, data)
+		f.evLogger.Log(events.FolderWatchStateChanged, data)
 	}
 	if err == nil {
 		return
@@ -800,7 +811,7 @@ func (f *folder) updateLocals(fs []protocol.FileInfo) {
 		filenames[i] = file.Name
 	}
 
-	events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{
+	f.evLogger.Log(events.LocalIndexUpdated, map[string]interface{}{
 		"folder":    f.ID,
 		"items":     len(fs),
 		"filenames": filenames,
@@ -839,7 +850,7 @@ func (f *folder) emitDiskChangeEvents(fs []protocol.FileInfo, typeOfEvent events
 		}
 
 		// Two different events can be fired here based on what EventType is passed into function
-		events.Default.Log(typeOfEvent, map[string]string{
+		f.evLogger.Log(typeOfEvent, map[string]string{
 			"folder":     f.ID,
 			"folderID":   f.ID, // incorrect, deprecated, kept for historical compliance
 			"label":      f.Label,

lib/model/folder_recvonly.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/ignore"
 	"github.com/syncthing/syncthing/lib/protocol"
@@ -56,8 +57,8 @@ type receiveOnlyFolder struct {
 	*sendReceiveFolder
 }
 
-func newReceiveOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service {
-	sr := newSendReceiveFolder(model, fset, ignores, cfg, ver, fs).(*sendReceiveFolder)
+func newReceiveOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem, evLogger events.Logger) service {
+	sr := newSendReceiveFolder(model, fset, ignores, cfg, ver, fs, evLogger).(*sendReceiveFolder)
 	sr.localFlags = protocol.FlagLocalReceiveOnly // gets propagated to the scanner, and set on locally changed files
 	return &receiveOnlyFolder{sr}
 }

lib/model/folder_recvonly_test.go

@@ -321,6 +321,7 @@ func setupROFolder() (*model, *sendOnlyFolder) {
 
 	f := &sendOnlyFolder{
 		folder: folder{
+			stateTracker:        newStateTracker(fcfg.ID, m.evLogger),
 			fset:                m.folderFiles[fcfg.ID],
 			FolderConfiguration: fcfg,
 		},

lib/model/folder_sendonly.go

@@ -9,6 +9,7 @@ package model
 import (
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/ignore"
 	"github.com/syncthing/syncthing/lib/protocol"
@@ -24,9 +25,9 @@ type sendOnlyFolder struct {
 	folder
 }
 
-func newSendOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, _ versioner.Versioner, _ fs.Filesystem) service {
+func newSendOnlyFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, _ versioner.Versioner, _ fs.Filesystem, evLogger events.Logger) service {
 	f := &sendOnlyFolder{
-		folder: newFolder(model, fset, ignores, cfg),
+		folder: newFolder(model, fset, ignores, cfg, evLogger),
 	}
 	f.folder.puller = f
 	f.folder.Service = util.AsService(f.serve)

lib/model/folder_sendrecv.go

@@ -108,9 +108,9 @@ type sendReceiveFolder struct {
 	pullErrorsMut sync.Mutex
 }
 
-func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service {
+func newSendReceiveFolder(model *model, fset *db.FileSet, ignores *ignore.Matcher, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem, evLogger events.Logger) service {
 	f := &sendReceiveFolder{
-		folder:        newFolder(model, fset, ignores, cfg),
+		folder:        newFolder(model, fset, ignores, cfg, evLogger),
 		fs:            fs,
 		versioner:     ver,
 		queue:         newJobQueue(),
@@ -176,7 +176,6 @@ func (f *sendReceiveFolder) pull() bool {
 
 	l.Debugf("%v pulling", f)
 
-	f.setState(FolderSyncing)
 	f.clearPullErrors()
 
 	scanChan := make(chan string)
@@ -194,6 +193,10 @@ func (f *sendReceiveFolder) pull() bool {
 		default:
 		}
 
+		// Needs to be set on every loop, as the puller might have set
+		// it to FolderSyncing during the last iteration.
+		f.setState(FolderSyncPreparing)
+
 		changed := f.pullerIteration(scanChan)
 
 		l.Debugln(f, "changed", changed, "on try", tries+1)
@@ -211,7 +214,7 @@ func (f *sendReceiveFolder) pull() bool {
 	// errors preventing us. Flag this with a warning and
 	// wait a bit longer before retrying.
 	if errors := f.Errors(); len(errors) > 0 {
-		events.Default.Log(events.FolderErrors, map[string]interface{}{
+		f.evLogger.Log(events.FolderErrors, map[string]interface{}{
 			"folder": f.folderID,
 			"errors": errors,
 		})
@@ -544,7 +547,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan<
 
 	f.resetPullError(file.Name)
 
-	events.Default.Log(events.ItemStarted, map[string]string{
+	f.evLogger.Log(events.ItemStarted, map[string]string{
 		"folder": f.folderID,
 		"item":   file.Name,
 		"type":   "dir",
@@ -552,7 +555,7 @@ func (f *sendReceiveFolder) handleDir(file protocol.FileInfo, dbUpdateChan chan<
 	})
 
 	defer func() {
-		events.Default.Log(events.ItemFinished, map[string]interface{}{
+		f.evLogger.Log(events.ItemFinished, map[string]interface{}{
 			"folder": f.folderID,
 			"item":   file.Name,
 			"error":  events.Error(err),
@@ -700,7 +703,7 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan c
 
 	f.resetPullError(file.Name)
 
-	events.Default.Log(events.ItemStarted, map[string]string{
+	f.evLogger.Log(events.ItemStarted, map[string]string{
 		"folder": f.folderID,
 		"item":   file.Name,
 		"type":   "symlink",
@@ -708,7 +711,7 @@ func (f *sendReceiveFolder) handleSymlink(file protocol.FileInfo, dbUpdateChan c
 	})
 
 	defer func() {
-		events.Default.Log(events.ItemFinished, map[string]interface{}{
+		f.evLogger.Log(events.ItemFinished, map[string]interface{}{
 			"folder": f.folderID,
 			"item":   file.Name,
 			"error":  events.Error(err),
@@ -782,7 +785,7 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, dbUpdateChan chan<
 	// care not declare another err.
 	var err error
 
-	events.Default.Log(events.ItemStarted, map[string]string{
+	f.evLogger.Log(events.ItemStarted, map[string]string{
 		"folder": f.folderID,
 		"item":   file.Name,
 		"type":   "dir",
@@ -790,7 +793,7 @@ func (f *sendReceiveFolder) deleteDir(file protocol.FileInfo, dbUpdateChan chan<
 	})
 
 	defer func() {
-		events.Default.Log(events.ItemFinished, map[string]interface{}{
+		f.evLogger.Log(events.ItemFinished, map[string]interface{}{
 			"folder": f.folderID,
 			"item":   file.Name,
 			"error":  events.Error(err),
@@ -822,7 +825,7 @@ func (f *sendReceiveFolder) deleteFileWithCurrent(file, cur protocol.FileInfo, h
 
 	f.resetPullError(file.Name)
 
-	events.Default.Log(events.ItemStarted, map[string]string{
+	f.evLogger.Log(events.ItemStarted, map[string]string{
 		"folder": f.folderID,
 		"item":   file.Name,
 		"type":   "file",
@@ -833,7 +836,7 @@ func (f *sendReceiveFolder) deleteFileWithCurrent(file, cur protocol.FileInfo, h
 		if err != nil {
 			f.newPullError(file.Name, errors.Wrap(err, "delete file"))
 		}
-		events.Default.Log(events.ItemFinished, map[string]interface{}{
+		f.evLogger.Log(events.ItemFinished, map[string]interface{}{
 			"folder": f.folderID,
 			"item":   file.Name,
 			"error":  events.Error(err),
@@ -844,10 +847,17 @@ func (f *sendReceiveFolder) deleteFileWithCurrent(file, cur protocol.FileInfo, h
 
 	if !hasCur {
 		// We should never try to pull a deletion for a file we don't have in the DB.
-		l.Debugln(f, "not deleting file we don't have", file.Name)
+		l.Debugln(f, "not deleting file we don't have, but update db", file.Name)
 		dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile}
 		return
 	}
+
+	if err = osutil.TraversesSymlink(f.fs, filepath.Dir(file.Name)); err != nil {
+		l.Debugln(f, "not deleting file behind symlink on disk, but update db", file.Name)
+		dbUpdateChan <- dbUpdateJob{file, dbUpdateDeleteFile}
+		return
+	}
+
 	if err = f.checkToBeDeleted(cur, scanChan); err != nil {
 		return
 	}
@@ -897,13 +907,13 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db
 	// care not declare another err.
 	var err error
 
-	events.Default.Log(events.ItemStarted, map[string]string{
+	f.evLogger.Log(events.ItemStarted, map[string]string{
 		"folder": f.folderID,
 		"item":   source.Name,
 		"type":   "file",
 		"action": "delete",
 	})
-	events.Default.Log(events.ItemStarted, map[string]string{
+	f.evLogger.Log(events.ItemStarted, map[string]string{
 		"folder": f.folderID,
 		"item":   target.Name,
 		"type":   "file",
@@ -911,14 +921,14 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db
 	})
 
 	defer func() {
-		events.Default.Log(events.ItemFinished, map[string]interface{}{
+		f.evLogger.Log(events.ItemFinished, map[string]interface{}{
 			"folder": f.folderID,
 			"item":   source.Name,
 			"error":  events.Error(err),
 			"type":   "file",
 			"action": "delete",
 		})
-		events.Default.Log(events.ItemFinished, map[string]interface{}{
+		f.evLogger.Log(events.ItemFinished, map[string]interface{}{
 			"folder": f.folderID,
 			"item":   target.Name,
 			"error":  events.Error(err),
@@ -982,8 +992,9 @@ func (f *sendReceiveFolder) renameFile(cur, source, target protocol.FileInfo, db
 	}
 
 	blockStatsMut.Lock()
-	blockStats["total"] += len(target.Blocks)
-	blockStats["renamed"] += len(target.Blocks)
+	minBlocksPerBlock := target.BlockSize() / protocol.MinBlockSize
+	blockStats["total"] += len(target.Blocks) * minBlocksPerBlock
+	blockStats["renamed"] += len(target.Blocks) * minBlocksPerBlock
 	blockStatsMut.Unlock()
 
 	// The file was renamed, so we have handled both the necessary delete
@@ -1045,7 +1056,6 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
 	populateOffsets(file.Blocks)
 
 	blocks := make([]protocol.BlockInfo, 0, len(file.Blocks))
-	var blocksSize int64
 	reused := make([]int32, 0, len(file.Blocks))
 
 	// Check for an old temporary file which might have some blocks we could
@@ -1066,7 +1076,6 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
 			_, ok := existingBlocks[block.String()]
 			if !ok {
 				blocks = append(blocks, block)
-				blocksSize += int64(block.Size)
 			} else {
 				reused = append(reused, int32(i))
 			}
@@ -1083,19 +1092,12 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
 	} else {
 		// Copy the blocks, as we don't want to shuffle them on the FileInfo
 		blocks = append(blocks, file.Blocks...)
-		blocksSize = file.Size
-	}
-
-	if err := f.CheckAvailableSpace(blocksSize); err != nil {
-		f.newPullError(file.Name, err)
-		f.queue.Done(file.Name)
-		return
 	}
 
 	// Shuffle the blocks
 	rand.Shuffle(blocks)
 
-	events.Default.Log(events.ItemStarted, map[string]string{
+	f.evLogger.Log(events.ItemStarted, map[string]string{
 		"folder": f.folderID,
 		"item":   file.Name,
 		"type":   "file",
@@ -1178,7 +1180,7 @@ func (f *sendReceiveFolder) shortcutFile(file, curFile protocol.FileInfo, dbUpda
 
 	f.resetPullError(file.Name)
 
-	events.Default.Log(events.ItemStarted, map[string]string{
+	f.evLogger.Log(events.ItemStarted, map[string]string{
 		"folder": f.folderID,
 		"item":   file.Name,
 		"type":   "file",
@@ -1186,7 +1188,7 @@ func (f *sendReceiveFolder) shortcutFile(file, curFile protocol.FileInfo, dbUpda
 	})
 
 	var err error
-	defer events.Default.Log(events.ItemFinished, map[string]interface{}{
+	defer f.evLogger.Log(events.ItemFinished, map[string]interface{}{
 		"folder": f.folderID,
 		"item":   file.Name,
 		"error":  events.Error(err),
@@ -1221,6 +1223,13 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
 	}()
 
 	for state := range in {
+		if err := f.CheckAvailableSpace(state.file.Size); err != nil {
+			state.fail(err)
+			// Nothing more to do for this failed file, since it would use to much disk space
+			out <- state.sharedPullerState
+			continue
+		}
+
 		dstFd, err := state.tempFile()
 		if err != nil {
 			// Nothing more to do for this failed file, since we couldn't create a temporary for it.
@@ -1397,6 +1406,8 @@ func (f *sendReceiveFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *
 			continue
 		}
 
+		f.setState(FolderSyncing) // Does nothing if already FolderSyncing
+
 		// The requestLimiter limits how many pending block requests we have
 		// ongoing at any given time, based on the size of the blocks
 		// themselves.
@@ -1561,21 +1572,22 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState, dbUpda
 			if err != nil {
 				f.newPullError(state.file.Name, err)
 			} else {
+				minBlocksPerBlock := state.file.BlockSize() / protocol.MinBlockSize
 				blockStatsMut.Lock()
-				blockStats["total"] += state.reused + state.copyTotal + state.pullTotal
-				blockStats["reused"] += state.reused
-				blockStats["pulled"] += state.pullTotal
+				blockStats["total"] += (state.reused + state.copyTotal + state.pullTotal) * minBlocksPerBlock
+				blockStats["reused"] += state.reused * minBlocksPerBlock
+				blockStats["pulled"] += state.pullTotal * minBlocksPerBlock
 				// copyOriginShifted is counted towards copyOrigin due to progress bar reasons
 				// for reporting reasons we want to separate these.
-				blockStats["copyOrigin"] += state.copyOrigin - state.copyOriginShifted
-				blockStats["copyOriginShifted"] += state.copyOriginShifted
-				blockStats["copyElsewhere"] += state.copyTotal - state.copyOrigin
+				blockStats["copyOrigin"] += (state.copyOrigin - state.copyOriginShifted) * minBlocksPerBlock
+				blockStats["copyOriginShifted"] += state.copyOriginShifted * minBlocksPerBlock
+				blockStats["copyElsewhere"] += (state.copyTotal - state.copyOrigin) * minBlocksPerBlock
 				blockStatsMut.Unlock()
 			}
 
 			f.model.progressEmitter.Deregister(state)
 
-			events.Default.Log(events.ItemFinished, map[string]interface{}{
+			f.evLogger.Log(events.ItemFinished, map[string]interface{}{
 				"folder": f.folderID,
 				"item":   state.file.Name,
 				"error":  events.Error(err),
@@ -1834,6 +1846,10 @@ func (f *sendReceiveFolder) deleteItemOnDisk(item protocol.FileInfo, scanChan ch
 // deleteDirOnDisk attempts to delete a directory. It checks for files/dirs inside
 // the directory and removes them if possible or returns an error if it fails
 func (f *sendReceiveFolder) deleteDirOnDisk(dir string, scanChan chan<- string) error {
+	if err := osutil.TraversesSymlink(f.fs, filepath.Dir(dir)); err != nil {
+		return err
+	}
+
 	files, _ := f.fs.DirNames(dir)
 
 	toBeDeleted := make([]string, 0, len(files))

lib/model/folder_sendrecv_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/fs"
 	"github.com/syncthing/syncthing/lib/ignore"
+	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/scanner"
 	"github.com/syncthing/syncthing/lib/sync"
@@ -96,7 +97,7 @@ func setupSendReceiveFolder(files ...protocol.FileInfo) (*model, *sendReceiveFol
 
 	f := &sendReceiveFolder{
 		folder: folder{
-			stateTracker:        newStateTracker("default"),
+			stateTracker:        newStateTracker("default", model.evLogger),
 			model:               model,
 			fset:                model.folderFiles[fcfg.ID],
 			initialScanFinished: make(chan struct{}),
@@ -121,6 +122,12 @@ func setupSendReceiveFolder(files ...protocol.FileInfo) (*model, *sendReceiveFol
 	return model, f
 }
 
+func cleanupSRFolder(f *sendReceiveFolder, m *model) {
+	m.evLogger.Stop()
+	os.Remove(m.cfg.ConfigPath())
+	os.RemoveAll(f.Filesystem().URI())
+}
+
 // Layout of the files: (indexes from the above array)
 // 12345678 - Required file
 // 02005008 - Existing file (currently in the index)
@@ -137,10 +144,7 @@ func TestHandleFile(t *testing.T) {
 	requiredFile.Blocks = blocks[1:]
 
 	m, f := setupSendReceiveFolder(existingFile)
-	defer func() {
-		os.Remove(m.cfg.ConfigPath())
-		os.Remove(f.Filesystem().URI())
-	}()
+	defer cleanupSRFolder(f, m)
 
 	copyChan := make(chan copyBlocksState, 1)
 	dbUpdateChan := make(chan dbUpdateJob, 1)
@@ -183,10 +187,7 @@ func TestHandleFileWithTemp(t *testing.T) {
 	requiredFile.Blocks = blocks[1:]
 
 	m, f := setupSendReceiveFolder(existingFile)
-	defer func() {
-		os.Remove(m.cfg.ConfigPath())
-		os.Remove(f.Filesystem().URI())
-	}()
+	defer cleanupSRFolder(f, m)
 
 	if _, err := prepareTmpFile(f.Filesystem()); err != nil {
 		t.Fatal(err)
@@ -236,10 +237,7 @@ func TestCopierFinder(t *testing.T) {
 	requiredFile.Name = "file2"
 
 	m, f := setupSendReceiveFolder(existingFile)
-	defer func() {
-		os.Remove(m.cfg.ConfigPath())
-		os.Remove(f.Filesystem().URI())
-	}()
+	defer cleanupSRFolder(f, m)
 
 	if _, err := prepareTmpFile(f.Filesystem()); err != nil {
 		t.Fatal(err)
@@ -255,8 +253,23 @@ func TestCopierFinder(t *testing.T) {
 
 	f.handleFile(requiredFile, copyChan, dbUpdateChan)
 
-	pulls := []pullBlockState{<-pullChan, <-pullChan, <-pullChan, <-pullChan}
-	finish := <-finisherChan
+	timeout := time.After(10 * time.Second)
+	pulls := make([]pullBlockState, 4)
+	for i := 0; i < 4; i++ {
+		select {
+		case pulls[i] = <-pullChan:
+		case <-timeout:
+			t.Fatalf("Timed out before receiving all 4 states on pullChan (already got %v)", i)
+		}
+	}
+	var finish *sharedPullerState
+	select {
+	case finish = <-finisherChan:
+	case <-timeout:
+		t.Fatal("Timed out before receiving 4 states on pullChan")
+	}
+
+	defer cleanupSharedPullerState(finish)
 
 	select {
 	case <-pullChan:
@@ -296,17 +309,13 @@ func TestCopierFinder(t *testing.T) {
 			t.Errorf("Block %d mismatch: %s != %s", eq, blks[eq-1].String(), blocks[eq].String())
 		}
 	}
-	finish.fd.Close()
 }
 
 func TestWeakHash(t *testing.T) {
 	// Setup the model/pull environment
 	model, fo := setupSendReceiveFolder()
+	defer cleanupSRFolder(fo, model)
 	ffs := fo.Filesystem()
-	defer func() {
-		os.Remove(model.cfg.ConfigPath())
-		os.Remove(ffs.URI())
-	}()
 
 	tempFile := fs.TempName("weakhash")
 	var shift int64 = 10
@@ -395,7 +404,7 @@ func TestWeakHash(t *testing.T) {
 	default:
 	}
 
-	finish.fd.Close()
+	cleanupSharedPullerState(finish)
 	if err := ffs.Remove(tempFile); err != nil {
 		t.Fatal(err)
 	}
@@ -415,7 +424,7 @@ func TestWeakHash(t *testing.T) {
 	}
 
 	finish = <-finisherChan
-	finish.fd.Close()
+	cleanupSharedPullerState(finish)
 
 	expectShifted := expectBlocks - expectPulls
 	if finish.copyOriginShifted != expectShifted {
@@ -432,10 +441,7 @@ func TestCopierCleanup(t *testing.T) {
 	// Create a file
 	file := setupFile("test", []int{0})
 	m, f := setupSendReceiveFolder(file)
-	defer func() {
-		os.Remove(m.cfg.ConfigPath())
-		os.Remove(f.Filesystem().URI())
-	}()
+	defer cleanupSRFolder(f, m)
 
 	file.Blocks = []protocol.BlockInfo{blocks[1]}
 	file.Version = file.Version.Update(myID.Short())
@@ -468,13 +474,10 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
 	file := setupFile("filex", []int{0, 2, 0, 0, 5, 0, 0, 8})
 
 	m, f := setupSendReceiveFolder()
-	defer func() {
-		os.Remove(m.cfg.ConfigPath())
-		os.Remove(f.Filesystem().URI())
-	}()
+	defer cleanupSRFolder(f, m)
 
 	// Set up our evet subscription early
-	s := events.Default.Subscribe(events.ItemFinished)
+	s := m.evLogger.Subscribe(events.ItemFinished)
 
 	// queue.Done should be called by the finisher routine
 	f.queue.Push("filex", 0, time.Time{})
@@ -528,9 +531,9 @@ func TestDeregisterOnFailInCopy(t *testing.T) {
 		t.Log("event took", time.Since(t0))
 
 		state.mut.Lock()
-		stateFd := state.fd
+		stateWriter := state.writer
 		state.mut.Unlock()
-		if stateFd != nil {
+		if stateWriter != nil {
 			t.Fatal("File not closed?")
 		}
 
@@ -558,13 +561,10 @@ func TestDeregisterOnFailInPull(t *testing.T) {
 	file := setupFile("filex", []int{0, 2, 0, 0, 5, 0, 0, 8})
 
 	m, f := setupSendReceiveFolder()
-	defer func() {
-		os.Remove(m.cfg.ConfigPath())
-		os.Remove(f.Filesystem().URI())
-	}()
+	defer cleanupSRFolder(f, m)
 
 	// Set up our evet subscription early
-	s := events.Default.Subscribe(events.ItemFinished)
+	s := m.evLogger.Subscribe(events.ItemFinished)
 
 	// queue.Done should be called by the finisher routine
 	f.queue.Push("filex", 0, time.Time{})
@@ -609,9 +609,9 @@ func TestDeregisterOnFailInPull(t *testing.T) {
 		t.Log("event took", time.Since(t0))
 
 		state.mut.Lock()
-		stateFd := state.fd
+		stateWriter := state.writer
 		state.mut.Unlock()
-		if stateFd != nil {
+		if stateWriter != nil {
 			t.Fatal("File not closed?")
 		}
 
@@ -636,12 +636,9 @@ func TestDeregisterOnFailInPull(t *testing.T) {
 
 func TestIssue3164(t *testing.T) {
 	m, f := setupSendReceiveFolder()
+	defer cleanupSRFolder(f, m)
 	ffs := f.Filesystem()
 	tmpDir := ffs.URI()
-	defer func() {
-		os.Remove(m.cfg.ConfigPath())
-		os.Remove(tmpDir)
-	}()
 
 	ignDir := filepath.Join("issue3164", "oktodelete")
 	subDir := filepath.Join(ignDir, "foobar")
@@ -728,11 +725,8 @@ func TestDiffEmpty(t *testing.T) {
 // in the db.
 func TestDeleteIgnorePerms(t *testing.T) {
 	m, f := setupSendReceiveFolder()
+	defer cleanupSRFolder(f, m)
 	ffs := f.Filesystem()
-	defer func() {
-		os.Remove(m.cfg.ConfigPath())
-		os.Remove(ffs.URI())
-	}()
 	f.IgnorePerms = true
 
 	name := "deleteIgnorePerms"
@@ -778,7 +772,7 @@ func TestCopyOwner(t *testing.T) {
 	// filesystem.
 
 	m, f := setupSendReceiveFolder()
-	defer os.Remove(m.cfg.ConfigPath())
+	defer cleanupSRFolder(f, m)
 	f.folder.FolderConfiguration = config.NewFolderConfiguration(m.id, f.ID, f.Label, fs.FilesystemTypeFake, "/TestCopyOwner")
 	f.folder.FolderConfiguration.CopyOwnershipFromParent = true
 
@@ -867,11 +861,8 @@ func TestCopyOwner(t *testing.T) {
 // is replaced with a directory and versions are conflicting
 func TestSRConflictReplaceFileByDir(t *testing.T) {
 	m, f := setupSendReceiveFolder()
+	defer cleanupSRFolder(f, m)
 	ffs := f.Filesystem()
-	defer func() {
-		os.Remove(m.cfg.ConfigPath())
-		os.Remove(ffs.URI())
-	}()
 
 	name := "foo"
 
@@ -902,11 +893,8 @@ func TestSRConflictReplaceFileByDir(t *testing.T) {
 // is replaced with a link and versions are conflicting
 func TestSRConflictReplaceFileByLink(t *testing.T) {
 	m, f := setupSendReceiveFolder()
+	defer cleanupSRFolder(f, m)
 	ffs := f.Filesystem()
-	defer func() {
-		os.Remove(m.cfg.ConfigPath())
-		os.Remove(ffs.URI())
-	}()
 
 	name := "foo"
 
@@ -933,3 +921,66 @@ func TestSRConflictReplaceFileByLink(t *testing.T) {
 		t.Fatal("Expected request to scan", confls[0], "got", scan)
 	}
 }
+
+// TestDeleteBehindSymlink checks that we don't delete or schedule a scan
+// when trying to delete a file behind a symlink.
+func TestDeleteBehindSymlink(t *testing.T) {
+	m, f := setupSendReceiveFolder()
+	defer cleanupSRFolder(f, m)
+	ffs := f.Filesystem()
+
+	destDir := createTmpDir()
+	defer os.RemoveAll(destDir)
+	destFs := fs.NewFilesystem(fs.FilesystemTypeBasic, destDir)
+
+	link := "link"
+	file := filepath.Join(link, "file")
+
+	must(t, ffs.MkdirAll(link, 0755))
+	fi := createFile(t, file, ffs)
+	f.updateLocalsFromScanning([]protocol.FileInfo{fi})
+	must(t, osutil.RenameOrCopy(ffs, destFs, file, "file"))
+	must(t, ffs.RemoveAll(link))
+
+	if err := osutil.DebugSymlinkForTestsOnly(destFs.URI(), filepath.Join(ffs.URI(), link)); err != nil {
+		if runtime.GOOS == "windows" {
+			// Probably we require permissions we don't have.
+			t.Skip("Need admin permissions or developer mode to run symlink test on Windows: " + err.Error())
+		} else {
+			t.Fatal(err)
+		}
+	}
+
+	fi.Deleted = true
+	fi.Version = fi.Version.Update(device1.Short())
+	scanChan := make(chan string, 1)
+	dbUpdateChan := make(chan dbUpdateJob, 1)
+	f.deleteFile(fi, dbUpdateChan, scanChan)
+	select {
+	case f := <-scanChan:
+		t.Fatalf("Received %v on scanChan", f)
+	case u := <-dbUpdateChan:
+		if u.jobType != dbUpdateDeleteFile {
+			t.Errorf("Expected jobType %v, got %v", dbUpdateDeleteFile, u.jobType)
+		}
+		if u.file.Name != fi.Name {
+			t.Errorf("Expected update for %v, got %v", fi.Name, u.file.Name)
+		}
+	default:
+		t.Fatalf("No db update received")
+	}
+	if _, err := destFs.Stat("file"); err != nil {
+		t.Errorf("Expected no error when stating file behind symlink, got %v", err)
+	}
+}
+
+func cleanupSharedPullerState(s *sharedPullerState) {
+	s.mut.Lock()
+	defer s.mut.Unlock()
+	if s.writer == nil {
+		return
+	}
+	s.writer.mut.Lock()
+	s.writer.fd.Close()
+	s.writer.mut.Unlock()
+}

lib/model/folder_summary.go

@@ -36,6 +36,7 @@ type folderSummaryService struct {
 	cfg       config.Wrapper
 	model     Model
 	id        protocol.DeviceID
+	evLogger  events.Logger
 	immediate chan string
 
 	// For keeping track of folders to recalculate for
@@ -47,7 +48,7 @@ type folderSummaryService struct {
 	lastEventReqMut sync.Mutex
 }
 
-func NewFolderSummaryService(cfg config.Wrapper, m Model, id protocol.DeviceID) FolderSummaryService {
+func NewFolderSummaryService(cfg config.Wrapper, m Model, id protocol.DeviceID, evLogger events.Logger) FolderSummaryService {
 	service := &folderSummaryService{
 		Supervisor: suture.New("folderSummaryService", suture.Spec{
 			PassThroughPanics: true,
@@ -55,6 +56,7 @@ func NewFolderSummaryService(cfg config.Wrapper, m Model, id protocol.DeviceID)
 		cfg:             cfg,
 		model:           m,
 		id:              id,
+		evLogger:        evLogger,
 		immediate:       make(chan string),
 		folders:         make(map[string]struct{}),
 		foldersMut:      sync.NewMutex(),
@@ -144,8 +146,8 @@ func (c *folderSummaryService) OnEventRequest() {
 // listenForUpdates subscribes to the event bus and makes note of folders that
 // need their data recalculated.
 func (c *folderSummaryService) listenForUpdates(stop chan struct{}) {
-	sub := events.Default.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged | events.RemoteDownloadProgress | events.DeviceConnected | events.FolderWatchStateChanged | events.DownloadProgress)
-	defer events.Default.Unsubscribe(sub)
+	sub := c.evLogger.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged | events.RemoteDownloadProgress | events.DeviceConnected | events.FolderWatchStateChanged | events.DownloadProgress)
+	defer sub.Unsubscribe()
 
 	for {
 		// This loop needs to be fast so we don't miss too many events.
@@ -291,7 +293,7 @@ func (c *folderSummaryService) sendSummary(folder string) {
 	if err != nil {
 		return
 	}
-	events.Default.Log(events.FolderSummary, map[string]interface{}{
+	c.evLogger.Log(events.FolderSummary, map[string]interface{}{
 		"folder":  folder,
 		"summary": data,
 	})
@@ -311,6 +313,6 @@ func (c *folderSummaryService) sendSummary(folder string) {
 		comp := c.model.Completion(devCfg.DeviceID, folder).Map()
 		comp["folder"] = folder
 		comp["device"] = devCfg.DeviceID.String()
-		events.Default.Log(events.FolderCompletion, comp)
+		c.evLogger.Log(events.FolderCompletion, comp)
 	}
 }

lib/model/folderstate.go

@@ -19,6 +19,7 @@ const (
 	FolderIdle folderState = iota
 	FolderScanning
 	FolderScanWaiting
+	FolderSyncPreparing
 	FolderSyncing
 	FolderError
 )
@@ -31,6 +32,8 @@ func (s folderState) String() string {
 		return "scanning"
 	case FolderScanWaiting:
 		return "scan-waiting"
+	case FolderSyncPreparing:
+		return "sync-preparing"
 	case FolderSyncing:
 		return "syncing"
 	case FolderError:
@@ -42,6 +45,7 @@ func (s folderState) String() string {
 
 type stateTracker struct {
 	folderID string
+	evLogger events.Logger
 
 	mut     sync.Mutex
 	current folderState
@@ -49,9 +53,10 @@ type stateTracker struct {
 	changed time.Time
 }
 
-func newStateTracker(id string) stateTracker {
+func newStateTracker(id string, evLogger events.Logger) stateTracker {
 	return stateTracker{
 		folderID: id,
+		evLogger: evLogger,
 		mut:      sync.NewMutex(),
 	}
 }
@@ -63,29 +68,32 @@ func (s *stateTracker) setState(newState folderState) {
 	}
 
 	s.mut.Lock()
-	if newState != s.current {
-		/* This should hold later...
-		if s.current != FolderIdle && (newState == FolderScanning || newState == FolderSyncing) {
-			panic("illegal state transition " + s.current.String() + " -> " + newState.String())
-		}
-		*/
+	defer s.mut.Unlock()
 
-		eventData := map[string]interface{}{
-			"folder": s.folderID,
-			"to":     newState.String(),
-			"from":   s.current.String(),
-		}
-
-		if !s.changed.IsZero() {
-			eventData["duration"] = time.Since(s.changed).Seconds()
-		}
-
-		s.current = newState
-		s.changed = time.Now()
-
-		events.Default.Log(events.StateChanged, eventData)
+	if newState == s.current {
+		return
 	}
-	s.mut.Unlock()
+
+	/* This should hold later...
+	if s.current != FolderIdle && (newState == FolderScanning || newState == FolderSyncing) {
+		panic("illegal state transition " + s.current.String() + " -> " + newState.String())
+	}
+	*/
+
+	eventData := map[string]interface{}{
+		"folder": s.folderID,
+		"to":     newState.String(),
+		"from":   s.current.String(),
+	}
+
+	if !s.changed.IsZero() {
+		eventData["duration"] = time.Since(s.changed).Seconds()
+	}
+
+	s.current = newState
+	s.changed = time.Now()
+
+	s.evLogger.Log(events.StateChanged, eventData)
 }
 
 // getState returns the current state, the time when it last changed, and the
@@ -124,5 +132,5 @@ func (s *stateTracker) setError(err error) {
 	s.err = err
 	s.changed = time.Now()
 
-	events.Default.Log(events.StateChanged, eventData)
+	s.evLogger.Log(events.StateChanged, eventData)
 }

lib/model/model.go

@@ -128,6 +128,7 @@ type model struct {
 	shortID           protocol.ShortID
 	cacheIgnoredFiles bool
 	protectedFiles    []string
+	evLogger          events.Logger
 
 	clientName    string
 	clientVersion string
@@ -152,7 +153,7 @@ type model struct {
 	foldersRunning int32 // for testing only
 }
 
-type folderFactory func(*model, *db.FileSet, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, fs.Filesystem) service
+type folderFactory func(*model, *db.FileSet, *ignore.Matcher, config.FolderConfiguration, versioner.Versioner, fs.Filesystem, events.Logger) service
 
 var (
 	folderFactories = make(map[config.FolderType]folderFactory)
@@ -175,7 +176,7 @@ var (
 // NewModel creates and starts a new model. The model starts in read-only mode,
 // where it sends index information to connected peers and responds to requests
 // for file data without altering the local folder in any way.
-func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Lowlevel, protectedFiles []string) Model {
+func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersion string, ldb *db.Lowlevel, protectedFiles []string, evLogger events.Logger) Model {
 	m := &model{
 		Supervisor: suture.New("model", suture.Spec{
 			Log: func(line string) {
@@ -186,11 +187,12 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersio
 		cfg:                 cfg,
 		db:                  ldb,
 		finder:              db.NewBlockFinder(ldb),
-		progressEmitter:     NewProgressEmitter(cfg),
+		progressEmitter:     NewProgressEmitter(cfg, evLogger),
 		id:                  id,
 		shortID:             id.Short(),
 		cacheIgnoredFiles:   cfg.Options().CacheIgnoredFiles,
 		protectedFiles:      protectedFiles,
+		evLogger:            evLogger,
 		clientName:          clientName,
 		clientVersion:       clientVersion,
 		folderCfgs:          make(map[string]config.FolderConfiguration),
@@ -208,6 +210,9 @@ func NewModel(cfg config.Wrapper, id protocol.DeviceID, clientName, clientVersio
 		fmut:                sync.NewRWMutex(),
 		pmut:                sync.NewRWMutex(),
 	}
+	for devID := range cfg.Devices() {
+		m.deviceStatRefs[devID] = stats.NewDeviceStatisticsReference(m.db, devID.String())
+	}
 	m.Add(m.progressEmitter)
 	scanLimiter.setCapacity(cfg.Options().MaxConcurrentScans)
 	cfg.Subscribe(m)
@@ -310,7 +315,7 @@ func (m *model) startFolderLocked(cfg config.FolderConfiguration) {
 	ffs.Hide(".stversions")
 	ffs.Hide(".stignore")
 
-	p := folderFactory(m, fset, m.folderIgnores[folder], cfg, ver, ffs)
+	p := folderFactory(m, fset, m.folderIgnores[folder], cfg, ver, ffs, m.evLogger)
 
 	m.folderRunners[folder] = p
 
@@ -387,8 +392,17 @@ func (m *model) RemoveFolder(cfg config.FolderConfiguration) {
 	m.fmut.Lock()
 	defer m.fmut.Unlock()
 
-	// Delete syncthing specific files
-	cfg.Filesystem().RemoveAll(config.DefaultMarkerName)
+	isPathUnique := true
+	for folderID, folderCfg := range m.folderCfgs {
+		if folderID != cfg.ID && folderCfg.Path == cfg.Path {
+			isPathUnique = false
+			break
+		}
+	}
+	if isPathUnique {
+		// Delete syncthing specific files
+		cfg.Filesystem().RemoveAll(config.DefaultMarkerName)
+	}
 
 	m.tearDownFolderLocked(cfg, fmt.Errorf("removing folder %v", cfg.Description()))
 	// Remove it from the database
@@ -642,9 +656,11 @@ func (m *model) ConnectionStats() map[string]interface{} {
 
 // DeviceStatistics returns statistics about each device
 func (m *model) DeviceStatistics() map[string]stats.DeviceStatistics {
-	res := make(map[string]stats.DeviceStatistics)
-	for id := range m.cfg.Devices() {
-		res[id.String()] = m.deviceStatRef(id).GetStatistics()
+	m.fmut.RLock()
+	defer m.fmut.RUnlock()
+	res := make(map[string]stats.DeviceStatistics, len(m.deviceStatRefs))
+	for id, sr := range m.deviceStatRefs {
+		res[id.String()] = sr.GetStatistics()
 	}
 	return res
 }
@@ -1023,7 +1039,7 @@ func (m *model) handleIndex(deviceID protocol.DeviceID, folder string, fs []prot
 	}
 	files.Update(deviceID, fs)
 
-	events.Default.Log(events.RemoteIndexUpdated, map[string]interface{}{
+	m.evLogger.Log(events.RemoteIndexUpdated, map[string]interface{}{
 		"device":  deviceID.String(),
 		"folder":  folder,
 		"items":   len(fs),
@@ -1077,7 +1093,7 @@ func (m *model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
 			}
 			m.cfg.AddOrUpdatePendingFolder(folder.ID, folder.Label, deviceID)
 			changed = true
-			events.Default.Log(events.FolderRejected, map[string]string{
+			m.evLogger.Log(events.FolderRejected, map[string]string{
 				"folder":      folder.ID,
 				"folderLabel": folder.Label,
 				"device":      deviceID.String(),
@@ -1180,6 +1196,7 @@ func (m *model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
 			fset:         fs,
 			prevSequence: startSequence,
 			dropSymlinks: dropSymlinks,
+			evLogger:     m.evLogger,
 		}
 		is.Service = util.AsService(is.serve)
 		// The token isn't tracked as the service stops when the connection
@@ -1417,12 +1434,11 @@ func (m *model) Closed(conn protocol.Connection, err error) {
 	device := conn.ID()
 
 	m.pmut.Lock()
-	defer m.pmut.Unlock()
 	conn, ok := m.conn[device]
 	if !ok {
+		m.pmut.Unlock()
 		return
 	}
-	m.progressEmitter.temporaryIndexUnsubscribe(conn)
 	delete(m.conn, device)
 	delete(m.connRequestLimiters, device)
 	delete(m.helloMessages, device)
@@ -1430,9 +1446,12 @@ func (m *model) Closed(conn protocol.Connection, err error) {
 	delete(m.remotePausedFolders, device)
 	closed := m.closed[device]
 	delete(m.closed, device)
+	m.pmut.Unlock()
+
+	m.progressEmitter.temporaryIndexUnsubscribe(conn)
 
 	l.Infof("Connection to %s at %s closed: %v", device, conn.Name(), err)
-	events.Default.Log(events.DeviceDisconnected, map[string]string{
+	m.evLogger.Log(events.DeviceDisconnected, map[string]string{
 		"id":    device.String(),
 		"error": err.Error(),
 	})
@@ -1444,14 +1463,14 @@ func (m *model) Closed(conn protocol.Connection, err error) {
 func (m *model) closeConns(devs []protocol.DeviceID, err error) config.Waiter {
 	conns := make([]connections.Connection, 0, len(devs))
 	closed := make([]chan struct{}, 0, len(devs))
-	m.pmut.Lock()
+	m.pmut.RLock()
 	for _, dev := range devs {
 		if conn, ok := m.conn[dev]; ok {
 			conns = append(conns, conn)
 			closed = append(closed, m.closed[dev])
 		}
 	}
-	m.pmut.Unlock()
+	m.pmut.RUnlock()
 	for _, conn := range conns {
 		conn.Close(err)
 	}
@@ -1773,7 +1792,7 @@ func (m *model) OnHello(remoteID protocol.DeviceID, addr net.Addr, hello protoco
 	if !ok {
 		m.cfg.AddOrUpdatePendingDevice(remoteID, hello.DeviceName, addr.String())
 		_ = m.cfg.Save() // best effort
-		events.Default.Log(events.DeviceRejected, map[string]string{
+		m.evLogger.Log(events.DeviceRejected, map[string]string{
 			"name":    hello.DeviceName,
 			"device":  remoteID.String(),
 			"address": addr.String(),
@@ -1859,7 +1878,7 @@ func (m *model) AddConnection(conn connections.Connection, hello protocol.HelloR
 		event["addr"] = addr.String()
 	}
 
-	events.Default.Log(events.DeviceConnected, event)
+	m.evLogger.Log(events.DeviceConnected, event)
 
 	l.Infof(`Device %s client is "%s %s" named "%s" at %s`, deviceID, hello.ClientName, hello.ClientVersion, hello.DeviceName, conn)
 
@@ -1894,28 +1913,20 @@ func (m *model) DownloadProgress(device protocol.DeviceID, folder string, update
 	downloads.Update(folder, updates)
 	state := downloads.GetBlockCounts(folder)
 
-	events.Default.Log(events.RemoteDownloadProgress, map[string]interface{}{
+	m.evLogger.Log(events.RemoteDownloadProgress, map[string]interface{}{
 		"device": device.String(),
 		"folder": folder,
 		"state":  state,
 	})
 }
 
-func (m *model) deviceStatRef(deviceID protocol.DeviceID) *stats.DeviceStatisticsReference {
-	m.fmut.Lock()
-	defer m.fmut.Unlock()
-
-	if sr, ok := m.deviceStatRefs[deviceID]; ok {
-		return sr
-	}
-
-	sr := stats.NewDeviceStatisticsReference(m.db, deviceID.String())
-	m.deviceStatRefs[deviceID] = sr
-	return sr
-}
-
 func (m *model) deviceWasSeen(deviceID protocol.DeviceID) {
-	m.deviceStatRef(deviceID).WasSeen()
+	m.fmut.RLock()
+	sr, ok := m.deviceStatRefs[deviceID]
+	m.fmut.RUnlock()
+	if ok {
+		sr.WasSeen()
+	}
 }
 
 type indexSender struct {
@@ -1926,6 +1937,7 @@ type indexSender struct {
 	fset         *db.FileSet
 	prevSequence int64
 	dropSymlinks bool
+	evLogger     events.Logger
 	connClosed   chan struct{}
 }
 
@@ -1941,8 +1953,8 @@ func (s *indexSender) serve(stop chan struct{}) {
 	// Subscribe to LocalIndexUpdated (we have new information to send) and
 	// DeviceDisconnected (it might be us who disconnected, so we should
 	// exit).
-	sub := events.Default.Subscribe(events.LocalIndexUpdated | events.DeviceDisconnected)
-	defer events.Default.Unsubscribe(sub)
+	sub := s.evLogger.Subscribe(events.LocalIndexUpdated | events.DeviceDisconnected)
+	defer sub.Unsubscribe()
 
 	evChan := sub.C()
 	ticker := time.NewTicker(time.Minute)
@@ -2119,9 +2131,9 @@ func (m *model) ScanFolderSubdirs(folder string, subs []string) error {
 }
 
 func (m *model) DelayScan(folder string, next time.Duration) {
-	m.fmut.Lock()
+	m.fmut.RLock()
 	runner, ok := m.folderRunners[folder]
-	m.fmut.Unlock()
+	m.fmut.RUnlock()
 	if !ok {
 		return
 	}
@@ -2131,10 +2143,10 @@ func (m *model) DelayScan(folder string, next time.Duration) {
 // numHashers returns the number of hasher routines to use for a given folder,
 // taking into account configuration and available CPU cores.
 func (m *model) numHashers(folder string) int {
-	m.fmut.Lock()
+	m.fmut.RLock()
 	folderCfg := m.folderCfgs[folder]
 	numFolders := len(m.folderCfgs)
-	m.fmut.Unlock()
+	m.fmut.RUnlock()
 
 	if folderCfg.Hashers > 0 {
 		// Specific value set in the config, use that.
@@ -2531,7 +2543,7 @@ func (m *model) CommitConfiguration(from, to config.Configuration) bool {
 			if toCfg.Paused {
 				eventType = events.FolderPaused
 			}
-			events.Default.Log(eventType, map[string]string{"id": toCfg.ID, "label": toCfg.Label})
+			m.evLogger.Log(eventType, map[string]string{"id": toCfg.ID, "label": toCfg.Label})
 		}
 	}
 
@@ -2547,7 +2559,15 @@ func (m *model) CommitConfiguration(from, to config.Configuration) bool {
 	toDevices := to.DeviceMap()
 	for deviceID, toCfg := range toDevices {
 		fromCfg, ok := fromDevices[deviceID]
-		if !ok || fromCfg.Paused == toCfg.Paused {
+		if !ok {
+			sr := stats.NewDeviceStatisticsReference(m.db, deviceID.String())
+			m.fmut.Lock()
+			m.deviceStatRefs[deviceID] = sr
+			m.fmut.Unlock()
+			continue
+		}
+		delete(fromDevices, deviceID)
+		if fromCfg.Paused == toCfg.Paused {
 			continue
 		}
 
@@ -2559,11 +2579,16 @@ func (m *model) CommitConfiguration(from, to config.Configuration) bool {
 		if toCfg.Paused {
 			l.Infoln("Pausing", deviceID)
 			m.closeConn(deviceID, errDevicePaused)
-			events.Default.Log(events.DevicePaused, map[string]string{"device": deviceID.String()})
+			m.evLogger.Log(events.DevicePaused, map[string]string{"device": deviceID.String()})
 		} else {
-			events.Default.Log(events.DeviceResumed, map[string]string{"device": deviceID.String()})
+			m.evLogger.Log(events.DeviceResumed, map[string]string{"device": deviceID.String()})
 		}
 	}
+	m.fmut.Lock()
+	for deviceID := range fromDevices {
+		delete(m.deviceStatRefs, deviceID)
+	}
+	m.fmut.Unlock()
 
 	scanLimiter.setCapacity(to.Options.MaxConcurrentScans)
 

lib/model/model_test.go

@@ -110,7 +110,7 @@ func createTmpWrapper(cfg config.Configuration) config.Wrapper {
 	if err != nil {
 		panic(err)
 	}
-	wrapper := config.Wrap(tmpFile.Name(), cfg)
+	wrapper := config.Wrap(tmpFile.Name(), cfg, events.NoopLogger)
 	tmpFile.Close()
 	return wrapper
 }
@@ -303,7 +303,7 @@ func TestDeviceRename(t *testing.T) {
 			DeviceID: device1,
 		},
 	}
-	cfg := config.Wrap("testdata/tmpconfig.xml", rawCfg)
+	cfg := config.Wrap("testdata/tmpconfig.xml", rawCfg, events.NoopLogger)
 
 	db := db.OpenMemory()
 	m := newModel(cfg, myID, "syncthing", "dev", db, nil)
@@ -339,7 +339,7 @@ func TestDeviceRename(t *testing.T) {
 		t.Errorf("Device name got overwritten")
 	}
 
-	cfgw, err := config.Load("testdata/tmpconfig.xml", myID)
+	cfgw, err := config.Load("testdata/tmpconfig.xml", myID, events.NoopLogger)
 	if err != nil {
 		t.Error(err)
 		return
@@ -3224,7 +3224,7 @@ func TestRequestLimit(t *testing.T) {
 	go func() {
 		second, err := m.Request(device1, "default", file, 2000, 0, nil, 0, false)
 		if err != nil {
-			t.Fatalf("Second request failed: %v", err)
+			t.Errorf("Second request failed: %v", err)
 		}
 		close(returned)
 		second.Close()
@@ -3358,12 +3358,12 @@ func TestModTimeWindow(t *testing.T) {
 }
 
 func TestDevicePause(t *testing.T) {
-	sub := events.Default.Subscribe(events.DevicePaused)
-	defer events.Default.Unsubscribe(sub)
-
 	m, _, fcfg := setupModelWithConnection()
 	defer cleanupModelAndRemoveDir(m, fcfg.Filesystem().URI())
 
+	sub := m.evLogger.Subscribe(events.DevicePaused)
+	defer sub.Unsubscribe()
+
 	m.pmut.RLock()
 	closed := m.closed[device1]
 	m.pmut.RUnlock()
@@ -3384,3 +3384,16 @@ func TestDevicePause(t *testing.T) {
 		t.Fatal("Timed out before device was paused")
 	}
 }
+
+func TestDeviceWasSeen(t *testing.T) {
+	m, _, fcfg := setupModelWithConnection()
+	defer cleanupModelAndRemoveDir(m, fcfg.Filesystem().URI())
+
+	m.deviceWasSeen(device1)
+
+	stats := m.DeviceStatistics()
+	entry := stats[device1.String()]
+	if time.Since(entry.LastSeen) > time.Second {
+		t.Error("device should have been seen now")
+	}
+}